Merge branch 'develop' into hotfix_config

CHANGES.rst

@@ -1,6 +1,10 @@
 1.13.1dev (6 June 2023)
 ------------------------
 
+- Add support for Gemini/GNIRS (IFU)
+- Added a script to convert a wavelength solution into something that can be placed in the reid archive.
+- Hotfix for GTC/OSIRIS lamp list
+- Hotfix for Arc1D stats annotations on the QA
 - Hotfix for metadata (correctly set config_independent frames when multiple configurations are being setup)
 - Hotfix for rebin (speed-up and conserves flux)
 - Hotfix for skysub regions GUI that used np.bool

doc/include/spectrographs_table.rst

@@ -8,8 +8,9 @@ gemini_gmos_north_e2v     :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNE
 gemini_gmos_north_ham     :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNHamSpectrograph`         GEMINI-N   GMOS-N        `Link <http://www.gemini.edu/instrumentation/gmos>`__                                                                            MultiSlit  True       False      Hamamatsu detector (R400, B600, R831); Used since Feb 2017; see :doc:`gemini_gmos`           
 gemini_gmos_north_ham_ns  :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSNHamNSSpectrograph`       GEMINI-N   GMOS-N        `Link <http://www.gemini.edu/instrumentation/gmos>`__                                                                            MultiSlit  True       False      Same as gemini_gmos_north_ham when used in nod-and-shuffle mode; see :doc:`gemini_gmos`      
 gemini_gmos_south_ham     :class:`~pypeit.spectrographs.gemini_gmos.GeminiGMOSSHamSpectrograph`         GEMINI-S   GMOS-S        `Link <http://www.gemini.edu/instrumentation/gmos>`__                                                                            MultiSlit  True       False      Hamamatsu detector (R400, B600, R831); see :doc:`gemini_gmos`                                
-gemini_gnirs              :class:`~pypeit.spectrographs.gemini_gnirs.GeminiGNIRSSpectrograph`           GEMINI-N   GNIRS         `Link <https://www.gemini.edu/instrumentation/gnirs>`__                                                                          Echelle    True       False                                                                                                   
-gtc_maat                  :class:`~pypeit.spectrographs.gtc_osiris.GTCMAATSpectrograph`                 GTC        OSIRIS        `Link <http://www.gtc.iac.es/instruments/osiris/>`__                                                                             IFU        True       False      See :doc:`gtc_osiris`                                                                        
+gemini_gnirs_echelle      :class:`~pypeit.spectrographs.gemini_gnirs.GeminiGNIRSEchelleSpectrograph`    GEMINI-N   GNIRS         `Link <https://www.gemini.edu/instrumentation/gnirs>`__                                                                          Echelle    True       False
+gemini_gnirs_ifu          :class:`~pypeit.spectrographs.gemini_gnirs.GNIRSIFUSpectrograph`              GEMINI-N   GNIRS         `Link <https://www.gemini.edu/instrumentation/gnirs>`__                                                                          IFU        True       False      Support for LR-IFU and HR-IFU mode. Further testing still required...
+gtc_maat                  :class:`~pypeit.spectrographs.gtc_osiris.GTCMAATSpectrograph`                 GTC        OSIRIS        `Link <http://www.gtc.iac.es/instruments/osiris/>`__                                                                             IFU        True       False      See :doc:`gtc_osiris`
 gtc_osiris                :class:`~pypeit.spectrographs.gtc_osiris.GTCOSIRISSpectrograph`               GTC        OSIRIS        `Link <http://www.gtc.iac.es/instruments/osiris/>`__                                                                             MultiSlit  True       False      See :doc:`gtc_osiris`                                                                        
 gtc_osiris_plus           :class:`~pypeit.spectrographs.gtc_osiris.GTCOSIRISPlusSpectrograph`           GTC        OSIRIS        `Link <http://www.gtc.iac.es/instruments/osiris/>`__                                                                             MultiSlit  True       False      See :doc:`gtc_osiris`                                                                        
 jwst_nircam               :class:`~pypeit.spectrographs.jwst_nircam.JWSTNIRCamSpectrograph`             JWST       NIRCAM        `Link <https://jwst-docs.stsci.edu/jwst-near-infrared-camera/nircam-observing-modes/nircam-wide-field-slitless-spectroscopy>`__  MultiSlit  False      False                                                                                                   

pypeit/calibrations.py

@@ -730,7 +730,7 @@ def get_slits(self):
             self.slits = frame['class'].from_file(cal_file)
             self.slits.mask = self.slits.mask_init.copy()
             if self.user_slits is not None:
-                self.slits.user_mask(detname, self.user_slits)            
+                self.slits.user_mask(detname, self.user_slits)
             return self.slits
 
         # Slits don't exist or we're not resusing them.  See if the Edges
@@ -744,7 +744,7 @@ def get_slits(self):
             # Write the slits calibration file
             self.slits.to_file()
             if self.user_slits is not None:
-                self.slits.user_mask(detname, self.user_slits)            
+                self.slits.user_mask(detname, self.user_slits)
             return self.slits
 
         # Need to build everything from scratch.  Start with the trace image.
@@ -791,7 +791,7 @@ def get_slits(self):
         edges = None
         self.slits.to_file()
         if self.user_slits is not None:
-            self.slits.user_mask(detname, self.user_slits)            
+            self.slits.user_mask(detname, self.user_slits)
         return self.slits
 
     def get_wv_calib(self):
@@ -1039,7 +1039,7 @@ def get_association(fitstbl, spectrograph, caldir, setup, calib_ID, det, must_ex
                               'dark': [buildimage.DarkImage],
                               'pixelflat': [flatfield.FlatImages],
                               'illumflat': [flatfield.FlatImages],
-                              'lampoffflats': [flatfield.FlatImages], 
+                              'lampoffflats': [flatfield.FlatImages],
                               'trace': [edgetrace.EdgeTraceSet, slittrace.SlitTraceSet],
                               'tilt': [buildimage.TiltImage, wavetilts.WaveTilts]
                              }

pypeit/core/datacube.py

@@ -1109,34 +1109,34 @@ def generate_image_subpixel(image_wcs, all_ra, all_dec, all_wave, all_sci, all_i
     Args:
         image_wcs (`astropy.wcs.wcs.WCS`_):
             World coordinate system to use for the white light images.
-        all_ra (`numpy.ndarray`_)
+        all_ra (`numpy.ndarray`_):
             1D flattened array containing the right ascension of each pixel (units = degrees)
-        all_dec (`numpy.ndarray`_)
+        all_dec (`numpy.ndarray`_):
             1D flattened array containing the declination of each pixel (units = degrees)
-        all_wave (`numpy.ndarray`_)
+        all_wave (`numpy.ndarray`_):
             1D flattened array containing the wavelength of each pixel (units = Angstroms)
         all_sci (`numpy.ndarray`_):
             1D flattened array containing the counts of each pixel from all spec2d files
         all_ivar (`numpy.ndarray`_):
             1D flattened array containing the inverse variance of each pixel from all spec2d files
         all_wghts (`numpy.ndarray`_):
             1D flattened array containing the weights of each pixel to be used in the combination
-        all_spatpos (`numpy.ndarray`_)
+        all_spatpos (`numpy.ndarray`_):
             1D flattened array containing the detector pixel location in the spatial direction
-        all_specpos (`numpy.ndarray`_)
+        all_specpos (`numpy.ndarray`_):
             1D flattened array containing the detector pixel location in the spectral direction
-        all_spatid (`numpy.ndarray`_)
+        all_spatid (`numpy.ndarray`_):
             1D flattened array containing the spatid of each pixel
-        tilts (`numpy.ndarray`_, list)
+        tilts (`numpy.ndarray`_, list):
             2D wavelength tilts frame, or a list of tilt frames (see all_idx)
-        slits (:class:`pypeit.slittrace.SlitTraceSet`_, list)
+        slits (:class:`pypeit.slittrace.SlitTraceSet`, list):
             Information stored about the slits, or a list of SlitTraceSet (see all_idx)
-        astrom_trans (:class:`pypeit.alignframe.AlignmentSplines`_, list):
+        astrom_trans (:class:`pypeit.alignframe.AlignmentSplines`, list):
             A Class containing the transformation between detector pixel coordinates
             and WCS pixel coordinates, or a list of Alignment Splines (see all_idx)
         bins (tuple):
             A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates
-        all_idx (`numpy.ndarray`_, optional)
+        all_idx (`numpy.ndarray`_, optional):
             If tilts, slits, and astrom_trans are lists, this should contain a 1D flattened array, of
             the same length as all_sci, containing the index the tilts, slits, and astrom_trans lists
             that corresponds to each pixel. Note that, in this case all of these lists need to be the same length.
@@ -1230,7 +1230,7 @@ def generate_cube_subpixel(outfile, output_wcs, all_ra, all_dec, all_wave, all_s
             and WCS pixel coordinates, or a list of Alignment Splines (see all_idx)
         bins (tuple):
             A 3-tuple (x,y,z) containing the histogram bin edges in x,y spatial and z wavelength coordinates
-        all_idx (`numpy.ndarray`_, optional)
+        all_idx (`numpy.ndarray`_, optional):
             If tilts, slits, and astrom_trans are lists, this should contain a 1D flattened array, of
             the same length as all_sci, containing the index the tilts, slits, and astrom_trans lists
             that corresponds to each pixel. Note that, in this case all of these lists need to be the same length.

pypeit/core/wavecal/autoid.py

@@ -176,8 +176,8 @@ def arc_fit_qa(waveFit, outfile=None, ids_only=False, title=None,
 
     # Stats
     wave_soln_fit = waveFit.pypeitfit.eval(waveFit.pixel_fit/waveFit.xnorm)#, 'legendre',minx=fit['fmin'], maxx=fit['fmax'])
-    ax_fit.text(0.1*len(arc_spec), 0.90*ymin+(ymax-ymin),r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(waveFit.cen_disp), size='small')
-    ax_fit.text(0.1*len(arc_spec), 0.80*ymin+(ymax-ymin),'RMS={:.3f} (pixels)'.format(waveFit.rms), size='small')
+    ax_fit.text(0.1, 0.9, r'$\Delta\lambda$={:.3f}$\AA$ (per pix)'.format(waveFit.cen_disp), size='small', transform=ax_fit.transAxes)
+    ax_fit.text(0.1, 0.8, 'RMS={:.3f} (pixels)'.format(waveFit.rms), size='small', transform=ax_fit.transAxes)
     # Arc Residuals
     ax_res = plt.subplot(gs[1,1])
     res = waveFit.wave_fit-wave_soln_fit
@@ -2763,14 +2763,14 @@ def report_prelim(self, slit, best_patt_dict, best_final_fit):
         # Report on the best preliminary result
         if best_final_fit is None:
             msgs.warn('---------------------------------------------------' + msgs.newline() +
-                      'Preliminary report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() +
+                      'Preliminary report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() +
                       '  No matches! Attempting to cross match.' + msgs.newline() +
                       '---------------------------------------------------')
             self._all_patt_dict[str(slit)] = None
             self._all_final_fit[str(slit)] = None
         elif best_final_fit['rms'] > wvutils.parse_param(self._par, 'rms_threshold', slit):
             msgs.warn('---------------------------------------------------' + msgs.newline() +
-                      'Preliminary report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() +
+                      'Preliminary report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() +
                       '  Poor RMS ({0:.3f})! Attempting to cross match.'.format(best_final_fit['rms']) + msgs.newline() +
                       '---------------------------------------------------')
             self._all_patt_dict[str(slit)] = None
@@ -2783,7 +2783,7 @@ def report_prelim(self, slit, best_patt_dict, best_final_fit):
                 signtxt = 'anitcorrelate'
             # Report
             msgs.info('---------------------------------------------------' + msgs.newline() +
-                      'Preliminary report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() +
+                      'Preliminary report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() +
                       '  Pixels {:s} with wavelength'.format(signtxt) + msgs.newline() +
                       '  Number of weak lines         = {:d}'.format(self._det_weak[str(slit)][0].size) + msgs.newline() +
                       '  Number of strong lines       = {:d}'.format(self._det_stro[str(slit)][0].size) + msgs.newline() +
@@ -2803,7 +2803,7 @@ def report_final(self):
         for slit in range(self._nslit):
             # Prepare a message for bad wavelength solutions
             badmsg = '---------------------------------------------------' + msgs.newline() +\
-                     'Final report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() +\
+                     'Final report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() +\
                      '  Wavelength calibration not performed!'
             if slit not in self._ok_mask:
                 msgs.warn(badmsg)
@@ -2822,7 +2822,7 @@ def report_final(self):
             centdisp = abs(centwave-tempwave)
             msgs.info(msgs.newline() +
                       '---------------------------------------------------' + msgs.newline() +
-                      'Final report for slit {0:d}/{1:d}:'.format(slit, self._nslit-1) + msgs.newline() +
+                      'Final report for slit {0:d}/{1:d}:'.format(slit+1, self._nslit) + msgs.newline() +
                       '  Pixels {:s} with wavelength'.format(signtxt) + msgs.newline() +
                       '  Number of weak lines         = {:d}'.format(self._det_weak[str(slit)][0].size) + msgs.newline() +
                       '  Number of strong lines       = {:d}'.format(self._det_stro[str(slit)][0].size) + msgs.newline() +

pypeit/find_objects.py

@@ -1184,10 +1184,12 @@ def joint_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0),
             counts = global_sky
             _scale = None if self.sciImg.img_scale is None else self.sciImg.img_scale[thismask]
             # NOTE: darkcurr must be a float for the call below to work.
-            var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask],
-                                         count_scale=_scale, noise_floor=adderr)
-            model_ivar[thismask] = utils.inverse(var)
-
+            if not self.bkg_redux:
+                var = procimg.variance_model(self.sciImg.base_var[thismask], counts=counts[thismask],
+                                             count_scale=_scale, noise_floor=adderr)
+                model_ivar[thismask] = utils.inverse(var)
+            else:
+                model_ivar[thismask] = self.sciImg.ivar[thismask]
             # RJC :: Recalculating the global sky and flexure is probably overkill... but please keep this code in for now
             # Recalculate the sky on each individual slit and redetermine the spectral flexure
             # global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask,
@@ -1227,7 +1229,8 @@ def global_skysub(self, skymask=None, update_crmask=True, trim_edg=(0,0),
         global_sky_sep = super().global_skysub(skymask=skymask, update_crmask=update_crmask,
                                                trim_edg=trim_edg, show_fit=show_fit, show=show,
                                                show_objs=show_objs)
-        if np.any(global_sky_sep[skymask] == 0):
+        # Check if any slits failed
+        if np.any(global_sky_sep[self.slitmask>=0] == 0) and not self.bkg_redux:
             # Cannot continue without a sky model for all slits
             msgs.error("Global sky subtraction has failed for at least one slit.")
 
@@ -1291,7 +1294,10 @@ def calculate_flexure(self, global_sky):
                                             mxshft=self.par['flexure']['spec_maxshift'],
                                             excess_shft=self.par['flexure']['excessive_shift'],
                                             method="slitcen")
-        this_slitshift = np.ones(self.slits.nslits) * flex_dict_ref['shift']
+        this_slitshift = np.zeros(self.slits.nslits)
+        if flex_dict_ref is not None:
+            msgs.warn("Only a relative spectral flexure correction will be performed")
+            this_slitshift = np.ones(self.slits.nslits) * flex_dict_ref['shift']
         # Now loop through all slits to calculate the additional shift relative to the reference slit
         flex_list = []
         for slit_idx, slit_spat in enumerate(self.slits.spat_id):

pypeit/scripts/arxiv_solution.py

@@ -0,0 +1,58 @@
+"""
+This script enables the user to convert a MasterWaveCalib wavelength solution fits file
+into a PypeIt arxiv solution that can be used with the full_template method.
+
+.. include common links, assuming primary doc root is up one directory
+.. include:: ../include/links.rst
+"""
+import time
+from pypeit import msgs
+from pypeit import par
+from pypeit import inputfiles
+from pypeit import utils
+from pypeit.scripts import scriptbase
+
+
+class ArxivSolution(scriptbase.ScriptBase):
+
+    @classmethod
+    def get_parser(cls, width=None):
+        parser = super().get_parser(description='Read in a MasterWaveCalib solution and convert it into the '
+                                                'format required for the PypeIt full template archive', width=width)
+        parser.add_argument('file', type = str, default=None, help='MasterWaveCalib file')
+        parser.add_argument('binning', type=int, help="Spectral binning")
+        parser.add_argument('-s', '--slit', default=0, type=int, help='Slit number to use')
+        parser.add_argument('-v', '--verbosity', type=int, default=1,
+                            help='Verbosity level between 0 [none] and 2 [all]. Default: 1. '
+                                 'Level 2 writes a log with filename make_arxiv_solution_YYYYMMDD-HHMM.log')
+        return parser
+
+    @staticmethod
+    def main(args):
+        import os
+        from pypeit.wavecalib import WaveCalib
+        from pypeit.core.wavecal import wvutils
+
+        # Set the verbosity, and create a logfile if verbosity == 2
+        msgs.set_logfile_and_verbosity('arxiv_solution', args.verbosity)
+
+        # Check that a file has been provided
+        if args.file is None:
+            msgs.error('You must input a MasterWaveCalib file')
+        elif not os.path.exists(args.file):
+            msgs.error("The following MasterWaveCalib file does not exist:" + msgs.newline() + args.file)
+
+        # Load the wavelength calibration file
+        wv_calib = WaveCalib.from_file(args.file)
+        wave = wv_calib['wv_fits'][args.slit]['wave_soln'].flatten()
+        spec = wv_calib['wv_fits'][args.slit]['spec'].flatten()
+        outname = args.file.replace(".fits", "_arXiv.fits")
+        wvutils.write_template(wave, spec, args.binning, './', outname)
+        print("")  # Empty line for clarity
+        msgs.info("To include the newly generated solution in the PypeIt archive," + msgs.newline() +
+                  "move (and appropriately rename) the following file: " + msgs.newline() +
+                  outname + msgs.newline() +
+                  "to the following directory:" + msgs.newline() +
+                  "pypeit/data/arc_lines/reid_arxiv/")
+        print("")  # Empty line for clarity
+        msgs.info("Please also consider sharing your solution with the PypeIt Developers.")