Merge branch 'develop' into xsh_std_waves

CHANGES.rst

@@ -6,13 +6,22 @@
 - Hotfix for GTC/OSIRIS lamp list
 - Hotfix for Arc1D stats annotations on the QA
 - Hotfix for metadata (correctly set config_independent frames when multiple configurations are being setup)
+- Hotfix for metadata (support lists in ``config_independent_frames()``)
 - Hotfix for rebin (speed-up and conserves flux)
 - Hotfix for skysub regions GUI that used np.bool
 - Hotfix to stop pypeit_setup from crashing on data from lbt_luci1, lbt_luci2, magellan_fire,
   magellan_fire_long, p200_tspec, or vlt_sinfoni.
 - Instrumental FWHM map is calculated and output in ``Calibrations`` and ``spec1d`` files.
 - Adds Keck/ESI to PypeIt
 - Add MDM/Modspec spectrograph
+- Store user-generated wavelength solution in pypeit cache
+- Fix a bug in ``spectrograph.select_detectors``, where a list of ``slitspatnum`` could not be used.
+- Improvements in 2D coaddition
+    - Fix a bug in `pypeit_setup_coadd2d` for the output file name of the .coadd2d file
+    - Added possibility to specify more than one Science folder in `pypeit_setup_coadd2d`
+    - Now ``only_slits`` parameter in `pypeit_coadd_2dspec` includes the detector number (similar to ``slitspatnum``)
+    - Added ``exclude_slits`` parameter in `pypeit_coadd_2dspec` to exclude specific slits
+    - Fix wrong RA & Dec for 2D coadded serendips
 
 1.13.0 (2 June 2023)
 --------------------

pypeit/core/gui/identify.py

@@ -18,7 +18,7 @@
 
 from pypeit.par import pypeitpar
 from pypeit.core.wavecal import wv_fitting, waveio, wvutils
-from pypeit import data, msgs
+from pypeit import msgs
 from astropy.io import ascii as ascii_io
 from astropy.table import Table
 
@@ -763,22 +763,7 @@ def store_solution(self, final_fit, binspec, rmstol=0.15,
                 # Instead of a generic name, save the wvarxiv with a unique identifier
                 date_str = datetime.now().strftime("%Y%m%dT%H%M")
                 wvarxiv_name = f"wvarxiv_{self.specname}_{date_str}.fits"
-                wvutils.write_template(wavelengths, self.specdata, binspec,
-                                         './', wvarxiv_name)
-
-                # Also copy the file to the cache for direct use
-                data.write_file_to_cache(wvarxiv_name,
-                                         wvarxiv_name,
-                                         "arc_lines/reid_arxiv")
-
-                msgs.info(f"Your arxiv solution has been written to ./{wvarxiv_name}\n")
-                msgs.info(f"Your arxiv solution has also been cached.{msgs.newline()}"
-                          f"To utilize this wavelength solution, insert the{msgs.newline()}"
-                          f"following block in your PypeIt Reduction File:{msgs.newline()}"
-                          f" [calibrations]{msgs.newline()}"
-                          f"   [[wavelengths]]{msgs.newline()}"
-                          f"     reid_arxiv = {wvarxiv_name}{msgs.newline()}"
-                          f"     method = full_template\n")
+                wvutils.write_template(wavelengths, self.specdata, binspec, './', wvarxiv_name, cache=True)
 
                 # Write the WVCalib file
                 outfname = "wvcalib.fits"

pypeit/core/wavecal/wvutils.py

@@ -17,7 +17,7 @@
 from astropy import constants
 
 from pypeit import msgs
-from pypeit import utils
+from pypeit import utils, data
 from pypeit.core import arc
 from pypeit.pypmsgs import PypeItError
 
@@ -771,7 +771,7 @@ def wavegrid(wave_min, wave_max, dwave, spec_samp_fact=1.0, log10=False):
 
 
 def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None,
-                   order=None, overwrite=True):
+                   order=None, overwrite=True, cache=False):
     """
     Write the template spectrum into a binary FITS table
 
@@ -783,14 +783,18 @@ def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None,
         binspec (int):
             Binning of the template
         outpath (str):
+            Directory to store the wavelength template file
         outroot (str):
+            Filename to use for the template
         det_cut (bool, optional):
             Cuts in wavelength for detector snippets
             Used primarily for DEIMOS
         order (`numpy.ndarray`_, optional):
             Echelle order numbers
         overwrite (bool, optional):
             If True, overwrite any existing file
+        cache (bool, optional):
+            Store the wavelength solution in the pypeit cache?
     """
     tbl = Table()
     tbl['wave'] = nwwv
@@ -809,4 +813,17 @@ def write_template(nwwv, nwspec, binspec, outpath, outroot, det_cut=None,
     # Write
     outfile = os.path.join(outpath, outroot)
     tbl.write(outfile, overwrite=overwrite)
-    print("Wrote: {}".format(outfile))
+    msgs.info(f"Your arxiv solution has been written to {outfile}\n")
+    if cache:
+        # Also copy the file to the cache for direct use
+        data.write_file_to_cache(outroot, outroot, "arc_lines/reid_arxiv")
+
+        msgs.info(f"Your arxiv solution has also been cached.{msgs.newline()}"
+                  f"To utilize this wavelength solution, insert the{msgs.newline()}"
+                  f"following block in your PypeIt Reduction File:{msgs.newline()}"
+                  f" [calibrations]{msgs.newline()}"
+                  f"   [[wavelengths]]{msgs.newline()}"
+                  f"     reid_arxiv = {outroot}{msgs.newline()}"
+                  f"     method = full_template\n")
+    print("")  # Empty line for clarity
+    msgs.info("Please consider sharing your solution with the PypeIt Developers.")

pypeit/find_objects.py

@@ -334,12 +334,9 @@ def run(self, std_trace=None, show_peaks=False, show_skysub_fit=False):
         initial_sky0 = self.global_skysub(skymask=self.initial_skymask, update_crmask=False,
                                           objs_not_masked=True, show_fit=show_skysub_fit)
         # First pass object finding
-        save_objfindQA = self.par['reduce']['findobj']['skip_second_find'] or self.std_redux \
-                            or self.initial_skymask is not None
         sobjs_obj, self.nobj = \
             self.find_objects(self.sciImg.image-initial_sky0, self.sciImg.ivar, std_trace=std_trace,
-                              show_peaks=show_peaks, show=self.findobj_show and not self.std_redux,
-                              save_objfindQA=save_objfindQA)
+                              show_peaks=show_peaks, show=self.findobj_show and not self.std_redux)
 
         if self.nobj == 0 or self.initial_skymask is not None:
             # Either no objects were found, or the initial sky mask was provided by the user.

pypeit/metadata.py

@@ -854,11 +854,21 @@ def set_configurations(self, configs=None, force=False, fill=None):
             for ftype, metakey in ignore_frames.items():
 
                 # TODO: For now, use this assert to check that the
-                # metakey is either not set or a string
-                assert metakey is None or isinstance(metakey, str), \
+                # metakey is either not set, or is a string/list
+                assert metakey is None or isinstance(metakey, str) or isinstance(metakey, list), \
                     'CODING ERROR: metadata keywords set by config_indpendent_frames are not ' \
                     'correctly defined for {0}; values must be None or a string.'.format(
                         self.spectrograph.__class__.__name__)
+                # If a list is input, check all elements of the list are strings
+                if isinstance(metakey, list):
+                    for ll in metakey:
+                        assert isinstance(ll, str), \
+                            'CODING ERROR: metadata keywords set by config_indpendent_frames are not ' \
+                            'correctly defined for {0}; values must be None or a string.'.format(
+                                self.spectrograph.__class__.__name__)
+                elif isinstance(metakey, str):
+                    # If metakey is a string, convert it to a one-element list
+                    metakey = [metakey]
 
                 # Get the list of frames of this type without a
                 # configuration
@@ -878,17 +888,20 @@ def set_configurations(self, configs=None, force=False, fill=None):
                     self.table['setup'][indx] = new_cfg_key
                     continue
 
-                # Find the unique values of meta for this configuration
-                uniq_meta = np.unique(self.table[metakey][in_cfg].data)
-                # Warn the user that the matching meta values are not
-                # unique for this configuration.
-                if uniq_meta.size != 1:
-                    msgs.warn('When setting the instrument configuration for {0} '.format(ftype)
-                              + 'frames, configuration {0} does not have unique '.format(cfg_key)
-                              + '{0} values.' .format(meta))
-                # Find the frames of this type that match any of the
-                # meta data values
-                indx &= np.isin(self.table[metakey], uniq_meta)
+                # Loop through the meta keys
+                for mkey in metakey:
+                    # Find the unique values of meta for this configuration
+                    uniq_meta = np.unique(self.table[mkey][in_cfg].data)
+                    # Warn the user that the matching meta values are not
+                    # unique for this configuration.
+                    if uniq_meta.size != 1:
+                        msgs.warn('When setting the instrument configuration for {0} '.format(ftype)
+                                  + 'frames, configuration {0} does not have unique '.format(cfg_key)
+                                  + '{0} values.' .format(meta))
+                    # Find the frames of this type that match any of the
+                    # meta data values
+                    indx &= np.isin(self.table[mkey], uniq_meta)
+
                 # assign
                 new_cfg_key = np.full(len(self.table['setup'][indx]), 'None', dtype=object)
                 for c in range(len(self.table['setup'][indx])):

pypeit/par/pypeitpar.py

@@ -1219,7 +1219,8 @@ class Coadd2DPar(ParSet):
     For a table with the current keywords, defaults, and descriptions,
     see :ref:`parameters`.
     """
-    def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None, user_obj=None,
+    def __init__(self, only_slits=None, exclude_slits=None, offsets=None,
+                 spat_toler=None, weights=None, user_obj=None,
                  use_slits4wvgrid=None, manual=None):
 
         # Grab the parameter names and values from the function
@@ -1233,11 +1234,17 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None,
         dtypes = OrderedDict.fromkeys(pars.keys())
         descr = OrderedDict.fromkeys(pars.keys())
 
-        # Offsets
         defaults['only_slits'] = None
-        dtypes['only_slits'] = [int, list]
-        descr['only_slits'] = 'Slit ID, or list of slit IDs that the user want to restrict the coadd to. ' \
-                              'I.e., only this/these slit/s will be coadded.'
+        dtypes['only_slits'] = [str, list]
+        descr['only_slits'] = 'Restrict coaddition to one or more of slits. Example syntax -- ' \
+                              'DET01:175,DET02:205 or MSC02:2234. This and ``exclude_slits`` ' \
+                              'are mutually exclusive. If both are provided, ``only_slits`` takes precedence.'
+
+        defaults['exclude_slits'] = None
+        dtypes['exclude_slits'] = [str, list]
+        descr['exclude_slits'] = 'Exclude one or more slits from the coaddition. Example syntax -- ' \
+                                 'DET01:175,DET02:205 or MSC02:2234. This and ``only_slits`` ' \
+                                 'are mutually exclusive. If both are provided, ``only_slits`` takes precedence.'
 
         defaults['offsets'] = 'auto'
         dtypes['offsets'] = [str, list]
@@ -1301,7 +1308,8 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None,
     @classmethod
     def from_dict(cls, cfg):
         k = np.array([*cfg.keys()])
-        parkeys = ['only_slits', 'offsets', 'spat_toler', 'weights', 'user_obj', 'use_slits4wvgrid', 'manual']
+        parkeys = ['only_slits', 'exclude_slits', 'offsets', 'spat_toler',
+                   'weights', 'user_obj', 'use_slits4wvgrid', 'manual']
 
         badkeys = np.array([pk not in parkeys for pk in k])
         if np.any(badkeys):

pypeit/scripts/arxiv_solution.py

@@ -47,12 +47,4 @@ def main(args):
         wave = wv_calib['wv_fits'][args.slit]['wave_soln'].flatten()
         spec = wv_calib['wv_fits'][args.slit]['spec'].flatten()
         outname = args.file.replace(".fits", "_arXiv.fits")
-        wvutils.write_template(wave, spec, args.binning, './', outname)
-        print("")  # Empty line for clarity
-        msgs.info("To include the newly generated solution in the PypeIt archive," + msgs.newline() +
-                  "move (and appropriately rename) the following file: " + msgs.newline() +
-                  outname + msgs.newline() +
-                  "to the following directory:" + msgs.newline() +
-                  "pypeit/data/arc_lines/reid_arxiv/")
-        print("")  # Empty line for clarity
-        msgs.info("Please also consider sharing your solution with the PypeIt Developers.")
+        wvutils.write_template(wave, spec, args.binning, './', outname, cache=True)

pypeit/scripts/coadd_2dspec.py

@@ -5,6 +5,7 @@
 .. include:: ../include/links.rst
 """
 from pypeit.scripts import scriptbase
+from pypeit.core import parse
 
 class CoAdd2DSpec(scriptbase.ScriptBase):
 
@@ -103,7 +104,7 @@ def main(args):
 
         # Get the paths
         coadd_scidir, qa_path = map(lambda x : Path(x).resolve(),
-                                    coadd2d.CoAdd2D.output_paths(spec2d_files, par))
+                                    coadd2d.CoAdd2D.output_paths(spec2d_files, par, coadd_dir=par['rdx']['redux_path']))
 
         # Get the output basename
         head2d = fits.getheader(spec2d_files[0])
@@ -112,6 +113,7 @@ def main(args):
 
         # Write the par to disk
         par_outfile = coadd_scidir.parent / f'{basename}_coadd2d.par'
+
         print(f'Writing full parameter set to {par_outfile}.')
         par.to_config(par_outfile, exclude_defaults=True, include_descr=False)
 
@@ -137,11 +139,9 @@ def main(args):
         sci_dict['meta']['find_negative'] = find_negative
 
         # Find the detectors to reduce
-        # TODO: Allow slitspatnum to be specified?  E.g.:
-#        detectors = spectrograph.select_detectors(
-#                subset=par['rdx']['detnum'] if par['rdx']['slitspatnum'] is None else
-#                        par['rdx']['slitspatnum'])
-        detectors = spectrograph.select_detectors(subset=par['rdx']['detnum'])
+        detectors = spectrograph.select_detectors(subset=par['rdx']['detnum'] if par['coadd2d']['only_slits'] is None
+                                                  else par['coadd2d']['only_slits'])
+
         msgs.info(f'Detectors to work on: {detectors}')
 
         # container for specobjs
@@ -152,14 +152,31 @@ def main(args):
         all_spec2d['meta']['bkg_redux'] = bkg_redux
         all_spec2d['meta']['find_negative'] = find_negative
 
+        # get only_slits and exclude_slits if they are set
+        only_dets, only_spat_ids, exclude_dets, exclude_spat_ids = None, None, None, None
+        if par['coadd2d']['only_slits'] is not None:
+            only_dets, only_spat_ids = parse.parse_slitspatnum(par['coadd2d']['only_slits'])
+        if par['coadd2d']['exclude_slits'] is not None:
+            if par['coadd2d']['only_slits'] is not None:
+                msgs.warn('Both `only_slits` and `exclude_slits` are provided. They are mutually exclusive. '
+                          'Using `only_slits` and ignoring `exclude_slits`')
+            else:
+                exclude_dets, exclude_spat_ids = parse.parse_slitspatnum(par['coadd2d']['exclude_slits'])
+
         # Loop on detectors
         for det in detectors:
             msgs.info("Working on detector {0}".format(det))
 
+            detname = spectrograph.get_det_name(det)
+            this_only_slits = only_spat_ids[only_dets == detname] if only_dets is not None else None
+            this_exclude_slits = exclude_spat_ids[exclude_dets == detname] if exclude_dets is not None else None
+
             # Instantiate Coadd2d
             coadd = coadd2d.CoAdd2D.get_instance(spec2d_files, spectrograph, par, det=det,
                                                  offsets=par['coadd2d']['offsets'],
                                                  weights=par['coadd2d']['weights'],
+                                                 only_slits=this_only_slits,
+                                                 exclude_slits=this_exclude_slits,
                                                  spec_samp_fact=args.spec_samp_fact,
                                                  spat_samp_fact=args.spat_samp_fact,
                                                  bkg_redux=bkg_redux, find_negative=find_negative,
@@ -168,7 +185,7 @@ def main(args):
 
             # TODO Add this stuff to a run method in coadd2d
             # Coadd the slits
-            coadd_dict_list = coadd.coadd(only_slits=par['coadd2d']['only_slits'])
+            coadd_dict_list = coadd.coadd()
             # Create the pseudo images
             pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
             # Reduce

pypeit/scripts/ql.py

@@ -353,7 +353,10 @@ def generate_sci_pypeitfile(redux_path:str,
                 detname = slitTrace.detname
                 # Mosaic?
                 mosaic = True if detname[0:3] == 'MSC' else False
-                det_id = np.where(ps_sci.spectrograph.list_detectors(mosaic=mosaic) == detname)[0]
+                # if mosaic is False list_detectors() returns a 2D array for some spectrographs, therefore we flatten it
+                spec_list_dets = ps_sci.spectrograph.list_detectors(mosaic=mosaic) if mosaic \
+                    else ps_sci.spectrograph.list_detectors().flatten()
+                det_id = np.where(spec_list_dets == detname)[0]
                 if len(det_id) == 0:
                     detname = None  # Reset
                     continue        # TODO: or fault?