Merge branch 'develop' of https://github.com/pypeit/PypeIt into sensf…

…unc_blaze_jwst_lists

# Conflicts:
#	CHANGES.rst
#	pypeit/coadd2d.py
#	pypeit/par/pypeitpar.py

CHANGES.rst

@@ -25,8 +25,13 @@
 - Changes to how masking is dealt with in extraction to fix a bug in how masks
   were being treated for echelle data
 - Various fixes and changes required to add more support for Keck/HIRES and JWST
-
-
+- Fix a bug in ``spectrograph.select_detectors``, where a list of ``slitspatnum`` could not be used.
+- Improvements in 2D coaddition
+    - Fix a bug in `pypeit_setup_coadd2d` for the output file name of the .coadd2d file
+    - Added possibility to specify more than one Science folder in `pypeit_setup_coadd2d`
+    - Now ``only_slits`` parameter in `pypeit_coadd_2dspec` includes the detector number (similar to ``slitspatnum``)
+    - Added ``exclude_slits`` parameter in `pypeit_coadd_2dspec` to exclude specific slits
+    - Fix wrong RA & Dec for 2D coadded serendips
 
 
 1.13.0 (2 June 2023)

pypeit/par/pypeitpar.py

@@ -1211,7 +1211,7 @@ class Coadd2DPar(ParSet):
     For a table with the current keywords, defaults, and descriptions,
     see :ref:`parameters`.
     """
-    def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None, user_obj=None,
+    def __init__(self, only_slits=None, exclude_slits=None, offsets=None, spat_toler=None, weights=None, user_obj=None,
                  use_slits4wvgrid=None, manual=None, wave_method=None):
 
         # Grab the parameter names and values from the function
@@ -1225,11 +1225,17 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None,
         dtypes = OrderedDict.fromkeys(pars.keys())
         descr = OrderedDict.fromkeys(pars.keys())
 
-        # Offsets
         defaults['only_slits'] = None
-        dtypes['only_slits'] = [int, list]
-        descr['only_slits'] = 'Slit ID, or list of slit IDs that the user want to restrict the coadd to. ' \
-                              'I.e., only this/these slit/s will be coadded.'
+        dtypes['only_slits'] = [str, list]
+        descr['only_slits'] = 'Restrict coaddition to one or more of slits. Example syntax -- ' \
+                              'DET01:175,DET02:205 or MSC02:2234. This and ``exclude_slits`` ' \
+                              'are mutually exclusive. If both are provided, ``only_slits`` takes precedence.'
+
+        defaults['exclude_slits'] = None
+        dtypes['exclude_slits'] = [str, list]
+        descr['exclude_slits'] = 'Exclude one or more slits from the coaddition. Example syntax -- ' \
+                                 'DET01:175,DET02:205 or MSC02:2234. This and ``only_slits`` ' \
+                                 'are mutually exclusive. If both are provided, ``only_slits`` takes precedence.'
 
         defaults['offsets'] = 'auto'
         dtypes['offsets'] = [str, list]
@@ -1309,7 +1315,8 @@ def __init__(self, only_slits=None, offsets=None, spat_toler=None, weights=None,
     @classmethod
     def from_dict(cls, cfg):
         k = np.array([*cfg.keys()])
-        parkeys = ['only_slits', 'offsets', 'spat_toler', 'weights', 'user_obj', 'use_slits4wvgrid', 'manual', 'wave_method']
+        parkeys = ['only_slits', 'exclude_slits', 'offsets', 'spat_toler', 'weights', 'user_obj', 'use_slits4wvgrid',
+                   'manual', 'wave_method']
 
         badkeys = np.array([pk not in parkeys for pk in k])
         if np.any(badkeys):

pypeit/scripts/coadd_2dspec.py

@@ -5,6 +5,7 @@
 .. include:: ../include/links.rst
 """
 from pypeit.scripts import scriptbase
+from pypeit.core import parse
 
 class CoAdd2DSpec(scriptbase.ScriptBase):
 
@@ -103,7 +104,7 @@ def main(args):
 
         # Get the paths
         coadd_scidir, qa_path = map(lambda x : Path(x).resolve(),
-                                    coadd2d.CoAdd2D.output_paths(spec2d_files, par))
+                                    coadd2d.CoAdd2D.output_paths(spec2d_files, par, coadd_dir=par['rdx']['redux_path']))
 
         # Get the output basename
         head2d = fits.getheader(spec2d_files[0])
@@ -112,6 +113,7 @@ def main(args):
 
         # Write the par to disk
         par_outfile = coadd_scidir.parent / f'{basename}_coadd2d.par'
+
         print(f'Writing full parameter set to {par_outfile}.')
         par.to_config(par_outfile, exclude_defaults=True, include_descr=False)
 
@@ -137,11 +139,9 @@ def main(args):
         sci_dict['meta']['find_negative'] = find_negative
 
         # Find the detectors to reduce
-        # TODO: Allow slitspatnum to be specified?  E.g.:
-#        detectors = spectrograph.select_detectors(
-#                subset=par['rdx']['detnum'] if par['rdx']['slitspatnum'] is None else
-#                        par['rdx']['slitspatnum'])
-        detectors = spectrograph.select_detectors(subset=par['rdx']['detnum'])
+        detectors = spectrograph.select_detectors(subset=par['rdx']['detnum'] if par['coadd2d']['only_slits'] is None
+                                                  else par['coadd2d']['only_slits'])
+
         msgs.info(f'Detectors to work on: {detectors}')
 
         # container for specobjs
@@ -152,14 +152,31 @@ def main(args):
         all_spec2d['meta']['bkg_redux'] = bkg_redux
         all_spec2d['meta']['find_negative'] = find_negative
 
+        # get only_slits and exclude_slits if they are set
+        only_dets, only_spat_ids, exclude_dets, exclude_spat_ids = None, None, None, None
+        if par['coadd2d']['only_slits'] is not None:
+            only_dets, only_spat_ids = parse.parse_slitspatnum(par['coadd2d']['only_slits'])
+        if par['coadd2d']['exclude_slits'] is not None:
+            if par['coadd2d']['only_slits'] is not None:
+                msgs.warn('Both `only_slits` and `exclude_slits` are provided. They are mutually exclusive. '
+                          'Using `only_slits` and ignoring `exclude_slits`')
+            else:
+                exclude_dets, exclude_spat_ids = parse.parse_slitspatnum(par['coadd2d']['exclude_slits'])
+
         # Loop on detectors
         for det in detectors:
             msgs.info("Working on detector {0}".format(det))
 
+            detname = spectrograph.get_det_name(det)
+            this_only_slits = only_spat_ids[only_dets == detname] if only_dets is not None else None
+            this_exclude_slits = exclude_spat_ids[exclude_dets == detname] if exclude_dets is not None else None
+
             # Instantiate Coadd2d
             coadd = coadd2d.CoAdd2D.get_instance(spec2d_files, spectrograph, par, det=det,
                                                  offsets=par['coadd2d']['offsets'],
                                                  weights=par['coadd2d']['weights'],
+                                                 only_slits=this_only_slits,
+                                                 exclude_slits=this_exclude_slits,
                                                  spec_samp_fact=args.spec_samp_fact,
                                                  spat_samp_fact=args.spat_samp_fact,
                                                  bkg_redux=bkg_redux, find_negative=find_negative,
@@ -168,7 +185,7 @@ def main(args):
 
             # TODO Add this stuff to a run method in coadd2d
             # Coadd the slits
-            coadd_dict_list = coadd.coadd(only_slits=par['coadd2d']['only_slits'])
+            coadd_dict_list = coadd.coadd()
             # Create the pseudo images
             pseudo_dict = coadd.create_pseudo_image(coadd_dict_list)
             # Reduce

pypeit/scripts/ql.py

@@ -353,7 +353,10 @@ def generate_sci_pypeitfile(redux_path:str,
                 detname = slitTrace.detname
                 # Mosaic?
                 mosaic = True if detname[0:3] == 'MSC' else False
-                det_id = np.where(ps_sci.spectrograph.list_detectors(mosaic=mosaic) == detname)[0]
+                # if mosaic is False list_detectors() returns a 2D array for some spectrographs, therefore we flatten it
+                spec_list_dets = ps_sci.spectrograph.list_detectors(mosaic=mosaic) if mosaic \
+                    else ps_sci.spectrograph.list_detectors().flatten()
+                det_id = np.where(spec_list_dets == detname)[0]
                 if len(det_id) == 0:
                     detname = None  # Reset
                     continue        # TODO: or fault?

pypeit/scripts/setup_coadd2d.py

@@ -18,8 +18,9 @@ def get_parser(cls, width=None):
         input_group = parser.add_mutually_exclusive_group(required=True)
         input_group.add_argument('-f', '--pypeit_file', type=str, default=None,
                                  help='PypeIt reduction file')
-        input_group.add_argument('-d', '--science_dir', type=str, default=None,
-                                 help='Directory with spec2d files to stack')
+        input_group.add_argument('-d', '--science_dir', type=str, nargs='+', default=None,
+                                 help='One or more directories with spec2d files to stack '
+                                      '(use wildcard to specify multiple directories).')
 
         parser.add_argument('--keep_par', dest='clean_par', default=True, action='store_false',
                             help='Propagate all parameters from the pypeit file to the coadd2d '
@@ -43,8 +44,19 @@ def get_parser(cls, width=None):
                                  'use --det 1 5; to coadd mosaics made up of detectors 1,5 and '
                                  '3,7, you would use --det 1,5 3,7')
         parser.add_argument('--only_slits', type=str, nargs='+',
-                            help='A space-separated set of slits to coadd.  If not provided, all '
-                                 'slits are coadded.')
+                            help='A space-separated set of slits to coadd. Example syntax -- '
+                                 '--only_slits DET01:175,DET02:205 or MSC02:2234.  If not provided, all '
+                                 'slits are coadded. If both --det and --only_slits are provided, '
+                                 '--det will be ignored. This and --exclude_slits are mutually exclusive. '
+                                 'If both are provided, --only_slits takes precedence.')
+        parser.add_argument('--exclude_slits', type=str, nargs='+',
+                            help='A space-separated set of slits to exclude in the coaddition. '
+                                 'This and --only_slits are mutually exclusive. '
+                                 'If both are provided, --only_slits takes precedence.')
+        parser.add_argument('--spat_toler', type=int, default=None,
+                            help='Desired tolerance in spatial pixel used to identify '
+                                 'slits in different exposures. If not provided, the default '
+                                 'value for the specific instrument/configuration is used.')
         parser.add_argument('--offsets', type=str, default=None,
                             help='Spatial offsets to apply to each image; see the '
                                  '[coadd2d][offsets] parameter.  Options are restricted here to '
@@ -82,7 +94,7 @@ def main(args):
             pypeitFile = None
             par = None
             spec_name = None
-            sci_dir = Path(args.science_dir).resolve()
+            sci_dirs = [Path(sc).resolve() for sc in args.science_dir]
         else:
             # Read the pypeit file
             pypeitFile = inputfiles.PypeItFile.from_file(args.pypeit_file)
@@ -97,17 +109,22 @@ def main(args):
             # based on the parameter value, then try to base it on the parent
             # directory of the provided pypeit file.  The latter is critical to the
             # vet_test in the dev-suite.
-            sci_dir = Path(par['rdx']['redux_path']).resolve() / par['rdx']['scidir']
-            if not sci_dir.exists():
-                sci_dir = Path(args.pypeit_file).resolve().parent / par['rdx']['scidir']
+            sci_dirs = [Path(par['rdx']['redux_path']).resolve() / par['rdx']['scidir']]
+            if not sci_dirs[0].exists():
+                sci_dirs = [Path(args.pypeit_file).resolve().parent / par['rdx']['scidir']]
 
-        if not sci_dir.exists():
-            msgs.error(f'Science directory not found: {sci_dir}\n')
+        sci_dirs_exist = [sc.exists() for sc in sci_dirs]
+        if not np.all(sci_dirs_exist):
+            msgs_string = 'The following science directories do not exist:' + msgs.newline()
+            for s in np.array(sci_dirs)[np.logical_not(sci_dirs_exist)]:
+                msgs_string += f'{s}' + msgs.newline()
+            msgs.error(msgs_string)
 
         # Find all the spec2d files:
-        spec2d_files = sorted(sci_dir.glob('spec2d*'))
+        spec2d_files = np.concatenate([sorted(sci_dir.glob('spec2d*')) for sci_dir in sci_dirs]).tolist()
+
         if len(spec2d_files) == 0:
-            msgs.error(f'No spec2d files found in {sci_dir}.')
+            msgs.error(f'No spec2d files.')
 
         if spec_name is None:
             with io.fits_open(spec2d_files[0]) as hdu:
@@ -146,31 +163,33 @@ def main(args):
         cfg = {} if args.clean_par or pypeitFile is None else dict(pypeitFile.config)
         if par is None:
             # This is only used to get the default offsets and weights (see below)
-            par = load_spectrograph(spec_name).default_pypeit_par()
-            utils.add_sub_dict(cfg, 'rdx')
-            cfg['rdx']['scidir'] = sci_dir.name
-        else:
-            utils.add_sub_dict(cfg, 'rdx')
-            cfg['rdx']['redux_path'] = par['rdx']['redux_path']
-            cfg['rdx']['scidir'] = par['rdx']['scidir']
-            cfg['rdx']['qadir'] = par['rdx']['qadir']
-            utils.add_sub_dict(cfg, 'calibrations')
-            cfg['calibrations']['calib_dir'] = par['calibrations']['calib_dir']
+            par = load_spectrograph(spec_name).config_specific_par(spec2d_files[0])
+        utils.add_sub_dict(cfg, 'rdx')
+        cfg['rdx']['redux_path'] = par['rdx']['redux_path']
+        cfg['rdx']['scidir'] = par['rdx']['scidir']
+        cfg['rdx']['qadir'] = par['rdx']['qadir']
+        utils.add_sub_dict(cfg, 'calibrations')
+        cfg['calibrations']['calib_dir'] = par['calibrations']['calib_dir']
         utils.add_sub_dict(cfg, 'coadd2d')
         cfg['coadd2d']['offsets'] = par['coadd2d']['offsets'] \
                 if args.offsets is None else args.offsets
         cfg['coadd2d']['weights'] = par['coadd2d']['weights'] \
                 if args.weights is None else args.weights
+        cfg['coadd2d']['spat_toler'] = par['coadd2d']['spat_toler'] \
+                if args.spat_toler is None else args.spat_toler
 
         # Build the default parameters
-        cfg = CoAdd2D.default_par(spec_name, inp_cfg=cfg, det=args.det, slits=args.only_slits)
+        cfg = CoAdd2D.default_par(spec_name, inp_cfg=cfg, det=args.det, only_slits=args.only_slits,
+                                  exclude_slits=args.exclude_slits)
 
         # Create a coadd2D file for each object
-        # NOTE: Below expect all spec2d files have the same path
         for obj, files in object_spec2d_files.items():
             tbl = Table()
             tbl['filename'] = [f.name for f in files]
-            ofile = args.pypeit_file.replace('.pypeit', f'_{obj}.coadd2d')
-            inputfiles.Coadd2DFile(config=cfg, file_paths=[str(sci_dir)],
+            ofile_name = f'{spec_name}_{obj}.coadd2d' if args.pypeit_file is None \
+                else Path(args.pypeit_file).name.replace('.pypeit', f'_{obj}.coadd2d')
+            ofile = str(Path(cfg['rdx']['redux_path']) / ofile_name)
+
+            inputfiles.Coadd2DFile(config=cfg, file_paths=[str(sc) for sc in sci_dirs],
                                    data_table=tbl).write(ofile)
 

pypeit/spectrographs/spectrograph.py

@@ -1054,16 +1054,20 @@ def select_detectors(self, subset=None):
         if subset is None:
             return np.arange(1, self.ndet+1).tolist()
 
-        if isinstance(subset, str):
-            _subset = parse.parse_slitspatnum(subset)[0].tolist()
+        # Parse subset if it's a string (single slitspatnum) or a list of slitspatnums
+        if isinstance(subset, str) or \
+                (isinstance(subset, list) and np.all([isinstance(ss, str) for ss in subset])
+                 and np.all([':' in ss for ss in subset])):
+            subset_list = [subset] if isinstance(subset, str) else subset
             # Convert detector to int/tuple
             new_dets = []
-            for item in _subset:
-                if 'DET' in item:
-                    idx = np.where(self.list_detectors() == item)[0][0]
+            for ss in subset_list:
+                parsed_det = parse.parse_slitspatnum(ss)[0][0]
+                if 'DET' in parsed_det:
+                    idx = np.where(self.list_detectors().flatten() == parsed_det)[0][0]
                     new_dets.append(idx+1)
-                elif 'MSC' in item:
-                    idx = np.where(self.list_detectors(mosaic=True) == item)[0][0]
+                elif 'MSC' in parsed_det:
+                    idx = np.where(self.list_detectors(mosaic=True) == parsed_det)[0][0]
                     new_dets.append(self.allowed_mosaics[idx])
             _subset = new_dets
         elif isinstance(subset, (int, tuple)):