diff --git a/doc/collate1d.rst b/doc/collate1d.rst index b77b3fc3e8..be44fe5526 100644 --- a/doc/collate1d.rst +++ b/doc/collate1d.rst @@ -144,6 +144,11 @@ followed by a list of spec1d files. An example configuration file is shown below # current directory. #outdir = /work/output + # Whether to check that spec1d files and archival sensfunc files have an + # up to date datamodel version. If false (the default) version numbers are + # not checked. + #chk_version = True + # A list of the spec1d files. Wildcards are allowed. # This follows the input file data block format. spec1d read diff --git a/doc/help/pypeit_collate_1d.rst b/doc/help/pypeit_collate_1d.rst index c8ae8adc92..d61fa3a918 100644 --- a/doc/help/pypeit_collate_1d.rst +++ b/doc/help/pypeit_collate_1d.rst @@ -8,7 +8,7 @@ [--flux] [--exclude_slit_bm EXCLUDE_SLIT_BM] [--exclude_serendip] [--wv_rms_thresh WV_RMS_THRESH] [--refframe {observed,heliocentric,barycentric}] - [-v VERBOSITY] + [--chk_version] [-v VERBOSITY] [input_file] Flux/Coadd multiple 1d spectra from multiple nights and prepare a directory for @@ -36,6 +36,8 @@ value are skipped, else all wavelength rms values are accepted. refframe Perform reference frame correction prior to coadding. Options are ['observed', 'heliocentric', 'barycentric']. Defaults to None. + chk_version If true, spec1ds and archival sensfuncs must match the currently + supported versions. If false (the default) version numbers are not checked. spec1d read @@ -86,6 +88,8 @@ --refframe {observed,heliocentric,barycentric} Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric + --chk_version Whether to check the data model versions of spec1d files + and sensfunc files. -v VERBOSITY, --verbosity VERBOSITY Verbosity level between 0 [none] and 2 [all]. Default: 1. Level 2 writes a log with filename diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index 62105a340a..945f287126 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -549,21 +549,22 @@ Collate1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` -========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== -``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. -``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. -``exclude_slit_trace_bm`` list, str .. [] A list of slit trace bitmask bits that should be excluded. -``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. -``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. -``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. -``outdir`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` The path where all coadded output files and report files will be placed. -``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric -``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. -``tolerance`` str, float .. ``1.0`` The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. -``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. -========================= ========== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +========================= ========== ======= =============================== ================================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +========================= ========== ======= =============================== ================================================================================================================================================================================================================================================================================================================================================================================================================== +``chk_version`` bool .. False Whether to check the data model versions of spec1d files and sensfunc files. +``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. +``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. +``exclude_slit_trace_bm`` list, str .. [] A list of slit trace bitmask bits that should be excluded. +``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. +``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. +``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. +``outdir`` str .. ``/home/dusty/work/PypeIt/doc`` The path where all coadded output files and report files will be placed. +``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric +``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. +``tolerance`` str, float .. ``1.0`` The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. +``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. +========================= ========== ======= =============================== ================================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -614,21 +615,21 @@ ReduxPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ReduxPar` -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== -``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame -``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` -``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). -``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). -``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. -``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. -``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. -``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. -``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. -``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. -``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. -====================== ============== ======= ============================================ =============================================================================================================================================================================================================================================================================================================================================================== +====================== ============== ======= =============================== =============================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +====================== ============== ======= =============================== =============================================================================================================================================================================================================================================================================================================================================================== +``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame +``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` +``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). +``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). +``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. +``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. +``redux_path`` str .. ``/home/dusty/work/PypeIt/doc`` Path to folder for performing reductions. Default is the current working directory. +``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. +``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. +``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. +``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. +====================== ============== ======= =============================== =============================================================================================================================================================================================================================================================================================================================================================== ---- @@ -733,7 +734,7 @@ Key Type Options Default Description ``find_extrap_npoly`` int .. 3 Polynomial order used for trace extrapolation ``find_fwhm`` int, float .. 5.0 Indicates roughly the fwhm of objects in pixels for object finding ``find_maxdev`` int, float .. 2.0 Maximum deviation of pixels from polynomial fit to trace used to reject bad pixels in trace fitting. -``find_min_max`` list .. .. It defines the minimum and maximum of your object in the spectral direction on the detector. It only used for object finding. This parameter is helpful if your object only has emission lines or at high redshift and the trace only shows in part of the detector. +``find_min_max`` list .. .. It defines the minimum and maximum of your object in pixels in the spectral direction on the detector. It only used for object finding. This parameter is helpful if your object only has emission lines or at high redshift and the trace only shows in part of the detector. ``find_negative`` bool .. .. Identify negative objects in object finding for spectra that are differenced. This is used to manually override the default behavior in PypeIt for object finding by setting this parameter to something other than None The default behavior is that PypeIt will search for negative object traces if background frames are present in the PypeIt file that are classified as "science" (i.e. via pypeit_setup -b, and setting bkg_id in the PypeIt file). If background frames are present that are classified as "sky", then PypeIt will NOT search for negative object traces. If one wishes to explicitly override this default behavior, set this parameter to True to find negative objects or False to ignore them. ``find_trim_edge`` list .. 5, 5 Trim the slit by this number of pixels left/right before finding objects ``maxnumber_sci`` int .. 10 Maximum number of objects to extract in a science frame. Use None for no limit. This parameter can be useful in situations where systematics lead to spurious extra objects. Setting this parameter means they will be trimmed. For mulitslit maxnumber applies per slit, for echelle observations this applies per order. Note that objects on a slit/order impact the sky-modeling and so maxnumber should never be lower than the true number of detectable objects on your slit. For image differenced observations with positive and negative object traces, maxnumber applies to the number of positive (or negative) traces individually. In other words, if you had two positive objects and one negative object, then you would set maxnumber to be equal to two (not three). Note that if manually extracted apertures are explicitly requested, they do not count against this maxnumber. If more than maxnumber objects are detected, then highest S/N ratio objects will be the ones that are kept. For multislit observations the choice here depends on the slit length. For echelle observations with short slits we set the default to be 1 diff --git a/pypeit/core/collate.py b/pypeit/core/collate.py index 97f9ba2018..caa1695634 100755 --- a/pypeit/core/collate.py +++ b/pypeit/core/collate.py @@ -68,23 +68,25 @@ def __init__(self, spec1d_obj, spec1d_header, spec1d_file, spectrograph, match_t self.coord = spec1d_obj['SPAT_PIXPOS'] @classmethod - def build_source_objects(cls, spec1d_files, match_type): + def build_source_objects(cls, specobjs_list, spec1d_files, match_type): """Build a list of SourceObjects from a list of spec1d files. There will be one SourceObject per SpecObj in the resulting list (i.e. no combining or collating is done by this method). Args: - spec1d_files (list of str): List of spec1d filenames + specobjs_list (list of :obj:`pypeit.specobjs.SpecObjs`): List of SpecObjs objects to build from. + + spec1d_files (list of str): List of spec1d filenames corresponding to each SpecObjs object. + match_type (str): What type of matching the SourceObjects will be configured for. Must be either 'ra/dec' or 'pixel' Returns: list of :obj:`SourceObject`: A list of uncollated SourceObjects with one SpecObj per SourceObject. """ result = [] - for spec1d_file in spec1d_files: - sobjs = specobjs.SpecObjs.from_fitsfile(spec1d_file) + for i, sobjs in enumerate(specobjs_list): spectrograph = load_spectrograph(sobjs.header['PYP_SPEC']) for sobj in sobjs: - result.append(SourceObject(sobj, sobjs.header, spec1d_file, spectrograph, match_type)) + result.append(SourceObject(sobj, sobjs.header, spec1d_files[i], spectrograph, match_type)) return result diff --git a/pypeit/fluxcalibrate.py b/pypeit/fluxcalibrate.py index 672c9a4f74..e07819e0be 100644 --- a/pypeit/fluxcalibrate.py +++ b/pypeit/fluxcalibrate.py @@ -13,7 +13,7 @@ from IPython import embed -def flux_calibrate(spec1dfiles, sensfiles, par=None, outfiles=None): +def flux_calibrate(spec1dfiles, sensfiles, par=None, outfiles=None, chk_version=True): """ Function for flux calibrating spectra. @@ -27,6 +27,8 @@ def flux_calibrate(spec1dfiles, sensfiles, par=None, outfiles=None): Parset object containing parameters governing the flux calibration. outfiles (list, optional): Names of the output files. If None, this is set to spec1dfiles and those are overwritten + chk_version (bool, optional): + Whether to check of the data model versions of spec1d and sens files. Defaults to True. """ @@ -41,10 +43,10 @@ def flux_calibrate(spec1dfiles, sensfiles, par=None, outfiles=None): sensf_last = None for spec1, sensf, outfile in zip(spec1dfiles, sensfiles, outfiles): # Read in the data - sobjs = specobjs.SpecObjs.from_fitsfile(spec1) + sobjs = specobjs.SpecObjs.from_fitsfile(spec1, chk_version=chk_version) history = History(sobjs.header) if sensf != sensf_last: - sens = sensfunc.SensFunc.from_file(sensf) + sens = sensfunc.SensFunc.from_file(sensf, chk_version=chk_version) sensf_last = sensf history.append(f'PypeIt Flux calibration "{sensf}"') sobjs.apply_flux_calib(par, spectrograph, sens) diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index fa6339c4d9..211be7ab29 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -4849,7 +4849,7 @@ class Collate1DPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, spec1d_outdir=None, refframe=None): + def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, match_using=None, exclude_slit_trace_bm=[], exclude_serendip=False, wv_rms_thresh=None, outdir=None, spec1d_outdir=None, refframe=None, chk_version=False): # Grab the parameter names and values from the function # arguments @@ -4928,6 +4928,10 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma descr['refframe'] = 'Perform reference frame correction prior to coadding. ' \ 'Options are: {0}'.format(', '.join(options['refframe'])) + defaults['chk_version'] = False + dtypes['chk_version'] = bool + descr['chk_version'] = "Whether to check the data model versions of spec1d files and sensfunc files." + # Instantiate the parameter set super(Collate1DPar, self).__init__(list(pars.keys()), values=list(pars.values()), @@ -4939,7 +4943,7 @@ def __init__(self, tolerance=None, dry_run=None, ignore_flux=None, flux=None, ma @classmethod def from_dict(cls, cfg): k = [*cfg.keys()] - parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', 'wv_rms_thresh', 'refframe'] + parkeys = ['tolerance', 'dry_run', 'ignore_flux', 'flux', 'match_using', 'exclude_slit_trace_bm', 'exclude_serendip', 'outdir', 'spec1d_outdir', 'wv_rms_thresh', 'refframe', 'chk_version'] badkeys = np.array([pk not in parkeys for pk in k]) if np.any(badkeys): diff --git a/pypeit/scripts/collate_1d.py b/pypeit/scripts/collate_1d.py index c7a5196f32..f2a342f938 100644 --- a/pypeit/scripts/collate_1d.py +++ b/pypeit/scripts/collate_1d.py @@ -124,7 +124,7 @@ def find_slits_to_exclude(spec2d_files, par): exclude_map = dict() for spec2d_file in spec2d_files: - allspec2d = AllSpec2DObj.from_fits(spec2d_file) + allspec2d = AllSpec2DObj.from_fits(spec2d_file, chk_version=par['collate1d']['chk_version']) for sobj2d in [allspec2d[det] for det in allspec2d.detectors]: for (slit_id, mask, slit_mask_id) in sobj2d['slits'].slit_info: for flag in exclude_flags: @@ -160,9 +160,7 @@ def exclude_source_objects(source_objects, exclude_map, par): :class:`~pypeit.core.collate.SourceObject` with any excluded ones removed. - - **missing_archive_msgs** (:obj:`list`): A list of messages - explaining why some source objects were excluded. - + - **excluded_messages** (:obj:`list`): A list of messages explaining why some source objects were excluded. """ filtered_objects = [] excluded_messages= [] @@ -195,21 +193,72 @@ def exclude_source_objects(source_objects, exclude_map, par): excluded_messages.append(msg) continue - if par['coadd1d']['ex_value'] == 'OPT' and sobj.OPT_COUNTS is None: - msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing OPT_COUNTS. Consider changing ex_value to "BOX".' - msgs.warn(msg) - excluded_messages.append(msg) - continue + if par['coadd1d']['ex_value'] == 'OPT': + msg = None + if sobj.OPT_COUNTS is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing OPT_COUNTS. Consider changing ex_value to "BOX".' + elif sobj.OPT_MASK is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing OPT_MASK. Consider changing ex_value to "BOX".' + else: + if len(sobj.OPT_COUNTS[sobj.OPT_MASK]) == 0: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because all of OPT_COUNTS was masked out. Consider changing ex_value to "BOX".' + + if msg is not None: + msgs.warn(msg) + excluded_messages.append(msg) + continue - if par['coadd1d']['ex_value'] == 'BOX' and sobj.BOX_COUNTS is None: - msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing BOX_COUNTS. Consider changing ex_value to "OPT".' - msgs.warn(msg) - excluded_messages.append(msg) - continue + if par['coadd1d']['ex_value'] == 'BOX': + msg = None + if sobj.BOX_COUNTS is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing BOX_COUNTS. Consider changing ex_value to "OPT".' + elif sobj.BOX_MASK is None: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because of missing BOX_MASK. Consider changing ex_value to "OPT".' + else: + if len(sobj.BOX_COUNTS[sobj.BOX_MASK]) == 0: + msg = f'Excluding {sobj.NAME} in {spec1d_file} because all of BOX_COUNTS was masked out. Consider changing ex_value to "OPT".' + + if msg is not None: + msgs.warn(msg) + excluded_messages.append(msg) + continue filtered_objects.append(source_object) return (filtered_objects, excluded_messages) +def read_spec1d_files(par, spec1d_files, failure_msgs): + """ + Read spec1d files. + + Args: + par (`obj`:pypeit.par.pypeitpar.PypeItPar): + Parameters for collating, fluxing, and coadding. + spec1d_files (list of str): + List of spec1d files to read. + failure_msgs(list of str): + Return parameter describing any failures that occurred when reading. + + Returns: + list of str: The SpecObjs objects that were successfully read. + list of str: The spec1d files that were successfully read. + """ + + specobjs_list = [] + good_spec1d_files = [] + for spec1d_file in spec1d_files: + try: + sobjs = SpecObjs.from_fitsfile(spec1d_file, chk_version = par['collate1d']['chk_version']) + specobjs_list.append(sobjs) + good_spec1d_files.append(spec1d_file) + except Exception as e: + formatted_exception = traceback.format_exc() + msgs.warn(formatted_exception) + msgs.warn(f"Failed to read {spec1d_file}, skipping it.") + failure_msgs.append(f"Failed to read {spec1d_file}, skipping it.") + failure_msgs.append(formatted_exception) + + return specobjs_list, good_spec1d_files + def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): """ Flux calibrate spec1d files using archived sens func files. @@ -251,7 +300,7 @@ def flux(par, spectrograph, spec1d_files, failed_fluxing_msgs): # Flux calibrate the spec1d file try: msgs.info(f"Running flux calibrate on {spec1d_file}") - FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib']) + FxCalib = fluxcalibrate.flux_calibrate([spec1d_file], [sens_file], par=par['fluxcalib'], chk_version=par['collate1d']['chk_version']) flux_calibrated_files.append(spec1d_file) except Exception: @@ -374,6 +423,9 @@ def coadd(par, coaddfile, source): # Set destination file for coadding par['coadd1d']['coaddfile'] = coaddfile + # Whether to be forgiving of data model versions + par['coadd1d']['chk_version'] = par['collate1d']['chk_version'] + # Determine if we should coadd flux calibrated data flux_key = par['coadd1d']['ex_value'] + "_FLAM" @@ -555,6 +607,9 @@ def build_parameters(args): if args.refframe is not None: params['collate1d']['refframe'] = args.refframe + if args.chk_version is True: + params['collate1d']['chk_version'] = True + return params, spectrograph, spec1d_files def create_report_archive(par): @@ -578,8 +633,8 @@ def create_report_archive(par): COADDED_SPEC1D_HEADER_KEYS = ['DISPNAME', 'DECKER', 'BINNING', 'MJD', 'AIRMASS', 'EXPTIME','GUIDFWHM', 'PROGPI', 'SEMESTER', 'PROGID'] COADDED_SPEC1D_COLUMN_NAMES = ['dispname', 'slmsknam', 'binning', 'mjd', 'airmass', 'exptime','guidfwhm', 'progpi', 'semester', 'progid'] - COADDED_SOBJ_KEYS = ['MASKDEF_OBJNAME', 'MASKDEF_ID', 'NAME', 'DET', 'RA', 'DEC', 'S2N', 'MASKDEF_EXTRACT', 'WAVE_RMS'] - COADDED_SOBJ_COLUMN_NAMES = ['maskdef_objname', 'maskdef_id', 'pypeit_name', 'det', 'objra', 'objdec', 's2n', 'maskdef_extract', 'wave_rms'] + COADDED_SOBJ_KEYS = ['MASKDEF_OBJNAME', 'MASKDEF_ID', 'NAME', 'DET', 'RA', 'DEC', 'MASKDEF_OBJMAG', 'MASKDEF_OBJMAG_BAND', 'S2N', 'MASKDEF_EXTRACT', 'WAVE_RMS'] + COADDED_SOBJ_COLUMN_NAMES = ['maskdef_objname', 'maskdef_id', 'pypeit_name', 'det', 'objra', 'objdec', 'maskdef_objmag', 'maskdef_objmag_band', 's2n', 'maskdef_extract', 'wave_rms'] report_names = ['filename'] + \ COADDED_SOBJ_COLUMN_NAMES + \ @@ -637,6 +692,8 @@ def get_parser(cls, width=None): 'F| value are skipped, else all wavelength rms values are accepted.\n' 'F| refframe Perform reference frame correction prior to coadding.\n' f'F| Options are {pypeitpar.WavelengthSolutionPar.valid_reference_frames()}. Defaults to None.\n' + 'F| chk_version If true, spec1ds and archival sensfuncs must match the currently\n' + 'F| supported versions. If false (the default) version numbers are not checked.\n' '\n' 'F|spec1d read\n' 'F|\n' @@ -663,6 +720,7 @@ def get_parser(cls, width=None): parser.add_argument("--wv_rms_thresh", type=float, default = None, help=blank_par.descr['wv_rms_thresh']) parser.add_argument("--refframe", type=str, default = None, choices = pypeitpar.WavelengthSolutionPar.valid_reference_frames(), help=blank_par.descr['refframe']) + parser.add_argument('--chk_version', action = 'store_true', help=blank_par.descr['chk_version']) parser.add_argument('-v', '--verbosity', type=int, default=1, help='Verbosity level between 0 [none] and 2 [all]. Default: 1. ' 'Level 2 writes a log with filename collate_1d_YYYYMMDD-HHMM.log') @@ -733,12 +791,14 @@ def main(args): refframe_correction(par, spectrograph, spec1d_files, spec1d_failure_msgs) + # Read in the spec1d files + specobjs_to_coadd, spec1d_files = read_spec1d_files(par, spec1d_files, spec1d_failure_msgs) + # Build source objects from spec1d file, this list is not collated - source_objects = SourceObject.build_source_objects(spec1d_files, + source_objects = SourceObject.build_source_objects(specobjs_to_coadd, spec1d_files, par['collate1d']['match_using']) - # Filter based on the coadding ex_value, and the exclude_serendip - # boolean + # Filter out unwanted SpecObj objects based on parameters (objects_to_coadd, excluded_obj_msgs) = exclude_source_objects(source_objects, exclude_map, par) # Collate the spectra diff --git a/pypeit/tests/test_collate_1d.py b/pypeit/tests/test_collate_1d.py index fdb6d11caf..d948ce260f 100644 --- a/pypeit/tests/test_collate_1d.py +++ b/pypeit/tests/test_collate_1d.py @@ -23,7 +23,7 @@ class MockSpecObj: - def __init__(self, MASKDEF_OBJNAME, MASKDEF_ID, DET, RA, DEC, SPAT_PIXPOS, NAME, WAVE_RMS, OPT_FLAM=None, OPT_COUNTS=None, BOX_COUNTS=None, VEL_CORR=None): + def __init__(self, MASKDEF_OBJNAME, MASKDEF_ID, DET, RA, DEC, SPAT_PIXPOS, NAME, WAVE_RMS, OPT_FLAM=None, OPT_COUNTS=None, BOX_COUNTS=None, OPT_MASK=None, BOX_MASK=None, VEL_CORR=None): self.MASKDEF_OBJNAME = MASKDEF_OBJNAME self.MASKDEF_ID = MASKDEF_ID self.DET = DetectorContainer.get_name(DET) @@ -34,6 +34,9 @@ def __init__(self, MASKDEF_OBJNAME, MASKDEF_ID, DET, RA, DEC, SPAT_PIXPOS, NAME, self.OPT_FLAM = OPT_FLAM self.OPT_COUNTS = OPT_COUNTS self.BOX_COUNTS = BOX_COUNTS + self.OPT_MASK = OPT_MASK + self.BOX_MASK = BOX_MASK + self.WAVE_RMS = WAVE_RMS self.VEL_CORR = VEL_CORR @@ -100,14 +103,14 @@ def __init__(self, file): # object4 also has boxcar counts and no opt_counts if file == "spec1d_file1": - self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=1, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1234.0, NAME='SPAT1234_SLIT1234_DET01', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100)), + self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=1, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1234.0, NAME='SPAT1234_SLIT1234_DET01', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.zeros(100, dtype=bool), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='1001', DET=1, RA=201.1522, DEC=27.3250, SPAT_PIXPOS=1334.0, NAME='SPAT1334_SLIT1234_DET01', WAVE_RMS=0.02, VEL_CORR=2.0, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object2', MASKDEF_ID='3002', DET=2, RA=201.0051, DEC=27.2228, SPAT_PIXPOS=5334.0, NAME='SPAT5334_SLIT4934_DET02', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3233.0, NAME='SPAT3233_SLIT3235_DET03', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), + MockSpecObj(MASKDEF_OBJNAME='object2', MASKDEF_ID='3002', DET=2, RA=201.0051, DEC=27.2228, SPAT_PIXPOS=5334.0, NAME='SPAT5334_SLIT4934_DET02', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100)), + MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3233.0, NAME='SPAT3233_SLIT3235_DET03', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3232.0, NAME='SPAT3232_SLIT3235_DET03', WAVE_RMS=0.03), - MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3236.0, NAME='SPAT3236_SLIT3245_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=7, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1233.0, NAME='SPAT1233_SLIT1235_DET07', WAVE_RMS=0.11, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='1001', DET=7, RA=201.1520, DEC=27.3249, SPAT_PIXPOS=1336.0, NAME='SPAT1336_SLIT1235_DET07', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_FLAM=np.zeros(100))] + MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3236.0, NAME='SPAT3236_SLIT3245_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100)), + MockSpecObj(MASKDEF_OBJNAME='object1', MASKDEF_ID='1001', DET=7, RA=201.1517, DEC=27.3246, SPAT_PIXPOS=1233.0, NAME='SPAT1233_SLIT1235_DET07', WAVE_RMS=0.11, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100), BOX_COUNTS=np.zeros(100), BOX_MASK=np.ones(100,dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='1001', DET=7, RA=201.1520, DEC=27.3249, SPAT_PIXPOS=1336.0, NAME='SPAT1336_SLIT1235_DET07', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool), OPT_FLAM=np.zeros(100))] elif file == "spec1d_file4": self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=None, DEC=None, SPAT_PIXPOS=3234.0, NAME='SPAT3234_SLIT3236_DET03', WAVE_RMS=0.01, VEL_CORR=2.0, OPT_FLAM=np.zeros(100), OPT_COUNTS=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=3, RA=None, DEC=None, SPAT_PIXPOS=6250.0, NAME='SPAT6250_SLIT6235_DET03', WAVE_RMS=0.02, BOX_COUNTS=np.zeros(100)), @@ -115,11 +118,11 @@ def __init__(self, file): MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='4004', DET=5, RA=None, DEC=None, SPAT_PIXPOS=6934.0, NAME='SPAT6934_SLIT6245_DET05', WAVE_RMS=0.20, BOX_COUNTS=np.zeros(100)), MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=None, DEC=None, SPAT_PIXPOS=3237.0, NAME='SPAT3237_SLIT3246_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100))] else: - self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3234.0, NAME='SPAT3234_SLIT3236_DET03', WAVE_RMS=0.01, OPT_FLAM=np.zeros(100), OPT_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=3, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6250.0, NAME='SPAT6250_SLIT6235_DET03', WAVE_RMS=0.02, BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=5, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6256.0, NAME='SPAT6256_SLIT6245_DET05', WAVE_RMS=0.01, BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='4004', DET=5, RA=201.0056, DEC=27.2419, SPAT_PIXPOS=6934.0, NAME='SPAT6934_SLIT6245_DET05', WAVE_RMS=0.20, BOX_COUNTS=np.zeros(100)), - MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3237.0, NAME='SPAT3237_SLIT3246_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100))] + self.specobjs = [MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=3, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3234.0, NAME='SPAT3234_SLIT3236_DET03', WAVE_RMS=0.01, OPT_FLAM=np.zeros(100), OPT_COUNTS=np.zeros(100), OPT_MASK=np.ones(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=3, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6250.0, NAME='SPAT6250_SLIT6235_DET03', WAVE_RMS=0.02, BOX_COUNTS=np.zeros(100), BOX_MASK=np.zeros(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='object4', MASKDEF_ID='4004', DET=5, RA=201.0052, DEC=27.2418, SPAT_PIXPOS=6256.0, NAME='SPAT6256_SLIT6245_DET05', WAVE_RMS=0.01, BOX_COUNTS=np.zeros(100), BOX_MASK=np.ones(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='SERENDIP', MASKDEF_ID='4004', DET=5, RA=201.0056, DEC=27.2419, SPAT_PIXPOS=6934.0, NAME='SPAT6934_SLIT6245_DET05', WAVE_RMS=0.20, BOX_COUNTS=np.zeros(100), BOX_MASK=np.ones(100, dtype=bool)), + MockSpecObj(MASKDEF_OBJNAME='object3', MASKDEF_ID='3003', DET=5, RA=201.2517, DEC=27.3333, SPAT_PIXPOS=3237.0, NAME='SPAT3237_SLIT3246_DET05', WAVE_RMS=0.01, OPT_COUNTS=np.zeros(100), BOX_MASK=np.zeros(100, dtype=bool))] def __getitem__(self, idx): return self.specobjs[idx] @@ -127,14 +130,14 @@ def __getitem__(self, idx): def write_to_fits(self, *args, **kwargs): pass -def mock_specobjs(file): +def mock_specobjs(file, chk_version=False): return MockSpecObjs(file) def test_group_spectra_by_radec(monkeypatch): - monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs) file_list = ['spec1d_file1', 'spec1d_file2'] - uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec') + specobjs_list = [mock_specobjs(file) for file in file_list] + uncollated_list = SourceObject.build_source_objects(specobjs_list=specobjs_list, spec1d_files=file_list, match_type='ra/dec') source_list = collate_spectra_by_source(uncollated_list, 0.0003, u.deg) assert len(source_list) == 6 @@ -178,9 +181,10 @@ def test_group_spectra_by_pixel(monkeypatch): monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs) file_list = ['spec1d_file1', 'spec1d_file4'] - spectrograph = load_spectrograph('keck_deimos') + # Test matching by pixel and that unit argument is ignored - uncollated_list = SourceObject.build_source_objects(file_list, 'pixel') + specobjs_list = [mock_specobjs(file) for file in file_list] + uncollated_list = SourceObject.build_source_objects(specobjs_list=specobjs_list, spec1d_files=file_list, match_type='pixel') source_list = collate_spectra_by_source(uncollated_list, 5.0, u.deg) assert len(source_list) == 7 @@ -363,21 +367,34 @@ def test_exclude_source_objects(monkeypatch): monkeypatch.setattr(specobjs.SpecObjs, "from_fitsfile", mock_specobjs) file_list = ['spec1d_file1', 'spec1d_file2'] - uncollated_list = SourceObject.build_source_objects(file_list, 'ra/dec') + specobjs_list = [mock_specobjs(file) for file in file_list] + # Add another object to spec1d_file2 for the case where OPT_MASK is NONE + specobjs_list[1].specobjs.append(MockSpecObj(MASKDEF_OBJNAME='object2', + MASKDEF_ID='3002', + DET=2, + RA=201.0051, + DEC=27.2228, + SPAT_PIXPOS=5335.0, + NAME='SPAT5335_SLIT4934_DET02', + WAVE_RMS=0.01, + OPT_COUNTS=np.zeros(100), + OPT_FLAM=np.zeros(100))) + + uncollated_list = SourceObject.build_source_objects(specobjs_list=specobjs_list, spec1d_files=file_list, match_type='ra/dec') par = pypeitpar.PypeItPar() par['collate1d']['exclude_serendip'] = True par['collate1d']['wv_rms_thresh'] = 0.1 filtered_list, excluded_msgs = exclude_source_objects(uncollated_list, {'3003': 'Test Exclude`'}, par) - assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT1234_SLIT1234_DET01', 'SPAT5334_SLIT4934_DET02'] - assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1', 'spec1d_file1'] + assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT5334_SLIT4934_DET02'] + assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1'] par['collate1d']['exclude_serendip'] = False par['coadd1d']['ex_value'] = 'BOX' par['collate1d']['wv_rms_thresh'] = None filtered_list, excluded_msgs = exclude_source_objects(uncollated_list, dict(), par) - assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT1234_SLIT1234_DET01', 'SPAT1233_SLIT1235_DET07', 'SPAT6250_SLIT6235_DET03', 'SPAT6256_SLIT6245_DET05', 'SPAT6934_SLIT6245_DET05'] - assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1', 'spec1d_file1', 'spec1d_file2', 'spec1d_file2', 'spec1d_file2' ] + assert [so.spec_obj_list[0].NAME for so in filtered_list] == ['SPAT1233_SLIT1235_DET07', 'SPAT6256_SLIT6245_DET05', 'SPAT6934_SLIT6245_DET05'] + assert [so.spec1d_file_list[0] for so in filtered_list] == ['spec1d_file1', 'spec1d_file2', 'spec1d_file2' ] def test_find_spec2d_from_spec1d(tmp_path): @@ -456,7 +473,7 @@ def mock_get_header(file): else: return {"DISPNAME": "600ZD" } - def mock_flux_calib(spec1d_files, sens_files, par): + def mock_flux_calib(spec1d_files, sens_files, par, chk_version=False): if spec1d_files[0] == "fail_flux.fits": raise PypeItError("Test failure") else: