From 007979b7050174ccb453f53d491d50eb2ca4c937 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 6 May 2024 09:45:00 -0700 Subject: [PATCH 1/4] autogenerate wavelength calibration tables --- doc/Makefile | 1 + doc/calibrations/wave_calib.rst | 41 +---- doc/include/linelist_table.rst | 23 +++ doc/include/reid_arxiv_table.rst | 174 +++++++++++++++++++ doc/pypeit_par.rst | 16 +- doc/scripts/build_wvcal_tbl_rst.py | 121 +++++++++++++ pypeit/calibrations.py | 4 +- pypeit/data/arc_lines/reid_arxiv/summary.txt | 171 ++++++++++++++++++ pypeit/inputfiles.py | 4 +- pypeit/pypeit.py | 4 +- pypeit/pypmsgs.py | 4 +- pypeit/setup_gui/controller.py | 4 +- 12 files changed, 510 insertions(+), 57 deletions(-) create mode 100644 doc/include/linelist_table.rst create mode 100644 doc/include/reid_arxiv_table.rst create mode 100644 doc/scripts/build_wvcal_tbl_rst.py create mode 100644 pypeit/data/arc_lines/reid_arxiv/summary.txt diff --git a/doc/Makefile b/doc/Makefile index 00a9e71cc9..aa37b83042 100644 --- a/doc/Makefile +++ b/doc/Makefile @@ -71,6 +71,7 @@ apirst: python ./scripts/build_spectbl_rst.py python ./scripts/make_example_files.py python ./scripts/build_standardstbl_rst.py + python ./scripts/build_wvcal_tbl_rst.py html: make apirst diff --git a/doc/calibrations/wave_calib.rst b/doc/calibrations/wave_calib.rst index 9a20252781..721c3f9091 100644 --- a/doc/calibrations/wave_calib.rst +++ b/doc/calibrations/wave_calib.rst @@ -66,25 +66,7 @@ the `NIST database `_, `"arc_lines" directory `_ of the repository. Here are the available lamps: -.. TODO: THIS TABLE IS OUT OF DATE. WE NEED A WAY OF AUTOMATICALLY GENERATING -.. THIS TABLE - -====== ========== ================ -Lamp Range (Å) Last updated -====== ========== ================ -ArI 3100-11000 7 October 2018 -CdI 3000-6500 28 February 2022 -CuI 4200-6100 4 October 2018 -FeI 3000-10000 26 April 2020 -HeI 3800-6000 21 December 2016 -HgI 2900-12000 28 February 2022 -KrI 4000-10000 3 May 2018 -NeI 5000-12000 3 May 2018 -XeI 4000-12000 3 May 2018 -ZnI 3000-5000 6 Sep 2023 -ThAr 3000-11000 9 January 2018 -FeAr 3000-9000 6 Sep 2023 -====== ========== ================ +.. include:: ../include/linelist_table.rst In the case of the ThAr list, all of the lines are taken from the NIST database, and they are labeled with a 'MURPHY' flag if the line also appears in the list @@ -554,26 +536,7 @@ latter must be constructed by a developer, using summarizes the existing ones (all of which are in the ``data/arc_lines/reid_arxiv`` folder): -.. TODO: THIS IS WAY OUT OF DATE. WE NEED AN AUTOMATED WAY OF GENERATING THIS TABLE - -=============== ========================= ============================= -Instrument Setup Name -=============== ========================= ============================= -keck_deimos 600ZD grating, all lamps keck_deimos_600ZD.fits -keck_deimos 830G grating, all lamps keck_deimos_830G.fits -keck_deimos 1200G grating, all lamps keck_deimos_1200G.fits -keck_deimos 1200B grating, all lamps keck_deimos_1200B.fits -keck_deimos 900ZD grating, all lamps keck_deimos_900ZD.fits -keck_lris_blue B300 grism, all lamps keck_lris_blue_300_d680.fits -keck_lris_blue B400 grism, all lamps? keck_lris_blue_400_d560.fits -keck_lris_blue B600 grism, all lamps keck_lris_blue_600_d560.fits -keck_lris_blue B1200 grism, all lamps keck_lris_blue_1200_d460.fits -keck_lris_red R400 grating, all lamps keck_lris_red_400.fits -keck_lris_red R1200/9000 , all lamps keck_lris_red_1200_9000.fits -shane_kast_blue 452_3306 grism, all lamps shane_kast_blue_452.fits -shane_kast_blue 600_4310 grism, all lamps shane_kast_blue_600.fits -shane_kast_blue 830_3460 grism, all lamps shane_kast_blue_830.fits -=============== ========================= ============================= +.. include:: ../include/reid_arxiv_table.rst .. TODO: WE SHOULD CONSIDER ADDING SOME OF THESE NOTEBOOKS DIRECTLY TO THE DOCS USING .. NBSPHINX: https://nbsphinx.readthedocs.io/ diff --git a/doc/include/linelist_table.rst b/doc/include/linelist_table.rst new file mode 100644 index 0000000000..d795236127 --- /dev/null +++ b/doc/include/linelist_table.rst @@ -0,0 +1,23 @@ +=============== ============= ========== = ===================== ============= ========== +Lamp Range (Å) Last Mod Lamp Range (Å) Last Mod +=============== ============= ========== = ===================== ============= ========== +ArII 3244 - 9512 2021-08-16 OH_GMOS 6172 - 10976 2020-11-19 +ArI_DeVeny 3950 - 10884 2024-02-05 OH_GNIRS 8026 - 24725 2020-11-19 +ArI 3950 - 10884 2024-01-17 OH_MODS 6172 - 10976 2020-11-19 +Ar_IR_GNIRS 11081 - 40434 2020-11-19 OH_MOSFIRE_H 13008 - 18991 2022-06-01 +Ar_IR_MOSFIRE 8266 - 25669 2022-06-01 OH_MOSFIRE_J 10002 - 14932 2022-06-01 +CdI_DeVeny 3134 - 6441 2024-02-05 OH_MOSFIRE_K 18067 - 24991 2022-06-01 +CdI 3134 - 6441 2022-10-27 OH_MOSFIRE_Y 8016 - 11989 2022-06-01 +CuI 4260 - 6034 2020-11-19 OH_NIRES 8016 - 24981 2020-11-19 +FeAr 3021 - 8922 2024-01-17 OH_R24000 9479 - 23847 2020-11-19 +FeII 3000 - 9079 2020-11-19 OH_XSHOOTER 8016 - 24994 2020-11-19 +FeI 3021 - 9003 2021-08-16 OH_triplespec 8016 - 23535 2020-11-19 +HeI 3889 - 7068 2023-02-27 ThAr_MagE 3034 - 10481 2020-11-19 +HgI_DeVeny 2968 - 10143 2024-02-05 ThAr_XSHOOTER_UVB 3094 - 10481 2022-10-27 +HgI 2968 - 10143 2022-10-27 ThAr_XSHOOTER_VIS_air 5407 - 10479 2020-11-19 +KrI 4275 - 9859 2021-04-23 ThAr_XSHOOTER_VIS 5409 - 10481 2020-11-19 +NeI_DeVeny 5039 - 11147 2024-02-05 ThAr 3000 - 10994 2020-11-19 +NeI 5039 - 11147 2023-10-10 XeI 4079 - 11131 2021-04-23 +Ne_IR_MOSFIRE 8649 - 24483 2022-06-01 ZnI 3019 - 7802 2024-01-17 +OH_FIRE_Echelle 8016 - 24993 2020-11-19 +=============== ============= ========== = ===================== ============= ========== diff --git a/doc/include/reid_arxiv_table.rst b/doc/include/reid_arxiv_table.rst new file mode 100644 index 0000000000..dff164117e --- /dev/null +++ b/doc/include/reid_arxiv_table.rst @@ -0,0 +1,174 @@ +===================================================== ================= ========= ======== ============================================================= +File Instrument Disperser Approx R Comments +===================================================== ================= ========= ======== ============================================================= +Flamingos2_HK_HK.fits gemini_flamingos2 HK_HK dunno Is this the same as the json file? If so, we should pick one +Flamingos2_HK_HK.json +Flamingos2_JH_JH.fits +Flamingos2_JH_JH.json +bok_bc_300.fits +gemini_gmos_b600_ham.fits +gemini_gmos_r400_e2v.fits +gemini_gmos_r400_e2v_mosaic.fits +gemini_gmos_r400_ham.fits +gemini_gmos_r831_ham.fits +gemini_gmos_south_ham_b600_compiled.fits +gemini_gnirs.fits +gemini_gnirs.json +gemini_gnirs_10mm_LBSX.fits +gemini_gnirs_idl.json +gemini_gnirs_lrifu_H.fits +gemini_gnirs_lrifu_K.fits +gtc_osiris_R1000B.fits +gtc_osiris_R1000R.fits +gtc_osiris_R2000B.fits +gtc_osiris_R2500I.fits +gtc_osiris_R2500R.fits +gtc_osiris_R2500U.fits +gtc_osiris_R2500V.fits +gtc_osiris_R300B.fits +gtc_osiris_R300R.fits +gtc_osiris_R500B.fits +gtc_osiris_R500R.fits +keck_deimos_1200B.fits +keck_deimos_1200G.fits +keck_deimos_600ZD.fits +keck_deimos_830G.fits +keck_deimos_900ZD.fits +keck_esi_ECH.fits +keck_hires_angle_fits.fits +keck_hires_composite_arc.fits +keck_kcrm_RH3.fits +keck_kcrm_RL.fits +keck_kcrm_RM1.fits +keck_kcrm_RM2.fits +keck_kcwi_BH2.fits +keck_kcwi_BL.fits +keck_kcwi_BM.fits +keck_lris_blue_1200_d460.fits +keck_lris_blue_300_d680.fits +keck_lris_blue_400_d560.fits +keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits +keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits +keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits +keck_lris_blue_B600_4000_d560_ArCdHgKrNeXeZn.fits +keck_lris_red_400.fits +keck_lris_red_600_10000.fits +keck_lris_red_600_5000.fits +keck_lris_red_600_7500.fits +keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits +keck_lris_red_R1200_9000.fits +keck_lris_red_R150_7500_ArCdHgNeZn.fits +keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits +keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits +keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits +keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits +keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits +keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits +keck_lris_red_R900_5500_ArCdHgNeZn.fits +keck_lris_red_mark4_R400.fits +keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits +keck_lris_red_orig_R150_7500_ArHgNe.fits +keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits +keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits +keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits +keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits +keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits +keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits +keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits +keck_mosfire_OH_H.fits +keck_mosfire_OH_J.fits +keck_mosfire_OH_J2.fits +keck_mosfire_OH_K.fits +keck_mosfire_OH_Y.fits +keck_mosfire_arcs_H.fits +keck_mosfire_arcs_J.fits +keck_mosfire_arcs_J2.fits +keck_mosfire_arcs_K.fits +keck_mosfire_arcs_Y.fits +keck_nires.fits +keck_nires.json +lbt_mods1b_blue.fits +lbt_mods1r_red.fits +lbt_mods1r_red.json +lbt_mods2r_red.fits +lbt_mods2r_red.json +ldt_deveny_1200_HgCdAr.fits +ldt_deveny_150_HgCdAr.fits +ldt_deveny_300_HgCdAr.fits +ldt_deveny_500_HgCdAr.fits +ldt_deveny_600_HgCdAr.fits +magellan_fire_echelle.fits +magellan_fire_long.fits +magellan_mage.fits +mdm_modspec_1200_5100.fits +mdm_modspec_echelle_Ar.fits +mdm_modspec_echelle_Ar_Only.fits +mdm_modspec_echelle_Ar_REAL.fits +mdm_modspec_echelle_NeXeAr.fits +mdm_osmos_mdm4k.fits +mdm_osmos_r4k.fits +mmt_binospec_1000.fits +mmt_binospec_270.fits +mmt_binospec_600.fits +mmt_bluechannel_1200GPM.fits +mmt_bluechannel_300GPM.fits +mmt_bluechannel_500GPM.fits +mmt_bluechannel_800GPM.fits +mmt_bluechannel_832GPM_order1.fits +mmt_bluechannel_832GPM_order2.fits +mmt_mmirs_HK_zJ.fits +mmt_mmirs_HK_zJ.json +mmt_mmirs_J_zJ.fits +mmt_mmirs_J_zJ.json +mmt_mmirs_K3000_Kspec.fits +mmt_mmirs_K3000_Kspec.json +not_alfosc_grism10.fits +not_alfosc_grism11.fits +not_alfosc_grism17.fits +not_alfosc_grism18.fits +not_alfosc_grism19.fits +not_alfosc_grism20.fits +not_alfosc_grism3.fits +not_alfosc_grism4.fits +not_alfosc_grism5.fits +not_alfosc_grism7.fits +not_alfosc_grism8.fits +ntt_efosc2_Gr4.fits +ntt_efosc2_Gr5.fits +ntt_efosc2_Gr6.fits +p200_dbsp_blue_1200_5000_d55_4700.fits +p200_dbsp_blue_1200_5000_d68_6000.fits +p200_dbsp_blue_300_3990_d55.fits +p200_dbsp_blue_600_4000_d55.fits +p200_dbsp_blue_600_4000_d68.fits +p200_dbsp_red_1200_7100_d68.fits +p200_dbsp_red_1200_9400_d55_8800.fits +p200_dbsp_red_316_7500_d55.fits +p200_dbsp_red_600_10000_d55.fits +p200_triplespec.fits +p200_triplespec_MasterWaveCalib.fits +shane_kast_blue_452.fits +shane_kast_blue_600.fits +shane_kast_blue_830.fits +shane_kast_red_1200_5000.fits +shane_kast_red_300_7500.fits +shane_kast_red_300_7500_NoNe.fits +shane_kast_red_600_7500.fits +soar_goodman_blue_400_SYZY.fits +soar_goodman_red_400_SYZY.fits +soar_goodman_red_600_SYZY_OLD.fits +tng_dolores_LR-B_arx.fits +tng_dolores_LR-B_arx_v2.fits +vlt_fors2_300I.fits +vlt_fors2_300V.fits +vlt_sinfoni_K.fits +vlt_xshooter_nir.fits +vlt_xshooter_nir.json +vlt_xshooter_nir_iraf.json +vlt_xshooter_uvb1x1.fits +vlt_xshooter_vis1x1.fits +vlt_xshooter_vis1x1.json +wht_isis_blue_1200_4800.fits +wht_isis_red_1200_6000.fits +wvarxiv_mdm_modspec_echelle_20220727T1201.fits +===================================================== ================= ========= ======== ============================================================= diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index 912a0e1d1d..1772f15ddc 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -2795,6 +2795,7 @@ Alterations to the default parameters are: detnum = (1, 2, 3), [calibrations] [[biasframe]] + exprng = None, 0.001, [[[process]]] overscan_method = median combine = median @@ -2822,6 +2823,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] + exprng = None, 60, [[[process]]] overscan_method = median satpix = nothing @@ -2829,11 +2831,10 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pinholeframe]] + exprng = 999999, None, [[[process]]] overscan_method = median use_biasimage = False - use_pixelflat = False - use_illumflat = False [[alignframe]] [[[process]]] overscan_method = median @@ -2842,12 +2843,14 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] + exprng = None, 60, [[[process]]] overscan_method = median use_biasimage = False use_pixelflat = False use_illumflat = False [[illumflatframe]] + exprng = None, 60, [[[process]]] overscan_method = median satpix = nothing @@ -2874,18 +2877,16 @@ Alterations to the default parameters are: mask_cr = True use_biasimage = False noise_floor = 0.01 - use_pixelflat = False - use_illumflat = False [[standardframe]] + exprng = 1, 600, [[[process]]] overscan_method = median mask_cr = True use_biasimage = False noise_floor = 0.01 - use_pixelflat = False - use_illumflat = False [[flatfield]] tweak_slits_thresh = 0.9 + slit_illum_finecorr = False [[wavelengths]] method = echelle echelle = True @@ -2916,13 +2917,12 @@ Alterations to the default parameters are: tracethresh = 15 spec_order = 5 [scienceframe] + exprng = 601, None, [[process]] overscan_method = median mask_cr = True use_biasimage = False noise_floor = 0.01 - use_pixelflat = False - use_illumflat = False [reduce] [[findobj]] find_trim_edge = 3, 3, diff --git a/doc/scripts/build_wvcal_tbl_rst.py b/doc/scripts/build_wvcal_tbl_rst.py new file mode 100644 index 0000000000..c4c36e2fce --- /dev/null +++ b/doc/scripts/build_wvcal_tbl_rst.py @@ -0,0 +1,121 @@ +""" +Dynamically build tables listing available wavelength calibration line list and +reid_arxiv spectra. +""" + +from importlib import resources +import time +import datetime + +# TODO: datetime.UTC is not defined in python 3.10. Remove this when we decide +# to no longer support it. +try: + __UTC__ = datetime.UTC +except AttributeError as e: + from datetime import timezone + __UTC__ = timezone.utc + +import numpy +import astropy.table + +from pypeit.utils import to_string, string_table +from pypeit import data + +from IPython import embed + + +def write_linelist_table(output_root): + + line_list_dir = resources.files('pypeit') / 'data'/ 'arc_lines' / 'lists' + files = sorted(line_list_dir.glob('*_lines.dat')) + nfiles = len(files) + + lamp = numpy.empty(nfiles, dtype=object) + wv_rng = numpy.empty(nfiles, dtype=object) + mod = numpy.empty(nfiles, dtype=object) + + for i,f in enumerate(files): + lamp[i] = f.name.replace('_lines.dat', '') + + tbl = astropy.table.Table.read(f, format='ascii.fixed_width', comment='#') + wv_rng[i] = f'{numpy.floor(numpy.amin(tbl['wave'])):.0f} - ' \ + f'{numpy.ceil(numpy.amax(tbl['wave'])):.0f}' + mod[i] = datetime.datetime.fromtimestamp(f.stat().st_mtime, tz=__UTC__).strftime('%Y-%m-%d') + + nrows = int(numpy.ceil(nfiles / 2)) + data_table = numpy.full((nrows+1, 7), '', dtype=object) + data_table[0,:] = ['Lamp', 'Range (Å)', 'Last Mod', ' ', + 'Lamp', 'Range (Å)', 'Last Mod'] + + data_table[1:,0] = lamp[:nrows] + data_table[1:,1] = wv_rng[:nrows] + data_table[1:,2] = mod[:nrows] + + data_table[1:nfiles-nrows+1,4] = lamp[nrows:] + data_table[1:nfiles-nrows+1,5] = wv_rng[nrows:] + data_table[1:nfiles-nrows+1,6] = mod[nrows:] + + ofile = output_root / 'linelist_table.rst' + lines = string_table(data_table, delimeter='rst') + with open(ofile, 'w') as f: + f.write(lines) + print(f'Wrote: {ofile}') + + +def write_reid_arxiv_table(output_root): + + reid_dir = resources.files('pypeit') / 'data'/ 'arc_lines' / 'reid_arxiv' + + # Find all the files + files = sorted(list(reid_dir.glob('*.fits')) + list(reid_dir.glob('*.json'))) + + # Read the summary file + tbl = astropy.table.Table.read(reid_dir / 'summary.txt', format='ascii', delimiter='|') + + # Check that all the files are in the table and vice versa + names = numpy.array([f.name for f in files]) + indx = numpy.isin(names, tbl['File']) + if not numpy.all(indx): + raise ValueError('Files found in reid_arxiv directory that are *not* in the summary.txt ' + f'file. Include summary entries for the following (or delete them): ' + f'{names[numpy.logical_not(indx)]}') + indx = numpy.isin(tbl['File'], names) + if not numpy.all(indx): + raise ValueError('Files found in the summary.txt file that are *not* in the reid_arxiv ' + 'directory. Remove summary entries for the following: ' + f'{tbl["File"][numpy.logical_not(indx)]}') + + nrow = len(tbl) + ncol = len(tbl.keys()) + data_table = numpy.full((nrow+1, ncol), '', dtype=object) + data_table[0,:] = tbl.keys() + for i,key in enumerate(tbl.keys()): + try: + data_table[1:,i] = tbl[key].data.astype(str).filled(' ') + except: + data_table[1:,i] = tbl[key].data.astype(str) + + ofile = output_root / 'reid_arxiv_table.rst' + lines = string_table(data_table, delimeter='rst') + with open(ofile, 'w') as f: + f.write(lines) + print(f'Wrote: {ofile}') + + +def main(): + t = time.perf_counter() + + output_root = resources.files('pypeit').parent / 'doc' / 'include' + if not output_root.is_dir(): + output_root.mkdir(parents=True) + + write_linelist_table(output_root) + write_reid_arxiv_table(output_root) + + print('Elapsed time: {0} seconds'.format(time.perf_counter() - t)) + + +if __name__ == '__main__': + main() + + diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 4b3f24f719..167ba89e0c 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -5,7 +5,7 @@ .. include:: ../include/links.rst """ from pathlib import Path -from datetime import datetime +import datetime from copy import deepcopy from abc import ABCMeta from collections import Counter @@ -1342,7 +1342,7 @@ def association_summary(ofile, fitstbl, spectrograph, caldir, subset=None, det=N with open(_ofile, 'w') as ff: ff.write('# Auto-generated calibration association file using PypeIt version: ' f' {__version__}\n') - ff.write(f'# UTC {datetime.now(__UTC__).isoformat(timespec="milliseconds")}\n') + ff.write(f'# UTC {datetime.datetime.now(__UTC__).isoformat(timespec="milliseconds")}\n') if det is None: ff.write(f'# NOTE: {detname} is a placeholder for the reduced detectors/mosaics\n') ff.write(yaml.dump(utils.yamlify(asn))) diff --git a/pypeit/data/arc_lines/reid_arxiv/summary.txt b/pypeit/data/arc_lines/reid_arxiv/summary.txt new file mode 100644 index 0000000000..60243f3735 --- /dev/null +++ b/pypeit/data/arc_lines/reid_arxiv/summary.txt @@ -0,0 +1,171 @@ +File | Instrument | Disperser | Approx R | Comments +Flamingos2_HK_HK.fits | gemini_flamingos2 | HK_HK | dunno | Is this the same as the json file? If so, we should pick one +Flamingos2_HK_HK.json | | | | +Flamingos2_JH_JH.fits | | | | +Flamingos2_JH_JH.json | | | | +bok_bc_300.fits | | | | +gemini_gmos_b600_ham.fits | | | | +gemini_gmos_r400_e2v.fits | | | | +gemini_gmos_r400_e2v_mosaic.fits | | | | +gemini_gmos_r400_ham.fits | | | | +gemini_gmos_r831_ham.fits | | | | +gemini_gmos_south_ham_b600_compiled.fits | | | | +gemini_gnirs.fits | | | | +gemini_gnirs.json | | | | +gemini_gnirs_10mm_LBSX.fits | | | | +gemini_gnirs_idl.json | | | | +gemini_gnirs_lrifu_H.fits | | | | +gemini_gnirs_lrifu_K.fits | | | | +gtc_osiris_R1000B.fits | | | | +gtc_osiris_R1000R.fits | | | | +gtc_osiris_R2000B.fits | | | | +gtc_osiris_R2500I.fits | | | | +gtc_osiris_R2500R.fits | | | | +gtc_osiris_R2500U.fits | | | | +gtc_osiris_R2500V.fits | | | | +gtc_osiris_R300B.fits | | | | +gtc_osiris_R300R.fits | | | | +gtc_osiris_R500B.fits | | | | +gtc_osiris_R500R.fits | | | | +keck_deimos_1200B.fits | | | | +keck_deimos_1200G.fits | | | | +keck_deimos_600ZD.fits | | | | +keck_deimos_830G.fits | | | | +keck_deimos_900ZD.fits | | | | +keck_esi_ECH.fits | | | | +keck_hires_angle_fits.fits | | | | +keck_hires_composite_arc.fits | | | | +keck_kcrm_RH3.fits | | | | +keck_kcrm_RL.fits | | | | +keck_kcrm_RM1.fits | | | | +keck_kcrm_RM2.fits | | | | +keck_kcwi_BH2.fits | | | | +keck_kcwi_BL.fits | | | | +keck_kcwi_BM.fits | | | | +keck_lris_blue_1200_d460.fits | | | | +keck_lris_blue_300_d680.fits | | | | +keck_lris_blue_400_d560.fits | | | | +keck_lris_blue_B1200_3400_d560_ArCdHgNeZn.fits | | | | +keck_lris_blue_B300_5000_d680_ArCdHgKrNeXeZnFeAr.fits | | | | +keck_lris_blue_B400_3400_d560_ArCdHgNeZnFeAr.fits | | | | +keck_lris_blue_B600_4000_d560_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_400.fits | | | | +keck_lris_red_600_10000.fits | | | | +keck_lris_red_600_5000.fits | | | | +keck_lris_red_600_7500.fits | | | | +keck_lris_red_R1200_7500_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R1200_9000.fits | | | | +keck_lris_red_R150_7500_ArCdHgNeZn.fits | | | | +keck_lris_red_R300_5000_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R400_8500_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R600_10000_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R600_5000_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R600_7500_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R831_8200_ArCdHgKrNeXeZn.fits | | | | +keck_lris_red_R900_5500_ArCdHgNeZn.fits | | | | +keck_lris_red_mark4_R400.fits | | | | +keck_lris_red_orig_R1200_7500_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R150_7500_ArHgNe.fits | | | | +keck_lris_red_orig_R300_5000_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R400_8500_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R600_10000_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R600_5000_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R600_7500_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R831_8200_ArCdHgNeZn.fits | | | | +keck_lris_red_orig_R900_5500_ArCdHgNeZn.fits | | | | +keck_mosfire_OH_H.fits | | | | +keck_mosfire_OH_J.fits | | | | +keck_mosfire_OH_J2.fits | | | | +keck_mosfire_OH_K.fits | | | | +keck_mosfire_OH_Y.fits | | | | +keck_mosfire_arcs_H.fits | | | | +keck_mosfire_arcs_J.fits | | | | +keck_mosfire_arcs_J2.fits | | | | +keck_mosfire_arcs_K.fits | | | | +keck_mosfire_arcs_Y.fits | | | | +keck_nires.fits | | | | +keck_nires.json | | | | +lbt_mods1b_blue.fits | | | | +lbt_mods1r_red.fits | | | | +lbt_mods1r_red.json | | | | +lbt_mods2r_red.fits | | | | +lbt_mods2r_red.json | | | | +ldt_deveny_1200_HgCdAr.fits | | | | +ldt_deveny_150_HgCdAr.fits | | | | +ldt_deveny_300_HgCdAr.fits | | | | +ldt_deveny_500_HgCdAr.fits | | | | +ldt_deveny_600_HgCdAr.fits | | | | +magellan_fire_echelle.fits | | | | +magellan_fire_long.fits | | | | +magellan_mage.fits | | | | +mdm_modspec_1200_5100.fits | | | | +mdm_modspec_echelle_Ar.fits | | | | +mdm_modspec_echelle_Ar_Only.fits | | | | +mdm_modspec_echelle_Ar_REAL.fits | | | | +mdm_modspec_echelle_NeXeAr.fits | | | | +mdm_osmos_mdm4k.fits | | | | +mdm_osmos_r4k.fits | | | | +mmt_binospec_1000.fits | | | | +mmt_binospec_270.fits | | | | +mmt_binospec_600.fits | | | | +mmt_bluechannel_1200GPM.fits | | | | +mmt_bluechannel_300GPM.fits | | | | +mmt_bluechannel_500GPM.fits | | | | +mmt_bluechannel_800GPM.fits | | | | +mmt_bluechannel_832GPM_order1.fits | | | | +mmt_bluechannel_832GPM_order2.fits | | | | +mmt_mmirs_HK_zJ.fits | | | | +mmt_mmirs_HK_zJ.json | | | | +mmt_mmirs_J_zJ.fits | | | | +mmt_mmirs_J_zJ.json | | | | +mmt_mmirs_K3000_Kspec.fits | | | | +mmt_mmirs_K3000_Kspec.json | | | | +not_alfosc_grism10.fits | | | | +not_alfosc_grism11.fits | | | | +not_alfosc_grism17.fits | | | | +not_alfosc_grism18.fits | | | | +not_alfosc_grism19.fits | | | | +not_alfosc_grism20.fits | | | | +not_alfosc_grism3.fits | | | | +not_alfosc_grism4.fits | | | | +not_alfosc_grism5.fits | | | | +not_alfosc_grism7.fits | | | | +not_alfosc_grism8.fits | | | | +ntt_efosc2_Gr4.fits | | | | +ntt_efosc2_Gr5.fits | | | | +ntt_efosc2_Gr6.fits | | | | +p200_dbsp_blue_1200_5000_d55_4700.fits | | | | +p200_dbsp_blue_1200_5000_d68_6000.fits | | | | +p200_dbsp_blue_300_3990_d55.fits | | | | +p200_dbsp_blue_600_4000_d55.fits | | | | +p200_dbsp_blue_600_4000_d68.fits | | | | +p200_dbsp_red_1200_7100_d68.fits | | | | +p200_dbsp_red_1200_9400_d55_8800.fits | | | | +p200_dbsp_red_316_7500_d55.fits | | | | +p200_dbsp_red_600_10000_d55.fits | | | | +p200_triplespec.fits | | | | +p200_triplespec_MasterWaveCalib.fits | | | | +shane_kast_blue_452.fits | | | | +shane_kast_blue_600.fits | | | | +shane_kast_blue_830.fits | | | | +shane_kast_red_1200_5000.fits | | | | +shane_kast_red_300_7500.fits | | | | +shane_kast_red_300_7500_NoNe.fits | | | | +shane_kast_red_600_7500.fits | | | | +soar_goodman_blue_400_SYZY.fits | | | | +soar_goodman_red_400_SYZY.fits | | | | +soar_goodman_red_600_SYZY_OLD.fits | | | | +tng_dolores_LR-B_arx.fits | | | | +tng_dolores_LR-B_arx_v2.fits | | | | +vlt_fors2_300I.fits | | | | +vlt_fors2_300V.fits | | | | +vlt_sinfoni_K.fits | | | | +vlt_xshooter_nir.fits | | | | +vlt_xshooter_nir.json | | | | +vlt_xshooter_nir_iraf.json | | | | +vlt_xshooter_uvb1x1.fits | | | | +vlt_xshooter_vis1x1.fits | | | | +vlt_xshooter_vis1x1.json | | | | +wht_isis_blue_1200_4800.fits | | | | +wht_isis_red_1200_6000.fits | | | | +wvarxiv_mdm_modspec_echelle_20220727T1201.fits | | | | diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index ee6f4a730d..ca6cc5d742 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -7,7 +7,7 @@ import glob import numpy as np import yaml -from datetime import datetime +import datetime import io import warnings from collections.abc import Sequence @@ -528,7 +528,7 @@ def write(self, input_file, version_override=None, date_override=None): documentation purposes only!** """ _version = __version__ if version_override is None else version_override - _date = datetime.now(__UTC__).isoformat(timespec='milliseconds') \ + _date = datetime.datetime.now(__UTC__).isoformat(timespec='milliseconds') \ if date_override is None else date_override # Here we go diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index 389639a613..2056ee7759 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -9,7 +9,7 @@ import time import os import copy -from datetime import datetime +import datetime # TODO: datetime.UTC is not defined in python 3.10. Remove this when we decide # to no longer support it. @@ -119,7 +119,7 @@ def __init__(self, pypeit_file, verbosity=2, overwrite=True, reuse_calibs=False, # Write the full parameter set here # -------------------------------------------------------------- par_file = pypeit_file.replace( - '.pypeit', f"_UTC_{datetime.now(__UTC__).date()}.par") + '.pypeit', f"_UTC_{datetime.datetime.now(__UTC__).date()}.par") self.par.to_config(par_file, include_descr=False) # -------------------------------------------------------------- diff --git a/pypeit/pypmsgs.py b/pypeit/pypmsgs.py index 6383dbe950..71d205e320 100644 --- a/pypeit/pypmsgs.py +++ b/pypeit/pypmsgs.py @@ -5,7 +5,7 @@ Why not use pythons native logging package? """ -from datetime import datetime +import datetime import sys import os import getpass @@ -413,7 +413,7 @@ def set_logfile_and_verbosity(self, scriptname, verbosity): Verbosity level between 0 [none] and 2 [all] """ # Create a UT timestamp (to the minute) for the log filename - timestamp = datetime.now(__UTC__).strftime("%Y%m%d-%H%M") + timestamp = datetime.datetime.now(__UTC__).strftime("%Y%m%d-%H%M") # Create a logfile only if verbosity == 2 logname = f"{scriptname}_{timestamp}.log" if verbosity == 2 else None # Set the verbosity in msgs diff --git a/pypeit/setup_gui/controller.py b/pypeit/setup_gui/controller.py index 8e2d8643e6..bb144eeeaa 100644 --- a/pypeit/setup_gui/controller.py +++ b/pypeit/setup_gui/controller.py @@ -7,7 +7,7 @@ import traceback import signal import sys -from datetime import datetime +import datetime import re import io from pathlib import Path @@ -602,7 +602,7 @@ def __init__(self, args): if args.logfile is not None: logpath = Path(args.logfile) if logpath.exists(): - timestamp = datetime.now(__UTC__).strftime("%Y%m%d-%H%M%S") + timestamp = datetime.datetime.now(__UTC__).strftime("%Y%m%d-%H%M%S") old_log=logpath.parent / (logpath.stem + f".{timestamp}" + logpath.suffix) logpath.rename(old_log) From 58c8cc2b6127565cac846a88ec7577032632fa63 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Mon, 6 May 2024 14:59:54 -0700 Subject: [PATCH 2/4] add todo --- doc/scripts/build_wvcal_tbl_rst.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/scripts/build_wvcal_tbl_rst.py b/doc/scripts/build_wvcal_tbl_rst.py index c4c36e2fce..7e8bb0f55e 100644 --- a/doc/scripts/build_wvcal_tbl_rst.py +++ b/doc/scripts/build_wvcal_tbl_rst.py @@ -64,6 +64,8 @@ def write_linelist_table(output_root): def write_reid_arxiv_table(output_root): + # TODO: Pull wavelength range (and resolution?) from files + reid_dir = resources.files('pypeit') / 'data'/ 'arc_lines' / 'reid_arxiv' # Find all the files From d06b3fb1025521f80ad8536c0ce44196233cfe0c Mon Sep 17 00:00:00 2001 From: "Timothy P. Ellsworth Bowers" Date: Tue, 25 Jun 2024 12:13:24 -0700 Subject: [PATCH 3/4] Use ``dataPaths`` infrastructure modified: doc/scripts/build_wvcal_tbl_rst.py --- doc/scripts/build_wvcal_tbl_rst.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/doc/scripts/build_wvcal_tbl_rst.py b/doc/scripts/build_wvcal_tbl_rst.py index 7e8bb0f55e..7d3ede1e0c 100644 --- a/doc/scripts/build_wvcal_tbl_rst.py +++ b/doc/scripts/build_wvcal_tbl_rst.py @@ -12,22 +12,20 @@ try: __UTC__ = datetime.UTC except AttributeError as e: - from datetime import timezone - __UTC__ = timezone.utc + __UTC__ = datetime.timezone.utc import numpy import astropy.table -from pypeit.utils import to_string, string_table -from pypeit import data +from pypeit.utils import string_table +from pypeit import dataPaths from IPython import embed def write_linelist_table(output_root): - line_list_dir = resources.files('pypeit') / 'data'/ 'arc_lines' / 'lists' - files = sorted(line_list_dir.glob('*_lines.dat')) + files = sorted(dataPaths.linelist.glob('*_lines.dat')) nfiles = len(files) lamp = numpy.empty(nfiles, dtype=object) @@ -66,7 +64,7 @@ def write_reid_arxiv_table(output_root): # TODO: Pull wavelength range (and resolution?) from files - reid_dir = resources.files('pypeit') / 'data'/ 'arc_lines' / 'reid_arxiv' + reid_dir = dataPaths.reid_arxiv # Find all the files files = sorted(list(reid_dir.glob('*.fits')) + list(reid_dir.glob('*.json'))) From 824c6f286cf12c74b71608a53fdfc5ef7275c872 Mon Sep 17 00:00:00 2001 From: Kyle Westfall Date: Fri, 6 Sep 2024 16:29:06 -0700 Subject: [PATCH 4/4] doc update, but they fail... --- doc/help/run_pypeit.rst | 2 +- doc/include/linelist_table.rst | 48 ++++++++++++++++++---------------- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 4cb290e4fa..77d8e84bbc 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,7 +4,7 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev336+gdf3013372.d20240827 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev468+g832ee84e4 ## ## Available spectrographs include: ## aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, diff --git a/doc/include/linelist_table.rst b/doc/include/linelist_table.rst index d795236127..d8eae92aaf 100644 --- a/doc/include/linelist_table.rst +++ b/doc/include/linelist_table.rst @@ -1,23 +1,25 @@ -=============== ============= ========== = ===================== ============= ========== -Lamp Range (Å) Last Mod Lamp Range (Å) Last Mod -=============== ============= ========== = ===================== ============= ========== -ArII 3244 - 9512 2021-08-16 OH_GMOS 6172 - 10976 2020-11-19 -ArI_DeVeny 3950 - 10884 2024-02-05 OH_GNIRS 8026 - 24725 2020-11-19 -ArI 3950 - 10884 2024-01-17 OH_MODS 6172 - 10976 2020-11-19 -Ar_IR_GNIRS 11081 - 40434 2020-11-19 OH_MOSFIRE_H 13008 - 18991 2022-06-01 -Ar_IR_MOSFIRE 8266 - 25669 2022-06-01 OH_MOSFIRE_J 10002 - 14932 2022-06-01 -CdI_DeVeny 3134 - 6441 2024-02-05 OH_MOSFIRE_K 18067 - 24991 2022-06-01 -CdI 3134 - 6441 2022-10-27 OH_MOSFIRE_Y 8016 - 11989 2022-06-01 -CuI 4260 - 6034 2020-11-19 OH_NIRES 8016 - 24981 2020-11-19 -FeAr 3021 - 8922 2024-01-17 OH_R24000 9479 - 23847 2020-11-19 -FeII 3000 - 9079 2020-11-19 OH_XSHOOTER 8016 - 24994 2020-11-19 -FeI 3021 - 9003 2021-08-16 OH_triplespec 8016 - 23535 2020-11-19 -HeI 3889 - 7068 2023-02-27 ThAr_MagE 3034 - 10481 2020-11-19 -HgI_DeVeny 2968 - 10143 2024-02-05 ThAr_XSHOOTER_UVB 3094 - 10481 2022-10-27 -HgI 2968 - 10143 2022-10-27 ThAr_XSHOOTER_VIS_air 5407 - 10479 2020-11-19 -KrI 4275 - 9859 2021-04-23 ThAr_XSHOOTER_VIS 5409 - 10481 2020-11-19 -NeI_DeVeny 5039 - 11147 2024-02-05 ThAr 3000 - 10994 2020-11-19 -NeI 5039 - 11147 2023-10-10 XeI 4079 - 11131 2021-04-23 -Ne_IR_MOSFIRE 8649 - 24483 2022-06-01 ZnI 3019 - 7802 2024-01-17 -OH_FIRE_Echelle 8016 - 24993 2020-11-19 -=============== ============= ========== = ===================== ============= ========== +================ ============= ========== = ===================== ============= ========== +Lamp Range (Å) Last Mod Lamp Range (Å) Last Mod +================ ============= ========== = ===================== ============= ========== +ArII 3244 - 9512 2021-08-16 OH_FIRE_Echelle 8016 - 24993 2020-11-19 +ArI_DeVeny 3950 - 10884 2024-02-05 OH_GMOS 6172 - 10976 2020-11-19 +ArI 3950 - 10884 2024-01-17 OH_GNIRS 8026 - 24725 2020-11-19 +Ar_IR_GNIRS 11081 - 40434 2020-11-19 OH_MODS 6172 - 10976 2020-11-19 +Ar_IR_MOSFIRE 8266 - 25669 2022-06-01 OH_MOSFIRE_H 13008 - 18991 2022-06-01 +CdI_DeVeny 3134 - 6441 2024-02-05 OH_MOSFIRE_J 10002 - 14932 2022-06-01 +CdI 3134 - 6441 2022-10-27 OH_MOSFIRE_K 18067 - 24991 2022-06-01 +CuI 4260 - 6034 2020-11-19 OH_MOSFIRE_Y 8016 - 11989 2022-06-01 +FeAr 3021 - 8922 2024-01-17 OH_NIRES 8016 - 24981 2020-11-19 +FeII 3000 - 9079 2020-11-19 OH_NIRSPEC_Y 8016 - 24998 2024-06-12 +FeI 3021 - 9003 2021-08-16 OH_R24000 9479 - 23847 2020-11-19 +HeI 3889 - 7068 2023-02-27 OH_XSHOOTER 8016 - 24994 2020-11-19 +HgI_DeVeny 2968 - 10143 2024-02-05 OH_triplespec 8016 - 23535 2020-11-19 +HgI 2968 - 10143 2022-10-27 ThAr_MagE 3034 - 10481 2020-11-19 +KrI 4275 - 9859 2021-04-23 ThAr_XSHOOTER_UVB 3094 - 10481 2022-10-27 +NIRSPEC-ArNeKrXe 9489 - 33923 2024-06-12 ThAr_XSHOOTER_VIS_air 5407 - 10479 2020-11-19 +NIRSPEC_K_OH 18433 - 23847 2024-06-12 ThAr_XSHOOTER_VIS 5409 - 10481 2020-11-19 +NIRSPEC_OH 9479 - 23847 2024-06-12 ThAr 3000 - 10994 2020-11-19 +NeI_DeVeny 5039 - 11147 2024-02-05 XeI 4079 - 11131 2021-04-23 +NeI 5039 - 11147 2023-10-10 ZnI 3019 - 7802 2024-01-17 +Ne_IR_MOSFIRE 8649 - 24483 2022-06-01 +================ ============= ========== = ===================== ============= ==========