diff --git a/.github/workflows/ci_cron.yml b/.github/workflows/ci_cron.yml index 530acb47ad..f57c373739 100644 --- a/.github/workflows/ci_cron.yml +++ b/.github/workflows/ci_cron.yml @@ -15,7 +15,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python: ['3.10', '3.11', '3.12'] + python: ['3.11', '3.12'] toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev] steps: - name: Check out repository diff --git a/.github/workflows/ci_tests.yml b/.github/workflows/ci_tests.yml index 9c2f37ec7d..0a9d9a0a1b 100644 --- a/.github/workflows/ci_tests.yml +++ b/.github/workflows/ci_tests.yml @@ -17,8 +17,8 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python: ['3.10', '3.11', '3.12'] - toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev] + python: ['3.11', '3.12'] + toxenv: [test, test-alldeps-cov, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev] steps: - name: Check out repository uses: actions/checkout@v3 @@ -48,7 +48,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, macos-latest] - python: ['3.10', '3.11', '3.12'] + python: ['3.11', '3.12'] toxenv: [test-alldeps] steps: - name: Check out repository @@ -71,7 +71,7 @@ jobs: - name: Conda environment check uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.12' - name: Install base dependencies run: | python -m pip install --upgrade pip tox @@ -86,7 +86,7 @@ jobs: - name: Python codestyle check uses: actions/setup-python@v4 with: - python-version: '3.11' + python-version: '3.12' - name: Install base dependencies run: | python -m pip install --upgrade pip diff --git a/deprecated/arc_old.py b/deprecated/arc_old.py index edecb94dd2..79b58ded49 100644 --- a/deprecated/arc_old.py +++ b/deprecated/arc_old.py @@ -855,3 +855,53 @@ def saturation_mask(a, satlevel): return mask.astype(int) +def mask_around_peaks(spec, inbpm): + """ + Find peaks in the input spectrum and mask pixels around them. + + All pixels to the left and right of a peak is masked until + a pixel has a lower value than the adjacent pixel. At this + point, we assume that spec has reached the noise level. + + Parameters + ---------- + spec: `numpy.ndarray`_ + Spectrum (1D array) in counts + inbpm: `numpy.ndarray`_ + Input bad pixel mask + + Returns + ------- + outbpm: `numpy.ndarray`_ + Bad pixel mask with pixels around peaks masked + """ + # Find the peak locations + pks = detect_peaks(spec) + + # Initialise some useful variables and the output bpm + xarray = np.arange(spec.size) + specdiff = np.append(np.diff(spec), 0.0) + outbpm = inbpm.copy() + + # Loop over the peaks and mask pixels around them + for i in range(len(pks)): + # Find all pixels to the left of the peak that are above the noise level + wl = np.where((xarray <= pks[i]) & (specdiff > 0.0))[0] + ww = (pks[i]-wl)[::-1] + # Find the first pixel to the left of the peak that is below the noise level + nmask = np.where(np.diff(ww) > 1)[0] + if nmask.size != 0 and nmask[0] > 5: + # Mask all pixels to the left of the peak + mini = max(0,wl.size-nmask[0]-1) + outbpm[wl[mini]:pks[i]] = True + # Find all pixels to the right of the peak that are above the noise level + ww = np.where((xarray >= pks[i]) & (specdiff < 0.0))[0] + # Find the first pixel to the right of the peak that is below the noise level + nmask = np.where(np.diff(ww) > 1)[0] + if nmask.size != 0 and nmask[0] > 5: + # Mask all pixels to the right of the peak + maxi = min(nmask[0], ww.size) + outbpm[pks[i]:ww[maxi]+2] = True + # Return the output bpm + return outbpm + diff --git a/deprecated/sensfunc.py b/deprecated/sensfunc.py new file mode 100644 index 0000000000..08f162231c --- /dev/null +++ b/deprecated/sensfunc.py @@ -0,0 +1,56 @@ +def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, + min_blaze_value=1e-3, debug=False): + """ + Compute the blaze function from a flat field image. + + Args: + wave (`numpy.ndarray`_): + Wavelength array. Shape = (nspec, norddet) + trace_spec (`numpy.ndarray`_): + Spectral pixels for the trace of the spectrum. Shape = (nspec, norddet) + trace_spat (`numpy.ndarray`_): + Spatial pixels for the trace of the spectrum. Shape = (nspec, norddet) + flatfile (:obj:`str`): + Filename for the flat field calibration image + box_radius (:obj:`float`, optional): + Radius of the boxcar extraction region used to extract the blaze function in pixels + min_blaze_value (:obj:`float`, optional): + Minimum value of the blaze function. Values below this are clipped and set to this value. Default=1e-3 + debug (:obj:`bool`, optional): + Show plots useful for debugging. Default=False + + Returns: + `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) + if norddet > 1, else shape = (nspec,) + """ + flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) + + pixelflat_raw = flatImages.pixelflat_raw + pixelflat_norm = flatImages.pixelflat_norm + pixelflat_proc, flat_bpm = flat.flatfield(pixelflat_raw, pixelflat_norm) + + flux_box = moment1d(pixelflat_proc * np.logical_not(flat_bpm), trace_spat, 2 * box_radius, row=trace_spec)[0] + + pixtot = moment1d(pixelflat_proc * 0 + 1.0, trace_spat, 2 * box_radius, row=trace_spec)[0] + pixmsk = moment1d(flat_bpm, trace_spat, 2 * box_radius, row=trace_spec)[0] + + mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) + + # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way + blaze_function = (np.clip(flux_box * mask_box, 1e-3, 1e9)).reshape(-1, 1) \ + if flux_box.ndim == 1 else flux_box * mask_box + wave_debug = wave.reshape(-1, 1) if wave.ndim == 1 else wave + log10_blaze_function = np.zeros_like(blaze_function) + norddet = log10_blaze_function.shape[1] + for iorddet in range(norddet): + blaze_function_smooth = utils.fast_running_median(blaze_function[:, iorddet], 5) + blaze_function_norm = blaze_function_smooth / blaze_function_smooth.max() + log10_blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) + if debug: + plt.plot(wave_debug[:, iorddet], log10_blaze_function[:, iorddet]) + if debug: + plt.show() + + # TODO It would probably better to just return an array of shape (nspec, norddet) even if norddet = 1, i.e. + # to get rid of this .squeeze() + return log10_blaze_function.squeeze() diff --git a/doc/api/pypeit.scripts.chk_flexure.rst b/doc/api/pypeit.scripts.chk_flexure.rst new file mode 100644 index 0000000000..90d71dfdb8 --- /dev/null +++ b/doc/api/pypeit.scripts.chk_flexure.rst @@ -0,0 +1,8 @@ +pypeit.scripts.chk\_flexure module +================================== + +.. automodule:: pypeit.scripts.chk_flexure + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.extract_datacube.rst b/doc/api/pypeit.scripts.extract_datacube.rst new file mode 100644 index 0000000000..49f3a9d13e --- /dev/null +++ b/doc/api/pypeit.scripts.extract_datacube.rst @@ -0,0 +1,8 @@ +pypeit.scripts.extract\_datacube module +======================================= + +.. automodule:: pypeit.scripts.extract_datacube + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.scripts.rst b/doc/api/pypeit.scripts.rst index 38fbcf5584..e9d732cc80 100644 --- a/doc/api/pypeit.scripts.rst +++ b/doc/api/pypeit.scripts.rst @@ -12,6 +12,7 @@ Submodules pypeit.scripts.chk_alignments pypeit.scripts.chk_edges pypeit.scripts.chk_flats + pypeit.scripts.chk_flexure pypeit.scripts.chk_for_calibs pypeit.scripts.chk_noise_1dspec pypeit.scripts.chk_noise_2dspec @@ -26,6 +27,7 @@ Submodules pypeit.scripts.compare_sky pypeit.scripts.compile_wvarxiv pypeit.scripts.edge_inspector + pypeit.scripts.extract_datacube pypeit.scripts.flux_calib pypeit.scripts.flux_setup pypeit.scripts.identify @@ -50,6 +52,7 @@ Submodules pypeit.scripts.show_1dspec pypeit.scripts.show_2dspec pypeit.scripts.show_arxiv + pypeit.scripts.show_pixflat pypeit.scripts.show_wvcalib pypeit.scripts.skysub_regions pypeit.scripts.tellfit diff --git a/doc/api/pypeit.scripts.show_pixflat.rst b/doc/api/pypeit.scripts.show_pixflat.rst new file mode 100644 index 0000000000..b66038acae --- /dev/null +++ b/doc/api/pypeit.scripts.show_pixflat.rst @@ -0,0 +1,8 @@ +pypeit.scripts.show\_pixflat module +=================================== + +.. automodule:: pypeit.scripts.show_pixflat + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.aat_uhrf.rst b/doc/api/pypeit.spectrographs.aat_uhrf.rst new file mode 100644 index 0000000000..2f4e657404 --- /dev/null +++ b/doc/api/pypeit.spectrographs.aat_uhrf.rst @@ -0,0 +1,8 @@ +pypeit.spectrographs.aat\_uhrf module +===================================== + +.. automodule:: pypeit.spectrographs.aat_uhrf + :members: + :private-members: + :undoc-members: + :show-inheritance: diff --git a/doc/api/pypeit.spectrographs.rst b/doc/api/pypeit.spectrographs.rst index c53968bcbb..239d591126 100644 --- a/doc/api/pypeit.spectrographs.rst +++ b/doc/api/pypeit.spectrographs.rst @@ -7,6 +7,7 @@ Submodules .. toctree:: :maxdepth: 4 + pypeit.spectrographs.aat_uhrf pypeit.spectrographs.bok_bc pypeit.spectrographs.gemini_flamingos pypeit.spectrographs.gemini_gmos diff --git a/doc/calibrations/flat.rst b/doc/calibrations/flat.rst index d224400051..7bc244b8a8 100644 --- a/doc/calibrations/flat.rst +++ b/doc/calibrations/flat.rst @@ -51,7 +51,7 @@ spectrograph that we have not included example screen-shots. Raw Flat --------- +++++++++ This is the processed and combined ``pixelflat`` image. Despite the name, it is not completely raw. @@ -60,7 +60,7 @@ This image should look like any one of your input ``pixelflat`` images. Pixel Flat ----------- +++++++++++ This is the normalized to unity image which is used to correct for pixel-to-pixel variations across the detector. @@ -77,17 +77,32 @@ true if there is limited flux at these ends (e.g. the data goes below the atmospheric cutoff). Illumination Flat ------------------ ++++++++++++++++++ This image should also have most values near unity, but there will be vertical coherence. And the edges (left/right) may fall off well below unity. Flat Model ----------- +++++++++++ This image should largely resemble the `Raw Flat`_. +pypeit_show_pixflat +------------------- + +In addition to ``pypeit_chk_flats``, if a custom pixel flat is provided by the user, +another script ``pypeit_show_pixflat`` is available to inspect it. The script is called as follows: + +.. code-block:: console + + pypeit_show_pixflat PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + +The script usage can be displayed by calling the script with the ``-h`` option: + +.. include:: ../help/pypeit_show_pixflat.rst + + Troubleshooting =============== diff --git a/doc/calibrations/flat_fielding.rst b/doc/calibrations/flat_fielding.rst index 904239b05b..134714c3db 100644 --- a/doc/calibrations/flat_fielding.rst +++ b/doc/calibrations/flat_fielding.rst @@ -204,7 +204,8 @@ need to provide the matching flat field images in your In short, if ``use_pixelflat=True`` for *any* of your frame types, at least one of the data files in the :ref:`pypeit_file` :ref:`data_block` must -be labelled as ``pixelflat`` (unless you `Feed a PixelFlat`_). +be labelled as ``pixelflat``, or ``slitless_pixflat`` +(unless you `Feed a PixelFlat`_). And, if ``use_illumflat=True`` for *any* of your frame types, at least one of the data files in the @@ -216,26 +217,47 @@ frames for the pixel and illumination corrections. This is supported, but we recommend that you set the ``trace`` frames to be the same as the ``illumflat`` frames. +.. _generate-pixflat: + +Generate a Slitless PixelFlat +----------------------------- + +If a set of ``slitless_pixflat`` frames are available in the +:ref:`data_block` of the :ref:`pypeit_file`, PypeIt will generate +a slitless pixel flat (unless you `Feed a PixelFlat`_ instead) +during the main :ref:`run-pypeit`, and will apply it to frame +types that have ``use_pixelflat=True``. +The slitless pixel flat is generated separately for each detector +(even in the case of a mosaic reduction) and it is stored in a FITS +file in the reduction directory, with one extension per detector. +In addition to saving the file in your reduction directory, +the constructed pixelflat is saved to the PypeIt cache (see ref:`data_installation`). +This allows you to use the file for both current and future reductions. +To use this file in future reductions, the user should add the slitless +pixel flat file name to the :ref:`pypeit_file` as shown in `Feed a PixelFlat`_. + +If you generate your own slitless pixel flat, and you think it is generally +applicable for your instrument, please consider sharing it with the PypeIt Developers. + + Feed a PixelFlat ---------------- If you have generated your own pixel flat (or were provided one) and it is trimmed and oriented following the expected :ref:`pypeit-orientation`, -then you may feed this into PypeIt. This is the recommended approach -at present for :ref:`lrisb`. +then you may feed this into PypeIt. -And you perform this by modifying the :ref:`parameter_block`: - -.. TODO: IS THIS STILL THE CORRECT APPROACH? WHAT DO PEOPLE DO IF THEY DON'T -.. HAVE THE DEV SUITE? +To use the available PixelFlat, you need to modify the :ref:`parameter_block` like, e.g.: .. code-block:: ini - [calibrations] - [[flatfield]] - pixelflat_file = /Users/joe/python/PypeIt-development-suite/CALIBS/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009.fits.gz + [calibrations] + [[flatfield]] + pixelflat_file = PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + +If any of the frames in the :ref:`data_block` are labelled as ``pixelflat``, or ``slitless_pixflat``, +the provided pixel flat file will still be used instead of generating a new one. -None of the frames in the :ref:`data_block` should be labelled as ``pixelflat``. Algorithms ---------- diff --git a/doc/calibrations/image_proc.rst b/doc/calibrations/image_proc.rst index f64d467908..b80caecf80 100644 --- a/doc/calibrations/image_proc.rst +++ b/doc/calibrations/image_proc.rst @@ -42,8 +42,9 @@ where: - the quantity :math:`C=N_{\rm frames}\ c/s\prime=c/s` is the number of electron counts excited by photons hitting the detector, - :math:`1/s=N_{\rm frames}/s\prime` is a factor that accounts for the number - of frames contributing to the electron counts, and the relative - throughput factors (see below) that can be measured from flat-field frames, + of frames contributing to the electron counts (`N_{\rm frames}`), and (`s\prime`) the relative + throughput factors (see below) that can be measured from flat-field frames plus a scaling factor + applied if the counts of each frame are scaled to the mean counts of all frames, - :math:`D` is the dark-current, i.e., the rate at which the detector generates thermal electrons, in e-/pixel/s, - :math:`N_{\rm bin}` is the number of pixels in a binned pixel, diff --git a/doc/coadd1d.rst b/doc/coadd1d.rst index 76d27d7781..0c4f2b8da4 100644 --- a/doc/coadd1d.rst +++ b/doc/coadd1d.rst @@ -303,3 +303,40 @@ and launches a GUI from the `linetools`_ package. e.g.: lt_xspec J1217p3905_coadd.fits +UVES_popler coaddition +====================== + +If you prefer to use a GUI for the coaddition (to manually remove +bad pixels, ghosts, cosmic rays etc.), then you can use the UVES_popler tool. +This tool is developed by Michael Murphy and is available at +`this link `__. +Here is an example of a coadded spectrum using UVES_popler: + +.. image:: ../figures/uves_popler.png + :scale: 60% + +UVES_popler was originally written to coadd ESO/UVES echelle spectra +that were reduced by the ESO pipeline, and it has been recently modified +to support the reduction of PypeIt longslit and echelle data. +For details on how to use the tool, please refer to the +`UVES_popler documentation `__. +To get you started with reading in PypeIt :doc:`out_spec1D` files, +you need to generate a text file that lists the absolute paths to the +:doc:`out_spec1D` files. Here is an example of how to generate this file: + +.. code-block:: console + + ls -1 /path/to/your/pypeit_output/Science/spec1d/*.fits > /path/to/your/pypeit_output/pypeit_spec1d_files.txt + +Then you can use this file as input to UVES_popler, by using the following command: + +.. code-block:: console + + cd /path/to/your/pypeit_output/ + UVES_popler -disp 50 -filetype 11 pypeit_spec1d_files.txt + +This will launch the GUI, where you can interactively coadd your spectra. The +``-disp 50`` option is used to set the pixel sampling of the spectra to 50 km/s, +and the ``-filetype 11`` option is used to specify that the input files are PypeIt +:doc:`out_spec1D` files. For more information on the options available, you can +specify the ``-h`` option. diff --git a/doc/coadd3d.rst b/doc/coadd3d.rst index 5651dd0ee2..4e16390484 100644 --- a/doc/coadd3d.rst +++ b/doc/coadd3d.rst @@ -155,7 +155,7 @@ There are several recommended steps of the coadd3d process that can be run separ pypeit_sensfunc spec1d_StandardStarName.fits -o sens_StandardStarName.fits - For further details, see :doc:`_sensitivity_function`. + For further details, see :ref:`sensitivity_function`. #. Step 4 - Generate a datacube of the science exposures. This is done by running the following command: @@ -164,7 +164,7 @@ There are several recommended steps of the coadd3d process that can be run separ pypeit_coadd_datacube ScienceName.coadd3d -o Note that you will need to specify the sensitivity function file using the ``sensfile`` option in the - :doc:`coadd3d_file` file. For further details, see :ref:`coadd3d_fluxing`. + :ref:`coadd3d_file` file. For further details, see :ref:`coadd3d_fluxing`. Combination options =================== @@ -239,7 +239,7 @@ If you would like to flux calibrate your datacube, you need to produce your standard star datacube first. Then extract the spectrum of the standard star using the ``pypeit_extract_datacube`` script. This will produce a ``spec1d`` file that you will need to use to generate a -sensitivity function in the usual way (see :doc:`_sensitivity_function`). +sensitivity function in the usual way (see :ref:`sensitivity_function`). Then, when generating the datacube of the science frame you must include the name of the sensitivity function in your ``coadd3d`` file as follows: @@ -289,7 +289,7 @@ then you can specify the ``skysub_frame`` in the ``spec2d`` block of the above. If you have dedicated sky frames, then it is generally recommended to reduce these frames as if they are regular science frames, but add the following keyword arguments at the top of your -:doc:`coadd3d_file`: +:ref:`coadd3d_file`: .. code-block:: ini diff --git a/doc/cookbook.rst b/doc/cookbook.rst index 915aed4865..19b397281e 100644 --- a/doc/cookbook.rst +++ b/doc/cookbook.rst @@ -70,7 +70,8 @@ what we recommend: - We will refer to that folder as ``RAWDIR`` The raw images can be gzip-compressed, although this means opening files will be -slower. +slower. See :ref:`setup-file-searching` for specific comments about the files +in your raw directory. A word on calibration data -------------------------- diff --git a/doc/dev/hiresconfig.rst b/doc/dev/hiresconfig.rst new file mode 100644 index 0000000000..38d07f705c --- /dev/null +++ b/doc/dev/hiresconfig.rst @@ -0,0 +1,118 @@ +.. include:: ../include/links.rst + +.. _hires_config: + +Automated sorting of HIRES frames by instrument configuration +============================================================= + +Version History +--------------- + + +========= ================ =========== =========== +*Version* *Author* *Date* ``PypeIt`` +========= ================ =========== =========== +1.0 Debora Pelliccia 10 Aug 2024 1.16.1.dev +========= ================ =========== =========== + +---- + +Basics +------ + +To prepare for the data reduction, PypeIt, first, automatically associates fits +files to specific :ref:`frame_types` (see :ref:`hires_frames`) and, then, +collects groups of frames in unique instrument configurations (see below). This is performed +by the :ref:`pypeit_setup` script, which sorts the frames and writes a +:ref:`pypeit_file` for each unique configuration. See :ref:`setup_doc`. + + +HIRES configuration identification +---------------------------------- + +The HIRES instrument configurations are determined by the function +:func:`pypeit.metadata.PypeItMetaData.unique_configurations`, +which finds unique combinations of the following keywords: + +=============== ============ +``fitstbl`` key Header Key +=============== ============ +``dispname`` ``XDISPERS`` +``decker`` ``DECKNAME`` +``binning`` ``BINNING`` +``filter1`` ``FIL1NAME`` +``echangle`` ``ECHANGL`` +``xdangle`` ``XDANGL`` +=============== ============ + +The unique configurations are determined by collating the relevant metadata from the headers +of all frames found by a run of :ref:`pypeit_setup`, *except* those that are designated as +bias and slitless_pixflat frames. Bias and slitless_pixflat frames can have header data (e.g., ``filter1``) +that do not match the instrument configuration that an observer intended for their use. +Therefore, PypeIt uses the ``dispname`` and ``binning`` keys to match the bias and +slitless_pixflat frames to the configurations with frames taken with the same cross-disperser +and same binning. +Note that when using the ``echangle`` and ``xdangle`` keys to identify configurations, PypeIt +uses a relative tolerance of 1e-3 and absolute tolerance of 1e-2 for ``echangle``, and a relative +tolerance of 1e-2 for ``xdangle``, to account for small differences in the values of these angles. + +After that, :func:`pypeit.metadata.PypeItMetaData.set_configurations` associates each frame +to the relevant unique configuration ("setup"), by assigning a setup identifier +(e.g., A,B,C,D...) to every frames for which the values of the above keywords match the +values of the specific unique configuration. + +HIRES calibration groups +------------------------ + +PypeIt uses the concept of a "calibration group" to define a complete set of +calibration frames (e.g., arcs, flats) and the science frames to which these calibration +frames should be applied. + +By default, :ref:`pypeit_setup` uses the setup identifier to assign frames to a single +calibration group. Frames that are in the same calibration group will have the same PypeIt +keyword ``calib``. No automated procedure exists to do anything except this. +However, the user can edit the :ref:`pypeit_file` to, within a given configuration, assign +specific calibration frames to specific science frames using the data in the ``calib`` column +of the :ref:`data_block`. + +Testing +------- + +To test that PypeIt can successfully identify multiple +configurations among a set of files, we have added the +``test_setup_keck_hires_multiconfig()`` test to +``${PYPEIT_DEV}/unit_tests/test_setups.py``. + +Here is an example of how to run the test: + +.. code-block:: bash + + cd ${PYPEIT_DEV}/unit_tests + pytest test_setup.py::test_setup_keck_hires_multiconfig -W ignore + +The tests require that you have downloaded the PypeIt +:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental +variable that points to the relevant directory. + +The algorithm for this test is as follows: + + 1. Collect the names of all files in selected HIRES directories. + + 2. Use :class:`~pypeit.pypeitsetup.PypeItSetup` to automatically + identify the configurations for these files. + + 3. Check that the code found two configurations and wrote the + pypeit files for each. + + 4. For each configuration: + + a. Read the pypeit file + + b. Check that the name for the setup is correct ('A' or 'B') + + c. Check that the calibration group is the same for all frames ('0' or '1') + + +Because these tests are now included in the PypeIt +:ref:`unit-tests`, these configuration checks are performed by the +developers for every new version of the code. diff --git a/doc/dev/hiresframes.rst b/doc/dev/hiresframes.rst new file mode 100644 index 0000000000..d303081a9a --- /dev/null +++ b/doc/dev/hiresframes.rst @@ -0,0 +1,128 @@ +.. include:: ../include/links.rst + +.. _hires_frames: + +Automated typing of HIRES frames +================================ + +Version History +--------------- + + +========= ================ =========== =========== +*Version* *Author* *Date* ``PypeIt`` +========= ================ =========== =========== +1.0 Debora Pelliccia 10 Aug 2024 1.16.1.dev +========= ================ =========== =========== + +---- + +Basics +------ + +The general procedure used to assign frames a given type is described +here: :ref:`frame_types`. + +HIRES frame typing +------------------ + +The primary typing of HIRES frames is performed by +:func:`pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph.check_frame_type`. +This function checks the values of various header keywords against a +set of criteria used to classify the frame type. +The header cards required for the frame-typing and their associated keyword in the +:class:`~pypeit.metadata.PypeItMetaData` object are: + +=============== ============ +``fitstbl`` key Header Key +=============== ============ +``exptime`` ``ELAPTIME`` +``hatch`` ``HATOPEN`` +``lampstat01`` See below +No key ``XCOVOPEN`` +No key ``AUTOSHUT`` +=============== ============ + +``lampstat01`` is defined using a combination of header keywords, which include +``LAMPCAT1``, ``LAMPCAT2``, ``LAMPQTZ2``, ``LAMPNAME``. If ``LAMPCAT1 = True`` or +``LAMPCAT2 = True``, ``lampstat01`` will be equal to ``'ThAr1'`` or ``'ThAr2'``, respectively. +If ``LAMPQTZ2 = True`` or ``LAMPNAME = 'quartz1'``, ``lampstat01`` will be equal to ``'on'``. + + +The criteria used to select each frame type are as follows: + +==================== ============ ============ ============ ====================================== ====================================================== +Frame ``hatch`` ``AUTOSHUT`` ``XCOVOPEN`` ``lampstat01`` ``exptime`` +==================== ============ ============ ============ ====================================== ====================================================== +``science`` ``True`` ``True`` ``True`` ``'off'`` ``>601s`` +``standard`` ``'open'`` ``True`` ``True`` ``'off'`` ``>1s`` & ``<600s`` +``bias`` ``False`` ``False`` ``True`` ``'off'`` ``<0.001s`` +``dark`` ``False`` ``True`` ``True`` ``'off'`` Not used +``slitless_pixflat`` ``False`` ``True`` ``False`` ``'off'`` ``<60s`` +``pixelflat`` ``False`` ``True`` ``True`` ``'on'`` ``<60s`` +``trace`` ``False`` ``True`` ``True`` ``'on'`` ``<60s`` +``illumflat`` ``False`` ``True`` ``True`` ``'on'`` ``<60s`` +``arc`` ``False`` ``True`` ``True`` ``'ThAr1'`` or ``'ThAr2'`` Not used +``tilt`` ``False`` ``True`` ``True`` ``'ThAr1'`` or ``'ThAr2'`` Not used +==================== ============ ============ ============ ====================================== ====================================================== + +Note that PypeIt employs commonly used value of ``exptime`` to distinguish frame type; +however, if needed, the user can specify a different value by +using the ``exprng`` parameter in the :ref:`pypeit_file`; see also :ref:`frame_types`. + +The ``science`` and ``standard`` frames have identical selection criteria, except for the +``exptime`` value. In order to better distinguish between the two types, the ``RA`` and ``DEC`` header +keywords are also used to assign the ``standard`` type to frames with ``RA`` and ``DEC`` values that are +within 10 arcmin of one of the standard stars available in PypeIt (see :ref:`standards`). + +The criteria used to select ``arc`` and ``tilt`` frames are identical; the same is true for +``pixelflat``, ``trace``, and ``illumflat`` frames. Note that if both ``pixelflat`` and +``slitless_pixflat`` frames are identified, the ``pixelflat`` assignment will be removed +so that the ``slitless_pixflat`` frames will be used for the flat fielding. + +Finally, note that a HIRES frame is never given a ``pinhole`` type. + + +Testing +------- + +To test that PypeIt can successfully identify HIRES framt types +among a set of files, we have added the +``test_hires()`` test to ``${PYPEIT_DEV}/unit_tests/test_frametype.py``. + +Here is an example of how to run the test: + +.. code-block:: bash + + cd ${PYPEIT_DEV}/unit_tests + pytest test_frametype.py::test_hires -W ignore + +The tests requires that you have downloaded the PypeIt +:ref:`dev-suite` and defined the ``PYPEIT_DEV`` environmental +variable that points to the relevant directory. The algorithm for +all these tests is the same and is as follows: + + 1. Find the directories in the :ref:`dev-suite` with Keck + HIRES data. + + 2. For each directory (i.e., instrument setup): + + a. Make sure there is a "by-hand" version of the pypeit file + for this setup where a human (one of the pypeit + developers) has ensured the frame types are correct. + + b. Effectively run :ref:`pypeit_setup` on each of the + instrument setups to construct a new pypeit file with the + automatically generated frame types. + + c. Read both the by-hand and automatically generated frame + types from these two pypeit files and check that they are + identical. This check is *only* performed for the + calibration frames, not any ``science`` or ``standard`` + frames. + +Because this test is now included in the ``PypeIt`` +:ref:`unit-tests`, this frame-typing check is performed by the +developers for every new version of the code. + + diff --git a/doc/dev/lrisconfig.rst b/doc/dev/lrisconfig.rst index 3e73cedcb5..aa0638ffe9 100644 --- a/doc/dev/lrisconfig.rst +++ b/doc/dev/lrisconfig.rst @@ -13,6 +13,7 @@ Version History *Version* *Author* *Date* ``PypeIt`` ========= ================ =========== =========== 1.0 Debora Pelliccia 6 Sep 2023 1.13.1.dev +1.1 Debora Pelliccia 10 Aug 2024 1.16.1.dev ========= ================ =========== =========== ---- @@ -50,12 +51,15 @@ which finds unique combinations of the following keywords: The unique configurations are determined by collating the relevant metadata from the headers of all frames found by a run of :ref:`pypeit_setup`, *except* those that are designated as -bias frames. The reason is that bias frames can have header data (e.g., ``dispangle``) +bias and slitless_pixflat frames. Bias frames can have header data (e.g., ``dispangle``) that do not match the instrument configuration that an observer intended for their use; e.g., the frames were taken before the instrument was fully configured for the night's observations. Therefore, PypeIt uses the ``dateobs``, ``binning``, ``amp`` keys to match the bias frames to the configurations with frames taken on the same date, with -the same binning and on the same amplifier. +the same binning and on the same amplifier. Similarly, slitless_pixflat frames are taken +without a slitmask (i.e., ``decker`` is different from the other frames), therefore +PypeIt uses the ``dateobs``, ``binning``, ``amp``, ``dispname``, ``dichroic`` keys to match +the slitless_pixflat frames to a specific configuration. After that, :func:`pypeit.metadata.PypeItMetaData.set_configurations` associates each frame to the relevant unique configuration ("setup"), by assigning a setup identifier diff --git a/doc/dev/lrisframes.rst b/doc/dev/lrisframes.rst index ef7c845273..ae52ee3c1c 100644 --- a/doc/dev/lrisframes.rst +++ b/doc/dev/lrisframes.rst @@ -13,6 +13,7 @@ Version History *Version* *Author* *Date* ``PypeIt`` ========= ================ =========== =========== 1.0 Debora Pelliccia 6 Sep 2023 1.13.1.dev +1.1 Debora Pelliccia 10 Aug 2024 1.16.1.dev ========= ================ =========== =========== ---- @@ -40,6 +41,7 @@ The header cards required for the frame-typing and their associated keyword in t =============== ====================================================== ``exptime`` ``ELAPTIME`` (``TELAPSE`` for ``keck_lris_red_mark4``) ``hatch`` ``TRAPDOOR`` +``decker`` ``SLITNAME`` ``lampstat01`` See below =============== ====================================================== @@ -51,32 +53,43 @@ of keywords used to define ``lampstat01`` varies depending on the available head The criteria used to select each frame type are as follows: -============= ============ ====================================== =========================================== -Frame ``hatch`` ``lampstat01`` ``exptime`` -============= ============ ====================================== =========================================== -``science`` ``'open'`` ``'off'`` ``>61s`` -``standard`` ``'open'`` ``'off'`` ``>1s`` & ``<61s`` -``bias`` ``'closed'`` ``'off'`` ``<0.001s`` -``pixelflat`` ``'closed'`` ``'Halogen' or '2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``pixelflat`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``trace`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``trace`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``illumflat`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``illumflat`` ``'open'`` ``'on'`` ``<60s``(LRIS RED) or ``<300s`` (LRIS BLUE) -``arc`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` Not used -``tilt`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` Not used -============= ============ ====================================== =========================================== +==================== ============ ====================================== ================= ====================================================== +Frame ``hatch`` ``lampstat01`` ``decker`` ``exptime`` +==================== ============ ====================================== ================= ====================================================== +``science`` ``'open'`` ``'off'`` ``!= 'GOH_LRIS'`` ``>61s`` +``standard`` ``'open'`` ``'off'`` ``!= 'GOH_LRIS'`` ``>1s`` & ``<61s`` (LRIS RED) or ``<900s`` (LRIS BLUE) +``bias`` ``'closed'`` ``'off'`` ``!= 'GOH_LRIS'`` ``<1s`` +``slitless_pixflat`` ``'open'`` ``'off'`` ``== 'direct'`` ``<60s`` +``pixelflat`` ``'closed'`` ``'Halogen' or '2H'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``pixelflat`` ``'open'`` ``'on'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``trace`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``trace`` ``'open'`` ``'on'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``illumflat`` ``'closed'`` ``'Halogen'`` or ``'2H'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``illumflat`` ``'open'`` ``'on'`` ``!= 'GOH_LRIS'`` ``<60s`` (LRIS RED) or ``<300s`` (LRIS BLUE) +``arc`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` ``!= 'GOH_LRIS'`` Not used +``tilt`` ``'closed'`` ``!= 'Halogen', '2H', 'on', 'off'`` ``!= 'GOH_LRIS'`` Not used +==================== ============ ====================================== ================= ====================================================== Note that PypeIt employs commonly used value of ``exptime`` to distinguish frame type; however, if needed, the user can specify a different value by using the ``exprng`` parameter in the :ref:`pypeit_file`; see also :ref:`frame_types`. +The ``science`` and ``standard`` frames have identical selection criteria, except for the +``exptime`` value. In order to better distinguish between the two types, the ``RA`` and ``DEC`` header +keywords are also used to assign the ``standard`` type to frames with ``RA`` and ``DEC`` values that are +within 10 arcmin of one of the standard stars available in PypeIt (see :ref:`standards`). + The criteria used to select ``arc`` and ``tilt`` frames are identical; the same is true for -``pixelflat``, ``trace``, and ``illumflat`` frames. However, it's important to note that +``pixelflat``, ``trace``, and ``illumflat`` frames. It's important to note that PypeIt is able to correctly assign the ``pixelflat``, ``trace``, and ``illumflat`` types -to the internal and dome flat frames, but the twilight flats will generally have the -``science`` or ``standard`` type. Therefore, the user should manually change their frame type -in the :ref:`pypeit_file`. +to the internal and dome flat frames, and it tries to do the same for the twilight flats, by selecting +frames that looks like ``science`` frames and include the following words in the ``OBJECT`` +or ``TARGNAME`` header keywords: 'sky', 'blank', 'twilight', 'twiflat', 'twi flat'. This way of +identifying twilight flats is not robust, therefore the user should always check the frame types assigned +and manually change them if needed in the :ref:`pypeit_file`. + +Note, also, that if both ``pixelflat`` and ``slitless_pixflat`` frames are identified, the ``pixelflat`` +assignment will be removed so that the ``slitless_pixflat`` frames will be used for the flat fielding. Finally, note that a LRIS frame is never given a ``pinhole`` or ``dark`` type. diff --git a/doc/figures/uves_popler.png b/doc/figures/uves_popler.png new file mode 100644 index 0000000000..eb4129d343 Binary files /dev/null and b/doc/figures/uves_popler.png differ diff --git a/doc/frametype.rst b/doc/frametype.rst index 83387add34..148f9cf078 100644 --- a/doc/frametype.rst +++ b/doc/frametype.rst @@ -42,24 +42,25 @@ description can be listed as follows: More detailed descriptions are given in the table below. -================ ============================================================= -Frame Type Description -================ ============================================================= -``align`` Used to align spatial positions in multiple slits. This frame is particularly useful for slit-based IFU, such as Keck KCWI. -``arc`` Spectrum of one or more calibration arc lamps -``bias`` Bias frame; typically a 0s exposure with the shutter closed -``dark`` Dark frame; typically a >0s exposure to assess dark current (shutter closed) -``illumflat`` Spectrum taken to correct illumination profile of the slit(s). This is often the same as the trace flat (below). -``lampoffflats`` Spectrum taken to remove persistence from lamp on flat exposures and/or thermal emission from the telescope and dome. Usually this is an exposure using a flat with lamps OFF -``pinhole`` Spectrum taken through a pinhole slit (i.e. a very short slit length), and is used to define the centre if a slit (currently, this frame is only used for echelle data reduction). Often this is an exposure using a flat lamp, but one can in principle use a standard star frame too (or a science frame if the spectrum is uniform). -``pixelflat`` Spectrum taken to correct for pixel-to-pixel detector variations Often an exposure using a dome (recommended) or internal flat lamp, but for observations in the very blue, this may be on-sky -``science`` Spectrum of one or more science targets -``standard`` Spectrum of spectrophotometric standard star PypeIt includes a list of pre-defined standards -``trace`` Spectrum taken to define the slit edges. Often this is an exposure using a flat lamp, but for observations in the very blue, this may be on-sky. The slit length of a trace frame should be the same as the science slit. -``tilt`` Exposure used to trace the tilt in the wavelength solution. Often the same file(s) as the arc. -``sky`` On-sky observation of the sky used for background subtraction -``None`` File could not be automatically identified by PypeIt -================ ============================================================= +==================== ============================================================= +Frame Type Description +==================== ============================================================= +``align`` Used to align spatial positions in multiple slits. This frame is particularly useful for slit-based IFU, such as Keck KCWI. +``arc`` Spectrum of one or more calibration arc lamps +``bias`` Bias frame; typically a 0s exposure with the shutter closed +``dark`` Dark frame; typically a >0s exposure to assess dark current (shutter closed) +``illumflat`` Spectrum taken to correct illumination profile of the slit(s). This is often the same as the trace flat (below). +``lampoffflats`` Spectrum taken to remove persistence from lamp on flat exposures and/or thermal emission from the telescope and dome. Usually this is an exposure using a flat with lamps OFF +``pinhole`` Spectrum taken through a pinhole slit (i.e. a very short slit length), and is used to define the centre if a slit (currently, this frame is only used for echelle data reduction). Often this is an exposure using a flat lamp, but one can in principle use a standard star frame too (or a science frame if the spectrum is uniform). +``pixelflat`` Spectrum taken to correct for pixel-to-pixel detector variations. Often an exposure using a dome (recommended) or internal flat lamp, but for observations in the very blue, this may be on-sky +``slitless_pixflat`` Spectrum taken without a slitmask or longslit to correct for pixel-to-pixel detector variations. This is often an exposure taken on-sky +``science`` Spectrum of one or more science targets +``standard`` Spectrum of spectrophotometric standard star PypeIt includes a list of pre-defined standards +``trace`` Spectrum taken to define the slit edges. Often this is an exposure using a flat lamp, but for observations in the very blue, this may be on-sky. The slit length of a trace frame should be the same as the science slit. +``tilt`` Exposure used to trace the tilt in the wavelength solution. Often the same file(s) as the arc. +``sky`` On-sky observation of the sky used for background subtraction +``None`` File could not be automatically identified by PypeIt +==================== ============================================================= .. TODO: Need to check that "sky" frametype is correct and/or used! diff --git a/doc/help/pypeit_cache_github_data.rst b/doc/help/pypeit_cache_github_data.rst index 804e0656f2..3ab386e7fd 100644 --- a/doc/help/pypeit_cache_github_data.rst +++ b/doc/help/pypeit_cache_github_data.rst @@ -9,7 +9,7 @@ Script to download/cache PypeIt github data positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -36,14 +36,15 @@ --exclude EXCLUDE [EXCLUDE ...] A subset of the directories to *exclude* from the list of files to download. Options are: tests, reid_arxiv, - nist, standards, skisim, sensfunc. This option is - mutually exclusive with --include. (default: ['tests']) + nist, standards, skisim, sensfunc, pixelflat. This + option is mutually exclusive with --include. (default: + ['tests']) --include INCLUDE [INCLUDE ...] The directories to *include* in the list of files to download. Use "--include all" to include all directories. Options are: all, tests, reid_arxiv, nist, - standards, skisim, sensfunc. This option is mutually - exclusive with --exclude. (default: None) + standards, skisim, sensfunc, pixelflat. This option is + mutually exclusive with --exclude. (default: None) --spec_dependent_only Only include files that are specific to the provided list of spectrographs. By default, the script also diff --git a/doc/help/pypeit_chk_flexure.rst b/doc/help/pypeit_chk_flexure.rst new file mode 100644 index 0000000000..d7110d32d4 --- /dev/null +++ b/doc/help/pypeit_chk_flexure.rst @@ -0,0 +1,18 @@ +.. code-block:: console + + $ pypeit_chk_flexure -h + usage: pypeit_chk_flexure [-h] (--spec | --spat) [--try_old] + input_file [input_file ...] + + Print QA on flexure to the screen + + positional arguments: + input_file One or more PypeIt spec2d or spec1d file + + options: + -h, --help show this help message and exit + --spec Check the spectral flexure (default: False) + --spat Check the spatial flexure (default: False) + --try_old Attempt to load old datamodel versions. A crash may ensue.. + (default: False) + \ No newline at end of file diff --git a/doc/help/pypeit_chk_for_calibs.rst b/doc/help/pypeit_chk_for_calibs.rst index 06cc2d6357..653b5d1e81 100644 --- a/doc/help/pypeit_chk_for_calibs.rst +++ b/doc/help/pypeit_chk_for_calibs.rst @@ -13,7 +13,7 @@ options: -h, --help show this help message and exit -s SPECTROGRAPH, --spectrograph SPECTROGRAPH - A valid spectrograph identifier: bok_bc, + A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -35,8 +35,11 @@ vlt_xshooter_nir, vlt_xshooter_uvb, vlt_xshooter_vis, wht_isis_blue, wht_isis_red (default: None) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) --save_setups If not toggled, remove setup_files/ folder and its files. (default: False) \ No newline at end of file diff --git a/doc/help/pypeit_extract_datacube.rst b/doc/help/pypeit_extract_datacube.rst new file mode 100644 index 0000000000..f4b16eab69 --- /dev/null +++ b/doc/help/pypeit_extract_datacube.rst @@ -0,0 +1,29 @@ +.. code-block:: console + + $ pypeit_extract_datacube -h + usage: pypeit_extract_datacube [-h] [-e EXT_FILE] [-s SAVE] [-o] + [-b BOXCAR_RADIUS] [-v VERBOSITY] + file + + Read in a datacube, extract a spectrum of a point source,and save it as a spec1d + file. + + positional arguments: + file spec3d.fits DataCube file + + options: + -h, --help show this help message and exit + -e EXT_FILE, --ext_file EXT_FILE + Configuration file with extraction parameters (default: + None) + -s SAVE, --save SAVE Output spec1d filename (default: None) + -o, --overwrite Overwrite any existing files/directories (default: + False) + -b BOXCAR_RADIUS, --boxcar_radius BOXCAR_RADIUS + Radius of the circular boxcar (in arcseconds) to use for + the extraction. (default: None) + -v VERBOSITY, --verbosity VERBOSITY + Verbosity level between 0 [none] and 2 [all]. Default: + 1. Level 2 writes a log with filename + extract_datacube_YYYYMMDD-HHMM.log (default: 1) + \ No newline at end of file diff --git a/doc/help/pypeit_obslog.rst b/doc/help/pypeit_obslog.rst index 3b4c807e8c..5b399fcee0 100644 --- a/doc/help/pypeit_obslog.rst +++ b/doc/help/pypeit_obslog.rst @@ -10,7 +10,7 @@ using PypeItMetaData. positional arguments: - spec A valid spectrograph identifier: bok_bc, + spec A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -75,8 +75,11 @@ -s SORT, --sort SORT Metadata keyword (pypeit-specific) to use to sort the output table. (default: mjd) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) -d OUTPUT_PATH, --output_path OUTPUT_PATH Path to top-level output directory. (default: current working directory) diff --git a/doc/help/pypeit_ql.rst b/doc/help/pypeit_ql.rst index b8991c356d..9302bc39c0 100644 --- a/doc/help/pypeit_ql.rst +++ b/doc/help/pypeit_ql.rst @@ -17,7 +17,7 @@ Script to produce quick-look PypeIt reductions positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, diff --git a/doc/help/pypeit_sensfunc.rst b/doc/help/pypeit_sensfunc.rst index 3a5b008e28..701944f327 100644 --- a/doc/help/pypeit_sensfunc.rst +++ b/doc/help/pypeit_sensfunc.rst @@ -1,9 +1,9 @@ .. code-block:: console $ pypeit_sensfunc -h - usage: pypeit_sensfunc [-h] [--algorithm {UVIS,IR}] [--multi MULTI] [-o OUTFILE] - [-s SENS_FILE] [-f FLATFILE] [--debug] - [--par_outfile PAR_OUTFILE] [-v VERBOSITY] + usage: pypeit_sensfunc [-h] [--extr {OPT,BOX}] [--algorithm {UVIS,IR}] + [--multi MULTI] [-o OUTFILE] [-s SENS_FILE] [-f FLATFILE] + [--debug] [--par_outfile PAR_OUTFILE] [-v VERBOSITY] spec1dfile Compute a sensitivity function @@ -14,6 +14,16 @@ options: -h, --help show this help message and exit + --extr {OPT,BOX} Override the default extraction method used for + computing the sensitivity function. Note that it is not + possible to set --extr and simultaneously use a .sens + file with the --sens_file option. If you are using a + .sens file, set the algorithm there via: + + [sensfunc] + extr = BOX + + The extraction options are: OPT or BOX --algorithm {UVIS,IR} Override the default algorithm for computing the sensitivity function. Note that it is not possible to @@ -62,11 +72,13 @@ -s SENS_FILE, --sens_file SENS_FILE Configuration file with sensitivity function parameters -f FLATFILE, --flatfile FLATFILE - Use the flat file for computing the sensitivity - function. Note that it is not possible to set - --flatfile and simultaneously use a .sens file with the - --sens_file option. If you are using a .sens file, set - the flatfile there via e.g.: + Use a flat calibration file to compute the blaze + function when generating the sensitivity function. This + is helpful to account for small scale undulations in the + sensitivity function. Note that it is not possible to + set --flatfile and simultaneously use a .sens file with + the --sens_file option. If you are using a .sens file, + set the flatfile there via e.g.: [sensfunc] flatfile = Calibrations/Flat_A_0_DET01.fits diff --git a/doc/help/pypeit_setup.rst b/doc/help/pypeit_setup.rst index 32f4da9915..01e0c078dd 100644 --- a/doc/help/pypeit_setup.rst +++ b/doc/help/pypeit_setup.rst @@ -11,7 +11,7 @@ options: -h, --help show this help message and exit -s SPECTROGRAPH, --spectrograph SPECTROGRAPH - A valid spectrograph identifier: bok_bc, + A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, @@ -39,8 +39,11 @@ --extension option to set the types of files to search for. (default: current working directory) -e EXTENSION, --extension EXTENSION - File extension; compression indicators (e.g. .gz) not - required. (default: .fits) + File extension to use. Must include the period (e.g., + ".fits") and it must be one of the allowed extensions + for this spectrograph. If None, root directory will be + searched for all files with any of the allowed + extensions. (default: None) -d OUTPUT_PATH, --output_path OUTPUT_PATH Path to top-level output directory. (default: current working directory) diff --git a/doc/help/pypeit_show_pixflat.rst b/doc/help/pypeit_show_pixflat.rst new file mode 100644 index 0000000000..d7394e32e3 --- /dev/null +++ b/doc/help/pypeit_show_pixflat.rst @@ -0,0 +1,17 @@ +.. code-block:: console + + $ pypeit_show_pixflat -h + usage: pypeit_show_pixflat [-h] [--det DET [DET ...]] file + + Show an archived Pixel Flat image in a ginga window. + + positional arguments: + file Pixel Flat filename, e.g. + pixelflat_keck_lris_blue.fits.gz + + options: + -h, --help show this help message and exit + --det DET [DET ...] Detector(s) to show. If more than one, list the detectors + as, e.g. --det 1 2 to show detectors 1 and 2. If not + provided, all detectors will be shown. (default: None) + \ No newline at end of file diff --git a/doc/help/pypeit_trace_edges.rst b/doc/help/pypeit_trace_edges.rst index dc265db21b..a9b5e21814 100644 --- a/doc/help/pypeit_trace_edges.rst +++ b/doc/help/pypeit_trace_edges.rst @@ -26,16 +26,16 @@ default mosaic. (default: None) -s SPECTROGRAPH, --spectrograph SPECTROGRAPH A valid spectrograph identifier, which is only used if - providing files directly: bok_bc, gemini_flamingos1, - gemini_flamingos2, gemini_gmos_north_e2v, - gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - gemini_gmos_south_ham, gemini_gnirs_echelle, - gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, - jwst_nircam, jwst_nirspec, keck_deimos, keck_esi, - keck_hires, keck_kcrm, keck_kcwi, keck_lris_blue, - keck_lris_blue_orig, keck_lris_red, keck_lris_red_mark4, - keck_lris_red_orig, keck_mosfire, keck_nires, - keck_nirspec_high, keck_nirspec_high_old, + providing files directly: aat_uhrf, bok_bc, + gemini_flamingos1, gemini_flamingos2, + gemini_gmos_north_e2v, gemini_gmos_north_ham, + gemini_gmos_north_ham_ns, gemini_gmos_south_ham, + gemini_gnirs_echelle, gemini_gnirs_ifu, gtc_maat, + gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, + keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + keck_lris_blue, keck_lris_blue_orig, keck_lris_red, + keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, + keck_nires, keck_nirspec_high, keck_nirspec_high_old, keck_nirspec_low, lbt_luci1, lbt_luci2, lbt_mods1b, lbt_mods1r, lbt_mods2b, lbt_mods2r, ldt_deveny, magellan_fire, magellan_fire_long, magellan_mage, diff --git a/doc/help/pypeit_view_fits.rst b/doc/help/pypeit_view_fits.rst index fb6bbd14e2..4ee223de61 100644 --- a/doc/help/pypeit_view_fits.rst +++ b/doc/help/pypeit_view_fits.rst @@ -9,7 +9,7 @@ View FITS files with ginga positional arguments: - spectrograph A valid spectrograph identifier: bok_bc, + spectrograph A valid spectrograph identifier: aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, gemini_gmos_north_ham, gemini_gmos_north_ham_ns, gemini_gmos_south_ham, diff --git a/doc/help/run_pypeit.rst b/doc/help/run_pypeit.rst index 2870d8a61a..4cb290e4fa 100644 --- a/doc/help/run_pypeit.rst +++ b/doc/help/run_pypeit.rst @@ -4,14 +4,14 @@ usage: run_pypeit [-h] [-v VERBOSITY] [-r REDUX_PATH] [-m] [-s] [-o] [-c] pypeit_file - ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev109+g885cb1823 + ## PypeIt : The Python Spectroscopic Data Reduction Pipeline v1.16.1.dev336+gdf3013372.d20240827 ## ## Available spectrographs include: - ## bok_bc, gemini_flamingos1, gemini_flamingos2, gemini_gmos_north_e2v, - ## gemini_gmos_north_ham, gemini_gmos_north_ham_ns, - ## gemini_gmos_south_ham, gemini_gnirs_echelle, gemini_gnirs_ifu, - ## gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, jwst_nirspec, - ## keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, + ## aat_uhrf, bok_bc, gemini_flamingos1, gemini_flamingos2, + ## gemini_gmos_north_e2v, gemini_gmos_north_ham, + ## gemini_gmos_north_ham_ns, gemini_gmos_south_ham, gemini_gnirs_echelle, + ## gemini_gnirs_ifu, gtc_maat, gtc_osiris, gtc_osiris_plus, jwst_nircam, + ## jwst_nirspec, keck_deimos, keck_esi, keck_hires, keck_kcrm, keck_kcwi, ## keck_lris_blue, keck_lris_blue_orig, keck_lris_red, ## keck_lris_red_mark4, keck_lris_red_orig, keck_mosfire, keck_nires, ## keck_nirspec_high, keck_nirspec_high_old, keck_nirspec_low, lbt_luci1, diff --git a/doc/include/class_datamodel_mosaic.rst b/doc/include/class_datamodel_mosaic.rst index ddc075089c..80c523ddef 100644 --- a/doc/include/class_datamodel_mosaic.rst +++ b/doc/include/class_datamodel_mosaic.rst @@ -1,5 +1,5 @@ -**Version**: 1.0.0 +**Version**: 1.0.1 ============== ================ ============================================================ =================================================================================== Attribute Type Array Type Description @@ -7,7 +7,7 @@ Attribute Type Array Type ``binning`` str On-chip binning ``detectors`` `numpy.ndarray`_ :class:`~pypeit.images.detector_container.DetectorContainer` List of objects with detector parameters. ``id`` int Mosaic ID number -``msc_order`` int Order of the interpolation used to construct the mosaic. +``msc_ord`` int Order of the interpolation used to construct the mosaic. ``platescale`` float Detector platescale in arcsec/pixel ``rot`` `numpy.ndarray`_ float Raw, hard-coded rotations (counter-clockwise in degrees) for each unbinned detector ``shape`` tuple Shape of each processed detector image diff --git a/doc/include/class_datamodel_pypeitimage.rst b/doc/include/class_datamodel_pypeitimage.rst index 1f21b5588b..9008ab7bc3 100644 --- a/doc/include/class_datamodel_pypeitimage.rst +++ b/doc/include/class_datamodel_pypeitimage.rst @@ -10,6 +10,7 @@ Attribute Type ``det_img`` `numpy.ndarray`_ `numpy.integer`_ If a detector mosaic, this image provides the detector that contributed to each pixel. ``detector`` :class:`~pypeit.images.detector_container.DetectorContainer`, :class:`~pypeit.images.mosaic.Mosaic` The detector (see :class:`~pypeit.images.detector_container.DetectorContainer`) or mosaic (see :class:`~pypeit.images.mosaic.Mosaic`) parameters ``exptime`` int, float Effective exposure time (s) +``filename`` str Filename for the image ``fullmask`` :class:`~pypeit.images.imagebitmask.ImageBitMaskArray` Image mask ``image`` `numpy.ndarray`_ `numpy.floating`_ Primary image data ``img_scale`` `numpy.ndarray`_ `numpy.floating`_ Image count scaling applied (e.g., 1/flat-field) diff --git a/doc/include/class_datamodel_sensfunc.rst b/doc/include/class_datamodel_sensfunc.rst index fbf98d1922..e57591681c 100644 --- a/doc/include/class_datamodel_sensfunc.rst +++ b/doc/include/class_datamodel_sensfunc.rst @@ -8,6 +8,7 @@ Attribute Type Array Type Description ``airmass`` float Airmass of the observation ``algorithm`` str Algorithm used for the sensitivity calculation. ``exptime`` float Exposure time +``extr`` str Extraction method used for the standard star (OPT or BOX) ``pypeline`` str PypeIt pipeline reduction path ``sens`` `astropy.table.table.Table`_ Table with the sensitivity function ``spec1df`` str PypeIt spec1D file used to for sensitivity function diff --git a/doc/include/data_dir.rst b/doc/include/data_dir.rst index adda09f16a..1c8e42c4d8 100644 --- a/doc/include/data_dir.rst +++ b/doc/include/data_dir.rst @@ -7,6 +7,7 @@ extinction extinction ... filters filters ... linelist arc_lines/lists ... nist arc_lines/NIST github +pixelflat pixelflats github reid_arxiv arc_lines/reid_arxiv github sensfunc sensfuncs github skisim skisim github diff --git a/doc/include/dependencies_table.rst b/doc/include/dependencies_table.rst index eedeadd17a..5156b259ec 100644 --- a/doc/include/dependencies_table.rst +++ b/doc/include/dependencies_table.rst @@ -1,5 +1,5 @@ -======================= ========================================================================================================================================================================================================================================================================================================================================================= -Python Version ``>=3.10,<3.13`` -Required for users ``IPython>=7.10.0``, ``PyERFA>=2.0.0``, ``PyYAML>=5.1``, ``astropy>=6.0``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=0.1``, ``fast-histogram>=0.11``, ``ginga>=5.1.0``, ``linetools>=0.3.1``, ``matplotlib>=3.7``, ``numpy>=1.23``, ``packaging>=0.19``, ``pygithub``, ``pyqt6``, ``qtpy>=2.0.1``, ``scikit-learn>=1.0``, ``scipy>=1.7`` -Required for developers ``coverage``, ``docutils<0.21``, ``psutil``, ``pygit2``, ``pytest-astropy``, ``pytest-cov``, ``pytest-qt``, ``pytest>=6.0.0``, ``scikit-image``, ``specutils>=1.13``, ``sphinx-automodapi``, ``sphinx>=1.6,<8``, ``sphinx_rtd_theme==2.0.0``, ``tox`` -======================= ========================================================================================================================================================================================================================================================================================================================================================= +======================= ======================================================================================================================================================================================================================================================================================================================================================== +Python Version ``>=3.11,<3.13`` +Required for users ``IPython>=8.0.0``, ``PyERFA>=2.0.0``, ``PyYAML>=6.0``, ``astropy>=6.0``, ``bottleneck``, ``configobj>=5.0.6``, ``extension-helpers>=1.0``, ``fast-histogram>=0.11``, ``ginga>=5.1.0``, ``linetools>=0.3.2``, ``matplotlib>=3.7``, ``numpy>=1.24``, ``packaging>=22.0``, ``pygithub``, ``pyqt6``, ``qtpy>=2.2.0``, ``scikit-learn>=1.2``, ``scipy>=1.9`` +Required for developers ``coverage``, ``docutils<0.21``, ``psutil``, ``pygit2``, ``pytest-astropy``, ``pytest-cov``, ``pytest-qt``, ``pytest>=7.0.0``, ``scikit-image>=0.23``, ``specutils>=1.13``, ``sphinx-automodapi``, ``sphinx>=1.6,<8``, ``sphinx_rtd_theme==2.0.0``, ``tox`` +======================= ======================================================================================================================================================================================================================================================================================================================================================== diff --git a/doc/include/inst_detector_table.rst b/doc/include/inst_detector_table.rst index cbb0df365a..41c4c8b132 100644 --- a/doc/include/inst_detector_table.rst +++ b/doc/include/inst_detector_table.rst @@ -1,6 +1,7 @@ ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== Instrument Det specaxis specflip spatflip namp gain RN darkcurr min sat nonlinear platescale ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== +``aat_uhrf`` 1 0 False False 1 1.0 0.0 0.0 -1.0e+10 65535.0 0.7600 0.0500 ``bok_bc`` 1 1 False False 1 1.5 3.0 5.4 -1.0e+10 65535.0 1.0000 0.2000 ``gemini_flamingos1`` 1 0 False False 1 3.8 6.0 1080.0 -1.0e+10 320000.0 0.8750 0.1500 ``gemini_flamingos2`` 1 0 True False 1 4.44 5.0 1800.0 -1.0e+10 700000.0 1.0000 0.1787 @@ -85,9 +86,9 @@ Instrument Det specaxis specflip spatflip namp gain ``vlt_fors2`` 1 1 False False 1 0.7 2.9 2.1 -1.0e+10 200000.0 0.8000 0.1260 ... 2 1 False False 1 0.7 3.15 1.4 -1.0e+10 200000.0 0.8000 0.1260 ``vlt_sinfoni`` 1 0 True False 1 2.42 7.0 540.0 -1.0e+10 1000000000.0 1.0000 0.0125 -``vlt_xshooter_nir`` 1 1 False False 1 2.12 8.0 0.0 -1.0e+10 200000.0 0.8600 0.1970 -``vlt_xshooter_uvb`` 1 0 True True 1 1.61 2.6 0.0 -1.0e+10 65000.0 0.8600 0.1610 -``vlt_xshooter_vis`` 1 0 False False 1 0.595 3.1 0.0 -1.0e+10 65535.0 0.8600 0.1600 +``vlt_xshooter_nir`` 1 1 False False 1 2.29 8.0 72.0 -1.0e+10 200000.0 0.8600 0.2450 +``vlt_xshooter_uvb`` 1 0 True True 1 ``None`` ``None`` 0.0 -1.0e+10 65000.0 0.8600 0.1640 +``vlt_xshooter_vis`` 1 0 False False 1 ``None`` ``None`` 0.0 -1.0e+10 65535.0 0.8600 0.1540 ``wht_isis_blue`` 1 0 False False 1 1.2 5.0 0.0 -1.0e+10 65535.0 0.7600 0.2000 ``wht_isis_red`` 1 0 False False 1 0.98 4.0 0.0 -1.0e+10 65535.0 0.7600 0.2200 ============================ === ======== ======== ======== ======== ========================== ====================== ======== ======== ============ ========= ========== diff --git a/doc/include/keck_deimos.sorted.rst b/doc/include/keck_deimos.sorted.rst index 4e41db2257..828519122f 100644 --- a/doc/include/keck_deimos.sorted.rst +++ b/doc/include/keck_deimos.sorted.rst @@ -12,17 +12,17 @@ filename | frametype | ra | dec | target | dispname | decker | binning | mjd | airmass | exptime | dispangle | amp | filter1 | lampstat01 | dateobs | utc | frameno | calib DE.20170527.06713.fits | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 d0527_0030.fits.gz | arc,tilt | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.077631 | 1.41291034 | 1.0 | 8099.98291016 | SINGLE:B | OG550 | Kr Xe Ar Ne | 2017-05-27 | 01:51:53.87 | 30 | 0 - DE.20170527.06790.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 d0527_0031.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 - d0527_0032.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 + DE.20170527.06790.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.07851 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:53:10.93 | 31 | 0 DE.20170527.06864.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 - DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + d0527_0032.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.079356 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:54:24.03 | 32 | 0 d0527_0033.fits.gz | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 + DE.20170527.06936.fits | pixelflat,illumflat,trace | 57.99999999999999 | 45.0 | DOME PHLAT | 830G | LongMirr | 1,1 | 57900.080211 | 1.41291034 | 4.0 | 8099.98291016 | SINGLE:B | OG550 | Qz | 2017-05-27 | 01:55:36.93 | 33 | 0 DE.20170527.37601.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.435131 | 1.03078874 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:26:41.61 | 80 | 0 - DE.20170527.38872.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 d0527_0081.fits.gz | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 - d0527_0083.fits.gz | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 + DE.20170527.38872.fits | science | 261.0363749999999 | 19.028166666666667 | P261_OFF | 830G | LongMirr | 1,1 | 57900.449842 | 1.01267696 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 10:47:52.92 | 81 | 0 DE.20170527.41775.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 + d0527_0083.fits.gz | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.483427 | 1.00093023 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:36:15.35 | 83 | 0 DE.20170527.43045.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.498135 | 1.00838805 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 11:57:25.35 | 84 | 0 DE.20170527.44316.fits | science | 261.0362916666666 | 19.028888888888886 | P261_OFF | 830G | LongMirr | 1,1 | 57900.512854 | 1.02377681 | 1200.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 12:18:36.71 | 85 | 0 DE.20170527.53184.fits | science | 349.99316666666664 | -5.16575 | Feige 110 | 830G | LongMirr | 1,1 | 57900.615484 | 1.42505162 | 45.0 | 8099.98291016 | SINGLE:B | OG550 | Off | 2017-05-27 | 14:46:24.88 | 93 | 0 diff --git a/doc/include/links.rst b/doc/include/links.rst index e4e1f1b84f..f1c47e7444 100644 --- a/doc/include/links.rst +++ b/doc/include/links.rst @@ -128,7 +128,7 @@ .. _pip: https://pip.pypa.io/en/stable/ .. _anaconda: https://www.anaconda.com/products/individual .. _conda: https://docs.conda.io/projects/conda/en/latest/index.html -.. _virtualenv: https://virtualenv.pypa.io/en/latest/ +.. _venv: https://docs.python.org/3/library/venv.html .. _pdb: https://docs.python.org/3/library/pdb.html .. _IPython.embed: https://ipython.readthedocs.io/en/stable/api/generated/IPython.terminal.embed.html#function .. _pytest: https://docs.pytest.org/en/latest/ diff --git a/doc/include/spectrographs_table.rst b/doc/include/spectrographs_table.rst index 1244f6e8e8..04b73891e7 100644 --- a/doc/include/spectrographs_table.rst +++ b/doc/include/spectrographs_table.rst @@ -1,6 +1,7 @@ ======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== ``PypeIt`` Name ``PypeIt`` Class Telescope Camera URL Pipeline Supported QL Tested Comments ======================== ============================================================================ ========= ============ =============================================================================================================================== ========= ========= ========= =============================================================================================== +aat_uhrf :class:`~pypeit.spectrographs.aat_uhrf.AATUHRFSpectrograph` AAT UHRF `Link `__ MultiSlit True False bok_bc :class:`~pypeit.spectrographs.bok_bc.BokBCSpectrograph` BOK BC `Link `__ MultiSlit True False Bok B&C spectrometer gemini_flamingos1 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS1Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit False False gemini_flamingos2 :class:`~pypeit.spectrographs.gemini_flamingos.GeminiFLAMINGOS2Spectrograph` GEMINI-S FLAMINGOS `Link `__ MultiSlit True False Flamingos-2 NIR spectrograph @@ -17,7 +18,7 @@ jwst_nircam :class:`~pypeit.spectrographs.jwst_nircam.JWSTNIRCamSp jwst_nirspec :class:`~pypeit.spectrographs.jwst_nirspec.JWSTNIRSpecSpectrograph` JWST NIRSPEC `Link `__ MultiSlit True False keck_deimos :class:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph` KECK DEIMOS `Link `__ MultiSlit True True Supported gratings: 600ZD, 830G, 900ZD, 1200B, 1200G; see :doc:`deimos` keck_esi :class:`~pypeit.spectrographs.keck_esi.KeckESISpectrograph` KECK ESI Echelle True False -keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False +keck_hires :class:`~pypeit.spectrographs.keck_hires.KECKHIRESSpectrograph` KECK HIRES `Link `__ Echelle False False Post detector upgrade (~ August 2004). See :doc:`keck_hires` keck_kcrm :class:`~pypeit.spectrographs.keck_kcwi.KeckKCRMSpectrograph` KECK KCRM `Link `__ SlicerIFU True False Supported setups: RL, RM1, RM2, RH3; see :doc:`keck_kcwi` keck_kcwi :class:`~pypeit.spectrographs.keck_kcwi.KeckKCWISpectrograph` KECK KCWI `Link `__ SlicerIFU True False Supported setups: BL, BM, BH2; see :doc:`keck_kcwi` keck_lris_blue :class:`~pypeit.spectrographs.keck_lris.KeckLRISBSpectrograph` KECK LRISb `Link `__ MultiSlit True False Blue camera; Current FITS file format; used from May 2009, see :doc:`lris` diff --git a/doc/installing.rst b/doc/installing.rst index 1772a7b8f7..bf190ca83e 100644 --- a/doc/installing.rst +++ b/doc/installing.rst @@ -36,7 +36,7 @@ Setup a clean python environment PypeIt is available from the `Python Package Index `_ (PyPI) and is installed via ``pip``. This process also installs and/or upgrades -PypeIt's :ref:`dependencies`, and for this reason, we highly (!!) recommend you +PypeIt's :ref:`dependencies`, and for this reason you should always first set up a clean python environment in which to install PypeIt. This mitigates any possible dependency conflicts with other packages you use. @@ -47,20 +47,18 @@ You can set up a new python environment using either `conda`_: conda create -n pypeit python=3.11 conda activate pypeit -or `virtualenv`_: +or `venv`_: .. code-block:: console - virtualenv pypeit + python -m venv pypeit source pypeit/bin/activate See the `Managing Environments with Conda -`_ -and/or `Virtualenv documentation `_ -for more details. See also `virtualenvwrapper -`_ as an option for more -easily managing `virtualenv`_ environments. The `conda`_ installation method described below -creates an environment for you. +`__ +and/or the `venv documentation `__ for +more details. The `conda`_ installation method described below creates an +environment for you. .. _installing-pip: @@ -222,29 +220,40 @@ for x86-64: Solutions/Recommendations/Feedback for these installation options are welcome; please `Submit an issue`_. +.. _install_windows: + User Installation on Windows --------------------------------------------- #. Download `Python for Windows `_. #. Run the installer. - - * Make sure "Add python.exe to Path" or "Add Python to environment variables" is selected before installing. - * If you have Admin privileges click "Disable path length limit" after the installation succeeds. + + * Make sure "Add python.exe to Path" or "Add Python to environment + variables" is selected before installing. -#. Downloand and run the `Visual Studio build tools `_ installer. + * If you have Admin privileges click "Disable path length limit" after the + installation succeeds. + +#. Download and run the `Visual Studio build tools + `_ installer. * Only "Desktop Development with C++" needs to be checked. + * Click install -#. Create a virtual environment as in `Setup a clean python environment `__ and install PypeIt as described above. +#. Create a virtual environment as in `Setup a clean python environment + `__ and install PypeIt as described above. -If running ``python`` on Windows brings up a window for the Microsoft Store you may want to change the application alias. -This is under ``Settings -> Apps -> App execution aliases`` on Windows 10 and ``Settings -> Apps -> Advanced app settings -> App execution aliases`` -on Windows 11. Disable the ``App Installer`` options for the ``python.exe`` and ``python3.exe`` executables. +If running ``python`` on Windows brings up a window for the Microsoft Store you +may want to change the application alias. This is under ``Settings -> Apps -> +App execution aliases`` on Windows 10 and ``Settings -> Apps -> Advanced app +settings -> App execution aliases`` on Windows 11. Disable the ``App Installer`` +options for the ``python.exe`` and ``python3.exe`` executables. -An alternative for running under Windows is to install the `Windows Subsystem for Linux (WSL) `_. -This in effect allows you to run PypeIt under Linux under Windows. +An alternative for running under Windows is to install the `Windows Subsystem +for Linux (WSL) `_. This +in effect allows you to run PypeIt under Linux under Windows. ---- @@ -751,4 +760,85 @@ In either case, over 100 tests should pass, nearly 100 will be skipped and none should fail. The skipped tests only run if the PypeIt development is installed and configured; see :ref:`dev-suite`. +---- + +.. _install_troubleshoot: + +Troubleshooting +=============== + +If you have trouble installing pypeit, you're encouraged to `join +`__ +our `PypeIt Users Slack `__ and post your issue +to the #installing channel. Here is an incomplete list of issues that users +have reported in the past. In addition to posting to the Users Slack if your +issue isn't among those listed below, *please let us know if these suggestions +do not work for you.* +**I am trying to install pypeit for the first time and it fails!**: The root +problem of this can be system dependent: + + - First, *always* make sure you install the code into a fresh environment. + + - If you're on Windows, make sure you follow the :ref:`install_windows` + instructions. If you're still having trouble, it may be because PypeIt + includes some C code to accelerate some computations. If the issue is + because the C compiler is not properly linking, you can try typing ``set + CC=help`` at the command prompt before running the ``pip install`` command. + + - Occasionally, the installation may fail because of incompatible dependencies. + This may be because of recent releases of one of PypeIt's dependencies; i.e., + updates to packages since the most recent PypeIt release. Please let us know + if this happens, and we will try to issue a new release asap that corrects + the incompatibility. In the short-term, we may ask you to install old + versions of packages that we know work. + +**I am trying to upgrade pypeit and it fails!**: First try uninstalling your +current pypeit version: + +.. code-block:: bash + + pip uninstall pypeit + +Then reinstall it. If that also fails, try creating a fresh environment and +reinstalling pypeit in that new environment. + +**The installation process succeeded, but the code is faulting!**: This could +be for a few reasons: + + - Recall that pypeit isn't necessarily backwards compatible. If you've + upgraded pypeit and tried to use it with data that was reduced by a previous + version, the fault may because of changes between versions. You will either + need to revert to your previous version or reprocess the data. + + - This may be because of dependency changes. A tell-tale signature of this is + if you get errors associate with missing or unknown keywords or arguments. + This is may be because of recent releases of one of PypeIt's dependencies; i.e., + updates to packages since the most recent PypeIt release. Please let us know + if this happens, and we will try to issue a new release asap that corrects + the incompatibility. In the short-term, we may ask you to install old + versions of packages that we know work. + +**The installation process succeeded and the code completes without faulting, +but the output looks wrong!**: This could happen for any number of reasons. +*We always welcome reports of failures!* Either `submit an issue +`__ or report it on the PypeIt Users +Slack. However, here are a few things to note and/or try: + + - Make sure you have checked your calibrations; see :ref:`calibrations`. The + issue may be related to a parameter that you can change. + + - If you don't see any ``spec1d`` files in your ``Science`` folder, this is + likely because the code didn't find any objects; see :ref:`object_finding`. + + - If you've recently upgraded the code, this may be related to changes in + dependencies that the developers didn't catch. PypeIt performs *a lot* of + testing before issuing a new release, but does not have complete test + coverage and performance validation. This means silent failures are the most + difficult to catch. + + - And, of course, the code will have bugs. If you find one, the more + information you provide the developers, the easier it will be for us to track + down the issue. Valuable information includes your OS, OS version, python + version, and pypeit version, as well as QA plots and ``ginga`` screen grabs + that illustrate the issue. diff --git a/doc/pypeit_par.rst b/doc/pypeit_par.rst index d8a7b4a747..ee49c46478 100644 --- a/doc/pypeit_par.rst +++ b/doc/pypeit_par.rst @@ -185,6 +185,9 @@ Current PypeItPar Parameter Hierarchy | ``[[lampoffflatsframe]]``: :ref:`framegrouppar` | ``[[[process]]]``: :ref:`processimagespar` | ``[[[[scattlight]]]]``: :ref:`scatteredlightpar` +| ``[[slitless_pixflatframe]]``: :ref:`framegrouppar` +| ``[[[process]]]``: :ref:`processimagespar` +| ``[[[[scattlight]]]]``: :ref:`scatteredlightpar` | ``[[scattlightframe]]``: :ref:`framegrouppar` | ``[[[process]]]``: :ref:`processimagespar` | ``[[[[scattlight]]]]``: :ref:`scatteredlightpar` @@ -252,32 +255,33 @@ CalibrationsPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.CalibrationsPar` -===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= -Key Type Options Default Description -===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= -``alignframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the align frames -``alignment`` :class:`~pypeit.par.pypeitpar.AlignPar` .. `AlignPar Keywords`_ Define the procedure for the alignment of traces -``arcframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength calibration -``biasframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the bias correction -``bpm_usebias`` bool .. False Make a bad pixel mask from bias frames? Bias frames must be provided. -``calib_dir`` str .. ``Calibrations`` The name of the directory for the processed calibration frames. The host path for the directory is set by the redux_path (see :class:`~pypeit.par.pypeitpar.ReduxPar`). Beware that success when changing the default value is not well tested! -``darkframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the dark-current correction -``flatfield`` :class:`~pypeit.par.pypeitpar.FlatFieldPar` .. `FlatFieldPar Keywords`_ Parameters used to set the flat-field procedure -``illumflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the illumination flat -``lampoffflatsframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the lamp off flats -``pinholeframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pinholes -``pixelflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pixel flat -``raise_chk_error`` bool .. True Raise an error if the calibration check fails -``scattlight_pad`` int .. 5 Number of unbinned pixels to extend the slit edges by when masking the slits. -``scattlightframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the scattered light frames -``skyframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the sky background observations -``slitedges`` :class:`~pypeit.par.pypeitpar.EdgeTracePar` .. `EdgeTracePar Keywords`_ Slit-edge tracing parameters -``standardframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the spectrophotometric standard observations -``tiltframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength tilts -``tilts`` :class:`~pypeit.par.pypeitpar.WaveTiltsPar` .. `WaveTiltsPar Keywords`_ Define how to trace the slit tilts using the trace frames -``traceframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for images used for slit tracing -``wavelengths`` :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` .. `WavelengthSolutionPar Keywords`_ Parameters used to derive the wavelength solution -===================== ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +========================= ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +Key Type Options Default Description +========================= ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= +``alignframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the align frames +``alignment`` :class:`~pypeit.par.pypeitpar.AlignPar` .. `AlignPar Keywords`_ Define the procedure for the alignment of traces +``arcframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength calibration +``biasframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the bias correction +``bpm_usebias`` bool .. False Make a bad pixel mask from bias frames? Bias frames must be provided. +``calib_dir`` str .. ``Calibrations`` The name of the directory for the processed calibration frames. The host path for the directory is set by the redux_path (see :class:`~pypeit.par.pypeitpar.ReduxPar`). Beware that success when changing the default value is not well tested! +``darkframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the dark-current correction +``flatfield`` :class:`~pypeit.par.pypeitpar.FlatFieldPar` .. `FlatFieldPar Keywords`_ Parameters used to set the flat-field procedure +``illumflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the illumination flat +``lampoffflatsframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the lamp off flats +``pinholeframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pinholes +``pixelflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the pixel flat +``raise_chk_error`` bool .. True Raise an error if the calibration check fails +``scattlight_pad`` int .. 5 Number of unbinned pixels to extend the slit edges by when masking the slits. +``scattlightframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the scattered light frames +``skyframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the sky background observations +``slitedges`` :class:`~pypeit.par.pypeitpar.EdgeTracePar` .. `EdgeTracePar Keywords`_ Slit-edge tracing parameters +``slitless_pixflatframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the slitless pixel flat +``standardframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the spectrophotometric standard observations +``tiltframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for the wavelength tilts +``tilts`` :class:`~pypeit.par.pypeitpar.WaveTiltsPar` .. `WaveTiltsPar Keywords`_ Define how to trace the slit tilts using the trace frames +``traceframe`` :class:`~pypeit.par.pypeitpar.FrameGroupPar` .. `FrameGroupPar Keywords`_ The frames and combination rules for images used for slit tracing +``wavelengths`` :class:`~pypeit.par.pypeitpar.WavelengthSolutionPar` .. `WavelengthSolutionPar Keywords`_ Parameters used to derive the wavelength solution +========================= ==================================================== ======= ================================= ================================================================================================================================================================================================================================================= ---- @@ -308,32 +312,33 @@ FlatFieldPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.FlatFieldPar` -========================== ================= ================================= =========== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ -Key Type Options Default Description -========================== ================= ================================= =========== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ -``fit_2d_det_response`` bool .. False Set this variable to True if you want to compute and account for the detector response in the flatfield image. Note that ``detector response`` refers to pixel sensitivity variations that primarily depend on (x,y) detector coordinates. In most cases, the default 2D bspline is sufficient to account for detector response (i.e. set this parameter to False). Note that this correction will _only_ be performed for the spectrographs that have a dedicated response correction implemented. Currently,this correction is only implemented for Keck+KCWI. -``illum_iter`` int .. 0 The number of rejection iterations to perform when constructing the slit-illumination profile. No rejection iterations are performed if 0. WARNING: Functionality still being tested. -``illum_rej`` int, float .. 5.0 The sigma threshold used in the rejection iterations used to refine the slit-illumination profile. Rejection iterations are only performed if ``illum_iter > 0``. -``method`` str ``bspline``, ``skip`` ``bspline`` Method used to flat field the data; use skip to skip flat-fielding. Options are: None, bspline, skip -``pixelflat_file`` str .. .. Filename of the image to use for pixel-level field flattening -``pixelflat_max_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths above this value. -``pixelflat_min_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths below this value. -``rej_sticky`` bool .. False Propagate the rejected pixels through the stages of the flat-field fitting (i.e, from the spectral fit, to the spatial fit, and finally to the 2D residual fit). If False, pixels rejected in each stage are included in each subsequent stage. -``saturated_slits`` str ``crash``, ``mask``, ``continue`` ``crash`` Behavior when a slit is encountered with a large fraction of saturated pixels in the flat-field. The options are: 'crash' - Raise an error and halt the data reduction; 'mask' - Mask the slit, meaning no science data will be extracted from the slit; 'continue' - ignore the flat-field correction, but continue with the reduction. -``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. -``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. -``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. -``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. -``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. -``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. -``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. -``spec_samp_coarse`` int, float .. 50.0 bspline break point spacing in units of pixels for 2-d bspline-polynomial fit to flat field image residuals. This should be a large number unless you are trying to fit a sky flat with lots of narrow spectral features. -``spec_samp_fine`` int, float .. 1.2 bspline break point spacing in units of pixels for spectral fit to flat field blaze function. -``tweak_slits`` bool .. True Use the illumination flat field to tweak the slit edges. This will work even if illumflatten is set to False -``tweak_slits_maxfrac`` float .. 0.1 If tweak_slit is True, this sets the maximum fractional amount (of a slits width) allowed for trimming each (i.e. left and right) slit boundary, i.e. the default is 10% which means slits would shrink or grow by at most 20% (10% on each side) -``tweak_slits_thresh`` float .. 0.93 If tweak_slits is True, this sets the illumination function threshold used to tweak the slit boundaries based on the illumination flat. It should be a number less than 1.0 -``twod_fit_npoly`` int .. .. Order of polynomial used in the 2D bspline-polynomial fit to flat-field image residuals. The code determines the order of these polynomials to each slit automatically depending on the slit width, which is why the default is None. Alter this paramter at your own risk! -========================== ================= ================================= =========== ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +========================== ================= ================================= ============= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +Key Type Options Default Description +========================== ================= ================================= ============= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ +``fit_2d_det_response`` bool .. False Set this variable to True if you want to compute and account for the detector response in the flatfield image. Note that ``detector response`` refers to pixel sensitivity variations that primarily depend on (x,y) detector coordinates. In most cases, the default 2D bspline is sufficient to account for detector response (i.e. set this parameter to False). Note that this correction will _only_ be performed for the spectrographs that have a dedicated response correction implemented. Currently,this correction is only implemented for Keck+KCWI. +``illum_iter`` int .. 0 The number of rejection iterations to perform when constructing the slit-illumination profile. No rejection iterations are performed if 0. WARNING: Functionality still being tested. +``illum_rej`` int, float .. 5.0 The sigma threshold used in the rejection iterations used to refine the slit-illumination profile. Rejection iterations are only performed if ``illum_iter > 0``. +``method`` str ``bspline``, ``skip`` ``bspline`` Method used to flat field the data; use skip to skip flat-fielding. Options are: None, bspline, skip +``pixelflat_file`` str .. .. Filename of the image to use for pixel-level field flattening +``pixelflat_max_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths above this value. +``pixelflat_min_wave`` int, float .. .. All values of the normalized pixel flat are set to 1 for wavelengths below this value. +``rej_sticky`` bool .. False Propagate the rejected pixels through the stages of the flat-field fitting (i.e, from the spectral fit, to the spatial fit, and finally to the 2D residual fit). If False, pixels rejected in each stage are included in each subsequent stage. +``saturated_slits`` str ``crash``, ``mask``, ``continue`` ``crash`` Behavior when a slit is encountered with a large fraction of saturated pixels in the flat-field. The options are: 'crash' - Raise an error and halt the data reduction; 'mask' - Mask the slit, meaning no science data will be extracted from the slit; 'continue' - ignore the flat-field correction, but continue with the reduction. +``slit_illum_finecorr`` bool .. True If True, a fine correction to the spatial illumination profile will be performed. The fine correction is a low order 2D polynomial fit to account for a gradual change to the spatial illumination profile as a function of wavelength. +``slit_illum_pad`` int, float .. 5.0 The number of pixels to pad the slit edges when constructing the slit-illumination profile. Single value applied to both edges. +``slit_illum_ref_idx`` int .. 0 The index of a reference slit (0-indexed) used for estimating the relative spectral sensitivity (or the relative blaze). This parameter is only used if ``slit_illum_relative = True``. +``slit_illum_relative`` bool .. False Generate an image of the relative spectral illumination for a multi-slit setup. If you set ``use_specillum = True`` for any of the frames that use the flatfield model, this *must* be set to True. Currently, this is only used for SlicerIFU reductions. +``slit_illum_smooth_npix`` int .. 10 The number of pixels used to determine smoothly varying relative weights is given by ``nspec/slit_illum_smooth_npix``, where nspec is the number of spectral pixels. +``slit_trim`` int, float, tuple .. 3.0 The number of pixels to trim each side of the slit when selecting pixels to use for fitting the spectral response function. Single values are used for both slit edges; a two-tuple can be used to trim the left and right sides differently. +``spat_samp`` int, float .. 5.0 Spatial sampling for slit illumination function. This is the width of the median filter in pixels used to determine the slit illumination function, and thus sets the minimum scale on which the illumination function will have features. +``spec_samp_coarse`` int, float .. 50.0 bspline break point spacing in units of pixels for 2-d bspline-polynomial fit to flat field image residuals. This should be a large number unless you are trying to fit a sky flat with lots of narrow spectral features. +``spec_samp_fine`` int, float .. 1.2 bspline break point spacing in units of pixels for spectral fit to flat field blaze function. +``tweak_method`` str ``threshold``, ``gradient`` ``threshold`` Method used to tweak the slit edges (when "tweak_slits" is set to True). Options include: threshold, gradient. The "threshold" method determines when the left and right slit edges fall below a threshold relative to the peak illumination. The "gradient" method determines where the gradient is the highest at the left and right slit edges. This method performs better when there is systematic vignetting in the spatial direction. +``tweak_slits`` bool .. True Use the illumination flat field to tweak the slit edges. This will work even if illumflatten is set to False +``tweak_slits_maxfrac`` float .. 0.1 If tweak_slit is True, this sets the maximum fractional amount (of a slits width) allowed for trimming each (i.e. left and right) slit boundary, i.e. the default is 10% which means slits would shrink or grow by at most 20% (10% on each side) +``tweak_slits_thresh`` float .. 0.93 If tweak_slits is True, this sets the illumination function threshold used to tweak the slit boundaries based on the illumination flat. It should be a number less than 1.0 +``twod_fit_npoly`` int .. .. Order of polynomial used in the 2D bspline-polynomial fit to flat-field image residuals. The code determines the order of these polynomials to each slit automatically depending on the slit width, which is why the default is None. Alter this paramter at your own risk! +========================== ================= ================================= ============= ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================ ---- @@ -582,21 +587,21 @@ Collate1DPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.Collate1DPar` -========================= =============== ======= =============================== ================================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -========================= =============== ======= =============================== ================================================================================================================================================================================================================================================================================================================================================================================================================== -``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. -``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. -``exclude_slit_trace_bm`` list, str .. A list of slit trace bitmask bits that should be excluded. -``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. -``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. -``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. -``outdir`` str .. ``/home/dusty/work/PypeIt/doc`` The path where all coadded output files and report files will be placed. -``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric -``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. -``tolerance`` str, float, int .. 1.0 The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. -``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. -========================= =============== ======= =============================== ================================================================================================================================================================================================================================================================================================================================================================================================================== +========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== +``dry_run`` bool .. False If set, the script will display the matching File and Object Ids but will not flux, coadd or archive. +``exclude_serendip`` bool .. False Whether to exclude SERENDIP objects from collating. +``exclude_slit_trace_bm`` list, str .. A list of slit trace bitmask bits that should be excluded. +``flux`` bool .. False If set, the script will flux calibrate using archived sensfuncs before coadding. +``ignore_flux`` bool .. False If set, the script will only coadd non-fluxed spectra even if flux data is present. Otherwise fluxed spectra are coadded if all spec1ds have been fluxed calibrated. +``match_using`` str .. ``ra/dec`` Determines how 1D spectra are matched as being the same object. Must be either 'pixel' or 'ra/dec'. +``outdir`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` The path where all coadded output files and report files will be placed. +``refframe`` str .. .. Perform reference frame correction prior to coadding. Options are: observed, heliocentric, barycentric +``spec1d_outdir`` str .. .. The path where all modified spec1d files are placed. These are only created if flux calibration or refframe correction are asked for. +``tolerance`` str, float, int .. 1.0 The tolerance used when comparing the coordinates of objects. If two objects are within this distance from each other, they are considered the same object. If match_using is 'ra/dec' (the default) this is an angular distance. The defaults units are arcseconds but other units supported by astropy.coordinates.Angle can be used (`e.g.`, '0.003d' or '0h1m30s'). If match_using is 'pixel' this is a float. +``wv_rms_thresh`` float .. .. If set, any objects with a wavelength RMS > this value are skipped, else all wavelength RMS values are accepted. +========================= =============== ======= ============================================ ================================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -617,7 +622,7 @@ Key Type Options ``multi_min_SN`` int, float .. 1 Minimum S/N for analyzing sky spectrum for flexure ``spec_maxshift`` int .. 20 Maximum allowed spectral flexure shift in pixels. ``spec_method`` str ``boxcar``, ``slitcen``, ``skip`` ``skip`` Method used to correct for flexure. Use skip for no correction. If slitcen is used, the flexure correction is performed before the extraction of objects (not recommended). Options are: None, boxcar, slitcen, skip -``spectrum`` str .. ``paranal_sky.fits`` Archive sky spectrum to be used for the flexure correction. +``spectrum`` str .. ``paranal_sky.fits`` Archive sky spectrum to be used for the flexure correction. See ``pypeit/data/sky_spec/`` for a list of available sky spectra. If ``model`` is used, a model sky spectrum will be generated using :func:`~pypeit.wavemodel.nearIR_modelsky` and the spectralresolution of the spectrum to be flexure corrected. =================== ========== ======================================================== ==================== ====================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================== @@ -649,22 +654,22 @@ ReduxPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ReduxPar` -====================== ============== ======= =============================== ========================================================================================================================================================================================================================================================================================================================================================================================================== -Key Type Options Default Description -====================== ============== ======= =============================== ========================================================================================================================================================================================================================================================================================================================================================================================================== -``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame -``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! -``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` -``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). -``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). -``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. -``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. -``redux_path`` str .. ``/home/dusty/work/PypeIt/doc`` Path to folder for performing reductions. Default is the current working directory. -``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. -``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. -``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. -``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. -====================== ============== ======= =============================== ========================================================================================================================================================================================================================================================================================================================================================================================================== +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== +``calwin`` int, float .. 0 The window of time in hours to search for calibration frames for a science frame +``chk_version`` bool .. True If True enforce strict PypeIt version checking to ensure that all files were created with the current version of PypeIt. If set to False, the code will attempt to read out-of-date files and keep going. Beware (!!) that this can lead to unforeseen bugs that either cause the code to crash or lead to erroneous results. I.e., you really need to know what you are doing if you set this to False! +``detnum`` int, list .. .. Restrict reduction to a list of detector indices. In case of mosaic reduction (currently only available for Gemini/GMOS and Keck/DEIMOS) ``detnum`` should be a list of tuples of the detector indices that are mosaiced together. E.g., for Gemini/GMOS ``detnum`` would be ``[(1,2,3)]`` and for Keck/DEIMOS it would be ``[(1, 5), (2, 6), (3, 7), (4, 8)]`` +``ignore_bad_headers`` bool .. False Ignore bad headers (NOT recommended unless you know it is safe). +``maskIDs`` str, int, list .. .. Restrict reduction to a set of slitmask IDs Example syntax -- ``maskIDs = 818006,818015`` This must be used with detnum (for now). +``qadir`` str .. ``QA`` Directory relative to calling directory to write quality assessment files. +``quicklook`` bool .. False Run a quick look reduction? This is usually good if you want to quickly reduce the data (usually at the telescope in real time) to get an initial estimate of the data quality. +``redux_path`` str .. ``/Users/westfall/Work/packages/pypeit/doc`` Path to folder for performing reductions. Default is the current working directory. +``scidir`` str .. ``Science`` Directory relative to calling directory to write science files. +``slitspatnum`` str, list .. .. Restrict reduction to a set of slit DET:SPAT values (closest slit is used). Example syntax -- slitspatnum = DET01:175,DET01:205 or MSC02:2234 If you are re-running the code, (i.e. modifying one slit) you *must* have the precise SPAT_ID index. +``sortroot`` str .. .. A filename given to output the details of the sorted files. If None, the default is the root name of the pypeit file. If off, no output is produced. +``spectrograph`` str .. .. Spectrograph that provided the data to be reduced. See :ref:`instruments` for valid options. +====================== ============== ======= ============================================ ========================================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -706,7 +711,6 @@ Key Type Options ``correct_dar`` bool .. True If True, the data will be corrected for differential atmospheric refraction (DAR). ``dec_max`` float .. .. Maximum DEC to use when generating the WCS. If None, the default is maximum DEC based on the WCS of all spaxels. Units should be degrees. ``dec_min`` float .. .. Minimum DEC to use when generating the WCS. If None, the default is minimum DEC based on the WCS of all spaxels. Units should be degrees. -``grating_corr`` bool .. True This option performs a small correction for the relative blaze function of all input frames that have (even slightly) different grating angles, or if you are flux calibrating your science data with a standard star that was observed with a slightly different setup. ``method`` str ``subpixel``, ``ngp`` ``subpixel`` What method should be used to generate the datacube. There are currently two options: (1) "subpixel" (default) - this algorithm divides each pixel in the spec2d frames into subpixels, and assigns each subpixel to a voxel of the datacube. Flux is conserved, but voxels are correlated, and the error spectrum does not account for covariance between adjacent voxels. See also, spec_subpixel and spat_subpixel. (2) "ngp" (nearest grid point) - this algorithm is effectively a 3D histogram. Flux is conserved, voxels are not correlated, however this option suffers the same downsides as any histogram; the choice of bin sizes can change how the datacube appears. This algorithm takes each pixel on the spec2d frame and puts the flux of this pixel into one voxel in the datacube. Depending on the binning used, some voxels may be empty (zero flux) while a neighboring voxel might contain the flux from two spec2d pixels. Note that all spec2d pixels that contribute to the same voxel are inverse variance weighted (e.g. if two pixels have the same variance, the voxel would be assigned the average flux of the two pixels). ``output_filename`` str .. .. If combining multiple frames, this string sets the output filename of the combined datacube. If combine=False, the output filenames will be prefixed with ``spec3d_*`` ``ra_max`` float .. .. Maximum RA to use when generating the WCS. If None, the default is maximum RA based on the WCS of all spaxels. Units should be degrees. @@ -714,13 +718,13 @@ Key Type Options ``reference_image`` str .. .. White light image of a previously combined datacube. The white light image will be used as a reference when calculating the offsets of the input spec2d files. Ideally, the reference image should have the same shape as the data to be combined (i.e. set the ra_min, ra_max etc. params so they are identical to the reference image). ``save_whitelight`` bool .. False Save a white light image of the combined datacube. The output filename will be given by the "output_filename" variable with a suffix "_whitelight". Note that the white light image collapses the flux along the wavelength axis, so some spaxels in the 2D white light image may have different wavelength ranges. To set the wavelength range, use the "whitelight_range" parameter. If combine=False, the individual spec3d files will have a suffix "_whitelight". ``scale_corr`` str .. .. This option performs a small correction for the relative spectral illumination scale of different spec2D files. Specify the relative path+file to the spec2D file that you would like to use for the relative scaling. If you want to perform this correction, it is best to use the spec2d file with the highest S/N sky spectrum. You should choose the same frame for both the standards and science frames. +``sensfile`` str .. .. Filename of a sensitivity function to use to flux calibrate your datacube. The sensitivity function file will also be used to correct the relative scales of the slits. ``skysub_frame`` str .. ``image`` Set the sky subtraction to be implemented. The default behaviour is to subtract the sky using the model that is derived from each individual image (i.e. set this parameter to "image"). To turn off sky subtraction completely, set this parameter to "none" (all lowercase). Finally, if you want to use a different frame for the sky subtraction, specify the relative path+file to the spec2D file that you would like to use for the sky subtraction. The model fit to the sky of the specified frame will be used. Note, the sky and science frames do not need to have the same exposure time; the sky model will be scaled to the science frame based on the relative exposure time. ``slice_subpixel`` int .. 5 When method=subpixel, slice_subpixel sets the subpixellation scale of each IFU slice. The default option is to divide each slice into 5 sub-slices during datacube creation. See also, spec_subpixel and spat_subpixel. ``slit_spec`` bool .. True If the data use slits in one spatial direction, set this to True. If the data uses fibres for all spaxels, set this to False. ``spat_subpixel`` int .. 5 When method=subpixel, spat_subpixel sets the subpixellation scale of each detector pixel in the spatial direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spec_subpixel and slice_subpixel. ``spatial_delta`` float .. .. The spatial size of each spaxel to use when generating the WCS (in arcsec). If None, the default is set by the spectrograph file. ``spec_subpixel`` int .. 5 When method=subpixel, spec_subpixel sets the subpixellation scale of each detector pixel in the spectral direction. The total number of subpixels in each pixel is given by spec_subpixel x spat_subpixel. The default option is to divide each spec2d pixel into 25 subpixels during datacube creation. See also, spat_subpixel and slice_subpixel. -``standard_cube`` str .. .. Filename of a standard star datacube. This cube will be used to correct the relative scales of the slits, and to flux calibrate the science datacube. ``wave_delta`` float .. .. The wavelength step to use when generating the WCS (in Angstroms). If None, the default is set by the wavelength solution. ``wave_max`` float .. .. Maximum wavelength to use when generating the WCS. If None, the default is maximum wavelength based on the WCS of all spaxels. Units should be Angstroms. ``wave_min`` float .. .. Minimum wavelength to use when generating the WCS. If None, the default is minimum wavelength based on the WCS of all spaxels. Units should be Angstroms. @@ -781,8 +785,9 @@ Key Type Options Default Description ``skip_second_find`` bool .. False Only perform one round of object finding (mainly for quick_look) ``skip_skysub`` bool .. False If True, do not sky subtract when performing object finding. This should be set to True for example when running on data that is already sky-subtracted. Note that for near-IR difference imaging one still wants to remove sky-residuals via sky-subtraction, and so this is typically set to False ``snr_thresh`` int, float .. 10.0 S/N threshold for object finding in wavelength direction smashed image. -``std_spec1d`` str .. .. A PypeIt spec1d file of a previously reduced standard star. The trace of the standard star spectrum is used as a crutch for tracing the object spectra, when a direct trace is not possible (i.e., faint sources). If provided, this overrides use of any standards included in your pypeit file; the standard exposures will still be reduced. +``std_spec1d`` str .. .. A PypeIt spec1d file of a previously reduced standard star. This can be used to trace the object spectra, but the ``use_std_trace`` parameter must be set to True. If provided, this overrides use of any standards included in your pypeit file; the standard exposures will still be reduced. ``trace_npoly`` int .. 5 Order of legendre polynomial fits to object traces. +``use_std_trace`` bool .. True If True, the trace of the standard star spectrum is used as a crutch for tracing the object spectra. This is useful when a direct trace is not possible (i.e., faint sources). Note that a standard star exposure must be included in your pypeit file, or the ``std_spec1d`` parameter must be set for this to work. =========================== ========== ======= ======= ============================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================= @@ -845,14 +850,14 @@ FrameGroupPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.FrameGroupPar` -============= =============================================== ============================================================================================================================================================================ ============================ =============================================================================================================================================================================================================================================================== -Key Type Options Default Description -============= =============================================== ============================================================================================================================================================================ ============================ =============================================================================================================================================================================================================================================================== -``exprng`` list .. None, None Used in identifying frames of this type. This sets the minimum and maximum allowed exposure times. There must be two items in the list. Use None to indicate no limit; i.e., to select exposures with any time greater than 30 sec, use exprng = [30, None]. -``frametype`` str ``align``, ``arc``, ``bias``, ``dark``, ``pinhole``, ``pixelflat``, ``illumflat``, ``lampoffflats``, ``scattlight``, ``science``, ``standard``, ``trace``, ``tilt``, ``sky`` ``science`` Frame type. Options are: align, arc, bias, dark, pinhole, pixelflat, illumflat, lampoffflats, scattlight, science, standard, trace, tilt, sky -``process`` :class:`~pypeit.par.pypeitpar.ProcessImagesPar` .. `ProcessImagesPar Keywords`_ Low level parameters used for basic image processing -``useframe`` str .. .. A calibrations file to use if it exists. -============= =============================================== ============================================================================================================================================================================ ============================ =============================================================================================================================================================================================================================================================== +============= =============================================== ================================================================================================================================================================================================== ============================ =============================================================================================================================================================================================================================================================== +Key Type Options Default Description +============= =============================================== ================================================================================================================================================================================================== ============================ =============================================================================================================================================================================================================================================================== +``exprng`` list .. None, None Used in identifying frames of this type. This sets the minimum and maximum allowed exposure times. There must be two items in the list. Use None to indicate no limit; i.e., to select exposures with any time greater than 30 sec, use exprng = [30, None]. +``frametype`` str ``align``, ``arc``, ``bias``, ``dark``, ``pinhole``, ``pixelflat``, ``illumflat``, ``lampoffflats``, ``slitless_pixflat``, ``scattlight``, ``science``, ``standard``, ``trace``, ``tilt``, ``sky`` ``science`` Frame type. Options are: align, arc, bias, dark, pinhole, pixelflat, illumflat, lampoffflats, slitless_pixflat, scattlight, science, standard, trace, tilt, sky +``process`` :class:`~pypeit.par.pypeitpar.ProcessImagesPar` .. `ProcessImagesPar Keywords`_ Low level parameters used for basic image processing +``useframe`` str .. .. A calibrations file to use if it exists. +============= =============================================== ================================================================================================================================================================================================== ============================ =============================================================================================================================================================================================================================================================== ---- @@ -864,43 +869,45 @@ ProcessImagesPar Keywords Class Instantiation: :class:`~pypeit.par.pypeitpar.ProcessImagesPar` -======================== ================================================ =================================================================== ============================= ============================================================================================================================================================================================================================================================================================================================================================ -Key Type Options Default Description -======================== ================================================ =================================================================== ============================= ============================================================================================================================================================================================================================================================================================================================================================ -``apply_gain`` bool .. True Convert the ADUs to electrons using the detector gain -``clip`` bool .. True Perform sigma clipping when combining. Only used with combine=mean -``comb_sigrej`` float .. .. Sigma-clipping level for when clip=True; Use None for automatic limit (recommended). -``combine`` str ``median``, ``mean`` ``mean`` Method used to combine multiple frames. Options are: median, mean -``dark_expscale`` bool .. False If designated dark frames are used and have a different exposure time than the science frames, scale the counts by the by the ratio in the exposure times to adjust the dark counts for the difference in exposure time. WARNING: You should always take dark frames that have the same exposure time as your science frames, so use this option with care! -``empirical_rn`` bool .. False If True, use the standard deviation in the overscan region to measure an empirical readnoise to use in the noise model. -``grow`` int, float .. 1.5 Factor by which to expand regions with cosmic rays detected by the LA cosmics routine. -``lamaxiter`` int .. 1 Maximum number of iterations for LA cosmics routine. -``mask_cr`` bool .. False Identify CRs and mask them -``n_lohi`` list .. 0, 0 Number of pixels to reject at the lowest and highest ends of the distribution; i.e., n_lohi = low, high. Use None for no limit. -``noise_floor`` float .. 0.0 Impose a noise floor by adding the provided fraction of the bias- and dark-subtracted electron counts to the error budget. E.g., a value of 0.01 means that the S/N of the counts in the image will never be greater than 100. -``objlim`` int, float .. 3.0 Object detection limit in LA cosmics routine -``orient`` bool .. True Orient the raw image into the PypeIt frame -``overscan_method`` str ``chebyshev``, ``polynomial``, ``savgol``, ``median``, ``odd_even`` ``savgol`` Method used to fit the overscan. Options are: chebyshev, polynomial, savgol, median, odd_even Note: Method "polynomial" is identical to "chebyshev"; the former is deprecated and will be removed. -``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. -``rmcompact`` bool .. True Remove compact detections in LA cosmics routine -``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing -``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. -``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget -``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine -``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. -``spat_flexure_correct`` bool .. False Correct slits, illumination flat, etc. for flexure -``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels -``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. -``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. -``trim`` bool .. True Trim the image to the detector supplied region -``use_biasimage`` bool .. True Use a bias image. If True, one or more must be supplied in the PypeIt file. -``use_darkimage`` bool .. False Subtract off a dark image. If True, one or more darks must be provided. -``use_illumflat`` bool .. True Use the illumination flat to correct for the illumination profile of each slit. -``use_overscan`` bool .. True Subtract off the overscan. Detector *must* have one or code will crash. -``use_pattern`` bool .. False Subtract off a detector pattern. This pattern is assumed to be sinusoidal along one direction, with a frequency that is constant across the detector. -``use_pixelflat`` bool .. True Use the pixel flat to make pixel-level corrections. A pixelflat image must be provied. -``use_specillum`` bool .. False Use the relative spectral illumination profiles to correct the spectral illumination profile of each slit. This is primarily used for slicer IFUs. To use this, you must set ``slit_illum_relative=True`` in the ``flatfield`` parameter set! -======================== ================================================ =================================================================== ============================= ============================================================================================================================================================================================================================================================================================================================================================ +======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +Key Type Options Default Description +======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== +``apply_gain`` bool .. True Convert the ADUs to electrons using the detector gain +``clip`` bool .. True Perform sigma clipping when combining. Only used with combine=mean +``comb_sigrej`` float .. .. Sigma-clipping level for when clip=True; Use None for automatic limit (recommended). +``combine`` str ``median``, ``mean`` ``mean`` Method used to combine multiple frames. Options are: median, mean +``correct_nonlinear`` list .. .. Correct for non-linear response of the detector. If None, no correction is performed. If a list, then the list should be the non-linear correction parameter (alpha), where the functional form is given by Ct = Cm (1 + alpha x Cm), with Ct and Cm the true and measured counts. This parameter is usually hard-coded for a given spectrograph, and should otherwise be left as None. +``dark_expscale`` bool .. False If designated dark frames are used and have a different exposure time than the science frames, scale the counts by the by the ratio in the exposure times to adjust the dark counts for the difference in exposure time. WARNING: You should always take dark frames that have the same exposure time as your science frames, so use this option with care! +``empirical_rn`` bool .. False If True, use the standard deviation in the overscan region to measure an empirical readnoise to use in the noise model. +``grow`` int, float .. 1.5 Factor by which to expand regions with cosmic rays detected by the LA cosmics routine. +``lamaxiter`` int .. 1 Maximum number of iterations for LA cosmics routine. +``mask_cr`` bool .. False Identify CRs and mask them +``n_lohi`` list .. 0, 0 Number of pixels to reject at the lowest and highest ends of the distribution; i.e., n_lohi = low, high. Use None for no limit. +``noise_floor`` float .. 0.0 Impose a noise floor by adding the provided fraction of the bias- and dark-subtracted electron counts to the error budget. E.g., a value of 0.01 means that the S/N of the counts in the image will never be greater than 100. +``objlim`` int, float .. 3.0 Object detection limit in LA cosmics routine +``orient`` bool .. True Orient the raw image into the PypeIt frame +``overscan_method`` str ``chebyshev``, ``polynomial``, ``savgol``, ``median``, ``odd_even`` ``savgol`` Method used to fit the overscan. Options are: chebyshev, polynomial, savgol, median, odd_even Note: Method "polynomial" is identical to "chebyshev"; the former is deprecated and will be removed. +``overscan_par`` int, list .. 5, 65 Parameters for the overscan subtraction. For 'chebyshev' or 'polynomial', set overcan_par = order; for 'savgol', set overscan_par = order, window size ; for 'median', set overscan_par = None or omit the keyword. +``rmcompact`` bool .. True Remove compact detections in LA cosmics routine +``satpix`` str ``reject``, ``force``, ``nothing`` ``reject`` Handling of saturated pixels. Options are: reject, force, nothing +``scale_to_mean`` bool .. False If True, scale the input images to have the same mean before combining. +``scattlight`` :class:`~pypeit.par.pypeitpar.ScatteredLightPar` .. `ScatteredLightPar Keywords`_ Scattered light subtraction parameters. +``shot_noise`` bool .. True Use the bias- and dark-subtracted image to calculate and include electron count shot noise in the image processing error budget +``sigclip`` int, float .. 4.5 Sigma level for rejection in LA cosmics routine +``sigfrac`` int, float .. 0.3 Fraction for the lower clipping threshold in LA cosmics routine. +``spat_flexure_correct`` bool .. False Correct slits, illumination flat, etc. for flexure +``spat_flexure_maxlag`` int .. 20 Maximum of possible spatial flexure correction, in pixels +``subtract_continuum`` bool .. False Subtract off the continuum level from an image. This parameter should only be set to True to combine arcs with multiple different lamps. For all other cases, this parameter should probably be False. +``subtract_scattlight`` bool .. False Subtract off the scattered light from an image. This parameter should only be set to True for spectrographs that have dedicated methods to subtract scattered light. For all other cases, this parameter should be False. +``trim`` bool .. True Trim the image to the detector supplied region +``use_biasimage`` bool .. True Use a bias image. If True, one or more must be supplied in the PypeIt file. +``use_darkimage`` bool .. False Subtract off a dark image. If True, one or more darks must be provided. +``use_illumflat`` bool .. True Use the illumination flat to correct for the illumination profile of each slit. +``use_overscan`` bool .. True Subtract off the overscan. Detector *must* have one or code will crash. +``use_pattern`` bool .. False Subtract off a detector pattern. This pattern is assumed to be sinusoidal along one direction, with a frequency that is constant across the detector. +``use_pixelflat`` bool .. True Use the pixel flat to make pixel-level corrections. A pixelflat image must be provied. +``use_specillum`` bool .. False Use the relative spectral illumination profiles to correct the spectral illumination profile of each slit. This is primarily used for slicer IFUs. To use this, you must set ``slit_illum_relative=True`` in the ``flatfield`` parameter set! +======================== ================================================ =================================================================== ============================= ======================================================================================================================================================================================================================================================================================================================================================================================== ---- @@ -938,6 +945,7 @@ Key Type Options ``IR`` :class:`~pypeit.par.pypeitpar.TelluricPar` .. `TelluricPar Keywords`_ Parameters for the IR sensfunc algorithm ``UVIS`` :class:`~pypeit.par.pypeitpar.SensfuncUVISPar` .. `SensfuncUVISPar Keywords`_ Parameters for the UVIS sensfunc algorithm ``algorithm`` str ``UVIS``, ``IR`` ``UVIS`` Specify the algorithm for computing the sensitivity function. The options are: (1) UVIS = Should be used for data with :math:`\lambda < 7000` A. No detailed model of telluric absorption but corrects for atmospheric extinction. (2) IR = Should be used for data with :math:`\lambda > 7000` A. Peforms joint fit for sensitivity function and telluric absorption using HITRAN models. +``extr`` str .. ``OPT`` Extraction method to use for the sensitivity function. Options are: 'OPT' (optimal extraction), 'BOX' (boxcar extraction). Default is 'OPT'. ``extrap_blu`` float .. 0.1 Fraction of minimum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the blue direction (`i.e.`, if the standard star spectrum cuts off at ``wave_min``) the sensfunc will be extrapolated to cover down to (1.0 - ``extrap_blu``) * ``wave_min`` ``extrap_red`` float .. 0.1 Fraction of maximum wavelength coverage to grow the wavelength coverage of the sensitivitity function in the red direction (`i.e.`, if the standard star spectrumcuts off at ``wave_max``) the sensfunc will be extrapolated to cover up to (1.0 + ``extrap_red``) * ``wave_max`` ``flatfile`` str .. .. Flat field file to be used if the sensitivity function model will utilize the blaze function computed from a flat field file in the Calibrations directory, e.g.Calibrations/Flat_A_0_DET01.fits @@ -1047,6 +1055,104 @@ these in the PypeIt file, you would be reproducing the effect of the `default_pypeit_par` method specific to each derived :class:`~pypeit.spectrographs.spectrograph.Spectrograph` class. +.. _instr_par-aat_uhrf: + +AAT UHRF (``aat_uhrf``) +----------------------- +Alterations to the default parameters are: + +.. code-block:: ini + + [rdx] + spectrograph = aat_uhrf + [calibrations] + [[biasframe]] + [[[process]]] + combine = median + use_biasimage = False + shot_noise = False + use_pixelflat = False + use_illumflat = False + [[darkframe]] + [[[process]]] + mask_cr = True + use_pixelflat = False + use_illumflat = False + [[arcframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[tiltframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[pixelflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[alignframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[traceframe]] + exprng = None, 60.0, + [[[process]]] + use_pixelflat = False + use_illumflat = False + [[illumflatframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[lampoffflatsframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[scattlightframe]] + [[[process]]] + satpix = nothing + use_pixelflat = False + use_illumflat = False + [[skyframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[standardframe]] + [[[process]]] + mask_cr = True + noise_floor = 0.01 + [[wavelengths]] + lamps = ThAr, + n_final = 3 + [[slitedges]] + sync_predict = nearest + bound_detector = True + [[tilts]] + spat_order = 4 + spec_order = 1 + [scienceframe] + exprng = 61, None, + [[process]] + mask_cr = True + sigclip = 10.0 + noise_floor = 0.01 + [reduce] + [[skysub]] + bspline_spacing = 3.0 + no_poly = True + user_regions = :10,75: + .. _instr_par-bok_bc: BOK BC (``bok_bc``) @@ -1129,6 +1235,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1257,6 +1371,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1385,6 +1507,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1497,6 +1627,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1586,6 +1722,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1675,6 +1817,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1764,6 +1912,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -1880,6 +2034,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2009,6 +2171,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2031,12 +2201,11 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [[flatfield]] + tweak_method = gradient tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_trim = 2 slit_illum_finecorr = False - [[slitedges]] - pad = 2 [[tilts]] spat_order = 1 spec_order = 1 @@ -2061,8 +2230,6 @@ Alterations to the default parameters are: [[extraction]] model_full_slit = True skip_extraction = True - [[cube]] - grating_corr = False [flexure] spec_maxshift = 0 [sensfunc] @@ -2137,6 +2304,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2152,6 +2325,9 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[flatfield]] + tweak_method = gradient + tweak_slits_thresh = 0.0 + tweak_slits_maxfrac = 0.0 slit_illum_finecorr = False [[wavelengths]] method = full_template @@ -2251,6 +2427,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2350,6 +2532,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2440,6 +2628,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2531,6 +2725,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2639,6 +2839,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2739,6 +2946,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -2872,6 +3085,15 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + scale_to_mean = True + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -2934,6 +3156,8 @@ Alterations to the default parameters are: [reduce] [[findobj]] find_trim_edge = 3, 3, + maxnumber_sci = 2 + maxnumber_std = 1 [[skysub]] global_sky_std = False [[extraction]] @@ -3008,6 +3232,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3023,18 +3253,16 @@ Alterations to the default parameters are: noise_floor = 0.01 [[flatfield]] spec_samp_coarse = 20.0 + tweak_method = gradient tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_illum_relative = True slit_illum_ref_idx = 14 - slit_illum_smooth_npix = 5 - fit_2d_det_response = True [[wavelengths]] fwhm_spat_order = 2 [[slitedges]] edge_thresh = 5 fit_order = 4 - pad = 2 [scienceframe] [[process]] mask_cr = True @@ -3085,16 +3313,18 @@ Alterations to the default parameters are: use_pattern = True [[arcframe]] [[[process]]] + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_pixelflat = False use_illumflat = False [[tiltframe]] [[[process]]] + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_pixelflat = False use_illumflat = False [[pixelflatframe]] [[[process]]] - combine = median satpix = nothing + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_pixelflat = False use_illumflat = False subtract_scattlight = True @@ -3112,6 +3342,7 @@ Alterations to the default parameters are: [[illumflatframe]] [[[process]]] satpix = nothing + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, use_illumflat = False use_pattern = True subtract_scattlight = True @@ -3120,6 +3351,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3132,32 +3369,32 @@ Alterations to the default parameters are: [[standardframe]] [[[process]]] mask_cr = True + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, noise_floor = 0.01 use_pattern = True [[flatfield]] spec_samp_coarse = 20.0 spat_samp = 1.0 + tweak_method = gradient tweak_slits_thresh = 0.0 tweak_slits_maxfrac = 0.0 slit_illum_relative = True slit_illum_ref_idx = 14 - slit_illum_smooth_npix = 5 fit_2d_det_response = True [[wavelengths]] fwhm_spat_order = 2 [[slitedges]] edge_thresh = 5 fit_order = 4 - pad = 2 [scienceframe] [[process]] mask_cr = True sigclip = 4.0 objlim = 1.5 + correct_nonlinear = -1.4e-07, -1.4e-07, -1.2e-07, -1.8e-07, noise_floor = 0.01 use_specillum = True use_pattern = True - subtract_scattlight = True [[[scattlight]]] finecorr_method = median [reduce] @@ -3183,7 +3420,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_blue [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3233,6 +3470,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3243,11 +3488,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61, + exprng = 1, 901, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.06 @@ -3285,7 +3532,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_blue_orig [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3335,6 +3582,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3345,11 +3600,13 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 [[standardframe]] - exprng = 1, 61, + exprng = 1, 901, [[[process]]] mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.06 @@ -3387,7 +3644,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3409,7 +3666,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3422,12 +3679,12 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3437,6 +3694,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3452,6 +3717,8 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.05 @@ -3500,7 +3767,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red_mark4 [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3522,7 +3789,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3535,12 +3802,12 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3550,6 +3817,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3565,6 +3840,8 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.05 @@ -3613,7 +3890,7 @@ Alterations to the default parameters are: spectrograph = keck_lris_red_orig [calibrations] [[biasframe]] - exprng = None, 0.001, + exprng = None, 1, [[[process]]] combine = median use_biasimage = False @@ -3635,7 +3912,7 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[pixelflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3648,12 +3925,12 @@ Alterations to the default parameters are: use_pixelflat = False use_illumflat = False [[traceframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] use_pixelflat = False use_illumflat = False [[illumflatframe]] - exprng = None, 60, + exprng = 0, 60, [[[process]]] satpix = nothing use_pixelflat = False @@ -3663,6 +3940,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + exprng = 0, 60, + [[[process]]] + combine = median + satpix = nothing + scale_to_mean = True + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3678,6 +3963,8 @@ Alterations to the default parameters are: mask_cr = True noise_floor = 0.01 spat_flexure_correct = True + [[flatfield]] + slit_illum_finecorr = False [[wavelengths]] sigdetect = 10.0 rms_thresh_frac_fwhm = 0.05 @@ -3792,6 +4079,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3920,6 +4215,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -3970,6 +4273,8 @@ Alterations to the default parameters are: noise_floor = 0.01 use_illumflat = False [reduce] + [[findobj]] + maxnumber_std = 1 [[skysub]] bspline_spacing = 0.8 [[extraction]] @@ -4063,6 +4368,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4208,6 +4521,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4352,6 +4673,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4482,6 +4811,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4602,6 +4939,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -4731,6 +5076,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -4856,6 +5209,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -4987,6 +5348,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -5112,6 +5481,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -5241,6 +5618,14 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = chebyshev + overscan_par = 1 + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = chebyshev @@ -5389,6 +5774,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5533,6 +5926,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5644,6 +6045,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5758,6 +6165,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5847,6 +6260,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -5954,6 +6373,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = odd_even + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = odd_even @@ -6058,6 +6485,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6176,6 +6610,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6296,6 +6737,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6428,6 +6877,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6536,6 +6992,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6624,6 +7087,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6723,6 +7192,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6816,6 +7291,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -6930,6 +7411,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7058,6 +7547,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7159,6 +7654,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7251,6 +7752,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7350,6 +7857,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7453,6 +7967,13 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7554,6 +8075,12 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7642,6 +8169,13 @@ Alterations to the default parameters are: satpix = nothing use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -7763,6 +8297,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7899,6 +8441,14 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_biasimage = False + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -7929,6 +8479,7 @@ Alterations to the default parameters are: ech_sigrej = 3.0 lamps = OH_XSHOOTER, sigdetect = 10.0 + fwhm_fromlines = False reid_arxiv = vlt_xshooter_nir.fits cc_thresh = 0.5 cc_local_thresh = 0.5 @@ -7957,7 +8508,7 @@ Alterations to the default parameters are: use_illumflat = False [reduce] [[findobj]] - trace_npoly = 8 + trace_npoly = 10 maxnumber_sci = 2 maxnumber_std = 1 [[skysub]] @@ -8055,6 +8606,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -8200,6 +8759,14 @@ Alterations to the default parameters are: use_biasimage = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + overscan_method = median + combine = median + satpix = nothing + use_biasimage = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] overscan_method = median @@ -8343,6 +8910,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing @@ -8448,6 +9022,13 @@ Alterations to the default parameters are: use_overscan = False use_pixelflat = False use_illumflat = False + [[slitless_pixflatframe]] + [[[process]]] + combine = median + satpix = nothing + use_overscan = False + use_pixelflat = False + use_illumflat = False [[scattlightframe]] [[[process]]] satpix = nothing diff --git a/doc/releases/1.16.1dev.rst b/doc/releases/1.16.1dev.rst index 4c00f878a2..10d9805408 100644 --- a/doc/releases/1.16.1dev.rst +++ b/doc/releases/1.16.1dev.rst @@ -14,6 +14,9 @@ Dependency Changes ------------------ - Support added for numpy>=2.0.0; numpy<=2.0.0 should still be supported +- Deprecated support for python 3.10 +- General update to dependencies to be roughly consistent with release of python + 3.11 Functionality/Performance Improvements and Additions ---------------------------------------------------- @@ -23,21 +26,40 @@ Functionality/Performance Improvements and Additions This is done by setting the parameter ``use_std_trace`` in FindObjPar. - Now PypeIt can handle the case where "Standard star trace does not match the number of orders in the echelle data" both in `run_pypeit` and in `pypeit_coadd_1dspec`. +- Added the functionality to use slitless flats to create pixelflats. Note: new frametype + `slitless_pixflat` is added to the PypeIt frametype list. +- The created pixelflats are stored in the reduction directory and in the PypeIt cache directory + `data/pixelflats`. +- Added a functionality that allows, when multiple frames are combined, to scale each + frame to have the same mean value before combining. To use this + functionality, the new parameter ``scale_mean`` should be set to ``True``. Instrument-specific Updates --------------------------- +- Improved LRIS frame typing, including the typing of slitless flats and sky flats. +- Improved HIRES frame typing and configuration setup. +- Added support for the (decommissioned) AAT/UHRF instrument +- Updated the requirements of a spectral flip for KCWI (blue) data. If all amplifiers are + used, the data will be flipped. If all amplifiers are used, the data will not be flipped. + Otherwise, the data will be flipped. - Updated X-Shooter detector gain and read noise to come from header, and updated plate scales to the most recent values from the manual. Detailed changes are: + - NIR arm: + - Platescale updated from 0.197 to 0.245 arcsec/pixel - Dark current updated from 0. to 72. e-/pixel/hr - Gain updated from 2.12 to 2.29 e-/DN + - VIS arm: + - Platescale updated from an order-dependent value, to being 0.154 arcsec/pixel for all orders + - UVB arm: + - Platescale updated from an order-dependent value, to being 0.164 arcsec/pixel for all orders @@ -60,10 +82,18 @@ Script Changes - Added the ``--extr`` parameter in the ``pypeit_sensfunc`` script (also as a SensFuncPar) to allow the user to specify the extraction method to use when computing the sensitivity function (before only optimal extraction was used). +- Added ``pypeit_show_pixflat`` script to inspect the (slitless) pixel flat + generated during the reduction and stored in ``data/pixelflats``. +- Added ``pypeit_chk_flexure`` script to check both spatial and spectral flexure applied to + the reduced data. +- Treatment of file names is now more formal. Compression signatures are now + considered, and filename matching is now more strict. Datamodel Changes ----------------- +- Adjusted spec1d datamodel to enable use with UVES_popler GUI tool + Under-the-hood Improvements --------------------------- @@ -85,6 +115,9 @@ Bug Fixes - Fix error "ValueError: setting an array element with a sequence. The requested array has an inhomogeneous shape after 1 dimensions..." occurring when unpacking the SpecObj spectrum but having an attribute of the SpecObj object that is None. +- Fixed an hidden bug that was causing the spatial flexure to fail. The bug was in the + `SlitTraceBitMask` class, where the function `exclude_for_flexure()` was not returning + the 'BOXSLIT' flag. diff --git a/doc/scripts.rst b/doc/scripts.rst index 0b37cc7b0c..e5c4aee3d6 100644 --- a/doc/scripts.rst +++ b/doc/scripts.rst @@ -566,6 +566,26 @@ The script usage can be displayed by calling the script with the .. include:: help/pypeit_chk_flats.rst +.. _pypeit_show_pixflat: + +pypeit_show_pixflat +------------------- + +Inspect in a Ginga window the (slitless) pixel flat produced by PypeIt and stored +in the PypeIt cache (see ref:`data_installation`). It displays each detector separately +in different channels. The script is useful for assessing the quality of the pixel-to-pixel +response of the detector. Typical call is: + +.. code-block:: console + + pypeit_show_pixflat PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + + +The script usage can be displayed by calling the script with the +``-h`` option: + +.. include:: help/pypeit_show_pixflat.rst + pypeit_show_2dspec ------------------ diff --git a/doc/setup.rst b/doc/setup.rst index 2a046b17e5..523ed2290d 100644 --- a/doc/setup.rst +++ b/doc/setup.rst @@ -22,6 +22,27 @@ preparatory script, :ref:`pypeit_obslog`, which provides a simple listing of the available data files; however, use of this script is optional in terms of setup for reducing your data. +.. _setup-file-searching: + +Raw File Searches +================= + +PypeIt scripts that search for raw files in a given directory base the search on +a list of file extensions. These are generally ``.fits`` and ``.fits.gz``, but +some spectrographs specify a different set. + +Some scripts allow you to specify the extension to use for the search, which +*must* be one of the allowed extensions for that spectrograph. E.g., +for a spectrograph that allows ``.fits`` and ``.fits.gz`` extension, you can +specify to only look for the ``.fits`` files, but you *cannot* have it look for +``.fits.bz2`` files. If your raw files have extensions that are currently not +allowed by the code, please `Submit an issue`_. + +If you have both compressed and uncompressed files in your directory, the search +function will generally find both. You are strongly encouraged to only include +one version (compressed or uncompressed) of each file in the directory with your +raw data. + .. _setup-metadata: Use of Metadata to Identify Instrument Configurations @@ -168,7 +189,9 @@ to be the same directory that holds the raw data. 1. First Execution ------------------ -We recommend you first execute ``pypeit_setup`` like this:: +We recommend you first execute ``pypeit_setup`` like this: + +.. code-block:: bash pypeit_setup -r path_to_your_raw_data/LB -s keck_lris_blue @@ -187,7 +210,9 @@ This execution of ``pypeit_setup`` searches for all `*.fits` and `*.fits.gz` files with the provided root directory. Generally, the provided path should **not** contain a wild-card and it is best if you provide the *full* path; however, you can search through multiple -directories as follows:: +directories as follows: + +.. code-block:: bash pypeit_setup -r "/Users/xavier/Keck/LRIS/data/2016apr06/Raw/*/LB" -s keck_lris_blue diff --git a/doc/spectrographs/aat_uhrf.rst b/doc/spectrographs/aat_uhrf.rst new file mode 100644 index 0000000000..ebef209ecf --- /dev/null +++ b/doc/spectrographs/aat_uhrf.rst @@ -0,0 +1,33 @@ +.. highlight:: rest + +******** +AAT UHRF +******** + + +Overview +======== + +This file summarizes several instrument specific +settings that are related to AAT/UHRF. + + +Wavelength Calibration +---------------------- + +UHRF has many wavelength setups, and the wavelength calibration +must be performed manually for each setup using :ref:`wvcalib-byhand` +approach and the :ref:`pypeit_identify` script. Since this spectrograph +is decommissioned, we do not expect to have a general solution +for this spectrograph. + +Object profile +-------------- + +UHRF is a slicer spectrograph, and the data are usually binned aggressively. +The object profile tends to be poorly characterised with the automated approach, +and you may need to generate your own wavelength dependent profile. Previously, +a Gaussian KDE profile was used, and this performed well, but is not available +by default. For users that are interested in this functionality, please contact +the PypeIt developers on the PypeIt User's Slack, or see the `aat_uhrf_rjc` +branch. diff --git a/doc/spectrographs/deimos.rst b/doc/spectrographs/deimos.rst index a6e4d8bc81..d9131773c8 100644 --- a/doc/spectrographs/deimos.rst +++ b/doc/spectrographs/deimos.rst @@ -15,9 +15,9 @@ settings that are related to the Keck/DEIMOS spectrograph. .. warning:: PypeIt currently *cannot* reduce images produced by reading - the DEIMOS CCDs with the A amplifier or those taken in imaging + the DEIMOS CCDs with the A+B amplifier or those taken in imaging mode. All image-handling assumes DEIMOS images have been read - with the B amplifier in the "Spectral" observing mode. PypeIt + with the A or B amplifier in the "Spectral" observing mode. PypeIt handles files that do not meet these criteria in two ways: - When running :ref:`pypeit_setup`, any frames not in diff --git a/doc/spectrographs/keck_hires.rst b/doc/spectrographs/keck_hires.rst index f8d3277554..f1628da181 100644 --- a/doc/spectrographs/keck_hires.rst +++ b/doc/spectrographs/keck_hires.rst @@ -6,13 +6,85 @@ Overview ======== This file summarizes several instrument specific settings that are related to the Keck/HIRES spectrograph. +Currently, PypeIt only supports the reduction of the post detector upgrade HIRES data (around August 2004). +Information on the frame typing and instrument configuration can be found in ref:`hiresframes` and :ref:`hires_config`. + +Default Settings +---------------- + +See :ref:`instr_par-keck_hires` for a list of modifications to the default settings. +*You do not have to add these changes to your PypeIt reduction file!* This is just a list of +how the parameters used for HIRES differ from the defaults listed in the preceding tables on that page. +Moreover, additional modifications may have been made for specific setups, e.g, for different binning, +etc. You can see a list of all the used parameters in the ``keck_hires_XXX.par`` +file generated by PypeIt at the beginning of the reduction. + +MOSAIC +====== + +PypeIt, by default, uses a mosaic approach for the reduction. It basically constructs a mosaic +of the blue, green, and red detector data and reduces it, instead of processing the detector data individually. +The mosaic reduction is switched on by setting the parameter ``detnum`` in :ref:`reduxpar` to be a +tuple of the detector indices that are mosaiced together. For HIRES, it looks like: + +.. code-block:: ini + + [rdx] + spectrograph = keck_hires + detnum = (1,2,3) + +This is already the default for HIRES, but the user can modify it in the :ref:`pypeit_file` to +turn off the mosaic reduction. + + +Calibrations +============ + +Flat Fielding +------------- +Special type of "slitless" flat images are sometime used for correcting +pixel-to-pixel variations in the HIRES data. These flats images are taken +with the cross disperser cover on, so that the light is somewhat diffused +and the orders are not visible. If this type of frames are present in the +:ref:`pypeit_file`, during the main reduction PypeIt will be able to identify them, +assign the correct frame type (`slitless_pixflat`), and use them for creating a "slitless" +pixel flat. Since, these frames are taken with different exposure time and +cross-disperser angle for each detector, PypeIt is able to select the correct +observations for each detector. The pixelflat is generated by first scaling +each flat frame to have the same mean counts, then median combining them. +A pixelflat file will be generated and stored in the reduction folder. +In addition, the constructed pixelflat is saved to the PypeIt cache (see ref:`data_installation`). +This allows you to use the file for both current and future reductions. +To use this file in future reductions, the user can add the +slitless pixelflat file name to the :ref:`pypeit_file` as shown below: + +.. code-block:: ini + + [calibrations] + [[flatfield]] + pixelflat_file = pixelflat_keck_hires_RED_1x2_20160330.fits.gz + +See more in :ref:`generate-pixflat`. + Wavelengths -=========== +----------- See :ref:`wvcalib-echelle` for details on the wavelength calibration. -We also note that several Orders from 40-45 are -frequently flagged as bad in the wavelength solution. -This is due, in part, to very bright ThAr line contamination. \ No newline at end of file + +Additional Reading +================== + +Here are additional docs related to Keck/HIRES. Note all of them are related +to the development of PypeIt for use with HIRES data: + +.. TODO: Generally useful information in these dev docs should be moved into +.. user-level doc pages, even if that means repeating information. + +.. toctree:: + :maxdepth: 1 + + ../dev/hiresframes + ../dev/hiresconfig \ No newline at end of file diff --git a/doc/spectrographs/lris.rst b/doc/spectrographs/lris.rst index f6b4b786c9..e28b7cf3d5 100644 --- a/doc/spectrographs/lris.rst +++ b/doc/spectrographs/lris.rst @@ -212,21 +212,45 @@ Pixel Flat It is recommend to correct for pixel-to-pixel variations using a slitless flat. If you did not take such calibration frames or cannot process them, -you may wish to use an archival. -`This link `__ -has the existing ones staged by the PypeIt team. +you may wish to use an archival one available in the PypeIt cache (see ref:`data_installation`). And then set the following in your :ref:`pypeit_file`: .. code-block:: ini - [calibrations] - [[flatfield]] - frame = path_to_the_file/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009.fits.gz - -.. warning:: - - Internal flats may be too bright and need to be tested. + [calibrations] + [[flatfield]] + pixelflat_file = PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz + + +If, instead, you have taken slitless pixelflat frames, PypeIt will be able to identify them +(see :ref:`lris_frames_report`) and process them during the reduction. The slitless pixelflat +is generated by first scaling each slitless flat frame to have the same mean counts, then +median combining them. A slitless pixelflat file will be generated +and stored in the reduction folder. In addition, the constructed pixelflat is saved to the +PypeIt cache. This allows you to use the file for both current and future reductions. +To use this file in future reductions, the user can add the +slitless pixelflat file name to the :ref:`pypeit_file` as shown above. Consider sharing +your slitless pixelflat file with the PypeIt Developers. See more in :ref:`generate-pixflat`. + +To generate slitless pixelflat, we recommend using twilight flats. As these are taken +on the twilight sky, it is important to dither between exposures to avoid bright stars +from landing in the same part of the detector. Related, it is best to take these as close +to sunset (or sunrise) as possible (about 10 minutes after/before sunset/sunrise) so that +one can use the shortest exposure time possible (3s for LRIS given its shutter). + +Internal flats may be too bright and need to be tested. +Our recommended recipe for taking twilight flats is: + + 1. Point away from the Galaxy (i.e. avoid stars) + 2. Select the desired setup for your spectroscopy + 3. Remove any selected slitmask/longslit + 4. Expose for 3s + 5. Check counts and make sure you have more than 40,000 + 6. Offset by ~1' and 1' in NE + 7. Repeat 4-6, increasing the exposure each time you dip below 30,000 counts or so + +About 5 exposures is enough but the more the better. Trace Flat ---------- diff --git a/doc/spectrographs/spectrographs.rst b/doc/spectrographs/spectrographs.rst index 275915a2c9..1fffa34730 100644 --- a/doc/spectrographs/spectrographs.rst +++ b/doc/spectrographs/spectrographs.rst @@ -33,6 +33,7 @@ instrument-specific details for running PypeIt. :caption: Spectrographs :maxdepth: 1 + aat_uhrf gemini_flamingos2 gemini_gmos gemini_gnirs diff --git a/environment.yml b/environment.yml index b52aa30b6e..c020c2fd15 100644 --- a/environment.yml +++ b/environment.yml @@ -2,7 +2,7 @@ name: pypeit channels: - defaults dependencies: - - python>=3.10,<3.13 + - python>=3.11,<3.13 - pip - pip: - pypeit diff --git a/presentations/py/users.py b/presentations/py/users.py index 9f3affcddd..73902997e4 100644 --- a/presentations/py/users.py +++ b/presentations/py/users.py @@ -21,9 +21,9 @@ def set_fontsize(ax, fsz): ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(fsz) -user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10"] +user_dates = ["2021-03-11", "2022-04-29", "2022-11-07", "2022-12-06", "2023-06-08", "2023-06-29", "2023-07-11", "2023-09-03", "2023-10-13", "2023-12-01", "2023-12-15", "2024-02-22", "2024-03-21", "2024-04-09", "2024-05-02", "2024-05-19", "2024-06-06", "2024-06-10", "2024-08-20"] user_dates = numpy.array([numpy.datetime64(date) for date in user_dates]) -user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620]) +user_number = numpy.array([125, 293, 390, 394, 477, 487, 506, 518, 531, 544, 551, 568, 579, 588, 596, 603, 616, 620, 643]) user_pred_dates = numpy.array([numpy.datetime64(date) for date in ["2024-06-10", "2024-12-31", "2025-12-31", "2026-12-31", diff --git a/pypeit/cache.py b/pypeit/cache.py index 52379cf6ad..b1d6555d1f 100644 --- a/pypeit/cache.py +++ b/pypeit/cache.py @@ -162,6 +162,7 @@ def fetch_remote_file( install_script: bool=False, force_update: bool=False, full_url: str=None, + return_none: bool=False, ) -> pathlib.Path: """ Use `astropy.utils.data`_ to fetch file from remote or cache @@ -191,6 +192,8 @@ def fetch_remote_file( full_url (:obj:`str`, optional): The full url. If None, use :func:`_build_remote_url`). Defaults to None. + return_none (:obj:`bool`, optional): + Return None if the file is not found. Defaults to False. Returns: `Path`_: The local path to the desired file in the cache @@ -248,6 +251,9 @@ def fetch_remote_file( "https://pypeit.readthedocs.io/en/latest/fluxing.html#extinction-correction" ) + elif return_none: + return None + else: err_msg = ( f"Error downloading {filename}: {error}{msgs.newline()}" diff --git a/pypeit/calibrations.py b/pypeit/calibrations.py index 13f60553c6..63c2d313b8 100644 --- a/pypeit/calibrations.py +++ b/pypeit/calibrations.py @@ -39,10 +39,13 @@ from pypeit.core import framematch from pypeit.core import parse from pypeit.core import scattlight as core_scattlight +from pypeit.core.mosaic import build_image_mosaic from pypeit.par import pypeitpar from pypeit.spectrographs.spectrograph import Spectrograph from pypeit import io from pypeit import utils +from pypeit import cache +from pypeit import dataPaths class Calibrations: @@ -732,21 +735,50 @@ def get_flats(self): # Check internals self._chk_set(['det', 'calib_ID', 'par']) - pixel_frame = {'type': 'pixelflat', 'class': flatfield.FlatImages} - raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ - = self.find_calibrations(pixel_frame['type'], pixel_frame['class']) - + # generate the slitless pixel flat (if frames available). + slitless_rows = self.fitstbl.find_frames('slitless_pixflat', calib_ID=self.calib_ID, index=True) + if len(slitless_rows) > 0: + sflat = flatfield.SlitlessFlat(self.fitstbl, slitless_rows, self.spectrograph, + self.par, qa_path=self.qa_path) + # A pixel flat will be saved to disc and self.par['flatfield']['pixelflat_file'] will be updated + self.par['flatfield']['pixelflat_file'] = \ + sflat.make_slitless_pixflat(msbias=self.msbias, msdark=self.msdark, calib_dir=self.calib_dir, + write_qa=self.write_qa, show=self.show) + + # get illumination flat frames illum_frame = {'type': 'illumflat', 'class': flatfield.FlatImages} raw_illum_files, illum_cal_file, illum_calib_key, illum_setup, illum_calib_id, detname \ = self.find_calibrations(illum_frame['type'], illum_frame['class']) + # get pixel flat frames + pixel_frame = {'type': 'pixelflat', 'class': flatfield.FlatImages} + raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ + = [], None, None, illum_setup, None, detname + # read in the raw pixelflat frames only if the user has not provided a pixelflat_file + if self.par['flatfield']['pixelflat_file'] is None: + raw_pixel_files, pixel_cal_file, pixel_calib_key, pixel_setup, pixel_calib_id, detname \ + = self.find_calibrations(pixel_frame['type'], pixel_frame['class']) + + # get lamp off flat frames raw_lampoff_files = self.fitstbl.find_frame_files('lampoffflats', calib_ID=self.calib_ID) + # Check if we have any calibration frames to work with if len(raw_pixel_files) == 0 and pixel_cal_file is None \ and len(raw_illum_files) == 0 and illum_cal_file is None: - msgs.warn(f'No raw {pixel_frame["type"]} or {illum_frame["type"]} frames found and ' - 'unable to identify a relevant processed calibration frame. Continuing...') - self.flatimages = None + # if no calibration frames are found, check if the user has provided a pixel flat file + if self.par['flatfield']['pixelflat_file'] is not None: + msgs.warn(f'No raw {pixel_frame["type"]} or {illum_frame["type"]} frames found but a ' + 'user-defined pixel flat file was provided. Using that file.') + self.flatimages = flatfield.FlatImages(PYP_SPEC=self.spectrograph.name, spat_id=self.slits.spat_id) + self.flatimages.calib_key = flatfield.FlatImages.construct_calib_key(self.fitstbl['setup'][self.frame], + self.calib_ID, detname) + self.flatimages = flatfield.load_pixflat(self.par['flatfield']['pixelflat_file'], self.spectrograph, + self.det, self.flatimages, calib_dir=self.calib_dir, + chk_version=self.chk_version) + else: + msgs.warn(f'No raw {pixel_frame["type"]} or {illum_frame["type"]} frames found and ' + 'unable to identify a relevant processed calibration frame. Continuing...') + self.flatimages = None return self.flatimages # If a processed calibration frame exists and we want to reuse it, do @@ -768,10 +800,9 @@ def get_flats(self): # Load user defined files if self.par['flatfield']['pixelflat_file'] is not None: # Load - msgs.info(f'Using user-defined file: {self.par["flatfield"]["pixelflat_file"]}') - with io.fits_open(self.par['flatfield']['pixelflat_file']) as hdu: - nrm_image = flatfield.FlatImages(pixelflat_norm=hdu[self.det].data) - self.flatimages = flatfield.merge(self.flatimages, nrm_image) + self.flatimages = flatfield.load_pixflat(self.par['flatfield']['pixelflat_file'], self.spectrograph, + self.det, self.flatimages, calib_dir=self.calib_dir, + chk_version=self.chk_version) # update slits self.slits.mask_flats(self.flatimages) return self.flatimages @@ -817,8 +848,8 @@ def get_flats(self): # Initialise the pixel flat pixelFlatField = flatfield.FlatField(pixel_flat, self.spectrograph, - self.par['flatfield'], self.slits, self.wavetilts, - self.wv_calib, qa_path=self.qa_path, + self.par['flatfield'], self.slits, wavetilts=self.wavetilts, + wv_calib=self.wv_calib, qa_path=self.qa_path, calib_key=calib_key) # Generate pixelflatImages = pixelFlatField.run(doqa=self.write_qa, show=self.show) @@ -863,8 +894,8 @@ def get_flats(self): # Initialise the illum flat illumFlatField = flatfield.FlatField(illum_flat, self.spectrograph, - self.par['flatfield'], self.slits, self.wavetilts, - self.wv_calib, spat_illum_only=True, + self.par['flatfield'], self.slits, wavetilts=self.wavetilts, + wv_calib=self.wv_calib, spat_illum_only=True, qa_path=self.qa_path, calib_key=calib_key) # Generate illumflatImages = illumFlatField.run(doqa=self.write_qa, show=self.show) @@ -899,10 +930,9 @@ def get_flats(self): # Should we allow that? if self.par['flatfield']['pixelflat_file'] is not None: # Load - msgs.info(f'Using user-defined file: {self.par["flatfield"]["pixelflat_file"]}') - with io.fits_open(self.par['flatfield']['pixelflat_file']) as hdu: - self.flatimages = flatfield.merge(self.flatimages, - flatfield.FlatImages(pixelflat_norm=hdu[self.det].data)) + self.flatimages = flatfield.load_pixflat(self.par['flatfield']['pixelflat_file'], self.spectrograph, + self.det, self.flatimages, calib_dir=self.calib_dir, + chk_version=self.chk_version) return self.flatimages @@ -1278,6 +1308,7 @@ def get_association(fitstbl, spectrograph, caldir, setup, calib_ID, det, must_ex 'pixelflat': [flatfield.FlatImages], 'illumflat': [flatfield.FlatImages], 'lampoffflats': [flatfield.FlatImages], + 'slitless_pixflat': [flatfield.FlatImages], 'trace': [edgetrace.EdgeTraceSet, slittrace.SlitTraceSet], 'tilt': [buildimage.TiltImage, wavetilts.WaveTilts] } @@ -1313,7 +1344,8 @@ def get_association(fitstbl, spectrograph, caldir, setup, calib_ID, det, must_ex indx = fitstbl.find_frames(frametype) & in_grp if not any(indx): continue - if not all(fitstbl['calib'][indx] == fitstbl['calib'][indx][0]): + if not (all(fitstbl['calib'][indx] == fitstbl['calib'][indx][0]) or + all([fitstbl['calib'][indx][0] in cc.split(',') for cc in fitstbl['calib'][indx]])): msgs.error(f'CODING ERROR: All {frametype} frames in group {calib_ID} ' 'are not all associated with the same subset of calibration ' 'groups; calib for the first file is ' @@ -1539,8 +1571,13 @@ def check_for_calibs(par, fitstbl, raise_error=True, cut_cfg=None): if ftype == 'pixelflat' \ and par['calibrations']['flatfield']['pixelflat_file'] is not None: continue + # Allow for no pixelflat but slitless_pixflat needs to exist + elif ftype == 'pixelflat' \ + and len(fitstbl.find_frame_files('slitless_pixflat', calib_ID=calib_ID)) > 0: + continue # Otherwise fail - msg = f'No frames of type={ftype} provide for the *{key}* processing ' \ + add_msg = ' or slitless_pixflat' if ftype == 'pixelflat' else '' + msg = f'No frames of type={ftype}{add_msg} provided for the *{key}* processing ' \ 'step. Add them to your PypeIt file!' pass_calib = False if raise_error: diff --git a/pypeit/coadd1d.py b/pypeit/coadd1d.py index e7ef072031..7084aacbc2 100644 --- a/pypeit/coadd1d.py +++ b/pypeit/coadd1d.py @@ -235,7 +235,7 @@ def load(self): msgs.error("Error in spec1d file for exposure {:d}: " "More than one object was identified with the OBJID={:s} in file={:s}".format( iexp, self.objids[iexp], self.spec1dfiles[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, _, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, blaze_iexp, _, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) waves.append(wave_iexp) fluxes.append(flux_iexp) @@ -486,7 +486,7 @@ def load_ech_arrays(self, spec1dfiles, objids, sensfuncfiles): indx = sobjs.name_indices(objids[iexp]) if not np.any(indx): msgs.error("No matching objects for {:s}. Odds are you input the wrong OBJID".format(objids[iexp])) - wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, _, _, meta_spec, header = \ + wave_iexp, flux_iexp, ivar_iexp, gpm_iexp, blaze_iexp, meta_spec, header = \ sobjs[indx].unpack_object(ret_flam=self.par['flux_value'], extract_type=self.par['ex_value']) # This np.atleast2d hack deals with the situation where we are wave_iexp is actually Multislit data, i.e. we are treating # it like an echelle spectrograph with a single order. This usage case arises when we want to use the diff --git a/pypeit/core/arc.py b/pypeit/core/arc.py index 086f7deb04..e41fc7fc8b 100644 --- a/pypeit/core/arc.py +++ b/pypeit/core/arc.py @@ -507,6 +507,7 @@ def get_censpec(slit_cen, slitmask, arcimg, gpm=None, box_rad=3.0, arc_spec[arc_spec_bpm] = 0.0 return arc_spec, arc_spec_bpm, np.all(arc_spec_bpm, axis=0) + def detect_peaks(x, mph=None, mpd=1, threshold=0, edge='rising', kpsh=False, valley=False, show=False, ax=None): """Detect peaks in data based on their amplitude and other features. @@ -783,7 +784,7 @@ def iter_continuum(spec, gpm=None, fwhm=4.0, sigthresh = 2.0, sigrej=3.0, niter_ max_nmask = int(np.ceil((max_mask_frac)*nspec_available)) for iter in range(niter_cont): spec_sub = spec - cont_now - mask_sigclip = np.invert(cont_mask & gpm) + mask_sigclip = np.logical_not(cont_mask & gpm) (mean, med, stddev) = stats.sigma_clipped_stats(spec_sub, mask=mask_sigclip, sigma_lower=sigrej, sigma_upper=sigrej, cenfunc='median', stdfunc=utils.nan_mad_std) # be very liberal in determining threshold for continuum determination diff --git a/pypeit/core/coadd.py b/pypeit/core/coadd.py index 3de7d319a8..3c1687c57a 100644 --- a/pypeit/core/coadd.py +++ b/pypeit/core/coadd.py @@ -688,12 +688,12 @@ def interp_spec(wave_new, waves, fluxes, ivars, gpms, log10_blaze_function=None, for ii in range(nexp): fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], log10_blazes_inter[:,ii] \ = interp_oned(wave_new, waves[:,ii], fluxes[:,ii], ivars[:,ii], gpms[:,ii], - log10_blaze_function = log10_blaze_function[:, ii], sensfunc=sensfunc, kind=kind) + log10_blaze_function=log10_blaze_function[:, ii], sensfunc=sensfunc, kind=kind) else: for ii in range(nexp): fluxes_inter[:,ii], ivars_inter[:,ii], gpms_inter[:,ii], _ \ = interp_oned(wave_new, waves[:,ii], fluxes[:,ii], ivars[:,ii], gpms[:,ii], sensfunc=sensfunc, kind=kind) - log10_blazes_inter=None + log10_blazes_inter = None return fluxes_inter, ivars_inter, gpms_inter, log10_blazes_inter diff --git a/pypeit/core/datacube.py b/pypeit/core/datacube.py index 9288e06695..aea12a2a5e 100644 --- a/pypeit/core/datacube.py +++ b/pypeit/core/datacube.py @@ -223,7 +223,7 @@ def extract_point_source(wave, flxcube, ivarcube, bpmcube, wcscube, exptime, Returns ------- - sobjs : `pypeit.specobjs.SpecObjs`_ + sobjs : :class:`pypeit.specobjs.SpecObjs` SpecObjs object containing the extracted spectrum """ if whitelight_range is None: diff --git a/pypeit/core/extract.py b/pypeit/core/extract.py index a820b27be5..83b2e1ad48 100644 --- a/pypeit/core/extract.py +++ b/pypeit/core/extract.py @@ -24,7 +24,8 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, - spec, min_frac_use=0.05, fwhmimg=None, base_var=None, count_scale=None, noise_floor=None): + spec, min_frac_use=0.05, fwhmimg=None, flatimg=None, + base_var=None, count_scale=None, noise_floor=None): r""" Perform optimal extraction `(Horne 1986) @@ -42,6 +43,7 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, - spec.OPT_COUNTS_NIVAR --> Optimally extracted noise variance (sky + read noise) only - spec.OPT_MASK --> Mask for optimally extracted flux - spec.OPT_FWHM --> Spectral FWHM (in A) for optimally extracted flux + - spec.OPT_FLAT --> Flat field spectrum, normalised at the peak value, for the optimally extracted flux - spec.OPT_COUNTS_SKY --> Optimally extracted sky - spec.OPT_COUNTS_SIG_DET --> Square root of optimally extracted read noise squared - spec.OPT_FRAC_USE --> Fraction of pixels in the object profile subimage used for this extraction @@ -88,6 +90,10 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, fwhmimg : `numpy.ndarray`_, None, optional: Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional: + Floating-point image containing the unnormalized flat-field image. This image + is used to extract the blaze function. Must have the same shape as ``sciimg``, + :math:`(N_{\rm spec}, N_{\rm spat})`. base_var : `numpy.ndarray`_, optional Floating-point "base-level" variance image set by the detector properties and the image processing steps. See :func:`~pypeit.core.procimg.base_variance`. @@ -145,6 +151,8 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, oprof_sub = oprof[:,mincol:maxcol] if fwhmimg is not None: fwhmimg_sub = fwhmimg[:,mincol:maxcol] + if flatimg is not None: + flatimg_sub = flatimg[:,mincol:maxcol] # enforce normalization and positivity of object profiles norm = np.nansum(oprof_sub,axis = 1) norm_oprof = np.outer(norm, np.ones(nsub)) @@ -190,6 +198,9 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, fwhm_opt = None if fwhmimg is not None: fwhm_opt = np.nansum(mask_sub*ivar_sub*fwhmimg_sub*oprof_sub, axis=1) * utils.inverse(tot_weight) + blaze_opt = None + if flatimg is not None: + blaze_opt = np.nansum(mask_sub*ivar_sub*flatimg_sub*oprof_sub, axis=1) * utils.inverse(tot_weight) # Interpolate wavelengths over masked pixels badwvs = (mivar_num <= 0) | np.invert(np.isfinite(wave_opt)) | (wave_opt <= 0.0) if badwvs.any(): @@ -221,6 +232,9 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, # Calculate the Angstroms/pixel and Spectral FWHM if fwhm_opt is not None: fwhm_opt *= np.gradient(wave_opt) # Convert pixel FWHM to Angstroms + # Normalize the blaze function + if blaze_opt is not None: + blaze_opt /= np.nanmax(blaze_opt) # Fill in the optimally extraction tags spec.OPT_WAVE = wave_opt # Optimally extracted wavelengths spec.OPT_COUNTS = flux_opt # Optimally extracted flux @@ -229,6 +243,8 @@ def extract_optimal(imgminsky, ivar, mask, waveimg, skyimg, thismask, oprof, spec.OPT_COUNTS_NIVAR = None if nivar_opt is None else nivar_opt*np.logical_not(badwvs) # Optimally extracted noise variance (sky + read noise) only spec.OPT_MASK = mask_opt*np.logical_not(badwvs) # Mask for optimally extracted flux spec.OPT_FWHM = fwhm_opt # Spectral FWHM (in Angstroms) for the optimally extracted spectrum + if blaze_opt is not None: + spec.OPT_FLAT = blaze_opt # Flat field spectrum, normalised to the peak value spec.OPT_COUNTS_SKY = sky_opt # Optimally extracted sky spec.OPT_COUNTS_SIG_DET = base_opt # Square root of optimally extracted read noise squared spec.OPT_FRAC_USE = frac_use # Fraction of pixels in the object profile subimage used for this extraction @@ -307,7 +323,7 @@ def extract_asym_boxcar(sciimg, left_trace, righ_trace, gpm=None, ivar=None): return flux_out, gpm_box, box_npix, ivar_out -def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, base_var=None, +def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, flatimg=None, base_var=None, count_scale=None, noise_floor=None): r""" Perform boxcar extraction for a single :class:`~pypeit.specobj.SpecObj`. @@ -325,6 +341,7 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b - spec.BOX_COUNTS_NIVAR --> Box car extracted noise variance - spec.BOX_MASK --> Box car extracted mask - spec.BOX_FWHM --> Box car extracted spectral FWHM + - spec.BOX_FLAT --> Box car extracted flatfield spectrum function (normalized to peak value) - spec.BOX_COUNTS_SKY --> Box car extracted sky - spec.BOX_COUNTS_SIG_DET --> Box car extracted read noise - spec.BOX_NPIX --> Number of pixels used in boxcar sum @@ -354,9 +371,12 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b Container that holds object, trace, and extraction information for the object in question. **This object is altered in place!** Note that this routine operates one object at a time. - fwhmimg : `numpy.ndarray`_, None, optional: + fwhmimg : `numpy.ndarray`_, None, optional Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional + Floating-point image containing the normalized flat-field. Must have the same shape as + ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. base_var : `numpy.ndarray`_, optional Floating-point "base-level" variance image set by the detector properties and the image processing steps. See :func:`~pypeit.core.procimg.base_variance`. @@ -365,7 +385,8 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b A scale factor, :math:`s`, that *has already been applied* to the provided science image. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that can be measured - from flat-field frames. For example, if the image has been flat-field + from flat-field frames plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``base_var``. The variance will be 0 @@ -406,6 +427,9 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b fwhm_box = None if fwhmimg is not None: fwhm_box = moment1d(fwhmimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] + blaze_box = None + if flatimg is not None: + blaze_box = moment1d(flatimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] varimg = 1.0/(ivar + (ivar == 0.0)) var_box = moment1d(varimg*mask, spec.TRACE_SPAT, 2*box_radius, row=spec.trace_spec)[0] nvar_box = None if var_no is None \ @@ -437,7 +461,11 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b # Calculate the Angstroms/pixel and the final spectral FWHM value if fwhm_box is not None: ang_per_pix = np.gradient(wave_box) - fwhm_box *= ang_per_pix / (pixtot - pixmsk) # Need to divide by total number of unmasked pixels + fwhm_box *= ang_per_pix * utils.inverse(pixtot - pixmsk) # Need to divide by total number of unmasked pixels + # Normalize the blaze function + if blaze_box is not None: + blaze_box *= utils.inverse(pixtot - pixmsk) # Need to divide by total number of unmasked pixels + blaze_box *= utils.inverse(np.nanmax(blaze_box[mask_box])) # Now normalize to the peak value # Fill em up! spec.BOX_WAVE = wave_box spec.BOX_COUNTS = flux_box*mask_box @@ -446,6 +474,8 @@ def extract_boxcar(imgminsky, ivar, mask, waveimg, skyimg, spec, fwhmimg=None, b spec.BOX_COUNTS_NIVAR = None if nivar_box is None else nivar_box*mask_box*np.logical_not(bad_box) spec.BOX_MASK = mask_box*np.logical_not(bad_box) spec.BOX_FWHM = fwhm_box # Spectral FWHM (in Angstroms) for the boxcar extracted spectrum + if blaze_box is not None: + spec.BOX_FLAT = blaze_box # Flat field spectrum, normalised to the peak value spec.BOX_COUNTS_SKY = sky_box spec.BOX_COUNTS_SIG_DET = base_box # TODO - Confirm this should be float, not int diff --git a/pypeit/core/findobj_skymask.py b/pypeit/core/findobj_skymask.py index f919f70bda..6e4698c844 100644 --- a/pypeit/core/findobj_skymask.py +++ b/pypeit/core/findobj_skymask.py @@ -1764,8 +1764,7 @@ def objs_in_slit(image, ivar, thismask, slit_left, slit_righ, flux_sum_smash = np.sum((image_rect*gpm_sigclip)[find_min_max_out[0]:find_min_max_out[1]], axis=0) flux_smash = flux_sum_smash*gpm_smash/(npix_smash + (npix_smash == 0.0)) flux_smash_mean, flux_smash_med, flux_smash_std = astropy.stats.sigma_clipped_stats( - flux_smash, mask=np.logical_not(gpm_smash), sigma_lower=3.0, sigma_upper=3.0 - ) + flux_smash, mask=np.logical_not(gpm_smash), sigma_lower=3.0, sigma_upper=3.0) flux_smash_recen = flux_smash - flux_smash_med # Return if none found and no hand extraction diff --git a/pypeit/core/flexure.py b/pypeit/core/flexure.py index 02265a3426..d4a816993f 100644 --- a/pypeit/core/flexure.py +++ b/pypeit/core/flexure.py @@ -17,6 +17,7 @@ from astropy import stats from astropy import units from astropy.io import ascii +from astropy.table import Table import scipy.signal import scipy.optimize as opt from scipy import interpolate @@ -36,7 +37,7 @@ from pypeit.datamodel import DataContainer from pypeit.images.detector_container import DetectorContainer from pypeit.images.mosaic import Mosaic -from pypeit import specobj, specobjs +from pypeit import specobj, specobjs, spec2dobj from pypeit import wavemodel from IPython import embed @@ -1394,6 +1395,83 @@ def sky_em_residuals(wave:np.ndarray, flux:np.ndarray, return dwave[m], diff[m], diff_err[m], los[m], los_err[m] +def flexure_diagnostic(file, file_type='spec2d', flexure_type='spec', chk_version=False): + """ + Print the spectral or spatial flexure of a spec2d or spec1d file + + Args: + file (:obj:`str`, `Path`_): + Filename of the spec2d or spec1d file to check + file_type (:obj:`str`, optional): + Type of the file to check. Options are 'spec2d' or 'spec1d'. Default + is 'spec2d'. + flexure_type (:obj:`str`, optional): + Type of flexure to check. Options are 'spec' or 'spat'. Default is + 'spec'. + chk_version (:obj:`bool`, optional): + If True, raise an error if the datamodel version or type check + failed. If False, throw a warning only. Default is False. + + Returns: + :obj:`astropy.table.Table`, :obj:`float`: If file_type is 'spec2d' and + flexure_type is 'spec', return a table with the spectral flexure. If + file_type is 'spec2d' and flexure_type is 'spat', return the spatial + flexure. If file_type is 'spec1d', return a table with the spectral + flexure. + """ + + # value to return + return_flex = None + + if file_type == 'spec2d': + # load the spec2d file + allspec2D = spec2dobj.AllSpec2DObj.from_fits(file, chk_version=chk_version) + # Loop on Detectors + for det in allspec2D.detectors: + print('') + print('=' * 50 + f'{det:^7}' + '=' * 51) + # get and print the spectral flexure + if flexure_type == 'spec': + spec_flex = allspec2D[det].sci_spec_flexure + spec_flex.rename_column('sci_spec_flexure', 'global_spec_shift') + if np.all(spec_flex['global_spec_shift'] != None): + spec_flex['global_spec_shift'].format = '0.3f' + # print the table + spec_flex.pprint_all() + # return the table + return_flex = spec_flex + # get and print the spatial flexure + if flexure_type == 'spat': + spat_flex = allspec2D[det].sci_spat_flexure + # print the value + print(f'Spat shift: {spat_flex}') + # return the value + return_flex = spat_flex + elif file_type == 'spec1d': + # no spat flexure in spec1d file + if flexure_type == 'spat': + msgs.error("Spat flexure not available in the spec1d file, try with a spec2d file") + # load the spec1d file + sobjs = specobjs.SpecObjs.from_fitsfile(file, chk_version=chk_version) + spec_flex = Table() + spec_flex['NAME'] = sobjs.NAME + spec_flex['global_spec_shift'] = sobjs.FLEX_SHIFT_GLOBAL + if np.all(spec_flex['global_spec_shift'] != None): + spec_flex['global_spec_shift'].format = '0.3f' + spec_flex['local_spec_shift'] = sobjs.FLEX_SHIFT_LOCAL + if np.all(spec_flex['local_spec_shift'] != None): + spec_flex['local_spec_shift'].format = '0.3f' + spec_flex['total_spec_shift'] = sobjs.FLEX_SHIFT_TOTAL + if np.all(spec_flex['total_spec_shift'] != None): + spec_flex['total_spec_shift'].format = '0.3f' + # print the table + spec_flex.pprint_all() + # return the table + return_flex = spec_flex + + return return_flex + + # TODO -- Consider separating the methods from the DataContainer as per calibrations class MultiSlitFlexure(DataContainer): """ diff --git a/pypeit/core/framematch.py b/pypeit/core/framematch.py index 4125f7011e..94200c405f 100644 --- a/pypeit/core/framematch.py +++ b/pypeit/core/framematch.py @@ -32,6 +32,7 @@ def __init__(self): ('lampoffflats', 'Flat-field exposure with lamps off used to remove ' 'persistence from lamp on flat exposures and/or thermal emission ' 'from the telescope and dome'), + ('slitless_pixflat', 'Flat-field exposure without slitmask used for pixel-to-pixel response'), ('scattlight', 'Frame (ideally with lots of counts) used to determine the scattered light model'), ('science', 'On-sky observation of a primary target'), ('standard', 'On-sky observation of a flux calibrator'), diff --git a/pypeit/core/procimg.py b/pypeit/core/procimg.py index c68be8d379..795a198446 100644 --- a/pypeit/core/procimg.py +++ b/pypeit/core/procimg.py @@ -1157,7 +1157,9 @@ def base_variance(rn_var, darkcurr=None, exptime=None, proc_var=None, count_scal - :math:`C` is the observed number of sky + object counts, - :math:`s=s\prime / N_{\rm frames}` is a scale factor derived from the (inverse of the) flat-field frames plus the number - of frames contributing to the object counts (see ``count_scale``), + of frames contributing to the object counts plus a scaling + factor applied if the counts of each frame are scaled to the + mean counts of all frames (see ``count_scale``), - :math:`D` is the dark current in electrons per **hour** (see ``darkcurr``), - :math:`t_{\rm exp}` is the effective exposure time in seconds (see @@ -1229,8 +1231,9 @@ def base_variance(rn_var, darkcurr=None, exptime=None, proc_var=None, count_scal A scale factor that *has already been applied* to the provided counts. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that - can be measured from flat-field frames. For example, if the image - has been flat-field corrected, this is the inverse of the flat-field counts. + can be measured from flat-field frames plus a scaling factor applied + if the counts of each frame are scaled to the mean counts of all frames. + For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``rn_var``. The variance will be 0 wherever :math:`s \leq 0`, modulo the provided ``noise_floor``. @@ -1287,7 +1290,9 @@ def variance_model(base, counts=None, count_scale=None, noise_floor=None): - :math:`C` is the observed number of sky + object counts, - :math:`s=s\prime / N_{\rm frames}` is a scale factor derived from the (inverse of the) flat-field frames plus the number - of frames contributing to the object counts (see ``count_scale``), + of frames contributing to the object counts plus a scaling factor + applied if the counts of each frame are scaled to the mean counts + of all frames (see ``count_scale``), - :math:`D` is the dark current in electrons per **hour**, - :math:`t_{\rm exp}` is the effective exposure time in seconds, - :math:`V_{\rm rn}` is the detector readnoise variance (i.e., @@ -1347,7 +1352,9 @@ def variance_model(base, counts=None, count_scale=None, noise_floor=None): A scale factor that *has already been applied* to the provided counts; see :math:`s` in the equations above. It accounts for the number of frames contributing to the provided counts, and - the relative throughput factors that can be measured from flat-field frames. + the relative throughput factors that can be measured from flat-field frames + plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, no scaling is expected, meaning ``counts`` are exactly the observed detector counts. If a single diff --git a/pypeit/core/skysub.py b/pypeit/core/skysub.py index d92a734b82..e1a44e53ca 100644 --- a/pypeit/core/skysub.py +++ b/pypeit/core/skysub.py @@ -248,7 +248,6 @@ def global_skysub(image, ivar, tilts, thismask, slit_left, slit_righ, inmask=Non return ythis - def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, fullbkpt=None): """ Utility routine used by local_skysub_extract that performs the joint b-spline fit for sky-background @@ -286,7 +285,7 @@ def skyoptimal(piximg, data, ivar, oprof, sigrej=3.0, npoly=1, spatial_img=None, fullbkpt : `numpy.ndarray`_, optional A 1d float array containing the breakpoints to be used for the B-spline fit. The breakpoints are arranged in the spectral - direction, i.e. along the directino of the piximg independent + direction, i.e. along the direction of the piximg independent variable. Returns @@ -543,7 +542,7 @@ def optimal_bkpts(bkpts_optimal, bsp_min, piximg, sampmask, samp_frac=0.80, def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, slit_left, slit_righ, sobjs, ingpm=None, bkg_redux_global_sky=None, - fwhmimg=None, spat_pix=None, adderr=0.01, bsp=0.6, + fwhmimg=None, flatimg=None, spat_pix=None, adderr=0.01, bsp=0.6, trim_edg=(3,3), std=False, prof_nsigma=None, niter=4, extract_good_frac=0.005, sigrej=3.5, bkpts_optimal=True, debug_bkpts=False, force_gauss=False, sn_gauss=4.0, model_full_slit=False, @@ -589,9 +588,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, Global sky model produced by global_skysub without the background subtraction. If sciimg is an A-B image, then this is the global sky modeled from the A image, while `global_sky` is the global modeled from the A-B image. This is None if the sciimg is not an A-B image. - fwhmimg : `numpy.ndarray`_, None, optional: + fwhmimg : `numpy.ndarray`_, None, optional Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional + Image containing the model of the flat field. If None, the + blaze function will not be calculated. spat_pix: `numpy.ndarray`_, optional Image containing the spatial location of pixels. If not input, it will be computed from ``spat_img = @@ -703,7 +705,8 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, A scale factor, :math:`s`, that *has already been applied* to the provided science image. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that can be measured - from flat-field frames. For example, if the image has been flat-field + from flat-field frames plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``base_var``. The variance will be 0 @@ -861,12 +864,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, objmask = ((spat_img >= (trace - 2.0 * sobjs[iobj].BOX_RADIUS)) & (spat_img <= (trace + 2.0 * sobjs[iobj].BOX_RADIUS))) # Boxcar extract.extract_boxcar(sciimg-skyimage, modelivar, (outmask & objmask), waveimg, - skyimage, sobjs[iobj], fwhmimg=fwhmimg, base_var=base_var, + skyimage, sobjs[iobj], fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # Optimal extract.extract_optimal(sciimg-skyimage, modelivar, (outmask & objmask), waveimg, skyimage, thismask, last_profile, sobjs[iobj], - fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, + fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # If the extraction is bad do not update if sobjs[iobj].OPT_MASK is not None: @@ -914,7 +917,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, isub, = np.where(localmask.flatten()) #sortpix = (piximg.flat[isub]).argsort() obj_profiles_flat = obj_profiles.reshape(nspec * nspat, objwork) - skymask = outmask & np.invert(edgmask) + skymask = outmask & np.logical_not(edgmask) sky_bmodel, obj_bmodel, outmask_opt = skyoptimal( piximg.flat[isub], sciimg.flat[isub], (modelivar * skymask).flat[isub], obj_profiles_flat[isub, :], spatial_img=spatial_img.flat[isub], @@ -973,10 +976,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, else: msgs.warn('ERROR: Bspline sky subtraction failed after 4 iterations of bkpt spacing') msgs.warn(' Moving on......') - obj_profiles = np.zeros_like(obj_profiles) + # obj_profiles = np.zeros_like(obj_profiles) isub, = np.where(localmask.flatten()) # Just replace with the global sky skyimage.flat[isub] = global_sky.flat[isub] + if iiter == niter: + msgs.warn('WARNING: LOCAL SKY SUBTRACTION NOT PERFORMED') outmask_extract = outmask if use_2dmodel_mask else inmask @@ -997,12 +1002,12 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, objmask = ((spat_img >= (trace - 2.0 * sobjs[iobj].BOX_RADIUS)) & (spat_img <= (trace + 2.0 * sobjs[iobj].BOX_RADIUS))) extract.extract_optimal(sciimg-skyimage, modelivar * thismask, (outmask_extract & objmask), waveimg, extract_sky, thismask, this_profile, sobjs[iobj], - fwhmimg=fwhmimg, base_var=base_var, count_scale=count_scale, + fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) # Boxcar extract.extract_boxcar(sciimg-skyimage, modelivar*thismask, (outmask_extract & objmask), waveimg, extract_sky, sobjs[iobj], - fwhmimg=fwhmimg, base_var=base_var, + fwhmimg=fwhmimg, flatimg=flatimg, base_var=base_var, count_scale=count_scale, noise_floor=adderr) sobjs[iobj].min_spat = min_spat sobjs[iobj].max_spat = max_spat @@ -1054,7 +1059,7 @@ def local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, global_sky, left, right, slitmask, sobjs, spat_pix=None, bkg_redux_global_sky=None, - fit_fwhm=False, + fit_fwhm=False, fwhmimg=None, flatimg=None, min_snr=2.0, bsp=0.6, trim_edg=(3,3), std=False, prof_nsigma=None, use_2dmodel_mask=True, niter=4, sigrej=3.5, bkpts_optimal=True, @@ -1127,6 +1132,12 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, fit_fwhm: bool, optional if True, perform a fit to the FWHM of the object profiles to use for non-detected sources + fwhmimg : `numpy.ndarray`_, None, optional + Floating-point image containing the modeled spectral FWHM (in pixels) at every pixel location. + Must have the same shape as ``sciimg``, :math:`(N_{\rm spec}, N_{\rm spat})`. + flatimg : `numpy.ndarray`_, None, optional + Image containing the model of the flat field. If None, the + blaze function will not be calculated. min_snr: float, optional FILL IN bsp : float, default = 0.6 @@ -1227,7 +1238,8 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, A scale factor that *has already been applied* to the provided science image. It accounts for the number of frames contributing to the provided counts, and the relative throughput factors that can be measured - from flat-field frames. For example, if the image has been flat-field corrected, + from flat-field frames plus a scaling factor applied if the counts of each frame are + scaled to the mean counts of all frames. For example, if the image has been flat-field corrected, this is the inverse of the flat-field counts. If None, set to 1. If a single float, assumed to be constant across the full image. If an array, the shape must match ``base_var``. The variance will be 0 wherever this @@ -1413,17 +1425,18 @@ def ech_local_skysub_extract(sciimg, sciivar, fullmask, tilts, waveimg, # Local sky subtraction and extraction skymodel[thismask], _this_bkg_redux_skymodel, objmodel[thismask], ivarmodel[thismask], extractmask[thismask] \ = local_skysub_extract(sciimg, sciivar, tilts, waveimg, global_sky, thismask, - left[:,iord], right[:,iord], sobjs[thisobj], - bkg_redux_global_sky=bkg_redux_global_sky, - spat_pix=spat_pix, ingpm=inmask, std=std, bsp=bsp, - trim_edg=trim_edg, prof_nsigma=prof_nsigma, niter=niter, - sigrej=sigrej, no_local_sky= no_local_sky, - use_2dmodel_mask=use_2dmodel_mask, - bkpts_optimal=bkpts_optimal, force_gauss=force_gauss, - sn_gauss=sn_gauss, model_full_slit=model_full_slit, - model_noise=model_noise, debug_bkpts=debug_bkpts, - show_resids=show_resids, show_profile=show_profile, - adderr=adderr, base_var=base_var, count_scale=count_scale) + left[:,iord], right[:,iord], sobjs[thisobj], + bkg_redux_global_sky=bkg_redux_global_sky, + fwhmimg=fwhmimg, flatimg=flatimg, + spat_pix=spat_pix, ingpm=inmask, std=std, bsp=bsp, + trim_edg=trim_edg, prof_nsigma=prof_nsigma, niter=niter, + sigrej=sigrej, no_local_sky=no_local_sky, + use_2dmodel_mask=use_2dmodel_mask, + bkpts_optimal=bkpts_optimal, force_gauss=force_gauss, + sn_gauss=sn_gauss, model_full_slit=model_full_slit, + model_noise=model_noise, debug_bkpts=debug_bkpts, + show_resids=show_resids, show_profile=show_profile, + adderr=adderr, base_var=base_var, count_scale=count_scale) if bkg_redux_skymodel is not None: bkg_redux_skymodel[thismask] = _this_bkg_redux_skymodel # update the FWHM fitting vector for the brighest object diff --git a/pypeit/core/telluric.py b/pypeit/core/telluric.py index 756dfc1189..de0e3c8e34 100644 --- a/pypeit/core/telluric.py +++ b/pypeit/core/telluric.py @@ -316,8 +316,11 @@ def conv_telluric(tell_model, dloglam, res): Resolution convolved telluric model. Shape = same size as input tell_model. """ - - + # Check the input values + if res <= 0.0: + msgs.error('Resolution must be positive.') + if dloglam == 0.0: + msgs.error('The telluric model grid has zero spacing in log wavelength. This is not supported.') pix_per_sigma = 1.0/res/(dloglam*np.log(10.0))/(2.0 * np.sqrt(2.0 * np.log(2))) # number of dloglam pixels per 1 sigma dispersion sig2pix = 1.0/pix_per_sigma # number of sigma per 1 pix if sig2pix > 2.0: @@ -334,6 +337,7 @@ def conv_telluric(tell_model, dloglam, res): conv_model = scipy.signal.convolve(tell_model,g,mode='same') return conv_model + def shift_telluric(tell_model, loglam, dloglam, shift, stretch): """ Routine to apply a shift to the telluric model. Note that the shift can be sub-pixel, i.e this routine interpolates. @@ -364,6 +368,7 @@ def shift_telluric(tell_model, loglam, dloglam, shift, stretch): tell_model_shift = np.interp(loglam_shift, loglam, tell_model) return tell_model_shift + def eval_telluric(theta_tell, tell_dict, ind_lower=None, ind_upper=None): """ Evaluate the telluric model. @@ -2410,18 +2415,12 @@ def __init__(self, wave, flux, ivar, gpm, telgridfile, obj_params, init_obj_mode self.disp = disp or debug self.sensfunc = sensfunc self.debug = debug - self.log10_blaze_func_in_arr = None # 2) Reshape all spectra to be (nspec, norders) - if log10_blaze_function is not None: + self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, self.log10_blaze_func_in_arr, \ + self.nspec_in, self.norders = utils.spec_atleast_2d( + wave, flux, ivar, gpm, log10_blaze_function=log10_blaze_function) - self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, self.log10_blaze_func_in_arr, \ - self.nspec_in, self.norders = utils.spec_atleast_2d( - wave, flux, ivar, gpm, log10_blaze_function=log10_blaze_function) - else: - self.wave_in_arr, self.flux_in_arr, self.ivar_in_arr, self.mask_in_arr, _, \ - self.nspec_in, self.norders = utils.spec_atleast_2d( - wave, flux, ivar, gpm) # 3) Read the telluric grid and initalize associated parameters wv_gpm = self.wave_in_arr > 1.0 if self.teltype == 'pca': diff --git a/pypeit/core/wavecal/wvutils.py b/pypeit/core/wavecal/wvutils.py index 219ae0ed03..d5b44af579 100644 --- a/pypeit/core/wavecal/wvutils.py +++ b/pypeit/core/wavecal/wvutils.py @@ -136,9 +136,13 @@ def get_sampling(waves, pix_per_R=3.0): wave_diff_flat += np.diff(wave_good).tolist() dloglam_flat += np.diff(np.log10(wave_good)).tolist() - + # Compute the median wavelength spacing dwave = np.median(wave_diff_flat) dloglam = np.median(dloglam_flat) + # Check that this won't introduce a divide by zero + if dloglam == 0.0: + msgs.error('The wavelength sampling has zero spacing in log wavelength. This is not supported.') + # Compute a guess of the resolution resln_guess = 1.0 / (pix_per_R* dloglam * np.log(10.0)) pix_per_sigma = 1.0 / resln_guess / (dloglam * np.log(10.0)) / (2.0 * np.sqrt(2.0 * np.log(2))) return dwave, dloglam, resln_guess, pix_per_sigma diff --git a/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits b/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits new file mode 100644 index 0000000000..4e2009a902 Binary files /dev/null and b/pypeit/data/arc_lines/reid_arxiv/aat_uhrf_3875.fits differ diff --git a/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B400_2x2_15apr2015_specflip.fits.gz b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B400_2x2_15apr2015_specflip.fits.gz new file mode 100644 index 0000000000..5fd3c06e3d Binary files /dev/null and b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B400_2x2_15apr2015_specflip.fits.gz differ diff --git a/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz new file mode 100644 index 0000000000..f22c724280 Binary files /dev/null and b/pypeit/data/pixelflats/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz differ diff --git a/pypeit/data/pixelflats/README b/pypeit/data/pixelflats/README new file mode 100644 index 0000000000..3617e4096c --- /dev/null +++ b/pypeit/data/pixelflats/README @@ -0,0 +1,11 @@ +Directory includes custom pixel flats to be used for Flat Fielding. + +The general naming structure is: +'pixelflat_{spec_name}{dispname}{dichroic}_{binning}_{date}.fits.gz' + +spec_name: spectrograph name (e.g. keck_lris_blue) +dispname: metadata dispname (if part of the spectrograph configuration keys) +dichroic: metadata dichroic (if part of the spectrograph configuration keys) +binning: spectral x spatial binning +date: date of the observations + diff --git a/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x2_20160330.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x2_20160330.fits.gz new file mode 100644 index 0000000000..e5bb58b7b6 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x2_20160330.fits.gz differ diff --git a/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x3_20170223.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x3_20170223.fits.gz new file mode 100644 index 0000000000..999703ae24 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_1x3_20170223.fits.gz differ diff --git a/pypeit/data/pixelflats/pixelflat_keck_hires_RED_2x2_20170614.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_2x2_20170614.fits.gz new file mode 100644 index 0000000000..e38d2d80f2 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_hires_RED_2x2_20170614.fits.gz differ diff --git a/pypeit/data/pixelflats/pixelflat_keck_lris_blue_600_4000_d560_2x2_20210411.fits.gz b/pypeit/data/pixelflats/pixelflat_keck_lris_blue_600_4000_d560_2x2_20210411.fits.gz new file mode 100644 index 0000000000..9eba3afdf1 Binary files /dev/null and b/pypeit/data/pixelflats/pixelflat_keck_lris_blue_600_4000_d560_2x2_20210411.fits.gz differ diff --git a/pypeit/extraction.py b/pypeit/extraction.py index 99f36f6432..0a818b1c22 100644 --- a/pypeit/extraction.py +++ b/pypeit/extraction.py @@ -64,7 +64,7 @@ class Extract: @classmethod def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_sky=None, bkg_redux_global_sky=None, waveTilts=None, tilts=None, wv_calib=None, waveimg=None, - bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): + flatimg=None, bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): """ Instantiate the Extract subclass appropriate for the provided spectrograph. @@ -101,6 +101,9 @@ def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, glob This is the waveCalib object which is optional, but either wv_calib or waveimg must be provided. waveimg (`numpy.ndarray`_, optional): Wave image. Either a wave image or wv_calib object (above) must be provided + flatimg (`numpy.ndarray`_, optional): + Flat image. This is optional, but if provided, it is used to extract the + normalized blaze profile. Same shape as ``sciImg``. bkg_redux (:obj:`bool`, optional): If True, the sciImg has been subtracted by a background image (e.g. standard treatment in the IR) @@ -128,12 +131,12 @@ def get_instance(cls, sciImg, slits, sobjs_obj, spectrograph, par, objtype, glob if c.__name__ == (spectrograph.pypeline + 'Extract'))( sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_sky=global_sky, bkg_redux_global_sky=bkg_redux_global_sky, waveTilts=waveTilts, tilts=tilts, - wv_calib=wv_calib, waveimg=waveimg, bkg_redux=bkg_redux, return_negative=return_negative, + wv_calib=wv_calib, waveimg=waveimg, flatimg=flatimg, bkg_redux=bkg_redux, return_negative=return_negative, std_redux=std_redux, show=show, basename=basename) def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_sky=None, bkg_redux_global_sky=None, waveTilts=None, tilts=None, wv_calib=None, waveimg=None, - bkg_redux=False, return_negative=False, std_redux=False, show=False, + flatimg=None, bkg_redux=False, return_negative=False, std_redux=False, show=False, basename=None): # Setup the parameters sets for this object. NOTE: This uses objtype, not frametype! @@ -146,6 +149,7 @@ def __init__(self, sciImg, slits, sobjs_obj, spectrograph, par, objtype, global_ self.par = par self.global_sky = global_sky if global_sky is not None else np.zeros_like(sciImg.image) self.bkg_redux_global_sky = bkg_redux_global_sky + self.flatimg = flatimg self.basename = basename # Parse @@ -776,7 +780,7 @@ def local_skysub_extract(self, global_sky, sobjs, bkg_redux_global_sky=None, self.slits_right[:, slit_idx], self.sobjs[thisobj], ingpm=ingpm, bkg_redux_global_sky=bkg_redux_global_sky, - fwhmimg=self.fwhmimg, spat_pix=spat_pix, + fwhmimg=self.fwhmimg, flatimg=self.flatimg, spat_pix=spat_pix, model_full_slit=model_full_slit, sigrej=sigrej, model_noise=model_noise, std=self.std_redux, bsp=bsp, @@ -888,6 +892,7 @@ def local_skysub_extract(self, global_sky, sobjs, bkg_redux_global_sky=None, self.slits_right[:, gdorders], self.slitmask, sobjs, bkg_redux_global_sky=bkg_redux_global_sky, spat_pix=spat_pix, + fwhmimg=self.fwhmimg, flatimg=self.flatimg, std=self.std_redux, fit_fwhm=fit_fwhm, min_snr=min_snr, bsp=bsp, sigrej=sigrej, force_gauss=force_gauss, sn_gauss=sn_gauss, diff --git a/pypeit/flatfield.py b/pypeit/flatfield.py index dd45ad026a..b11735ce51 100644 --- a/pypeit/flatfield.py +++ b/pypeit/flatfield.py @@ -5,11 +5,15 @@ .. include:: ../include/links.rst """ +from pathlib import Path +from copy import deepcopy import inspect import numpy as np from scipy import interpolate, ndimage +from astropy.io import fits + from matplotlib import pyplot as plt from matplotlib import gridspec @@ -22,13 +26,21 @@ from pypeit import datamodel from pypeit import calibframe +from pypeit import edgetrace +from pypeit import io from pypeit.display import display +from pypeit.images import buildimage from pypeit.core import qa from pypeit.core import flat from pypeit.core import tracewave from pypeit.core import basis from pypeit.core import fitting +from pypeit.core import parse +from pypeit.core.mosaic import build_image_mosaic +from pypeit.spectrographs.util import load_spectrograph from pypeit import slittrace +from pypeit import dataPaths +from pypeit import cache class FlatImages(calibframe.CalibFrame): @@ -483,10 +495,14 @@ class FlatField: corrections. If None, the default parameters are used. slits (:class:`~pypeit.slittrace.SlitTraceSet`): The current slit traces. - wavetilts (:class:`~pypeit.wavetilts.WaveTilts`): - The current wavelength tilt traces; see - wv_calib (:class:`~pypeit.wavecalib.WaveCalib`): - Wavelength calibration object + wavetilts (:class:`~pypeit.wavetilts.WaveTilts`, optional): + The current fit to the wavelength tilts. I can be None, + for example, if slitless is True. + wv_calib (:class:`~pypeit.wavecalib.WaveCalib`, optional): + Wavelength calibration object. It can be None, for example, if + slitless is True. + slitless (bool, optional): + True if the input rawflatimg is a slitless flat. Default is False. spat_illum_only (bool, optional): Only perform the spatial illumination calculation, and ignore the 2D bspline fit. This should only be set to true if you @@ -510,8 +526,8 @@ class FlatField: Image of the relative spectral illumination for a multislit spectrograph """ - def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts, wv_calib, - spat_illum_only=False, qa_path=None, calib_key=None): + def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts=None, wv_calib=None, + slitless=False, spat_illum_only=False, qa_path=None, calib_key=None): # Defaults self.spectrograph = spectrograph @@ -528,7 +544,8 @@ def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts, wv_calib self.wv_calib = wv_calib # Worth a check - self.wavetilts.is_synced(self.slits) + if self.wavetilts is not None and not slitless: + self.wavetilts.is_synced(self.slits) # Attributes unique to this Object self.rawflatimg = rawflatimg # Un-normalized pixel flat as a PypeItImage @@ -540,6 +557,13 @@ def __init__(self, rawflatimg, spectrograph, flatpar, slits, wavetilts, wv_calib self.spat_illum_only = spat_illum_only self.spec_illum = None # Relative spectral illumination image self.waveimg = None + self.slitless = slitless # is this a slitless flat? + + # get waveimg here if available + if self.wavetilts is None or self.wv_calib is None: + msgs.warn("Wavelength calib or tilts are not available. Wavelength image not generated.") + else: + self.build_waveimg() # this set self.waveimg # Completed steps self.steps = [] @@ -578,13 +602,20 @@ def run(self, doqa=False, debug=False, show=False): :class:`FlatImages`: Container with the results of the flat-field analysis. """ + + # check if self.wavetilts is available. It can be None if the flat is slitless, but it's needed otherwise + if self.wavetilts is None and not self.slitless: + msgs.warn("Wavelength tilts are not available. Cannot generate this flat image.") + return None + # Fit it # NOTE: Tilts do not change and self.slits is updated internally. if not self.flatpar['fit_2d_det_response']: # This spectrograph does not have a structure correction # implemented. Ignore detector structure. self.fit(spat_illum_only=self.spat_illum_only, doqa=doqa, debug=debug) - else: # Iterate on the pixelflat if required by the spectrograph + elif self.waveimg is not None: + # Iterate on the pixelflat if required by the spectrograph # User has requested a structure correction. # Note: This will only be performed if it is coded for each individual spectrograph. # Make a copy of the original flat @@ -671,11 +702,14 @@ def build_waveimg(self): Generate an image of the wavelength of each pixel. """ msgs.info("Generating wavelength image") - flex = self.wavetilts.spat_flexure - slitmask = self.slits.slit_img(initial=True, flexure=flex) - tilts = self.wavetilts.fit2tiltimg(slitmask, flexure=flex) - # Save to class attribute for inclusion in the Flat calibration frame - self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) + if self.wavetilts is None or self.wv_calib is None: + msgs.error("Wavelength calib or tilts are not available. Cannot generate wavelength image.") + else: + flex = self.wavetilts.spat_flexure + slitmask = self.slits.slit_img(initial=True, flexure=flex) + tilts = self.wavetilts.fit2tiltimg(slitmask, flexure=flex) + # Save to class attribute for inclusion in the Flat calibration frame + self.waveimg = self.wv_calib.build_waveimg(tilts, self.slits, spat_flexure=flex) def show(self, wcs_match=True): """ @@ -816,9 +850,6 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): npoly = self.flatpar['twod_fit_npoly'] saturated_slits = self.flatpar['saturated_slits'] - # Build wavelength image -- not always used, but for convenience done here - if self.waveimg is None: self.build_waveimg() - # Setup images nspec, nspat = self.rawflatimg.image.shape rawflat = self.rawflatimg.image @@ -833,14 +864,11 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): ivar_log = gpm_log.astype(float)/0.5**2 # Get the non-linear count level - # TODO: This is currently hacked to deal with Mosaics - try: + if self.rawflatimg.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in self.rawflatimg.detector.detectors]) + else: nonlinear_counts = self.rawflatimg.detector.nonlinear_counts() - except: - nonlinear_counts = 1e10 - # Other setup -# nonlinear_counts = self.spectrograph.nonlinear_counts(self.rawflatimg.detector) -# nonlinear_counts = self.rawflatimg.detector.nonlinear_counts() # TODO -- JFH -- CONFIRM THIS SHOULD BE ON INIT # It does need to be *all* of the slits @@ -962,10 +990,13 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): # TODO: Put this stuff in a self.spectral_fit method? # Create the tilts image for this slit - # TODO -- JFH Confirm the sign of this shift is correct! - _flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure - tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], - self.wavetilts['func2d'], spat_shift=-1*_flexure) + if self.slitless: + tilts = np.tile(np.arange(rawflat.shape[0]) / rawflat.shape[0], (rawflat.shape[1], 1)).T + else: + # TODO -- JFH Confirm the sign of this shift is correct! + _flexure = 0. if self.wavetilts.spat_flexure is None else self.wavetilts.spat_flexure + tilts = tracewave.fit2tilts(rawflat.shape, self.wavetilts['coeffs'][:,:,slit_idx], + self.wavetilts['func2d'], spat_shift=-1*_flexure) # Convert the tilt image to an image with the spectral pixel index spec_coo = tilts * (nspec-1) @@ -1283,19 +1314,35 @@ def fit(self, spat_illum_only=False, doqa=True, debug=False): * np.fmax(self.msillumflat[onslit_tweak], 0.05) \ * np.fmax(spec_model[onslit_tweak], 1.0) + # Check for infinities and NaNs in the flat-field model + winfnan = np.where(np.logical_not(np.isfinite(self.flat_model[onslit_tweak]))) + if winfnan[0].size != 0: + msgs.warn('There are {0:d} pixels with non-finite values in the flat-field model ' + 'for slit {1:d}!'.format(winfnan[0].size, slit_spat) + msgs.newline() + + 'These model pixel values will be set to the raw pixel value.') + self.flat_model[np.where(onslit_tweak)[0][winfnan]] = rawflat[np.where(onslit_tweak)[0][winfnan]] + # Check for unrealistically high or low values of the model + whilo = np.where((self.flat_model[onslit_tweak] >= nonlinear_counts) | + (self.flat_model[onslit_tweak] <= 0.0)) + if whilo[0].size != 0: + msgs.warn('There are {0:d} pixels with unrealistically high or low values in the flat-field model ' + 'for slit {1:d}!'.format(whilo[0].size, slit_spat) + msgs.newline() + + 'These model pixel values will be set to the raw pixel value.') + self.flat_model[np.where(onslit_tweak)[0][whilo]] = rawflat[np.where(onslit_tweak)[0][whilo]] + # Construct the pixel flat #trimmed_slitid_img_anew = self.slits.slit_img(pad=-trim, slitidx=slit_idx) #onslit_trimmed_anew = trimmed_slitid_img_anew == slit_spat - self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak]/self.flat_model[onslit_tweak] + self.mspixelflat[onslit_tweak] = rawflat[onslit_tweak] * utils.inverse(self.flat_model[onslit_tweak]) # TODO: Add some code here to treat the edges and places where fits # go bad? # Minimum wavelength? - if self.flatpar['pixelflat_min_wave'] is not None: + if self.flatpar['pixelflat_min_wave'] is not None and self.waveimg is not None: bad_wv = self.waveimg[onslit_tweak] < self.flatpar['pixelflat_min_wave'] self.mspixelflat[np.where(onslit_tweak)[0][bad_wv]] = 1. # Maximum wavelength? - if self.flatpar['pixelflat_max_wave'] is not None: + if self.flatpar['pixelflat_max_wave'] is not None and self.waveimg is not None: bad_wv = self.waveimg[onslit_tweak] > self.flatpar['pixelflat_max_wave'] self.mspixelflat[np.where(onslit_tweak)[0][bad_wv]] = 1. @@ -1378,7 +1425,8 @@ def spatial_fit(self, norm_spec, spat_coo, median_slit_width, spat_gpm, gpm, deb return exit_status, spat_coo_data, spat_flat_data, spat_bspl, spat_gpm_fit, \ spat_flat_fit, spat_flat_data_raw - def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, slit_trim=3, doqa=False): + def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, + slit_trim=3, tolerance=0.1, doqa=False): """ Generate a relative scaling image for a slicer IFU. All slits are scaled relative to a reference slit, specified in @@ -1402,6 +1450,10 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s Trim the slit edges by this number of pixels during the fitting. Note that the fit will be evaluated on the pixels indicated by onslit_tweak. A positive number trims the slit edges, a negative number pads the slit edges. + tolerance : float, optional + Tolerance for the relative scaling of the slits. A value of 0.1 means that the + relative scaling of the slits must be within 10% of unity. Any data outside of + this tolerance will be masked. doqa : :obj:`bool`, optional: Save the QA? @@ -1410,6 +1462,10 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s illumflat_finecorr: `numpy.ndarray`_ An image (same shape as normed) containing the fine correction to the spatial illumination profile """ + # check id self.waveimg is available + if self.waveimg is None: + msgs.warn("Cannot perform the fine correction to the spatial illumination without the wavelength image.") + return # TODO :: Include fit_order in the parset?? fit_order = np.array([3, 6]) slit_txt = self.slits.slitord_txt @@ -1421,7 +1477,8 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s onslit_tweak_trim = self.slits.slit_img(pad=-slit_trim, slitidx=slit_idx, initial=False) == slit_spat # Setup slitimg = (slit_spat + 1) * onslit_tweak.astype(int) - 1 # Need to +1 and -1 so that slitimg=-1 when off the slit - left, right, msk = self.slits.select_edges(flexure=self.wavetilts.spat_flexure) + + left, right, msk = self.slits.select_edges(flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0) this_left = left[:, slit_idx] this_right = right[:, slit_idx] slitlen = int(np.median(this_right - this_left)) @@ -1431,7 +1488,7 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s this_wave = self.waveimg[this_slit] xpos_img = self.slits.spatial_coordinate_image(slitidx=slit_idx, slitid_img=slitimg, - flexure_shift=self.wavetilts.spat_flexure) + flexure_shift=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0) # Generate the trimmed versions for fitting this_slit_trim = np.where(onslit_tweak_trim & self.rawflatimg.select_flag(invert=True)) this_wave_trim = self.waveimg[this_slit_trim] @@ -1449,6 +1506,10 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s if xfrac * slitlen < slit_trim: xfrac = slit_trim/slitlen gpmfit[np.where((xpos_fit < xfrac) | (xpos_fit > 1-xfrac))] = False + # If the data deviate too much from unity, mask them. We're only interested in a + # relative correction that's less than ~10% from unity. + gpmfit[np.where((normed[this_slit_trim] < 1-tolerance) | (normed[this_slit_trim] > 1 + tolerance))] = False + # Perform the full fit fullfit = fitting.robust_fit(xpos_fit, normed[this_slit_trim], fit_order, x2=ypos_fit, in_gpm=gpmfit, function='legendre2d', upper=2, lower=2, maxdev=1.0, minx=0.0, maxx=1.0, minx2=0.0, maxx2=1.0) @@ -1459,7 +1520,10 @@ def spatial_fit_finecorr(self, normed, onslit_tweak, slit_idx, slit_spat, gpm, s illumflat_finecorr[this_slit] = fullfit.eval(xpos, ypos) else: msgs.warn(f"Fine correction to the spatial illumination failed for {slit_txt} {slit_ordid}") - return + return illumflat_finecorr + + # If corrections exceed the tolerance, then clip them to the level of the tolerance + illumflat_finecorr = np.clip(illumflat_finecorr, 1-tolerance, 1+tolerance) # Prepare QA if doqa: @@ -1496,6 +1560,11 @@ def extract_structure(self, rawflat_orig, slit_trim=3): divided by the spectral and spatial illumination profile fits). """ msgs.info("Extracting flatfield structure") + + # check if the waveimg is available + if self.waveimg is None: + msgs.error("Cannot perform the extraction of the flatfield structure without the wavelength image.") + # Build the mask and make a temporary instance of FlatImages bpmflats = self.build_mask() # Initialise bad splines (for when the fit goes wrong) @@ -1519,7 +1588,7 @@ def extract_structure(self, rawflat_orig, slit_trim=3): scale_model = illum_profile_spectral(rawflat, self.waveimg, self.slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], model=None, gpmask=gpm, skymask=None, trim=self.flatpar['slit_trim'], - flexure=self.wavetilts.spat_flexure, + flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0, smooth_npix=self.flatpar['slit_illum_smooth_npix']) # Trim the edges by a few pixels to avoid edge effects onslits_trim = gpm & (self.slits.slit_img(pad=-slit_trim, initial=False) != -1) @@ -1565,8 +1634,10 @@ def spectral_illumination(self, gpm=None, debug=False): An image containing the appropriate scaling """ msgs.info("Deriving spectral illumination profile") - # Generate a wavelength image - if self.waveimg is None: self.build_waveimg() + # check if the waveimg is available + if self.waveimg is None: + msgs.warn("Cannot perform the spectral illumination without the wavelength image.") + return None msgs.info('Performing a joint fit to the flat-field response') # Grab some parameters trim = self.flatpar['slit_trim'] @@ -1580,12 +1651,13 @@ def spectral_illumination(self, gpm=None, debug=False): return illum_profile_spectral(rawflat, self.waveimg, self.slits, slit_illum_ref_idx=self.flatpar['slit_illum_ref_idx'], model=None, gpmask=gpm, skymask=None, trim=trim, - flexure=self.wavetilts.spat_flexure, + flexure=self.wavetilts.spat_flexure if self.wavetilts is not None else 0.0, smooth_npix=self.flatpar['slit_illum_smooth_npix'], debug=debug) - def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', thresh=0.93, maxfrac=0.1, debug=False): - """ + def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', thresh=0.93, + maxfrac=0.1, debug=False): + r""" Tweak the slit edges based on the normalized slit illumination profile. Args: @@ -1606,12 +1678,15 @@ def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', profile. Shape is :math:`(N_{\rm flat},)`. method (:obj:`str`, optional): Method to use for tweaking the slit edges. Options are: - - 'threshold': Use the threshold to set the slit edge - and then shift it to the left or right based on the - illumination profile. - - 'gradient': Use the gradient of the illumination - profile to set the slit edge and then shift it to - the left or right based on the illumination profile. + + - ``'threshold'``: Use the threshold to set the slit edge + and then shift it to the left or right based on the + illumination profile. + + - ``'gradient'``: Use the gradient of the illumination + profile to set the slit edge and then shift it to the left + or right based on the illumination profile. + thresh (:obj:`float`, optional): Threshold of the normalized flat profile at which to place the two slit edges. @@ -1630,13 +1705,14 @@ def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', tuple: Returns six objects: - The threshold used to set the left edge - - The fraction of the slit that the left edge is shifted to - the right + - The fraction of the slit that the left edge is shifted to the + right - The adjusted left edge - The threshold used to set the right edge - - The fraction of the slit that the right edge is shifted to - the left + - The fraction of the slit that the right edge is shifted to the + left - The adjusted right edge + """ # TODO :: Since this is just a wrapper, and not really "core", maybe it should be moved to pypeit.flatfield? # Tweak the edges via the specified method @@ -1649,6 +1725,208 @@ def tweak_slit_edges(self, left, right, spat_coo, norm_flat, method='threshold', msgs.error("Method for tweaking slit edges not recognized: {0}".format(method)) +class SlitlessFlat: + """ + Class to generate a slitless pixel flat-field calibration image. + + Args: + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + slitless_rows (`numpy.ndarray`_): + Boolean array selecting the rows in the fitstbl that + correspond to the slitless frames. + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): + The spectrograph object. + par (:class:`~pypeit.par.pypeitpar.CalibrationsPar`): + Parameter set defining optional parameters of PypeIt's algorithms + for Calibrations + qa_path (`Path`_): + Path for the QA diagnostics. + + """ + + def __init__(self, fitstbl, slitless_rows, spectrograph, par, qa_path=None): + + self.fitstbl = fitstbl + # Boolean array selecting the rows in the fitstbl that correspond to the slitless frames. + self.slitless_rows = slitless_rows + self.spectrograph = spectrograph + self.par = par + self.qa_path = qa_path + + def slitless_pixflat_fname(self): + """ + Generate the name of the slitless pixel flat file. + + Returns: + :obj:`str`: The name of the slitless pixel flat + + """ + if len(self.slitless_rows) == 0: + msgs.error('No slitless_pixflat frames found. Cannot generate the slitless pixel flat file name.') + + # generate the slitless pixel flat file name + spec_name = self.fitstbl.spectrograph.name + date = self.fitstbl.construct_obstime(self.slitless_rows[0]).iso.split(' ')[0].replace('-', '') if \ + self.fitstbl[self.slitless_rows][0]['mjd'] is not None else '00000000' + # setup info to add to the filename + dispname = '' if 'dispname' not in self.spectrograph.configuration_keys() else \ + f"_{self.fitstbl[self.slitless_rows[0]]['dispname'].replace('/', '_').replace(' ', '_').replace('(', '').replace(')', '').replace(':', '_').replace('+', '_')}" + dichroic = '' if 'dichroic' not in self.spectrograph.configuration_keys() else \ + f"_d{self.fitstbl[self.slitless_rows[0]]['dichroic']}" + binning = self.fitstbl[self.slitless_rows[0]]['binning'].replace(',', 'x') + # file name + return f'pixelflat_{spec_name}{dispname}{dichroic}_{binning}_{date}.fits' + + def make_slitless_pixflat(self, msbias=None, msdark=None, calib_dir=None, write_qa=False, show=False): + """ + Generate and save to disc a slitless pixel flat-field calibration images. + The pixel flat file will have one extension per detector, even in the case of a mosaic. + Contrary to the regular calibration flow, the slitless pixel flat is created for all detectors + of the current spectrograph at once, and not only the one for the current detector. + Since the slitless pixel flat images are saved to disc, this approach helps with the I/O + This is a method is used in `~pypeit.calibrations.get_flats()`. + + Note: par['flatfield']['pixelflat_file'] is updated in this method. + + Args: + msbias (:class:`~pypeit.images.buildimage.BiasImage`, optional): + Bias image for bias subtraction; passed to + :func:`~pypeit.images.buildimage.buildimage_fromlist()` + msdark (:class:`~pypeit.images.buildimage.DarkImage`, optional): + Dark-current image; passed to + :func:`~pypeit.images.buildimage.buildimage_fromlist()` + calib_dir (`Path`_): + Path for the processed calibration files. + write_qa (:obj:`bool`, optional): + Write QA plots to disk? + show (:obj:`bool`, optional): + Show the diagnostic plots? + + Returns: + :obj:`str`: The name of the slitless pixel flat file that was generated. + + """ + + # First thing first, check if the user has provided slitless_pixflat frames + if len(self.slitless_rows) == 0: + # return unchanged self.par['flatfield']['pixelflat_file'] + return self.par['flatfield']['pixelflat_file'] + + # all detectors of this spectrograph + _detectors = np.array(self.spectrograph.select_detectors()) + + # Check if a user-provided slitless pixelflat already exists for the current detectors + if self.par['flatfield']['pixelflat_file'] is not None: + _pixel_flat_file = dataPaths.pixelflat.get_file_path(self.par['flatfield']['pixelflat_file'], + return_none=True) + + if _pixel_flat_file is not None: + # get detector names + detnames = np.array([self.spectrograph.get_det_name(_det) for _det in _detectors]) + # open the file + with io.fits_open(_pixel_flat_file) as hdu: + # list of available detectors in the pixel flat file + file_detnames = [h.name.split('-')[0] for h in hdu] + # check if the current detnames are in the list + in_file = np.array([d in file_detnames for d in detnames]) + # if all detectors are in the file, return + if np.all(in_file): + msgs.info(f"Both slitless_pixflat frames and user-defined file found. " + f"The user-defined file will be used: {self.par['flatfield']['pixelflat_file']}") + # return unchanged self.par['flatfield']['pixelflat_file'] + return self.par['flatfield']['pixelflat_file'] + else: + # get the detectors that are not in the file + _detectors = _detectors[np.logical_not(in_file)] + detnames = detnames[np.logical_not(in_file)] + msgs.info(f'Both slitless_pixflat frames and user-defined file found, but the ' + f'following detectors are not in the file: {detnames}. Using the ' + f'slitless_pixflat frames to generate the missing detectors.') + + # make the slitless pixel flat + pixflat_norm_list = [] + detname_list = [] + for _det in _detectors: + # Parse the raw slitless pixelflat frames. Note that this is spectrograph dependent. + # If the method does not exist in the specific spectrograph class, nothing will happen + this_raw_idx = self.spectrograph.parse_raw_files(self.fitstbl[self.slitless_rows], det=_det, + ftype='slitless_pixflat') + if len(this_raw_idx) == 0: + msgs.warn(f'No raw slitless_pixflat frames found for {self.spectrograph.get_det_name(_det)}. ' + f'Continuing...') + continue + this_raw_files = self.fitstbl.frame_paths(self.slitless_rows[this_raw_idx]) + msgs.info(f'Creating slitless pixel-flat calibration frame ' + f'for {self.spectrograph.get_det_name(_det)} using files: ') + for f in this_raw_files: + msgs.prindent(f'{Path(f).name}') + + # Reset the BPM + msbpm = self.spectrograph.bpm(this_raw_files[0], _det, msbias=msbias if self.par['bpm_usebias'] else None) + + # trace image + traceimg = buildimage.buildimage_fromlist(self.spectrograph, _det, self.par['traceframe'], + [this_raw_files[0]], dark=msdark, bias=msbias, bpm=msbpm) + # slit edges + # we need to change some parameters for the slit edge tracing + edges_par = deepcopy(self.par['slitedges']) + # lower the threshold for edge detection + edges_par['edge_thresh'] = 50. + # this is used for longslit (i.e., no pca) + edges_par['sync_predict'] = 'nearest' + # remove spurious edges by setting a large minimum slit gap (20% of the detector size + platescale = parse.parse_binning(traceimg.detector.binning)[1] * traceimg.detector['platescale'] + edges_par['minimum_slit_gap'] = 0.2 * traceimg.image.shape[1] * platescale + # if no slits are found the bound_detector parameter add 2 traces at the detector edges + edges_par['bound_detector'] = True + # set the buffer to 0 + edges_par['det_buffer'] = 0 + _spectrograph = deepcopy(self.spectrograph) + # need to treat this as a MultiSlit spectrograph (no echelle parameters used) + _spectrograph.pypeline = 'MultiSlit' + edges = edgetrace.EdgeTraceSet(traceimg, _spectrograph, edges_par, auto=True) + slits = edges.get_slits() + if show: + edges.show(title='Slitless flat edge tracing') + # + # flat image + slitless_pixel_flat = buildimage.buildimage_fromlist(self.spectrograph, _det, self.par['slitless_pixflatframe'], + this_raw_files, dark=msdark, bias=msbias, bpm=msbpm) + + # increase saturation threshold (some hires slitless flats are very bright) + slitless_pixel_flat.detector.saturation *= 1.5 + # Initialise the pixel flat + flatpar = deepcopy(self.par['flatfield']) + # do not tweak the slits + flatpar['tweak_slits'] = False + pixelFlatField = FlatField(slitless_pixel_flat, self.spectrograph, flatpar, slits, wavetilts=None, + wv_calib=None, slitless=True, qa_path=self.qa_path) + + # Generate + pixelflatImages = pixelFlatField.run(doqa=write_qa, show=show) + pixflat_norm_list.append(pixelflatImages.pixelflat_norm) + detname_list.append(self.spectrograph.get_det_name(_det)) + + if len(detname_list) > 0: + # get the pixel flat file name + if self.par['flatfield']['pixelflat_file'] is not None and _pixel_flat_file is not None: + fname = self.par['flatfield']['pixelflat_file'] + else: + fname = self.slitless_pixflat_fname() + # file will be saved in the reduction directory, but also cached in the data/pixelflats folder + # therefore we update self.par['flatfield']['pixelflat_file'] to the new file, + # so that it can be used for the rest of the reduction and for the other files in the same run + self.par['flatfield']['pixelflat_file'] = fname + + # Save the result + write_pixflat_to_fits(pixflat_norm_list, detname_list, self.spectrograph.name, + calib_dir.parent if calib_dir is not None else Path('.').absolute(), + fname, to_cache=True) + + return self.par['flatfield']['pixelflat_file'] + + def spatillum_finecorr_qa(normed, finecorr, left, right, ypos, cut, outfile=None, title=None, half_slen=50): """ Plot the QA for the fine correction fits to the spatial illumination profile @@ -2095,4 +2373,203 @@ def merge(init_cls, merge_cls): return FlatImages(**dd) +def write_pixflat_to_fits(pixflat_norm_list, detname_list, spec_name, outdir, pixelflat_name, to_cache=True): + """ + Write the pixel-to-pixel flat-field images to a FITS file. + The FITS file will have an extension for each detector (never a mosaic). + The `load_pixflat()` method read this file and transform it into a mosaic if needed. + This image is generally used as a user-provided pixel flat-field image and ingested + in the reduction using the `pixelflat_file` parameter in the PypeIt file. + + Args: + pixflat_norm_list (:obj:`list`): + List of 2D `numpy.ndarray`_ arrays containing the pixel-to-pixel flat-field images. + detname_list (:obj:`list`): + List of detector names. + spec_name (:obj:`str`): + Name of the spectrograph. + outdir (:obj:`pathlib.Path`): + Path to the output directory. + pixelflat_name (:obj:`str`): + Name of the output file to be written. + to_cache (:obj:`bool`, optional): + If True, the file will be written to the cache directory pypeit/data/pixflats. + + """ + + msgs.info("Writing the pixel-to-pixel flat-field images to a FITS file.") + + # Check that the number of detectors matches the number of pixelflat_norm arrays + if len(pixflat_norm_list) != len(detname_list): + msgs.error("The number of detectors does not match the number of pixelflat_norm arrays. " + "The pixelflat file cannot be written.") + + # local output (reduction directory) + pixelflat_file = outdir / pixelflat_name + + # Check if the file already exists + old_hdus = [] + old_detnames = [] + old_hdr = None + if pixelflat_file.exists(): + msgs.warn("The pixelflat file already exists. It will be overwritten/updated.") + old_hdus = fits.open(pixelflat_file) + old_detnames = [h.name.split('-')[0] for h in old_hdus] # this has also 'PRIMARY' + old_hdr = old_hdus[0].header + + # load spectrograph + spec = load_spectrograph(spec_name) + + # Create the new HDUList + _hdr = io.initialize_header(hdr=old_hdr) + prihdu = fits.PrimaryHDU(header=_hdr) + prihdu.header['CALIBTYP'] = (FlatImages.calib_type, 'PypeIt: Calibration frame type') + new_hdus = [prihdu] + + extnum = 1 + for d in spec.select_detectors(): + detname = spec.get_det_name(d) + extname = f'{detname}-PIXELFLAT_NORM' + # update or add the detectors that we want to save + if detname in detname_list: + det_idx = detname_list.index(detname) + pixflat_norm = pixflat_norm_list[det_idx] + hdu = fits.ImageHDU(data=pixflat_norm, name=extname) + prihdu.header[f'EXT{extnum:04d}'] = hdu.name + new_hdus.append(hdu) + # keep the old detectors that were not updated + elif detname in old_detnames: + old_det_idx = old_detnames.index(detname) + hdu = old_hdus[old_det_idx] + prihdu.header[f'EXT{extnum:04d}'] = hdu.name + new_hdus.append(hdu) + extnum += 1 + + # Write the new HDUList + new_hdulist = fits.HDUList(new_hdus) + # Check if the directory exists + if not pixelflat_file.parent.is_dir(): + pixelflat_file.parent.mkdir(parents=True) + new_hdulist.writeto(pixelflat_file, overwrite=True) + msgs.info(f'A slitless Pixel Flat file for detectors {detname_list} has been saved to {msgs.newline()}' + f'{pixelflat_file}') + + # common msg + add_msgs = f"add the following to your PypeIt Reduction File:{msgs.newline()}" \ + f" [calibrations]{msgs.newline()}" \ + f" [[flatfield]]{msgs.newline()}" \ + f" pixelflat_file = {pixelflat_name}{msgs.newline()}{msgs.newline()}{msgs.newline()}" \ + f"Please consider sharing your Pixel Flat file with the PypeIt Developers.{msgs.newline()}" \ + + + if to_cache: + # NOTE that the file saved in the cache is gzipped, while the one saved in the outdir is not + # This prevents `dataPaths.pixelflat.get_file_path()` from returning the file saved in the outdir + cache.write_file_to_cache(pixelflat_file, pixelflat_name+'.gz', f"pixelflats") + msgs.info(f"The slitless Pixel Flat file has also been saved to the PypeIt cache directory {msgs.newline()}" + f"{str(dataPaths.pixelflat)} {msgs.newline()}" + f"It will be automatically used in this run. " + f"If you want to use this file in future runs, {add_msgs}") + else: + msgs.info(f"To use this file, move it to the PypeIt data directory {msgs.newline()}" + f"{str(dataPaths.pixelflat)} {msgs.newline()} and {add_msgs}") + + +def load_pixflat(pixel_flat_file, spectrograph, det, flatimages, calib_dir=None, chk_version=False): + """ + Load a pixel flat from a file and add it to the flatimages object. + The pixel flat file has one detector per extension, even in the case of a mosaic. + Therefore, if this is a mosaic reduction, this script will construct a pixel flat + mosaic. The Edges file needs to exist in the Calibration Folder, since the mosaic + parameters are pulled from it. + This is used in `~pypeit.calibrations.get_flats()`. + + Args: + pixel_flat_file (:obj:`str`): + Name of the pixel flat file. + spectrograph (:class:`~pypeit.spectrographs.spectrograph.Spectrograph`): + The spectrograph object. + det (:obj:`int`, :obj:`tuple`): + The single detector or set of detectors in a mosaic to process. + flatimages (:class:`~pypeit.flatfield.FlatImages`): + The flat field images object. + calib_dir (:obj:`str`, optional): + The path to the calibration directory. + chk_version (:obj:`bool`, optional): + Check the version of the file. + + Returns: + :class:`~pypeit.flatfield.FlatImages`: The flat images object with the pixel flat added. + + """ + # Check if the pixel flat file exists + if pixel_flat_file is None: + msgs.error('No pixel flat file defined. Cannot load the pixel flat!') + + # get the path + _pixel_flat_file = dataPaths.pixelflat.get_file_path(pixel_flat_file, return_none=True) + if _pixel_flat_file is None: + msgs.error(f'Cannot load the pixel flat file, {pixel_flat_file}. It is not a direct path, ' + f'a cached file, or a file that can be downloaded from a PypeIt repository.') + + # If this is a mosaic, we need to construct the pixel flat mosaic + if isinstance(det, tuple): + # We need to grab mosaic info from another existing calibration frame. + # We use EdgeTraceSet image to get `tform` and `msc_ord`. Check if EdgeTraceSet file exists. + edges_file = Path(edgetrace.EdgeTraceSet.construct_file_name(flatimages.calib_key, + calib_dir=calib_dir)).absolute() + if not edges_file.exists(): + msgs.error('Edges file not found in the Calibrations folder. ' + 'It is needed to grab the mosaic parameters to load and mosaic the input pixel flat!') + + # Load detector info from EdgeTraceSet file + traceimg = edgetrace.EdgeTraceSet.from_file(edges_file, chk_version=chk_version).traceimg + det_info = traceimg.detector + # check that the mosaic parameters are defined + if not np.all(np.in1d(['tform', 'msc_ord'], list(det_info.keys()))) or \ + det_info.tform is None or det_info.msc_ord is None: + msgs.error('Mosaic parameters are not defined in the Edges frame. Cannot load the pixel flat!') + + # read the file + with io.fits_open(_pixel_flat_file) as hdu: + # list of available detectors in the pixel flat file + file_dets = [int(h.name.split('-')[0].split('DET')[1]) for h in hdu[1:]] + # check if all detectors required for the mosaic are in the list + if not np.all(np.in1d(list(det), file_dets)): + msgs.error(f'Not all detectors in the mosaic are in the pixel flat file: ' + f'{pixel_flat_file}. Cannot load the pixel flat!') + + # get the pixel flat images of only the detectors in the mosaic + pixflat_images = np.concatenate([hdu[f'DET{d:02d}-PIXELFLAT_NORM'].data[None,:,:] for d in det]) + # construct the pixel flat mosaic + pixflat_msc, _,_,_ = build_image_mosaic(pixflat_images, det_info.tform, order=det_info.msc_ord) + # check that the mosaic has the correct shape + if pixflat_msc.shape != traceimg.image.shape: + msgs.error('The constructed pixel flat mosaic does not have the correct shape. ' + 'Cannot load this pixel flat as a mosaic!') + msgs.info(f'Using pixelflat file: {pixel_flat_file} ' + f'for {spectrograph.get_det_name(det)}.') + nrm_image = FlatImages(pixelflat_norm=pixflat_msc) + + # If this is not a mosaic, we can simply read the pixel flat for the current detector + else: + # current detector name + detname = spectrograph.get_det_name(det) + # read the file + with io.fits_open(_pixel_flat_file) as hdu: + # list of available detectors in the pixel flat file + file_detnames = [h.name.split('-')[0] for h in hdu] # this list has also the 'PRIMARY' extension + # check if the current detector is in the list + if detname in file_detnames: + # get the index of the current detector + idx = file_detnames.index(detname) + # get the pixel flat image + msgs.info(f'Using pixelflat file: {pixel_flat_file} for {detname}.') + nrm_image = FlatImages(pixelflat_norm=hdu[idx].data) + else: + msgs.error(f'{detname} not found in the pixel flat file: ' + f'{pixel_flat_file}. Cannot load the pixel flat!') + nrm_image = None + + return merge(flatimages, nrm_image) diff --git a/pypeit/images/combineimage.py b/pypeit/images/combineimage.py index 635cb4905d..99fb4f1629 100644 --- a/pypeit/images/combineimage.py +++ b/pypeit/images/combineimage.py @@ -22,9 +22,10 @@ class CombineImage: Process and combine detector images. Args: - rawImages (:obj:`list`, :class:`~pypeit.images.pypeitimage.PypeItImage): - Either a single :class:`~pypeit.images.pypeitimage.PypeItImage` object or a list of one or more - of these objects to be combined into a an image. + rawImages (:obj:`list`, :class:`~pypeit.images.pypeitimage.PypeItImage`): + Either a single :class:`~pypeit.images.pypeitimage.PypeItImage` + object or a list of one or more of these objects to be combined into + an image. par (:class:`~pypeit.par.pypeitpar.ProcessImagesPar`): Parameters that dictate the processing of the images. @@ -194,6 +195,33 @@ def run(self, ignore_saturation=False, maxiters=5): else: comb_texp = exptime[0] + # scale the images to their mean, if requested, before combining + if self.par['scale_to_mean']: + msgs.info("Scaling images to have the same mean before combining") + # calculate the mean of the images + [mean_img], _, mean_gpm, _ = combine.weighted_combine(np.ones(self.nimgs, dtype=float)/self.nimgs, + [img_stack], + [rn2img_stack], + # var_list is added because it is + # required by the function but not used + gpm_stack, sigma_clip=self.par['clip'], + sigma_clip_stack=img_stack, + sigrej=self.par['comb_sigrej'], maxiters=maxiters) + + # scale factor + # TODO: Chose the median over the whole frame to avoid outliers. Is this the right choice? + _mscale = np.nanmedian(mean_img[None, mean_gpm]/img_stack[:, mean_gpm], axis=1) + # reshape the scale factor + mscale = _mscale[:, None, None] + # scale the images + img_stack *= mscale + # scale the scales + scl_stack *= mscale + + # scale the variances + rn2img_stack *= mscale**2 + basev_stack *= mscale**2 + # Coadd them if self.par['combine'] == 'mean': weights = np.ones(self.nimgs, dtype=float)/self.nimgs diff --git a/pypeit/images/mosaic.py b/pypeit/images/mosaic.py index 9bf4d95355..1ab8dd5fdc 100644 --- a/pypeit/images/mosaic.py +++ b/pypeit/images/mosaic.py @@ -30,7 +30,7 @@ class Mosaic(datamodel.DataContainer): """ # Set the version of this class - version = '1.0.0' + version = '1.0.1' # WARNING: `binning` and `platescale` have the same names as datamodel # components in pypeit.images.detector_container.DetectorContainer. This is @@ -53,14 +53,14 @@ class Mosaic(datamodel.DataContainer): 'tform': dict(otype=np.ndarray, atype=float, descr='The full transformation matrix for each detector used to ' 'construct the mosaic.'), - 'msc_order': dict(otype=int, descr='Order of the interpolation used to construct the mosaic.')} + 'msc_ord': dict(otype=int, descr='Order of the interpolation used to construct the mosaic.')} name_prefix = 'MSC' """ Prefix for the name of the mosaic. """ - def __init__(self, id, detectors, shape, shift, rot, tform, msc_order): + def __init__(self, id, detectors, shape, shift, rot, tform, msc_ord): args, _, _, values = inspect.getargvalues(inspect.currentframe()) d = dict([(k,values[k]) for k in args[1:]]) @@ -107,8 +107,8 @@ def _bundle(self): tbl['rot'] = self.rot if self.tform is not None: tbl['tform'] = self.tform - if self.msc_order is not None: - tbl.meta['msc_order'] = self.msc_order + if self.msc_ord is not None: + tbl.meta['msc_ord'] = self.msc_ord if self.id is not None: tbl.meta['id'] = self.id if self.shape is not None: @@ -213,5 +213,5 @@ def copy(self): """ Return a (deep) copy of the object. """ - return Mosaic(id=self.id, detectors=np.array([det.copy() for det in self.detectors]), shape=self.shape, shift=self.shift.copy(), rot=self.rot.copy(), tform=self.tform.copy(), msc_order=self.msc_order) + return Mosaic(id=self.id, detectors=np.array([det.copy() for det in self.detectors]), shape=self.shape, shift=self.shift.copy(), rot=self.rot.copy(), tform=self.tform.copy(), msc_ord=self.msc_ord) diff --git a/pypeit/images/rawimage.py b/pypeit/images/rawimage.py index 0e8fb26758..8389cc0ac9 100644 --- a/pypeit/images/rawimage.py +++ b/pypeit/images/rawimage.py @@ -1358,7 +1358,7 @@ def build_mosaic(self): # Transform the image data to the mosaic frame. This call determines # the shape of the mosaic image and adjusts the relative transforms to # the absolute mosaic frame. - self.image, _, _img_npix, _tforms = build_image_mosaic(self.image, self.mosaic.tform, order=self.mosaic.msc_order) + self.image, _, _img_npix, _tforms = build_image_mosaic(self.image, self.mosaic.tform, order=self.mosaic.msc_ord) shape = self.image.shape # Maintain dimensionality self.image = np.expand_dims(self.image, 0) @@ -1369,7 +1369,7 @@ def build_mosaic(self): # Transform the BPM and maintain its type bpm_type = self.bpm.dtype - self._bpm = build_image_mosaic(self.bpm.astype(float), _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self._bpm = build_image_mosaic(self.bpm.astype(float), _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] # Include pixels that have no contribution from the original image in # the bad pixel mask of the mosaic. self._bpm[_img_npix < 1] = 1 @@ -1384,29 +1384,29 @@ def build_mosaic(self): # Get the pixels associated with each amplifier self.datasec_img = build_image_mosaic(self.datasec_img.astype(float), _tforms, - mosaic_shape=shape, order=self.mosaic.msc_order)[0] + mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.datasec_img = np.expand_dims(np.round(self.datasec_img).astype(int), 0) # Get the pixels associated with each detector self.det_img = build_image_mosaic(self.det_img.astype(float), _tforms, - mosaic_shape=shape, order=self.mosaic.msc_order)[0] + mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.det_img = np.expand_dims(np.round(self.det_img).astype(int), 0) # Transform all the variance arrays, as necessary if self.rn2img is not None: - self.rn2img = build_image_mosaic(self.rn2img, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.rn2img = build_image_mosaic(self.rn2img, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.rn2img = np.expand_dims(self.rn2img, 0) if self.dark is not None: - self.dark = build_image_mosaic(self.dark, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.dark = build_image_mosaic(self.dark, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.dark = np.expand_dims(self.dark, 0) if self.dark_var is not None: - self.dark_var = build_image_mosaic(self.dark_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.dark_var = build_image_mosaic(self.dark_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.dark_var = np.expand_dims(self.dark_var, 0) if self.proc_var is not None: - self.proc_var = build_image_mosaic(self.proc_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.proc_var = build_image_mosaic(self.proc_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.proc_var = np.expand_dims(self.proc_var, 0) if self.base_var is not None: - self.base_var = build_image_mosaic(self.base_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_order)[0] + self.base_var = build_image_mosaic(self.base_var, _tforms, mosaic_shape=shape, order=self.mosaic.msc_ord)[0] self.base_var = np.expand_dims(self.base_var, 0) # TODO: Mosaicing means that many of the internals are no longer diff --git a/pypeit/inputfiles.py b/pypeit/inputfiles.py index 952f40142b..52a41f7c11 100644 --- a/pypeit/inputfiles.py +++ b/pypeit/inputfiles.py @@ -1149,6 +1149,5 @@ def grab_rawfiles(file_of_files:str=None, list_of_files:list=None, raw_paths:lis return [str(p / f) for p in _raw_paths for f in list_of_files if (p / f).exists()] # Find all files that have the correct extension - return np.concatenate([files_from_extension(str(p), extension=extension) - for p in _raw_paths]).tolist() + return files_from_extension(_raw_paths, extension=extension) diff --git a/pypeit/io.py b/pypeit/io.py index 2e0c13b992..1d230ca5eb 100644 --- a/pypeit/io.py +++ b/pypeit/io.py @@ -851,40 +851,48 @@ def create_symlink(filename, symlink_dir, relative_symlink=False, overwrite=Fals os.symlink(olink_src, olink_dest) -def files_from_extension(raw_path, - extension:str='fits'): +def files_from_extension(raw_path, extension='.fits'): """ - Grab the list of files with a given extension + Find files from one or more paths with one or more extensions. - Args: - raw_path (str or list): - Path(s) to raw files, which may or may not include the prefix of the - files to search for. - - For a string input, for example, this can be the directory - ``'/path/to/files/'`` or the directory plus the file prefix - ``'/path/to/files/prefix'``, which yeilds the search strings - ``'/path/to/files/*fits'`` or ``'/path/to/files/prefix*fits'``, - respectively. - - For a list input, this can use wildcards for multiple directories. + This is a recursive function. If ``raw_path`` is a list, the function is + called for every item in the list and the results are concatenated. - extension (str, optional): - File extension to search on. + Args: + raw_path (:obj:`str`, `Path`_, :obj:`list`): + One or more paths to search for files, which may or may not include + the prefix of the files to search for. For string input, this can + be the directory ``'/path/to/files/'`` or the directory plus the + file prefix ``'/path/to/files/prefix'``, which yeilds the search + strings ``'/path/to/files/*fits'`` or + ``'/path/to/files/prefix*fits'``, respectively. For a list input, + this can use wildcards for multiple directories. + + extension (:obj:`str`, :obj:`list`, optional): + One or more file extensions to search on. Returns: - list: List of raw data filenames (sorted) with full path - """ - if isinstance(raw_path, str): - # Grab the list of files - dfname = os.path.join(raw_path, f'*{extension}*') \ - if os.path.isdir(raw_path) else f'{raw_path}*{extension}*' - return sorted(glob.glob(dfname)) + :obj:`list`: List of `Path`_ objects with the full path to the set of + unique raw data filenames that match the provided criteria search + strings. + """ + if isinstance(raw_path, (str, Path)): + _raw_path = Path(raw_path).absolute() + if _raw_path.is_dir(): + prefix = '' + else: + _raw_path, prefix = _raw_path.parent, _raw_path.name + if not _raw_path.is_dir(): + msgs.error(f'{_raw_path} does not exist!') + ext = [extension] if isinstance(extension, str) else extension + files = numpy.concatenate([sorted(_raw_path.glob(f'{prefix}*{e}')) for e in ext]) + return numpy.unique(files).tolist() if isinstance(raw_path, list): - return numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]).tolist() + files = numpy.concatenate([files_from_extension(p, extension=extension) for p in raw_path]) + return numpy.unique(files).tolist() - msgs.error(f"Incorrect type {type(raw_path)} for raw_path (must be str or list)") + msgs.error(f"Incorrect type {type(raw_path)} for raw_path; must be str, Path, or list.") diff --git a/pypeit/metadata.py b/pypeit/metadata.py index faabcea586..f72a7c20c9 100644 --- a/pypeit/metadata.py +++ b/pypeit/metadata.py @@ -20,7 +20,6 @@ from pypeit import msgs from pypeit import inputfiles from pypeit.core import framematch -from pypeit.core import flux_calib from pypeit.core import parse from pypeit.core import meta from pypeit.io import dict_to_lines @@ -1449,28 +1448,10 @@ def get_frame_types(self, flag_unknown=False, user=None, merge=True): indx = self.spectrograph.check_frame_type(ftype, self.table, exprng=exprng) # Turn on the relevant bits type_bits[indx] = self.type_bitmask.turn_on(type_bits[indx], flag=ftype) - - # Find the nearest standard star to each science frame - # TODO: Should this be 'standard' or 'science' or both? - if 'ra' not in self.keys() or 'dec' not in self.keys(): - msgs.warn('Cannot associate standard with science frames without sky coordinates.') - else: - # TODO: Do we want to do this here? - indx = self.type_bitmask.flagged(type_bits, flag='standard') - for b, f, ra, dec in zip(type_bits[indx], self['filename'][indx], self['ra'][indx], - self['dec'][indx]): - if ra == 'None' or dec == 'None': - msgs.warn('RA and DEC must not be None for file:' + msgs.newline() + f) - msgs.warn('The above file could be a twilight flat frame that was' - + msgs.newline() + 'missed by the automatic identification.') - b = self.type_bitmask.turn_off(b, flag='standard') - continue - # If an object exists within 20 arcmins of a listed standard, - # then it is probably a standard star - foundstd = flux_calib.find_standard_file(ra, dec, check=True) - b = self.type_bitmask.turn_off(b, flag='science' if foundstd else 'standard') - + # Vet assigned frame types (this can be spectrograph dependent) + self.spectrograph.vet_assigned_ftypes(type_bits, self) + # Find the files without any types indx = np.logical_not(self.type_bitmask.flagged(type_bits)) if np.any(indx): diff --git a/pypeit/par/pypeitpar.py b/pypeit/par/pypeitpar.py index c398366bd2..6f2251e2d5 100644 --- a/pypeit/par/pypeitpar.py +++ b/pypeit/par/pypeitpar.py @@ -73,6 +73,7 @@ def __init__(self, existing_par=None, foo=None): from pypeit.par import util from pypeit.core.framematch import FrameTypeBitMask from pypeit import msgs +from pypeit import dataPaths def tuple_force(par): @@ -209,6 +210,7 @@ def __init__(self, trim=None, apply_gain=None, orient=None, overscan_method=None, overscan_par=None, combine=None, satpix=None, mask_cr=None, clip=None, + scale_to_mean=None, #cr_sigrej=None, n_lohi=None, #replace=None, lamaxiter=None, grow=None, @@ -372,6 +374,10 @@ def __init__(self, trim=None, apply_gain=None, orient=None, dtypes['clip'] = bool descr['clip'] = 'Perform sigma clipping when combining. Only used with combine=mean' + defaults['scale_to_mean'] = False + dtypes['scale_to_mean'] = bool + descr['scale_to_mean'] = 'If True, scale the input images to have the same mean before combining.' + defaults['comb_sigrej'] = None dtypes['comb_sigrej'] = float descr['comb_sigrej'] = 'Sigma-clipping level for when clip=True; ' \ @@ -457,7 +463,7 @@ def from_dict(cls, cfg): 'overscan_method', 'overscan_par', 'use_darkimage', 'dark_expscale', 'spat_flexure_correct', 'spat_flexure_maxlag', 'use_illumflat', 'use_specillum', 'empirical_rn', 'shot_noise', 'noise_floor', 'use_pixelflat', 'combine', - 'correct_nonlinear', 'satpix', #'calib_setup_and_bit', + 'scale_to_mean', 'correct_nonlinear', 'satpix', #'calib_setup_and_bit', 'n_lohi', 'mask_cr', 'lamaxiter', 'grow', 'clip', 'comb_sigrej', 'rmcompact', 'sigclip', 'sigfrac', 'objlim'] @@ -833,9 +839,13 @@ def validate(self): return # Check the frame exists - if not os.path.isfile(self.data['pixelflat_file']): - raise ValueError('Provided frame file name does not exist: {0}'.format( - self.data['pixelflat_file'])) + # only the file name is provided, so we need to check if the file exists + # in the right place (data/pixelflats) + file_path = dataPaths.pixelflat.get_file_path(self.data['pixelflat_file'], return_none=True) + if file_path is None: + msgs.error( + f'Provided pixelflat file, {self.data["pixelflat_file"]} not found. It is not a direct path, ' + 'a cached file, or a file that can be downloaded from a PypeIt repository.') # Check that if tweak slits is true that illumflatten is alwo true # TODO -- We don't need this set, do we?? See the desc of tweak_slits above @@ -1939,7 +1949,7 @@ class SensFuncPar(ParSet): For a table with the current keywords, defaults, and descriptions, see :ref:`parameters`. """ - def __init__(self, flatfile=None, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_det=None, algorithm=None, UVIS=None, + def __init__(self, use_flat=None, extrap_blu=None, extrap_red=None, samp_fact=None, multi_spec_det=None, algorithm=None, UVIS=None, IR=None, polyorder=None, star_type=None, star_mag=None, star_ra=None, extr=None, star_dec=None, mask_hydrogen_lines=None, mask_helium_lines=None, hydrogen_mask_wid=None): # Grab the parameter names and values from the function arguments @@ -1952,11 +1962,9 @@ def __init__(self, flatfile=None, extrap_blu=None, extrap_red=None, samp_fact=No dtypes = OrderedDict.fromkeys(pars.keys()) descr = OrderedDict.fromkeys(pars.keys()) - defaults['flatfile'] = None - dtypes['flatfile'] = str - descr['flatfile'] = 'Flat field file to be used if the sensitivity function model will utilize the blaze ' \ - 'function computed from a flat field file in the Calibrations directory, e.g.' \ - 'Calibrations/Flat_A_0_DET01.fits' + defaults['use_flat'] = False + dtypes['use_flat'] = bool + descr['use_flat'] = 'If True, the flatfield spectrum will be used when computing the sensitivity function.' defaults['extr'] = 'OPT' dtypes['extr'] = str @@ -2059,7 +2067,7 @@ def from_dict(cls, cfg): k = np.array([*cfg.keys()]) # Single element parameters - parkeys = ['flatfile', 'extrap_blu', 'extrap_red', 'samp_fact', 'multi_spec_det', 'algorithm', + parkeys = ['use_flat', 'extrap_blu', 'extrap_red', 'samp_fact', 'multi_spec_det', 'algorithm', 'polyorder', 'star_type', 'star_mag', 'star_ra', 'star_dec', 'extr', 'mask_hydrogen_lines', 'mask_helium_lines', 'hydrogen_mask_wid'] @@ -4495,7 +4503,7 @@ class CalibrationsPar(ParSet): def __init__(self, calib_dir=None, bpm_usebias=None, biasframe=None, darkframe=None, arcframe=None, tiltframe=None, pixelflatframe=None, pinholeframe=None, alignframe=None, alignment=None, traceframe=None, illumflatframe=None, - lampoffflatsframe=None, scattlightframe=None, skyframe=None, standardframe=None, + lampoffflatsframe=None, slitless_pixflatframe=None, scattlightframe=None, skyframe=None, standardframe=None, scattlight_pad=None, flatfield=None, wavelengths=None, slitedges=None, tilts=None, raise_chk_error=None): @@ -4581,6 +4589,15 @@ def __init__(self, calib_dir=None, bpm_usebias=None, biasframe=None, darkframe=N dtypes['lampoffflatsframe'] = [ ParSet, dict ] descr['lampoffflatsframe'] = 'The frames and combination rules for the lamp off flats' + defaults['slitless_pixflatframe'] = FrameGroupPar(frametype='slitless_pixflat', + process=ProcessImagesPar(satpix='nothing', + use_pixelflat=False, + use_illumflat=False, + use_specillum=False, + combine='median')) + dtypes['slitless_pixflatframe'] = [ ParSet, dict ] + descr['slitless_pixflatframe'] = 'The frames and combination rules for the slitless pixel flat' + defaults['pinholeframe'] = FrameGroupPar(frametype='pinhole') dtypes['pinholeframe'] = [ ParSet, dict ] descr['pinholeframe'] = 'The frames and combination rules for the pinholes' @@ -4672,7 +4689,7 @@ def from_dict(cls, cfg): parkeys = [ 'calib_dir', 'bpm_usebias', 'raise_chk_error'] allkeys = parkeys + ['biasframe', 'darkframe', 'arcframe', 'tiltframe', 'pixelflatframe', - 'illumflatframe', 'lampoffflatsframe', 'scattlightframe', + 'illumflatframe', 'lampoffflatsframe', 'slitless_pixflatframe', 'scattlightframe', 'pinholeframe', 'alignframe', 'alignment', 'traceframe', 'standardframe', 'skyframe', 'scattlight_pad', 'flatfield', 'wavelengths', 'slitedges', 'tilts'] badkeys = np.array([pk not in allkeys for pk in k]) @@ -4698,6 +4715,8 @@ def from_dict(cls, cfg): kwargs[pk] = FrameGroupPar.from_dict('illumflat', cfg[pk]) if pk in k else None pk = 'lampoffflatsframe' kwargs[pk] = FrameGroupPar.from_dict('lampoffflats', cfg[pk]) if pk in k else None + pk = 'slitless_pixflatframe' + kwargs[pk] = FrameGroupPar.from_dict('slitless_pixflat', cfg[pk]) if pk in k else None pk = 'pinholeframe' kwargs[pk] = FrameGroupPar.from_dict('pinhole', cfg[pk]) if pk in k else None pk = 'scattlightframe' @@ -5272,7 +5291,7 @@ def valid_telescopes(): """ Return the valid telescopes. """ - return [ 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', + return ['AAT', 'GEMINI-N','GEMINI-S', 'KECK', 'SHANE', 'WHT', 'APF', 'TNG', 'VLT', 'MAGELLAN', 'LBT', 'MMT', 'KPNO', 'NOT', 'P200', 'BOK', 'GTC', 'SOAR', 'NTT', 'LDT', 'JWST', 'HILTNER'] def validate(self): diff --git a/pypeit/pypeit.py b/pypeit/pypeit.py index a32e85463d..432bfe76f5 100644 --- a/pypeit/pypeit.py +++ b/pypeit/pypeit.py @@ -1029,6 +1029,10 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk if not self.par['reduce']['extraction']['skip_extraction']: msgs.info(f"Extraction begins for {self.basename} on det={det}") + # set the flatimg, if it exists + flatimg = None if self.caliBrate.flatimages is None else self.caliBrate.flatimages.pixelflat_model + if flatimg is None: + msgs.warn("No flat image was found. A spectrum of the flatfield will not be extracted!") # Instantiate Reduce object # Required for pipeline specific object # At instantiation, the fullmask in self.sciImg is modified @@ -1036,7 +1040,7 @@ def extract_one(self, frames, det, sciImg, bkg_redux_sciimg, objFind, initial_sk self.exTract = extraction.Extract.get_instance( sciImg, slits, sobjs_obj, self.spectrograph, self.par, self.objtype, global_sky=final_global_sky, bkg_redux_global_sky=bkg_redux_global_sky, - waveTilts=self.caliBrate.wavetilts, wv_calib=self.caliBrate.wv_calib, + waveTilts=self.caliBrate.wavetilts, wv_calib=self.caliBrate.wv_calib, flatimg=flatimg, bkg_redux=self.bkg_redux, return_negative=self.par['reduce']['extraction']['return_negative'], std_redux=self.std_redux, basename=self.basename, show=self.show) # Perform the extraction diff --git a/pypeit/pypeitdata.py b/pypeit/pypeitdata.py index 5cfad9b8cc..779aeb4241 100644 --- a/pypeit/pypeitdata.py +++ b/pypeit/pypeitdata.py @@ -224,7 +224,7 @@ def _parse_format(f): return _f.suffix.replace('.','').lower() def get_file_path(self, data_file, force_update=False, to_pkg=None, return_format=False, - quiet=False): + return_none=False, quiet=False): """ Return the path to a file. @@ -266,6 +266,9 @@ def get_file_path(self, data_file, force_update=False, to_pkg=None, return_forma If True, the returned object is a :obj:`tuple` that includes the file path and its format (e.g., ``'fits'``). If False, only the file path is returned. + return_none (:obj:`bool`, optional): + If True, return None if the file does not exist. If False, an + error is raised if the file does not exist. quiet (:obj:`bool`, optional): Suppress messages @@ -300,7 +303,10 @@ def get_file_path(self, data_file, force_update=False, to_pkg=None, return_forma # if the file exists in the cache and force_update is False. subdir = str(self.path.relative_to(self.data)) _cached_file = cache.fetch_remote_file(data_file, subdir, remote_host=self.host, - force_update=force_update) + force_update=force_update, return_none=return_none) + if _cached_file is None: + msgs.warn(f'File {data_file} not found in the cache.') + return None # If we've made it this far, the file is being pulled from the cache. if to_pkg is None: @@ -359,6 +365,8 @@ class PypeItDataPaths: 'skisim': {'path': 'skisim', 'host': 'github'}, 'filters': {'path': 'filters', 'host': None}, 'sensfunc': {'path': 'sensfuncs', 'host': 'github'}, + # Pixel Flats + 'pixelflat': {'path': 'pixelflats', 'host': 'github'}, # Other 'sky_spec': {'path': 'sky_spec', 'host': None}, 'static_calibs': {'path': 'static_calibs', 'host': None}, diff --git a/pypeit/pypeitsetup.py b/pypeit/pypeitsetup.py index 74a82159dc..20b95b8c03 100644 --- a/pypeit/pypeitsetup.py +++ b/pypeit/pypeitsetup.py @@ -163,30 +163,40 @@ def from_pypeit_file(cls, filename): cfg_lines=pypeItFile.cfg_lines, pypeit_file=filename) - # TODO: Make the default here match the default used by - # io.files_from_extension? @classmethod - def from_file_root(cls, root, spectrograph, extension='.fits'): + def from_file_root(cls, root, spectrograph, extension=None): """ Instantiate the :class:`~pypeit.pypeitsetup.PypeItSetup` object by providing a file root. Args: - root (:obj:`str`): - String used to find the raw files; see + root (:obj:`str`, `Path`_, :obj:`list`): + One or more paths within which to search for files; see :func:`~pypeit.io.files_from_extension`. - spectrograph (:obj:`str`): + spectrograph (:obj:`str`, :class:`~pypeit.spectrographs.spectrograph.Spectrograph`): The PypeIt name of the spectrograph used to take the observations. This should be one of the available options in :attr:`~pypeit.spectrographs.available_spectrographs`. - extension (:obj:`str`, optional): + extension (:obj:`str`, :obj:`list`, optional): The extension common to all the fits files to reduce; see - :func:`~pypeit.io.files_from_extension`. - + :func:`~pypeit.io.files_from_extension`. If None, uses the + ``allowed_extensions`` of the spectrograph class. Otherwise, + this *must* be a subset of the allowed extensions for the + selected spectrograph. + Returns: :class:`PypeitSetup`: The instance of the class. """ - return cls.from_rawfiles(io.files_from_extension(root, extension=extension), spectrograph) + # NOTE: This works if `spectrograph` is either a string or a + # Spectrograph object + spec = load_spectrograph(spectrograph).__class__ + files = spec.find_raw_files(root, extension=extension) + nfiles = len(files) + if nfiles == 0: + msgs.error(f'Unable to find any raw files for {spec.name} in {root}!') + else: + msgs.info(f'Found {nfiles} {spec.name} raw files.') + return cls.from_rawfiles(files, spectrograph) @classmethod def from_rawfiles(cls, data_files:list, spectrograph:str, frametype=None): diff --git a/pypeit/scripts/__init__.py b/pypeit/scripts/__init__.py index ff0164f966..9f4bef02c6 100644 --- a/pypeit/scripts/__init__.py +++ b/pypeit/scripts/__init__.py @@ -9,6 +9,7 @@ from pypeit.scripts import chk_alignments from pypeit.scripts import chk_edges from pypeit.scripts import chk_flats +from pypeit.scripts import chk_flexure from pypeit.scripts import chk_tilts from pypeit.scripts import chk_for_calibs from pypeit.scripts import chk_noise_1dspec @@ -51,6 +52,7 @@ from pypeit.scripts import trace_edges from pypeit.scripts import view_fits from pypeit.scripts import compile_wvarxiv +from pypeit.scripts import show_pixflat # Build the list of script classes diff --git a/pypeit/scripts/chk_flexure.py b/pypeit/scripts/chk_flexure.py new file mode 100644 index 0000000000..d384dc8e29 --- /dev/null +++ b/pypeit/scripts/chk_flexure.py @@ -0,0 +1,63 @@ +""" +This script displays the flexure (spatial or spectral) applied to the science data. + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" + +from pypeit.scripts import scriptbase + + +class ChkFlexure(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Print QA on flexure to the screen', + width=width) + + parser.add_argument('input_file', type=str, nargs='+', help='One or more PypeIt spec2d or spec1d file') + inp = parser.add_mutually_exclusive_group(required=True) + inp.add_argument('--spec', default=False, action='store_true', help='Check the spectral flexure') + inp.add_argument('--spat', default=False, action='store_true', help='Check the spatial flexure') + parser.add_argument('--try_old', default=False, action='store_true', + help='Attempt to load old datamodel versions. A crash may ensue..') + return parser + + @staticmethod + def main(args): + + from IPython import embed + from astropy.io import fits + from pypeit import msgs + from pypeit.core import flexure + + chk_version = not args.try_old + + # Loop over the input files + for in_file in args.input_file: + + msgs.info(f'Checking fluxure for file: {in_file}') + + # What kind of file are we?? + hdul = fits.open(in_file) + head0 = hdul[0].header + file_type = None + if 'PYP_CLS' in head0.keys() and head0['PYP_CLS'].strip() == 'AllSpec2DObj': + file_type = 'spec2d' + elif 'DMODCLS' in head0.keys() and head0['DMODCLS'].strip() == 'SpecObjs': + file_type = 'spec1d' + else: + msgs.error("Bad file type input!") + + # Check the flexure + flexure.flexure_diagnostic(in_file, file_type=file_type, flexure_type='spat' if args.spat else 'spec', + chk_version=chk_version) + + # space between files for clarity + print('') + + + + + + diff --git a/pypeit/scripts/chk_for_calibs.py b/pypeit/scripts/chk_for_calibs.py index ed3000d992..9269529fa7 100644 --- a/pypeit/scripts/chk_for_calibs.py +++ b/pypeit/scripts/chk_for_calibs.py @@ -21,8 +21,11 @@ def get_parser(cls, width=None): parser.add_argument('-s', '--spectrograph', default=None, type=str, help='A valid spectrograph identifier: {0}'.format( ', '.join(available_spectrographs))) - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('--save_setups', default=False, action='store_true', help='If not toggled, remove setup_files/ folder and its files.') return parser diff --git a/pypeit/scripts/coadd_1dspec.py b/pypeit/scripts/coadd_1dspec.py index 7763f9b905..98306e1126 100644 --- a/pypeit/scripts/coadd_1dspec.py +++ b/pypeit/scripts/coadd_1dspec.py @@ -22,69 +22,6 @@ from pypeit.spectrographs.util import load_spectrograph -## TODO: This is basically the exact same code as read_fluxfile in the fluxing -## script. Consolidate them? Make this a standard method in parse or io. -#def read_coaddfile(ifile): -# """ -# Read a ``PypeIt`` coadd1d file, akin to a standard PypeIt file. -# -# The top is a config block that sets ParSet parameters. The name of the -# spectrograph is required. -# -# Args: -# ifile (:obj:`str`): -# Name of the coadd file -# -# Returns: -# :obj:`tuple`: Three objects are returned: a :obj:`list` with the -# configuration entries used to modify the relevant -# :class:`~pypeit.par.parset.ParSet` parameters, a :obj:`list` the names -# of spec1d files to be coadded, and a :obj:`list` with the object IDs -# aligned with each of the spec1d files. -# """ -# # Read in the pypeit reduction file -# msgs.info('Loading the coadd1d file') -# lines = inputfiles.read_pypeit_file_lines(ifile) -# is_config = np.ones(len(lines), dtype=bool) -# -# -# # Parse the fluxing block -# spec1dfiles = [] -# objids_in = [] -# s, e = inputfiles.InputFile.find_block(lines, 'coadd1d') -# if s >= 0 and e < 0: -# msgs.error("Missing 'coadd1d end' in {0}".format(ifile)) -# elif (s < 0) or (s==e): -# msgs.error("Missing coadd1d read or [coadd1d] block in in {0}. Check the input format for the .coadd1d file".format(ifile)) -# else: -# for ctr, line in enumerate(lines[s:e]): -# prs = line.split(' ') -# spec1dfiles.append(prs[0]) -# if ctr == 0 and len(prs) != 2: -# msgs.error('Invalid format for .coadd1d file.' + msgs.newline() + -# 'You must have specify a spec1dfile and objid on the first line of the coadd1d block') -# if len(prs) > 1: -# objids_in.append(prs[1]) -# is_config[s-1:e+1] = False -# -# # Chck the sizes of the inputs -# nspec = len(spec1dfiles) -# if len(objids_in) == 1: -# objids = nspec*objids_in -# elif len(objids_in) == nspec: -# objids = objids_in -# else: -# msgs.error('Invalid format for .flux file.' + msgs.newline() + -# 'You must specify a single objid on the first line of the coadd1d block,' + msgs.newline() + -# 'or specify am objid for every spec1dfile in the coadd1d block.' + msgs.newline() + -# 'Run pypeit_coadd_1dspec --help for information on the format') -# # Construct config to get spectrograph -# cfg_lines = list(lines[is_config]) -# -# # Return -# return cfg_lines, spec1dfiles, objids - - def build_coadd_file_name(spec1dfiles, spectrograph): """Build the output file name for coadding. The filename convention is coadd1d___.fits or @@ -116,6 +53,7 @@ def build_coadd_file_name(spec1dfiles, spectrograph): path = os.path.dirname(os.path.abspath(spec1dfiles[0])) return os.path.join(path, f'coadd1d_{target}_{instrument_name}_{date_portion}.fits') + class CoAdd1DSpec(scriptbase.ScriptBase): @classmethod diff --git a/pypeit/scripts/identify.py b/pypeit/scripts/identify.py index 64a688242d..f5d123ed62 100644 --- a/pypeit/scripts/identify.py +++ b/pypeit/scripts/identify.py @@ -180,12 +180,14 @@ def main(args): arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[slit_val]) - # Launch the identify window - # TODO -- REMOVE THIS HACK - try: + # Get the non-linear count level + if msarc.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in msarc.detector.detectors]) + else: nonlinear_counts = msarc.detector.nonlinear_counts() - except AttributeError: - nonlinear_counts = None + + # Launch the identify window arcfitter = Identify.initialise(arccen, lamps, slits, slit=int(slit_val), par=par, wv_calib_all=wv_calib_slit, wavelim=[args.wmin, args.wmax], nonlinear_counts=nonlinear_counts, @@ -272,12 +274,15 @@ def main(args): # If we just want the normal one-trace output else: arccen, arc_maskslit = wavecal.extract_arcs(slitIDs=[int(args.slits)]) - # Launch the identify window - # TODO -- REMOVE THIS HACK - try: + + # Get the non-linear count level + if msarc.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in msarc.detector.detectors]) + else: nonlinear_counts = msarc.detector.nonlinear_counts() - except AttributeError: - nonlinear_counts = None + + # Launch the identify window arcfitter = Identify.initialise(arccen, lamps, slits, slit=int(args.slits), par=par, wv_calib_all=wv_calib, wavelim=[args.wmin, args.wmax], nonlinear_counts=nonlinear_counts, diff --git a/pypeit/scripts/multislit_flexure.py b/pypeit/scripts/multislit_flexure.py index 98e836196b..1691361687 100644 --- a/pypeit/scripts/multislit_flexure.py +++ b/pypeit/scripts/multislit_flexure.py @@ -18,55 +18,6 @@ from pypeit.scripts import scriptbase -#def read_flexfile(ifile): -# """ -# Read a ``PypeIt`` flexure file, akin to a standard ``PypeIt`` file. -# -# The top is a config block that sets ParSet parameters. -# -# Args: -# ifile (:obj:`str`): -# Name of the flexure file -# -# Returns: -# :obj:`tuple`: Two objects are returned: a :obj:`list` with the -# configuration entries used to modify the relevant -# :class:`~pypeit.par.parset.ParSet` parameters and a :obj:`list` with the -# names of spec1d files to be flexure corrected. -# """ -# # Read in the pypeit reduction file -# msgs.info('Loading the flexure file') -# lines = inputfiles.read_pypeit_file_lines(ifile) -# is_config = np.ones(len(lines), dtype=bool) -# -# # Parse the fluxing block -# spec1dfiles = [] -# objids_in = [] -# s, e = inputfiles.InputFile.find_block(lines, 'flexure') -# if s >= 0 and e < 0: -# msgs.error("Missing 'flexure end' in {0}".format(ifile)) -# elif (s < 0) or (s == e): -# msgs.error( -# "Missing flexure read block in {0}. Check the input format for the .flex file".format(ifile)) -# else: -# for ctr, line in enumerate(lines[s:e]): -# prs = line.split(' ') -# spec1dfiles.append(prs[0]) -# if len(prs) > 1: -# msgs.error('Invalid format for .flex file.' + msgs.newline() + -# 'You must specify only spec1dfiles in the block ') -# is_config[s-1:e+1] = False -# -# # Chck the sizes of the inputs -# nspec = len(spec1dfiles) -# -# # Construct config to get spectrograph -# cfg_lines = list(lines[is_config]) -# -# # Return -# return cfg_lines, spec1dfiles - - # TODO: Maybe not a good idea to name this script the same as the # flexure.MultiSlitFlexure class, but it is technically okay... class MultiSlitFlexure(scriptbase.ScriptBase): diff --git a/pypeit/scripts/obslog.py b/pypeit/scripts/obslog.py index 693b670a0c..6e1e7a7657 100644 --- a/pypeit/scripts/obslog.py +++ b/pypeit/scripts/obslog.py @@ -65,8 +65,11 @@ def get_parser(cls, width=None): parser.add_argument('-s', '--sort', default='mjd', type=str, help='Metadata keyword (pypeit-specific) to use to sort the output ' 'table.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-d', '--output_path', default='current working directory', help='Path to top-level output directory.') parser.add_argument('-o', '--overwrite', default=False, action='store_true', @@ -115,8 +118,7 @@ def main(args): f'argument.') # Generate the metadata table - ps = PypeItSetup.from_file_root(args.root, args.spec, - extension=args.extension) + ps = PypeItSetup.from_file_root(args.root, args.spec, extension=args.extension) ps.run(setup_only=True, # This allows for bad headers groupings=args.groupings, clean_config=args.bad_frames) diff --git a/pypeit/scripts/sensfunc.py b/pypeit/scripts/sensfunc.py index 53ab98a3e0..6a8a1d58ae 100644 --- a/pypeit/scripts/sensfunc.py +++ b/pypeit/scripts/sensfunc.py @@ -66,17 +66,16 @@ def get_parser(cls, width=None): 'provided but with .fits trimmed off if it is in the filename.') parser.add_argument("-s", "--sens_file", type=str, help='Configuration file with sensitivity function parameters') - parser.add_argument("-f", "--flatfile", type=str, - help="R|Use a flat calibration file to compute the blaze function when generating " - "the sensitivity function. This is helpful to account for small scale undulations " - "in the sensitivity function. Note that it is not possible to set --flatfile and " - "simultaneously use a .sens file with the --sens_file option. If " - "you are using a .sens file, set the flatfile there via e.g.:\n\n" + parser.add_argument("-f", "--use_flat", default=False, action="store_true", + help="R|Use the extracted spectrum of the flatfield calibration to estimate the blaze " + "function when generating the sensitivity function. This is helpful to account for " + "small scale undulations in the sensitivity function. The spec1dfile must contain the " + "extracted flatfield response in order to use this option. This spectrum is extracted " + "by default, unless you did not compute a pixelflat frame. Note that it is not " + "possible to set --use_flat and simultaneously use a .sens file with the --sens_file " + "option. If you are using a .sens file, set the use_flat flag with the argument:\n\n" "F| [sensfunc]\n" - "F| flatfile = Calibrations/Flat_A_0_DET01.fits\n\n" - "Where Flat_A_0_DET01.fits is the flat file in your " - "Calibrations directory\n") - + "F| use_flat = True") parser.add_argument("--debug", default=False, action="store_true", help="show debug plots?") parser.add_argument("--par_outfile", default='sensfunc.par', @@ -111,13 +110,14 @@ def main(args): " [sensfunc]\n" " algorithm = IR\n" "\n") - if args.flatfile is not None and args.sens_file is not None: - msgs.error("It is not possible to set --flatfile and simultaneously use a .sens " + + if args.use_flat and args.sens_file is not None: + msgs.error("It is not possible to set --use_flat and simultaneously use a .sens " "file via the --sens_file option. If you are using a .sens file set the " - "flatfile there via:\n" + "use_flat flag in your .sens file using the argument:\n" "\n" " [sensfunc]\n" - " flatfile = Calibrations/Flat_A_0_DET01.fits'\n" + " use_flat = True\n" "\n") if args.multi is not None and args.sens_file is not None: @@ -172,10 +172,10 @@ def main(args): if args.algorithm is not None: par['sensfunc']['algorithm'] = args.algorithm - # If flatfile was provided override defaults. Note this does undo .sens + # If use_flat was flagged in the input, set use_flat to True. Note this does undo .sens # file since they cannot both be passed - if args.flatfile is not None: - par['sensfunc']['flatfile'] = args.flatfile + if args.use_flat: + par['sensfunc']['use_flat'] = True # If multi was set override defaults. Note this does undo .sens file # since they cannot both be passed diff --git a/pypeit/scripts/setup.py b/pypeit/scripts/setup.py index 9de645baa3..f417caf172 100644 --- a/pypeit/scripts/setup.py +++ b/pypeit/scripts/setup.py @@ -28,8 +28,11 @@ def get_parser(cls, width=None): 'directory (e.g., /data/Kast) or the search string up through ' 'the wildcard (.e.g, /data/Kast/b). Use the --extension option ' 'to set the types of files to search for.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-d', '--output_path', default='current working directory', help='Path to top-level output directory.') parser.add_argument('-o', '--overwrite', default=False, action='store_true', diff --git a/pypeit/scripts/setup_gui.py b/pypeit/scripts/setup_gui.py index d864eb0372..b8925be7e6 100644 --- a/pypeit/scripts/setup_gui.py +++ b/pypeit/scripts/setup_gui.py @@ -27,8 +27,11 @@ def get_parser(cls, width=None): 'the wildcard (.e.g, /data/Kast/b). Use the --extension option ' 'to set the types of files to search for. Default is the ' 'current working directory.') - parser.add_argument('-e', '--extension', default='.fits', - help='File extension; compression indicators (e.g. .gz) not required.') + parser.add_argument('-e', '--extension', default=None, + help='File extension to use. Must include the period (e.g., ".fits") ' + 'and it must be one of the allowed extensions for this ' + 'spectrograph. If None, root directory will be searched for ' + 'all files with any of the allowed extensions.') parser.add_argument('-l', '--logfile', type=str, default=None, help="Write the PypeIt logs to the given file. If the file exists it will be renamed.") parser.add_argument('-v', '--verbosity', type=int, default=2, diff --git a/pypeit/scripts/show_2dspec.py b/pypeit/scripts/show_2dspec.py index 07d9b6ae32..eb701fad1e 100644 --- a/pypeit/scripts/show_2dspec.py +++ b/pypeit/scripts/show_2dspec.py @@ -64,8 +64,11 @@ def show_trace(sobjs, det, viewer, ch): maskdef_extr_list.append(maskdef_extr_flag is True) manual_extr_list.append(manual_extr_flag is True) - display.show_trace(viewer, ch, np.swapaxes(trace_list, 1,0), np.array(trc_name_list), - maskdef_extr=np.array(maskdef_extr_list), manual_extr=np.array(manual_extr_list)) + if len(trace_list) > 0: + display.show_trace(viewer, ch, np.swapaxes(trace_list, 1,0), np.array(trc_name_list), + maskdef_extr=np.array(maskdef_extr_list), manual_extr=np.array(manual_extr_list)) + else: + msgs.warn('spec1d file found, but no objects were extracted for this detector.') class Show2DSpec(scriptbase.ScriptBase): diff --git a/pypeit/scripts/show_pixflat.py b/pypeit/scripts/show_pixflat.py new file mode 100644 index 0000000000..0240837691 --- /dev/null +++ b/pypeit/scripts/show_pixflat.py @@ -0,0 +1,62 @@ +""" +Show on a ginga window the archived pixel flat field image + +.. include common links, assuming primary doc root is up one directory +.. include:: ../include/links.rst +""" + +from pypeit.scripts import scriptbase +from IPython import embed + + +class ShowPixFlat(scriptbase.ScriptBase): + + @classmethod + def get_parser(cls, width=None): + parser = super().get_parser(description='Show an archived Pixel Flat image in a ginga window.', + width=width) + parser.add_argument("file", type=str, help="Pixel Flat filename, e.g. pixelflat_keck_lris_blue.fits.gz") + parser.add_argument('--det', default=None, type=int, nargs='+', + help='Detector(s) to show. If more than one, list the detectors as, e.g. --det 1 2 ' + 'to show detectors 1 and 2. If not provided, all detectors will be shown.') + return parser + + @staticmethod + def main(args): + import numpy as np + from pypeit import msgs + from pypeit import io + from pypeit.display import display + from pypeit import dataPaths + + # check if the file exists + file_path = dataPaths.pixelflat.get_file_path(args.file, return_none=True) + if file_path is None: + msgs.error(f'Provided pixelflat file, {args.file} not found. It is not a direct path, ' + f'a cached file, or a file that can be downloaded from a PypeIt repository.') + + # Load the image + with io.fits_open(file_path) as hdu: + # get all the available detectors in the file + file_dets = [int(h.name.split('-')[0].split('DET')[1]) for h in hdu[1:]] + # if detectors are provided, check if they are in the file + if args.det is not None: + in_file = np.isin(args.det, file_dets) + # if none of the provided detectors are in the file, raise an error + if not np.any(in_file): + msgs.error(f"Provided detector(s) not found in the file. Available detectors are {file_dets}") + # if some of the provided detectors are not in the file, warn the user + elif np.any(np.logical_not(in_file)): + det_not_in_file = np.array(args.det)[np.logical_not(in_file)] + msgs.warn(f"Detector(s) {det_not_in_file} not found in the file. Available detectors are {file_dets}") + + # show the image + display.connect_to_ginga(raise_err=True, allow_new=True) + for h in hdu[1:]: + det = int(h.name.split('-')[0].split('DET')[1]) + if args.det is not None and det not in args.det: + continue + display.show_image(h.data, chname=h.name, cuts=(0.9, 1.1), clear=False, wcs_match=True) + + + diff --git a/pypeit/sensfunc.py b/pypeit/sensfunc.py index 5f2360af95..e32c0d94ce 100644 --- a/pypeit/sensfunc.py +++ b/pypeit/sensfunc.py @@ -254,13 +254,9 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, msgs.error(f'There is a problem with your standard star spec1d file: {self.spec1df}') # Unpack standard - wave, counts, counts_ivar, counts_mask, trace_spec, trace_spat, self.meta_spec, header \ - = self.sobjs_std.unpack_object(ret_flam=False, extract_type=self.extr, remove_missing=True) - - # Compute the blaze function - # TODO Make the blaze function optional - log10_blaze_function = self.compute_blaze(wave, trace_spec, trace_spat, par['flatfile']) \ - if par['flatfile'] is not None else None + wave, counts, counts_ivar, counts_mask, log10_blaze_function, self.meta_spec, header \ + = self.sobjs_std.unpack_object(ret_flam=False, log10blaze=True, extract_blaze=par['use_flat'], + extract_type=self.extr, remove_missing=True) # Perform any instrument tweaks wave_twk, counts_twk, counts_ivar_twk, counts_mask_twk, log10_blaze_function_twk = \ @@ -281,64 +277,6 @@ def __init__(self, spec1dfile, sensfile, par, par_fluxcalib=None, debug=False, star_mag=self.par['star_mag'], ra=star_ra, dec=star_dec) - def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0, - min_blaze_value=1e-3, debug=False): - """ - Compute the blaze function from a flat field image. - - Args: - wave (`numpy.ndarray`_): - Wavelength array. Shape = (nspec, norddet) - trace_spec (`numpy.ndarray`_): - Spectral pixels for the trace of the spectrum. Shape = (nspec, norddet) - trace_spat (`numpy.ndarray`_): - Spatial pixels for the trace of the spectrum. Shape = (nspec, norddet) - flatfile (:obj:`str`): - Filename for the flat field calibration image - box_radius (:obj:`float`, optional): - Radius of the boxcar extraction region used to extract the blaze function in pixels - min_blaze_value (:obj:`float`, optional): - Minimum value of the blaze function. Values below this are clipped and set to this value. Default=1e-3 - debug (:obj:`bool`, optional): - Show plots useful for debugging. Default=False - - Returns: - `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet) - if norddet > 1, else shape = (nspec,) - """ - flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version) - - pixelflat_raw = flatImages.pixelflat_raw - pixelflat_norm = flatImages.pixelflat_norm - pixelflat_proc, flat_bpm = flat.flatfield(pixelflat_raw, pixelflat_norm) - - flux_box = moment1d(pixelflat_proc * np.logical_not(flat_bpm), trace_spat, 2 * box_radius, row=trace_spec)[0] - - pixtot = moment1d(pixelflat_proc*0 + 1.0, trace_spat, 2 * box_radius, row=trace_spec)[0] - pixmsk = moment1d(flat_bpm, trace_spat, 2 * box_radius, row=trace_spec)[0] - - mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0) - - # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way - blaze_function = (np.clip(flux_box*mask_box, 1e-3, 1e9)).reshape(-1,1) \ - if flux_box.ndim == 1 else flux_box*mask_box - wave_debug = wave.reshape(-1,1) if wave.ndim == 1 else wave - log10_blaze_function = np.zeros_like(blaze_function) - norddet = log10_blaze_function.shape[1] - for iorddet in range(norddet): - blaze_function_smooth = utils.fast_running_median(blaze_function[:, iorddet], 5) - blaze_function_norm = blaze_function_smooth/blaze_function_smooth.max() - log10_blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) - if debug: - plt.plot(wave_debug[:, iorddet], log10_blaze_function[:,iorddet]) - if debug: - plt.show() - - - # TODO It would probably better to just return an array of shape (nspec, norddet) even if norddet = 1, i.e. - # to get rid of this .squeeze() - return log10_blaze_function.squeeze() - def _bundle(self): """ Bundle the object for writing using @@ -471,8 +409,8 @@ def flux_std(self): # TODO assign this to the data model # Unpack the fluxed standard - _wave, _flam, _flam_ivar, _flam_mask, _, _, _, _ \ - = self.sobjs_std.unpack_object(ret_flam=True, extract_type=self.extr) + _wave, _flam, _flam_ivar, _flam_mask, _blaze, _, _ \ + = self.sobjs_std.unpack_object(ret_flam=True, extract_type=self.extr) # Reshape to 2d arrays wave, flam, flam_ivar, flam_mask, _, _, _ \ = utils.spec_atleast_2d(_wave, _flam, _flam_ivar, _flam_mask) diff --git a/pypeit/slittrace.py b/pypeit/slittrace.py index 9405b51ad2..e1367629f3 100644 --- a/pypeit/slittrace.py +++ b/pypeit/slittrace.py @@ -68,7 +68,7 @@ def exclude_for_reducing(self): def exclude_for_flexure(self): # Ignore these flags when performing a flexure calculation # Currently they are *all* of the flags.. - return ['SHORTSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', 'BADALIGNCALIB', + return ['SHORTSLIT', 'BOXSLIT', 'USERIGNORE', 'BADWVCALIB', 'BADTILTCALIB', 'BADALIGNCALIB', 'SKIPFLATCALIB', 'BADFLATCALIB', 'BADSKYSUB', 'BADEXTRACT'] diff --git a/pypeit/specobj.py b/pypeit/specobj.py index d69bd74f0a..eb28627b06 100644 --- a/pypeit/specobj.py +++ b/pypeit/specobj.py @@ -55,7 +55,7 @@ class SpecObj(datamodel.DataContainer): Running index for the order. """ - version = '1.1.10' + version = '1.1.11' """ Current datamodel version number. """ @@ -87,6 +87,8 @@ class SpecObj(datamodel.DataContainer): 'OPT_COUNTS_NIVAR': dict(otype=np.ndarray, atype=float, descr='Optimally extracted noise variance, sky+read ' 'noise only (counts^2)'), + 'OPT_FLAT': dict(otype=np.ndarray, atype=float, + descr='Optimally extracted flatfield spectrum, normalised to the peak value.'), 'OPT_MASK': dict(otype=np.ndarray, atype=np.bool_, descr='Mask for optimally extracted flux. True=good'), 'OPT_FWHM': dict(otype=np.ndarray, atype=float, @@ -120,6 +122,8 @@ class SpecObj(datamodel.DataContainer): 'BOX_COUNTS_NIVAR': dict(otype=np.ndarray, atype=float, descr='Boxcar extracted noise variance, sky+read noise ' 'only (counts^2)'), + 'BOX_FLAT': dict(otype=np.ndarray, atype=float, + descr='Boxcar extracted flatfield spectrum, normalized to the peak value.'), 'BOX_MASK': dict(otype=np.ndarray, atype=np.bool_, descr='Mask for boxcar extracted flux. True=good'), 'BOX_FWHM': dict(otype=np.ndarray, atype=float, @@ -251,11 +255,20 @@ def __init__(self, PYPELINE, DET, OBJTYPE='unknown', @classmethod def from_arrays(cls, PYPELINE:str, wave:np.ndarray, counts:np.ndarray, ivar:np.ndarray, - mode='OPT', DET='DET01', SLITID=0, **kwargs): + flat=None, mode='OPT', DET='DET01', SLITID=0, **kwargs): # Instantiate slf = cls(PYPELINE, DET, SLITID=SLITID) + # Check the type of the flat field if it's not None + if flat is not None: + if not isinstance(flat, np.ndarray): + msgs.error('Flat must be a numpy array') + if flat.shape != counts.shape: + msgs.error('Flat and counts must have the same shape') # Add in arrays - for item, attr in zip([wave, counts, ivar], ['_WAVE', '_COUNTS', '_COUNTS_IVAR']): + for item, attr in zip([wave, counts, ivar, flat], ['_WAVE', '_COUNTS', '_COUNTS_IVAR', '_FLAT']): + # Check if any of the arrays are None. If so, skip + if item is None: + continue setattr(slf, mode+attr, item.astype(float)) # Mask. Watch out for places where ivar is infinite due to a divide by 0 slf[mode+'_MASK'] = (slf[mode+'_COUNTS_IVAR'] > 0.) & np.isfinite(slf[mode+'_COUNTS_IVAR']) diff --git a/pypeit/specobjs.py b/pypeit/specobjs.py index 4b6da7ef36..1093687079 100644 --- a/pypeit/specobjs.py +++ b/pypeit/specobjs.py @@ -188,7 +188,8 @@ def nobj(self): """ return len(self.specobjs) - def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False): + def unpack_object(self, ret_flam=False, log10blaze=False, min_blaze_value=1e-3, extract_type='OPT', + extract_blaze=False, remove_missing=False): """ Utility function to unpack the sobjs for one object and return various numpy arrays describing the spectrum and meta @@ -196,11 +197,17 @@ def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False the relevant indices for the object. Args: - ret_flam (:obj:`bool`, optional): - If True return the FLAM, otherwise return COUNTS. - extract_type (:obj:`str`, optional): + ret_flam (:obj:`bool`, optional): + If True return the FLAM, otherwise return COUNTS. + log10blaze (:obj:`bool`, optional): + If True return the log10 of the blaze function. + min_blaze_value (:obj:`float`, optional): + Minimum value of the blaze function to consider as good. + extract_type (:obj:`str`, optional): Extraction type to use. Default is 'OPT'. - remove_missing (:obj:`bool`, optional): + extract_blaze (:obj:`bool`, optional): + If True, extract the blaze function. Default is False. + remove_missing (:obj:`bool`, optional): If True, remove any missing data (i.e. where the flux is None). Default is False. @@ -215,6 +222,7 @@ def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False Flambda or counts) - flux_gpm (`numpy.ndarray`_): Good pixel mask. True=Good + - blaze (`numpy.ndarray`_, None): Blaze function - meta_spec (dict:) Dictionary containing meta data. The keys are defined by spectrograph.parse_spec_header() @@ -225,6 +233,7 @@ def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False flux_attr = 'FLAM' if ret_flam else 'COUNTS' flux_key = '{}_{}'.format(extract_type, flux_attr) wave_key = '{}_WAVE'.format(extract_type) + blaze_key = '{}_FLAT'.format(extract_type) # Check for missing data none_flux = [f is None for f in getattr(self, flux_key)] @@ -249,8 +258,8 @@ def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False flux = np.zeros((nspec, norddet)) flux_ivar = np.zeros((nspec, norddet)) flux_gpm = np.zeros((nspec, norddet), dtype=bool) - trace_spec = np.zeros((nspec, norddet)) - trace_spat = np.zeros((nspec, norddet)) + if extract_blaze: + blaze = np.zeros((nspec, norddet), dtype=float) detector = [None]*norddet ech_orders = np.zeros(norddet, dtype=int) @@ -263,8 +272,19 @@ def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False ech_orders[iorddet] = self[iorddet].ECH_ORDER flux[:, iorddet] = getattr(self, flux_key)[iorddet] flux_ivar[:, iorddet] = getattr(self, flux_key+'_IVAR')[iorddet] - trace_spat[:, iorddet] = self[iorddet].TRACE_SPAT - trace_spec[:, iorddet] = self[iorddet].trace_spec + if extract_blaze: + blaze[:, iorddet] = getattr(self, blaze_key)[iorddet] + + # Log10 blaze + if extract_blaze: + blaze_function = np.copy(blaze) + if log10blaze: + for iorddet in range(norddet): + blaze_function_smooth = utils.fast_running_median(blaze[:, iorddet], 5) + blaze_function_norm = blaze_function_smooth / blaze_function_smooth.max() + blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None)) + else: + blaze_function = None # Populate meta data spectrograph = load_spectrograph(self.header['PYP_SPEC']) @@ -279,11 +299,12 @@ def unpack_object(self, ret_flam=False, extract_type='OPT', remove_missing=False # Return if self[0].PYPELINE in ['MultiSlit', 'SlicerIFU'] and self.nobj == 1: meta_spec['ECH_ORDERS'] = None + blaze_ret = blaze_function.reshape(nspec) if extract_blaze else None return wave.reshape(nspec), flux.reshape(nspec), flux_ivar.reshape(nspec), \ - flux_gpm.reshape(nspec), trace_spec.reshape(nspec), trace_spat.reshape(nspec), meta_spec, self.header + flux_gpm.reshape(nspec), blaze_ret, meta_spec, self.header else: meta_spec['ECH_ORDERS'] = ech_orders - return wave, flux, flux_ivar, flux_gpm, trace_spec, trace_spat, meta_spec, self.header + return wave, flux, flux_ivar, flux_gpm, blaze_function, meta_spec, self.header def get_std(self, multi_spec_det=None): """ @@ -791,6 +812,15 @@ def write_to_fits(self, subheader, outfile, overwrite=True, update_det=None, header[key.upper()] = line else: header[key.upper()] = subheader[key] + # Also store the datetime in ISOT format + if key.upper() == 'MJD': + if isinstance(subheader[key], (list, tuple)): + mjdval = subheader[key][0] + elif isinstance(subheader[key], float): + mjdval = subheader[key] + else: + raise ValueError('Header card must be a float or a FITS header tuple') + header['DATETIME'] = (Time(mjdval, format='mjd').isot, "Date and time of the observation in ISOT format") # Add calibration associations to Header if self.calibs is not None: for key, val in self.calibs.items(): diff --git a/pypeit/spectrographs/__init__.py b/pypeit/spectrographs/__init__.py index 3377156931..06efedfb57 100644 --- a/pypeit/spectrographs/__init__.py +++ b/pypeit/spectrographs/__init__.py @@ -3,11 +3,12 @@ # The import of all the spectrograph modules here is what enables the dynamic # compiling of all the available spectrographs below -from pypeit.spectrographs import gtc_osiris +from pypeit.spectrographs import aat_uhrf from pypeit.spectrographs import bok_bc from pypeit.spectrographs import gemini_flamingos from pypeit.spectrographs import gemini_gmos from pypeit.spectrographs import gemini_gnirs +from pypeit.spectrographs import gtc_osiris from pypeit.spectrographs import keck_esi from pypeit.spectrographs import keck_deimos from pypeit.spectrographs import keck_hires diff --git a/pypeit/spectrographs/aat_uhrf.py b/pypeit/spectrographs/aat_uhrf.py new file mode 100644 index 0000000000..ea71c1ed10 --- /dev/null +++ b/pypeit/spectrographs/aat_uhrf.py @@ -0,0 +1,257 @@ +""" +Module for Shane/Kast specific methods. + +.. include:: ../include/links.rst +""" +import os + +from IPython import embed + +import numpy as np + +from astropy.time import Time + +from pypeit import msgs +from pypeit import telescopes +from pypeit.core import framematch +from pypeit.spectrographs import spectrograph +from pypeit.images import detector_container +from pypeit import data + + +class AATUHRFSpectrograph(spectrograph.Spectrograph): + """ + Child to handle AAT/UHRF specific code + """ + ndet = 1 + telescope = telescopes.AATTelescopePar() + url = 'https://aat.anu.edu.au/science/instruments/decomissioned/uhrf/overview' + ql_supported = False + name = 'aat_uhrf' + camera = 'UHRF' + supported = True + header_name = 'uhrf' + allowed_extensions = [".FTS"] + + def get_detector_par(self, det, hdu=None): + """ + Return metadata for the selected detector. + + Args: + det (:obj:`int`): + 1-indexed detector number. + hdu (`astropy.io.fits.HDUList`_, optional): + The open fits file with the raw image of interest. If not + provided, frame-dependent parameters are set to a default. + + Returns: + :class:`~pypeit.images.detector_container.DetectorContainer`: + Object with the detector metadata. + """ + # Retrieve the binning + binning = '1,1' if hdu is None else self.compound_meta(self.get_headarr(hdu), "binning") + dsec = 1 + 1024//int(binning.split(',')[0]) + # Detector 1 + detector_dict = dict( + binning=binning, + det=1, + dataext=0, + specaxis=0, + specflip=False, + spatflip=False, + platescale=0.05, # Not sure about this value + saturation=65535., + mincounts=-1e10, + nonlinear=0.76, + numamplifiers=1, + gain=np.asarray([1.0]), # Not sure about this value + ronoise=np.asarray([0.0]), # Determine the read noise from the overscan region + xgap=0., + ygap=0., + ysize=1., + darkcurr=0.0, # e-/pixel/hour + # These are rows, columns on the raw frame, 1-indexed + datasec=np.asarray(['[:, 1:{:d}]'.format(dsec)]), + oscansec=np.asarray(['[:, {:d}:]'.format(dsec+1)]) + ) + return detector_container.DetectorContainer(**detector_dict) + + @classmethod + def default_pypeit_par(cls): + """ + Return the default parameters to use for this instrument. + + Returns: + :class:`~pypeit.par.pypeitpar.PypeItPar`: Parameters required by + all of PypeIt methods. + """ + par = super().default_pypeit_par() + + # Ignore PCA + par['calibrations']['slitedges']['sync_predict'] = 'nearest' + # Bound the detector with slit edges if no edges are found + par['calibrations']['slitedges']['bound_detector'] = True + + # Never correct for flexure - the sky is subdominant compared to the object and basically never detected. + par['flexure']['spec_method'] = 'skip' + + # Sky subtraction parameters - this instrument has no sky lines, but we still use the sky subtraction + # routine to subtract scattered light. + par['reduce']['skysub']['no_poly'] = True + par['reduce']['skysub']['bspline_spacing'] = 3.0 + par['reduce']['skysub']['user_regions'] = ':10,75:' # This is about right for most setups tested so far + par['scienceframe']['process']['sigclip'] = 10.0 + + # Set some parameters for the calibrations + # par['calibrations']['wavelengths']['reid_arxiv'] = 'None' + par['calibrations']['wavelengths']['lamps'] = ['ThAr'] + par['calibrations']['wavelengths']['n_final'] = 3 + par['calibrations']['tilts']['spat_order'] = 4 + par['calibrations']['tilts']['spec_order'] = 1 + + # Set the default exposure time ranges for the frame typing + # Trace frames should be the same as arc frames - it will force a bound detector and this + # allows the scattered light to be subtracted. A pixel-to-pixel sensitivity correction is + # not needed for this instrument, since it's a small slicer that projects the target onto + # multiple pixels. This instrument observes bright objects only, so sky subtraction is not + # important, but the sky subtraction routine is used to subtract scattered light, instead. + par['calibrations']['arcframe']['exprng'] = [None, 60.0] + par['calibrations']['tiltframe']['exprng'] = [None, 60.0] + par['calibrations']['traceframe']['exprng'] = [None, 60.0] + par['scienceframe']['exprng'] = [61, None] + + return par + + def init_meta(self): + """ + Define how metadata are derived from the spectrograph files. + + That is, this associates the PypeIt-specific metadata keywords + with the instrument-specific header cards using :attr:`meta`. + """ + self.meta = {} + # Required (core) + self.meta['ra'] = dict(ext=0, card='MEANRA') + self.meta['dec'] = dict(ext=0, card='MEANDEC') + self.meta['target'] = dict(ext=0, card='OBJECT') + # dispname is arm specific (blue/red) + self.meta['decker'] = dict(ext=0, card='WINDOW') + self.meta['dispname'] = dict(ext=0, card='WINDOW') + self.meta['binning'] = dict(ext=0, card=None, compound=True) + self.meta['mjd'] = dict(ext=0, card=None, compound=True) + self.meta['exptime'] = dict(ext=0, card='TOTALEXP') + self.meta['airmass'] = dict(ext=0, card=None, compound=True) + # Additional ones, generally for configuration determination or time + # self.meta['dichroic'] = dict(ext=0, card='BSPLIT_N') + # self.meta['instrument'] = dict(ext=0, card='VERSION') + + def compound_meta(self, headarr, meta_key): + """ + Methods to generate metadata requiring interpretation of the header + data, instead of simply reading the value of a header card. + + Args: + headarr (:obj:`list`): + List of `astropy.io.fits.Header`_ objects. + meta_key (:obj:`str`): + Metadata keyword to construct. + + Returns: + object: Metadata value read from the header(s). + """ + if meta_key == 'mjd': + date = headarr[0]['UTDATE'].replace(":","-") + time = headarr[0]['UTSTART'] + ttime = Time(f'{date}T{time}', format='isot') + return ttime.mjd + elif meta_key == 'binning': + binspat = int(np.ceil(1024/headarr[0]['NAXIS1'])) + binspec = int(np.ceil(1024/headarr[0]['NAXIS2'])) + return f'{binspat},{binspec}' + elif meta_key == 'airmass': + # Calculate the zenith distance + zendist = 0.5*(headarr[0]['ZDSTART']+headarr[0]['ZDEND']) + # Return the airmass based on the zenith distance + return 1./np.cos(np.deg2rad(zendist)) + msgs.error("Not ready for this compound meta") + + def configuration_keys(self): + """ + Return the metadata keys that define a unique instrument + configuration. + + This list is used by :class:`~pypeit.metadata.PypeItMetaData` to + identify the unique configurations among the list of frames read + for a given reduction. + + Returns: + :obj:`list`: List of keywords of data pulled from file headers + and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` + object. + """ + # decker is not included because arcs are often taken with a 0.5" slit + return ['dispname'] + + def check_frame_type(self, ftype, fitstbl, exprng=None): + """ + Check for frames of the provided type. + + Args: + ftype (:obj:`str`): + Type of frame to check. Must be a valid frame type; see + frame-type :ref:`frame_type_defs`. + fitstbl (`astropy.table.Table`_): + The table with the metadata for one or more frames to check. + exprng (:obj:`list`, optional): + Range in the allowed exposure time for a frame of type + ``ftype``. See + :func:`pypeit.core.framematch.check_frame_exptime`. + + Returns: + `numpy.ndarray`_: Boolean array with the flags selecting the + exposures in ``fitstbl`` that are ``ftype`` type frames. + """ + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + if ftype in ['science']: + return good_exp + if ftype in ['standard']: + return np.zeros(len(fitstbl), dtype=bool) + if ftype == 'bias': + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['pixelflat', 'trace', 'illumflat']: + # Flats and trace frames are typed together + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['pinhole', 'dark']: + # Don't type pinhole or dark frames + return np.zeros(len(fitstbl), dtype=bool) + if ftype in ['arc', 'tilt']: + return good_exp + + msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) + return np.zeros(len(fitstbl), dtype=bool) + + def config_specific_par(self, scifile, inp_par=None): + """ + Modify the PypeIt parameters to hard-wired values used for + specific instrument configurations. + + Args: + scifile (:obj:`str`): + File to use when determining the configuration and how + to adjust the input parameters. + inp_par (:class:`~pypeit.par.parset.ParSet`, optional): + Parameter set used for the full run of PypeIt. If None, + use :func:`default_pypeit_par`. + + Returns: + :class:`~pypeit.par.parset.ParSet`: The PypeIt parameter set + adjusted for configuration specific parameter values. + """ + par = super().config_specific_par(scifile, inp_par=inp_par) + + if par['calibrations']['wavelengths']['reid_arxiv'] is None: + msgs.warn("Wavelength setup not supported!" + msgs.newline() + msgs.newline() + + "Please perform your own wavelength calibration, and provide the path+filename using:" + msgs.newline() + + msgs.pypeitpar_text(['calibrations', 'wavelengths', 'reid_arxiv = '])) + # Return + return par diff --git a/pypeit/spectrographs/gemini_gmos.py b/pypeit/spectrographs/gemini_gmos.py index e00506920e..3c861056aa 100644 --- a/pypeit/spectrographs/gemini_gmos.py +++ b/pypeit/spectrographs/gemini_gmos.py @@ -108,6 +108,7 @@ class GeminiGMOSSpectrograph(spectrograph.Spectrograph): """ ndet = 3 url = 'http://www.gemini.edu/instrumentation/gmos' + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] def __init__(self): super().__init__() @@ -460,7 +461,7 @@ def get_rawimage(self, raw_file, det): return detectors[0], array[0], hdu, exptime, rawdatasec_img[0], oscansec_img[0] return mosaic, array, hdu, exptime, rawdatasec_img, oscansec_img - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -481,7 +482,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -535,7 +536,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): msc_tfm[i] = build_image_mosaic_transform(shape, msc_sft[i], msc_rot[i], tuple(reversed(binning))) return Mosaic(mosaic_id, detectors, shape, np.array(msc_sft), np.array(msc_rot), - np.array(msc_tfm), msc_order) + np.array(msc_tfm), msc_ord) @property def allowed_mosaics(self): @@ -864,7 +865,7 @@ def get_detector_par(self, det, hdu=None): # Return return detector_container.DetectorContainer(**detectors[det-1]) - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -885,7 +886,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -907,7 +908,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): else: self.detid = 'BI5-36-4k-2,BI11-33-4k-1,BI12-34-4k-1' - return super().get_mosaic_par(mosaic, hdu=hdu, msc_order=msc_order) + return super().get_mosaic_par(mosaic, hdu=hdu, msc_ord=msc_ord) @classmethod def default_pypeit_par(cls): @@ -1045,6 +1046,7 @@ class GeminiGMOSNSpectrograph(GeminiGMOSSpectrograph): telescope = telescopes.GeminiNTelescopePar() camera = 'GMOS-N' header_name = 'GMOS-N' + allowed_extensions = ['.fits', '.fits.bz2', '.fits.gz'] class GeminiGMOSNHamSpectrograph(GeminiGMOSNSpectrograph): @@ -1142,7 +1144,7 @@ def get_detector_par(self, det, hdu=None): # Return return detector_container.DetectorContainer(**detectors[det-1]) - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -1163,7 +1165,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -1174,7 +1176,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): # Detector ID (it is used to identify the correct mosaic geometry) self.detid = 'BI13-20-4k-1,BI12-09-4k-2,BI13-18-4k-2' - return super().get_mosaic_par(mosaic, hdu=hdu, msc_order=msc_order) + return super().get_mosaic_par(mosaic, hdu=hdu, msc_ord=msc_ord) def config_specific_par(self, scifile, inp_par=None): """ @@ -1410,7 +1412,7 @@ def get_detector_par(self, det, hdu=None): # Return return detector_container.DetectorContainer(**detectors[det-1]) - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -1431,7 +1433,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -1443,7 +1445,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): # TODO: Check this is correct self.detid = 'e2v 10031-23-05,10031-01-03,10031-18-04' - return super().get_mosaic_par(mosaic, hdu=hdu, msc_order=msc_order) + return super().get_mosaic_par(mosaic, hdu=hdu, msc_ord=msc_ord) def config_specific_par(self, scifile, inp_par=None): """ diff --git a/pypeit/spectrographs/gemini_gnirs.py b/pypeit/spectrographs/gemini_gnirs.py index 5a88298847..52e346661b 100644 --- a/pypeit/spectrographs/gemini_gnirs.py +++ b/pypeit/spectrographs/gemini_gnirs.py @@ -24,6 +24,7 @@ class GeminiGNIRSSpectrograph(spectrograph.Spectrograph): url = 'https://www.gemini.edu/instrumentation/gnirs' header_name = 'GNIRS' telescope = telescopes.GeminiNTelescopePar() + allowed_extensions = ['.fits', '.fits.bz2'] def __init__(self): super().__init__() diff --git a/pypeit/spectrographs/keck_deimos.py b/pypeit/spectrographs/keck_deimos.py index 250f09af3a..5549b388c2 100644 --- a/pypeit/spectrographs/keck_deimos.py +++ b/pypeit/spectrographs/keck_deimos.py @@ -824,7 +824,7 @@ def get_rawimage(self, raw_file, det): return detectors[0], image[0], hdu, exptime, rawdatasec_img[0], oscansec_img[0] return mosaic, image, hdu, exptime, rawdatasec_img, oscansec_img - def get_mosaic_par(self, mosaic, hdu=None, msc_order=5): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=5): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -845,7 +845,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=5): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -892,7 +892,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=5): msc_tfm[i] = build_image_mosaic_transform(shape, msc_sft[i], msc_rot[i], binning) return Mosaic(mosaic_id, detectors, shape, np.array(msc_sft), np.array(msc_rot), - np.array(msc_tfm), msc_order) + np.array(msc_tfm), msc_ord) @property def allowed_mosaics(self): @@ -1964,19 +1964,22 @@ def load_wmko_std_spectrum(fits_file:str, outfile=None, pad = False, split=True) # Generate SpecObj if not split: sobj1 = specobj.SpecObj.from_arrays('MultiSlit', idl_vac.value[0:npix], - idl_spec['COUNTS'].data[0:npix], - 1./(idl_spec['COUNTS'].data[0:npix]), - DET='MSC03') + idl_spec['COUNTS'].data[0:npix], + 1./(idl_spec['COUNTS'].data[0:npix]), + np.ones(idl_spec['COUNTS'].data[0:npix].size), + DET='MSC03') else: sobj1 = specobj.SpecObj.from_arrays('MultiSlit', idl_vac.value[0:npix], - idl_spec['COUNTS'].data[0:npix], - 1./(idl_spec['COUNTS'].data[0:npix]), - DET='DET03') + idl_spec['COUNTS'].data[0:npix], + 1./(idl_spec['COUNTS'].data[0:npix]), + np.ones(idl_spec['COUNTS'].data[0:npix].size), + DET='DET03') sobj2 = specobj.SpecObj.from_arrays('MultiSlit', idl_vac.value[npix:], - idl_spec['COUNTS'].data[npix:], - 1./(idl_spec['COUNTS'].data[npix:]), - DET='DET07') + idl_spec['COUNTS'].data[npix:], + 1./(idl_spec['COUNTS'].data[npix:]), + np.ones(idl_spec['COUNTS'].data[npix:].size), + DET='DET07') # SpecObjs sobjs = specobjs.SpecObjs() diff --git a/pypeit/spectrographs/keck_hires.py b/pypeit/spectrographs/keck_hires.py index 22c92e639c..7ba80d00be 100644 --- a/pypeit/spectrographs/keck_hires.py +++ b/pypeit/spectrographs/keck_hires.py @@ -62,9 +62,11 @@ class KECKHIRESSpectrograph(spectrograph.Spectrograph): ech_fixed_format = False supported = False # TODO before support = True - # 1. Implement flat fielding - # 2. Test on several different setups - # 3. Implement PCA extrapolation into the blue + # 1. Implement flat fielding - DONE + # 2. Test on several different setups - DONE + # 3. Implement PCA extrapolation into the blue + + comment = 'Post detector upgrade (~ August 2004). See :doc:`keck_hires`' # TODO: Place holder parameter set taken from X-shooter VIS for now. @@ -99,6 +101,9 @@ def default_pypeit_par(cls): par['calibrations']['standardframe']['exprng'] = [1, 600] par['scienceframe']['exprng'] = [601, None] + # Set default processing for slitless_pixflat + par['calibrations']['slitless_pixflatframe']['process']['scale_to_mean'] = True + # Slit tracing par['calibrations']['slitedges']['edge_thresh'] = 8.0 par['calibrations']['slitedges']['fit_order'] = 8 @@ -221,22 +226,21 @@ def init_meta(self): # Required (core) self.meta['ra'] = dict(ext=0, card='RA', required_ftypes=['science', 'standard']) self.meta['dec'] = dict(ext=0, card='DEC', required_ftypes=['science', 'standard']) - self.meta['target'] = dict(ext=0, card='OBJECT') + self.meta['target'] = dict(ext=0, card='TARGNAME') self.meta['decker'] = dict(ext=0, card='DECKNAME') self.meta['binning'] = dict(card=None, compound=True) self.meta['mjd'] = dict(card=None, compound=True) # This may depend on the old/new detector self.meta['exptime'] = dict(ext=0, card='ELAPTIME') self.meta['airmass'] = dict(ext=0, card='AIRMASS') - #self.meta['dispname'] = dict(ext=0, card='ECHNAME') + # Extras for config and frametyping self.meta['hatch'] = dict(ext=0, card='HATOPEN') self.meta['dispname'] = dict(ext=0, card='XDISPERS') self.meta['filter1'] = dict(ext=0, card='FIL1NAME') self.meta['echangle'] = dict(ext=0, card='ECHANGL', rtol=1e-3, atol=1e-2) self.meta['xdangle'] = dict(ext=0, card='XDANGL', rtol=1e-2) -# self.meta['idname'] = dict(ext=0, card='IMAGETYP') - # NOTE: This is the native keyword. IMAGETYP is from KOA. + self.meta['object'] = dict(ext=0, card='OBJECT') self.meta['idname'] = dict(card=None, compound=True) self.meta['frameno'] = dict(ext=0, card='FRAMENO') self.meta['instrument'] = dict(ext=0, card='INSTRUME') @@ -257,7 +261,6 @@ def compound_meta(self, headarr, meta_key): object: Metadata value read from the header(s). """ if meta_key == 'binning': - # TODO JFH Is this correct or should it be flipped? binspatial, binspec = parse.parse_binning(headarr[0]['BINNING']) binning = parse.binning2string(binspec, binspatial) return binning @@ -277,22 +280,28 @@ def compound_meta(self, headarr, meta_key): return 'off' elif meta_key == 'idname': - if not headarr[0].get('LAMPCAT1') and not headarr[0].get('LAMPCAT2') and \ + xcovopen = headarr[0].get('XCOVOPEN') + collcoveropen = (headarr[0].get('XDISPERS') == 'RED' and headarr[0].get('RCCVOPEN')) or \ + (headarr[0].get('XDISPERS') == 'UV' and headarr[0].get('BCCVOPEN')) + + if xcovopen and collcoveropen and \ + not headarr[0].get('LAMPCAT1') and not headarr[0].get('LAMPCAT2') and \ not headarr[0].get('LAMPQTZ2') and not (headarr[0].get('LAMPNAME') == 'quartz1'): if headarr[0].get('HATOPEN') and headarr[0].get('AUTOSHUT'): return 'Object' elif not headarr[0].get('HATOPEN'): return 'Bias' if not headarr[0].get('AUTOSHUT') else 'Dark' - elif headarr[0].get('AUTOSHUT') and (headarr[0].get('LAMPCAT1') or headarr[0].get('LAMPCAT2')): - if (headarr[0].get('XDISPERS') == 'RED' and not headarr[0].get('RCCVOPEN')) or \ - (headarr[0].get('XDISPERS') == 'UV' and not headarr[0].get('BCCVOPEN')): + elif xcovopen and collcoveropen and \ + headarr[0].get('AUTOSHUT') and (headarr[0].get('LAMPCAT1') or headarr[0].get('LAMPCAT2')): + return 'Line' + elif collcoveropen and \ + headarr[0].get('AUTOSHUT') and \ + (headarr[0].get('LAMPQTZ2') or (headarr[0].get('LAMPNAME') == 'quartz1')) and \ + not headarr[0].get('HATOPEN'): + if not xcovopen: return 'slitlessFlat' else: - return 'Line' - elif headarr[0].get('AUTOSHUT') and \ - (headarr[0].get('LAMPQTZ2') or (headarr[0].get('LAMPNAME') == 'quartz1')) \ - and not headarr[0].get('HATOPEN'): - return 'IntFlat' + return 'IntFlat' else: msgs.error("Not ready for this compound meta") @@ -311,7 +320,26 @@ def configuration_keys(self): and used to constuct the :class:`~pypeit.metadata.PypeItMetaData` object. """ - return ['decker', 'dispname', 'filter1', 'echangle', 'xdangle', 'binning'] + return ['dispname', 'decker', 'filter1', 'echangle', 'xdangle', 'binning'] + + def config_independent_frames(self): + """ + Define frame types that are independent of the fully defined + instrument configuration. + + Bias and dark frames are considered independent of a configuration, + but the DATE-OBS keyword is used to assign each to the most-relevant + configuration frame group. See + :func:`~pypeit.metadata.PypeItMetaData.set_configurations`. + + Returns: + :obj:`dict`: Dictionary where the keys are the frame types that + are configuration independent and the values are the metadata + keywords that can be used to assign the frames to a configuration + group. + """ + return {'bias': ['dispname', 'binning'], 'dark': ['dispname', 'binning'], + 'slitless_pixflat': ['dispname', 'binning']} def raw_header_cards(self): """ @@ -344,9 +372,6 @@ def pypeit_file_keys(self): """ return super().pypeit_file_keys() + ['hatch', 'lampstat01', 'frameno'] - - - def check_frame_type(self, ftype, fitstbl, exprng=None): """ Check for frames of the provided type. @@ -369,14 +394,14 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) # TODO: Allow for 'sky' frame type, for now include sky in # 'science' category - if ftype == 'science': - return good_exp & (fitstbl['idname'] == 'Object') - if ftype == 'standard': + if ftype in ['science', 'standard']: return good_exp & (fitstbl['idname'] == 'Object') if ftype == 'bias': return good_exp & (fitstbl['idname'] == 'Bias') if ftype == 'dark': return good_exp & (fitstbl['idname'] == 'Dark') + if ftype == 'slitless_pixflat': + return good_exp & (fitstbl['idname'] == 'slitlessFlat') if ftype in ['illumflat', 'pixelflat', 'trace']: # Flats and trace frames are typed together return good_exp & (fitstbl['idname'] == 'IntFlat') @@ -387,6 +412,117 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool) + def vet_assigned_ftypes(self, type_bits, fitstbl): + """ + + NOTE: this function should only be called when running pypeit_setup, + in order to not overwrite any user-provided frame types. + + This method checks the assigned frame types for consistency. + For frames that are assigned both the science and standard types, + this method chooses the one that is most likely, by checking if the + frames are within 10 arcmin of a listed standard star. + + In addition, for this instrument, if a frame is assigned both a + pixelflat and slitless_pixflat type, the pixelflat type is removed. + NOTE: if the same frame is assigned to multiple configurations, this + method will remove the pixelflat type for all configurations, i.e., + it is not possible to use slitless_pixflat type for one calibration group + and pixelflat for another. + + Args: + type_bits (`numpy.ndarray`_): + Array with the frame types assigned to each frame. + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + + Returns: + `numpy.ndarray`_: The updated frame types. + + """ + type_bits = super().vet_assigned_ftypes(type_bits, fitstbl) + + # If both pixelflat and slitless_pixflat are assigned to the same frame, remove pixelflat + + # where slitless_pixflat is assigned + slitless_idx = fitstbl.type_bitmask.flagged(type_bits, flag='slitless_pixflat') + # where pixelflat is assigned + pixelflat_idx = fitstbl.type_bitmask.flagged(type_bits, flag='pixelflat') + + # find configurations where both pixelflat and slitless_pixflat are assigned + pixflat_match = np.zeros(len(fitstbl), dtype=bool) + + for f, frame in enumerate(fitstbl): + if pixelflat_idx[f]: + match_config_values = [] + for slitless in fitstbl[slitless_idx]: + match_config_values.append(np.all([frame[c] == slitless[c] + for c in self.config_independent_frames()['slitless_pixflat']])) + pixflat_match[f] = np.any(match_config_values) + + # remove pixelflat from the type_bits + type_bits[pixflat_match] = fitstbl.type_bitmask.turn_off(type_bits[pixflat_match], 'pixelflat') + + return type_bits + + def parse_raw_files(self, fitstbl, det=1, ftype=None): + """ + Parse the list of raw files with given frame type and detector. + This is spectrograph-specific, and it is not defined for all + spectrographs. + Since different slitless_pixflat frames are usually taken for + each of the three detectors, this method parses the slitless_pixflat + frames and returns the correct one for the requested detector. + + Args: + fitstbl (`astropy.table.Table`_): + Table with metadata of the raw files to parse. + det (:obj:`int`, optional): + 1-indexed detector number to parse. + ftype (:obj:`str`, optional): + Frame type to parse. If None, no frames are parsed + and the indices of all frames are returned. + + Returns: + `numpy.ndarray`_: The indices of the raw files in the fitstbl that are parsed. + + """ + + if ftype == 'slitless_pixflat': + # Check for the required info + if len(fitstbl) == 0: + msgs.warn('Fitstbl provided is emtpy. No parsing done.') + # return empty array + return np.array([], dtype=int) + elif det is None: + msgs.warn('Detector number must be provided to parse slitless_pixflat frames. No parsing done.') + # return index array of length of fitstbl + return np.arange(len(fitstbl)) + + # how many unique xdangle values are there? + # If they are 3, then we have a different slitless flat file per detector + xdangles = np.unique(np.int32(fitstbl['xdangle'].value)) + if len(xdangles) == 3: + sort_xdagles = np.argsort(xdangles) + # xdagles: -5 for red(det=3), -4 for green (det=2), -3 for blue (det=1) dets + # select the corresponding files for the requested detector + if det == 1: + # blue detector + return np.where(np.int32(fitstbl['xdangle'].value) == -3)[0] + elif det == 2: + # green detector + return np.where(np.int32(fitstbl['xdangle'].value) == -4)[0] + elif det == 3: + # red detector + return np.where(np.int32(fitstbl['xdangle'].value) == -5)[0] + else: + msgs.warn('The provided list of slitless_pixflat frames does not have exactly 3 unique XDANGLE values. ' + 'Pypeit cannot determine which slitless_pixflat frame corresponds to the requested detector. ' + 'All frames will be used.') + return np.arange(len(fitstbl)) + + else: + return super().parse_raw_files(fitstbl, det=det, ftype=ftype) def get_rawimage(self, raw_file, det, spectrim=20): """ @@ -502,7 +638,7 @@ def get_rawimage(self, raw_file, det, spectrim=20): return mosaic, image, hdu, exptime, rawdatasec_img, oscansec_img - def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): + def get_mosaic_par(self, mosaic, hdu=None, msc_ord=0): """ Return the hard-coded parameters needed to construct detector mosaics from unbinned images. @@ -523,7 +659,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): default. BEWARE: If ``hdu`` is not provided, the binning is assumed to be `1,1`, which will cause faults if applied to binned images! - msc_order (:obj:`int`, optional): + msc_ord (:obj:`int`, optional): Order of the interpolation used to construct the mosaic. Returns: @@ -574,7 +710,7 @@ def get_mosaic_par(self, mosaic, hdu=None, msc_order=0): msc_tfm[i] = build_image_mosaic_transform(shape, msc_sft[i], msc_rot[i], tuple(reversed(binning))) return Mosaic(mosaic_id, detectors, shape, np.array(msc_sft), np.array(msc_rot), - np.array(msc_tfm), msc_order) + np.array(msc_tfm), msc_ord) @property diff --git a/pypeit/spectrographs/keck_kcwi.py b/pypeit/spectrographs/keck_kcwi.py index cdf4a20b10..abb0bd1179 100644 --- a/pypeit/spectrographs/keck_kcwi.py +++ b/pypeit/spectrographs/keck_kcwi.py @@ -860,7 +860,7 @@ def get_detector_par(self, det, hdu=None): # Some properties of the image binning = self.compound_meta(self.get_headarr(hdu), "binning") numamps = hdu[0].header['NVIDINP'] - specflip = True if hdu[0].header['AMPID1'] == 2 else False + specflip = False if hdu[0].header['AMPMODE'] == 'ALL' else True gainmul, gainarr = hdu[0].header['GAINMUL'], np.zeros(numamps) ronarr = np.zeros(numamps) # Set this to zero (determine the readout noise from the overscan regions) # dsecarr = np.array(['']*numamps) diff --git a/pypeit/spectrographs/keck_lris.py b/pypeit/spectrographs/keck_lris.py index ffc42afb87..8c807d052a 100644 --- a/pypeit/spectrographs/keck_lris.py +++ b/pypeit/spectrographs/keck_lris.py @@ -22,6 +22,7 @@ from pypeit import io from pypeit.core import parse from pypeit.core import framematch +from pypeit.core import flux_calib from pypeit.spectrographs import spectrograph from pypeit.spectrographs import slitmask from pypeit.images import detector_container @@ -80,14 +81,21 @@ def default_pypeit_par(cls): par['calibrations']['wavelengths']['n_first'] = 3 par['calibrations']['wavelengths']['n_final'] = 5 # Set the default exposure time ranges for the frame typing - par['calibrations']['biasframe']['exprng'] = [None, 0.001] + par['calibrations']['biasframe']['exprng'] = [None, 1] par['calibrations']['darkframe']['exprng'] = [999999, None] # No dark frames par['calibrations']['pinholeframe']['exprng'] = [999999, None] # No pinhole frames - par['calibrations']['pixelflatframe']['exprng'] = [None, 60] - par['calibrations']['traceframe']['exprng'] = [None, 60] - par['calibrations']['illumflatframe']['exprng'] = [None, 60] + par['calibrations']['pixelflatframe']['exprng'] = [0, 60] + par['calibrations']['traceframe']['exprng'] = [0, 60] + par['calibrations']['illumflatframe']['exprng'] = [0, 60] + par['calibrations']['slitless_pixflatframe']['exprng'] = [0, 60] par['calibrations']['standardframe']['exprng'] = [1, 61] + # Set default processing for slitless_pixflat + par['calibrations']['slitless_pixflatframe']['process']['scale_to_mean'] = True + + # Turn off flat illumination fine correction (it's more often bad than good) + par['calibrations']['flatfield']['slit_illum_finecorr'] = False + # Flexure # Always correct for spectral flexure, starting with default parameters par['flexure']['spec_method'] = 'boxcar' @@ -153,8 +161,8 @@ def init_meta(self): """ self.meta = {} # Required (core) - self.meta['ra'] = dict(ext=0, card='RA') - self.meta['dec'] = dict(ext=0, card='DEC') + self.meta['ra'] = dict(card=None, compound=True) + self.meta['dec'] = dict(card=None, compound=True) self.meta['target'] = dict(ext=0, card='TARGNAME') self.meta['decker'] = dict(ext=0, card='SLITNAME') self.meta['binning'] = dict(card=None, compound=True) @@ -174,6 +182,7 @@ def init_meta(self): # Extras for pypeit file self.meta['dateobs'] = dict(card=None, compound=True) self.meta['amp'] = dict(ext=0, card='NUMAMPS') + self.meta['object'] = dict(ext=0, card='OBJECT') # Lamps # similar approach to DEIMOS @@ -193,7 +202,20 @@ def compound_meta(self, headarr, meta_key): Returns: object: Metadata value read from the header(s). """ - if meta_key == 'binning': + # LRIS sometime misses RA and/or Dec in the header. When this happens, set them to 0 + if meta_key == 'ra': + if headarr[0].get('RA') is None: + msgs.warn('Keyword RA not found in header. Setting to 0') + return '00:00:00.00' + else: + return headarr[0]['RA'] + elif meta_key == 'dec': + if headarr[0].get('DEC') is None: + msgs.warn('Keyword DEC not found in header. Setting to 0') + return '+00:00:00.0' + else: + return headarr[0]['DEC'] + elif meta_key == 'binning': binspatial, binspec = parse.parse_binning(headarr[0]['BINNING']) binning = parse.binning2string(binspec, binspatial) return binning @@ -292,7 +314,8 @@ def config_independent_frames(self): keywords that can be used to assign the frames to a configuration group. """ - return {'bias': ['amp', 'binning', 'dateobs'], 'dark': ['amp', 'binning', 'dateobs']} + return {'bias': ['amp', 'binning', 'dateobs'], 'dark': ['amp', 'binning', 'dateobs'], + 'slitless_pixflat': ['amp', 'binning', 'dateobs', 'dispname', 'dichroic']} def pypeit_file_keys(self): """ @@ -324,20 +347,49 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): `numpy.ndarray`_: Boolean array with the flags selecting the exposures in ``fitstbl`` that are ``ftype`` type frames. """ - good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) + # good exposures + good_exp = framematch.check_frame_exptime(fitstbl['exptime'], exprng) & (fitstbl['decker'] != 'GOH_LRIS') + # no images no_img = np.array([d not in ['Mirror', 'mirror', 'clear'] for d in fitstbl['dispname']]) + + # Check frame type if ftype == 'science': return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') & no_img if ftype == 'standard': - return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') & no_img + std = np.zeros(len(fitstbl), dtype=bool) + if 'ra' in fitstbl.keys() and 'dec' in fitstbl.keys(): + std = np.array([flux_calib.find_standard_file(ra, dec, toler=10.*units.arcmin, check=True) + for ra, dec in zip(fitstbl['ra'], fitstbl['dec'])]) + return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') & no_img & std if ftype == 'bias': return good_exp & self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'closed') + if ftype == 'slitless_pixflat': + # these are sky flats, like science but without the slitmask + return (good_exp & self.lamps(fitstbl, 'off') & + (fitstbl['hatch'] == 'open') & no_img & (fitstbl['decker'] == 'direct')) if ftype in ['pixelflat', 'trace', 'illumflat']: # Allow for dome or internal good_dome = self.lamps(fitstbl, 'dome') & (fitstbl['hatch'] == 'open') good_internal = self.lamps(fitstbl, 'internal') & (fitstbl['hatch'] == 'closed') + # attempt at identifying sky flats (not robust, but better than nothing) + # they are basically science frames, so we look for "sky" words in the header + is_sky = self.lamps(fitstbl, 'off') & (fitstbl['hatch'] == 'open') + # look for specific words in the target or object header keywords + words_to_search = ['sky', 'blank', 'twilight', 'twiflat', 'twi flat'] + for i, row in enumerate(fitstbl): + in_target = False + if row['target'] is not None: + if np.any([w in row['target'].lower() for w in words_to_search]): + in_target = True + in_object = False + if row['object'] is not None: + if np.any([w in row['object'].lower() for w in words_to_search]): + in_object = True + is_sky[i] = in_target or in_object + # put together the sky flats requirement + sky_flat = is_sky & (fitstbl['decker'] != 'direct') # Flats and trace frames are typed together - return good_exp & (good_dome + good_internal) & no_img + return good_exp & (good_dome + good_internal + sky_flat) & no_img if ftype in ['pinhole', 'dark']: # Don't type pinhole or dark frames return np.zeros(len(fitstbl), dtype=bool) @@ -346,6 +398,59 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): msgs.warn('Cannot determine if frames are of type {0}.'.format(ftype)) return np.zeros(len(fitstbl), dtype=bool) + + def vet_assigned_ftypes(self, type_bits, fitstbl): + """ + + NOTE: this function should only be called when running pypeit_setup, + in order to not overwrite any user-provided frame types. + + This method checks the assigned frame types for consistency. + For frames that are assigned both the science and standard types, + this method chooses the one that is most likely, by checking if the + frames are within 10 arcmin of a listed standard star. + + In addition, for this instrument, if a frame is assigned both a + pixelflat and slitless_pixflat type, the pixelflat type is removed. + NOTE: if the same frame is assigned to multiple configurations, this + method will remove the pixelflat type for all configurations, i.e., + it is not possible to use slitless_pixflat type for one calibration group + and pixelflat for another. + + Args: + type_bits (`numpy.ndarray`_): + Array with the frame types assigned to each frame. + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + + Returns: + `numpy.ndarray`_: The updated frame types. + + """ + type_bits = super().vet_assigned_ftypes(type_bits, fitstbl) + + # If both pixelflat and slitless_pixflat are assigned to the same frame, remove pixelflat + + # where slitless_pixflat is assigned + slitless_idx = fitstbl.type_bitmask.flagged(type_bits, flag='slitless_pixflat') + # where pixelflat is assigned + pixelflat_idx = fitstbl.type_bitmask.flagged(type_bits, flag='pixelflat') + + # find configurations where both pixelflat and slitless_pixflat are assigned + pixflat_match = np.zeros(len(fitstbl), dtype=bool) + + for f, frame in enumerate(fitstbl): + if pixelflat_idx[f]: + match_config_values = [] + for slitless in fitstbl[slitless_idx]: + match_config_values.append(np.all([frame[c] == slitless[c] + for c in self.config_independent_frames()['slitless_pixflat']])) + pixflat_match[f] = np.any(match_config_values) + + # remove pixelflat from the type_bits + type_bits[pixflat_match] = fitstbl.type_bitmask.turn_off(type_bits[pixflat_match], 'pixelflat') + + return type_bits def lamps(self, fitstbl, status): """ @@ -853,6 +958,8 @@ def default_pypeit_par(cls): par['calibrations']['traceframe']['exprng'] = [None, 300] par['calibrations']['illumflatframe']['exprng'] = [None, 300] + par['calibrations']['standardframe']['exprng'] = [1, 901] + return par def config_specific_par(self, scifile, inp_par=None): @@ -1927,38 +2034,48 @@ def lris_read_amp(inp, ext): return data, predata, postdata, x1, y1 -def convert_lowredux_pixelflat(infil, outfil): +def convert_lowredux_pixelflat(infil, outfil, specflip=False, separate_extensions=False): """ Convert LowRedux pixelflat to PYPIT format Returns ------- """ # Read - hdu = io.fits_open(infil) - data = hdu[0].data + hdu0 = io.fits_open(infil) + data = hdu0[0].data # prihdu = fits.PrimaryHDU() hdus = [prihdu] - prihdu.header['FRAMETYP'] = 'pixelflat' + prihdu.header['CALIBTYP'] = ('Flat', 'PypeIt: Calibration frame type') # Detector 1 - img1 = data[:,:data.shape[1]//2] + if separate_extensions: + img1 = hdu0['DET1'].data + else: + img1 = data[:, :data.shape[1] // 2] + if specflip: + img1 = np.flip(img1, axis=0) hdu = fits.ImageHDU(img1) - hdu.name = 'DET1' - prihdu.header['EXT0001'] = 'DET1-pixelflat' + hdu.name = 'DET01-PIXELFLAT_NORM' + prihdu.header['EXT0001'] = hdu.name hdus.append(hdu) # Detector 2 - img2 = data[:,data.shape[1]//2:] + if separate_extensions: + img2 = hdu0['DET2'].data + else: + img2 = data[:, data.shape[1] // 2:] + if specflip: + img2 = np.flip(img2, axis=0) hdu = fits.ImageHDU(img2) - hdu.name = 'DET2' - prihdu.header['EXT0002'] = 'DET2-pixelflat' + hdu.name = 'DET02-PIXELFLAT_NORM' + prihdu.header['EXT0002'] = hdu.name hdus.append(hdu) # Finish hdulist = fits.HDUList(hdus) - hdulist.writeto(outfil, clobber=True) + hdulist.writeto(outfil, overwrite=True) print('Wrote {:s}'.format(outfil)) diff --git a/pypeit/spectrographs/keck_mosfire.py b/pypeit/spectrographs/keck_mosfire.py index e874baea83..44ffc91f1d 100644 --- a/pypeit/spectrographs/keck_mosfire.py +++ b/pypeit/spectrographs/keck_mosfire.py @@ -335,7 +335,7 @@ def compound_meta(self, headarr, meta_key): FLATSPEC = headarr[0].get('FLATSPEC') PWSTATA7 = headarr[0].get('PWSTATA7') PWSTATA8 = headarr[0].get('PWSTATA8') - if FLATSPEC == 0 and PWSTATA7 == 0 and PWSTATA8 == 0: + if FLATSPEC == 0 and PWSTATA7 == 0 and PWSTATA8 == 0 and headarr[0].get('FILTER') != 'Dark': if 'Flat' in headarr[0].get('OBJECT'): return 'flatlampoff' else: diff --git a/pypeit/spectrographs/mdm_modspec.py b/pypeit/spectrographs/mdm_modspec.py index 68b1dbe2c6..25f0f78077 100644 --- a/pypeit/spectrographs/mdm_modspec.py +++ b/pypeit/spectrographs/mdm_modspec.py @@ -31,6 +31,8 @@ class MDMModspecEchelleSpectrograph(spectrograph.Spectrograph): supported = True comment = 'MDM Modspec spectrometer; Only 1200l/mm disperser (so far)' + allowed_extensions = ['.fit'] + def get_detector_par(self, det, hdu=None): """ Return metadata for the selected detector. diff --git a/pypeit/spectrographs/spectrograph.py b/pypeit/spectrographs/spectrograph.py index 4c8de5d4a9..81bf5843fd 100644 --- a/pypeit/spectrographs/spectrograph.py +++ b/pypeit/spectrographs/spectrograph.py @@ -33,12 +33,14 @@ import numpy as np from astropy.io import fits +from astropy import units from pypeit import msgs from pypeit import io from pypeit.core import parse from pypeit.core import procimg from pypeit.core import meta +from pypeit.core import flux_calib from pypeit.par import pypeitpar from pypeit.images.detector_container import DetectorContainer from pypeit.images.mosaic import Mosaic @@ -145,7 +147,7 @@ class Spectrograph: Metadata model that is generic to all spectrographs. """ - allowed_extensions = None + allowed_extensions = ['.fits', '.fits.gz'] """ Defines the allowed extensions for the input fits files. """ @@ -269,6 +271,44 @@ def ql_par(): ) ) ) + + @classmethod + def find_raw_files(cls, root, extension=None): + """ + Find raw observations for this spectrograph in the provided directory. + + This is a wrapper for :func:`~pypeit.io.files_from_extension` that + handles the restrictions of the file extensions specific to this + spectrograph. + + Args: + root (:obj:`str`, `Path`_, :obj:`list`): + One or more paths to search for files, which may or may not include + the prefix of the files to search for. For string input, this can + be the directory ``'/path/to/files/'`` or the directory plus the + file prefix ``'/path/to/files/prefix'``, which yeilds the search + strings ``'/path/to/files/*fits'`` or + ``'/path/to/files/prefix*fits'``, respectively. For a list input, + this can use wildcards for multiple directories. + extension (:obj:`str`, :obj:`list`, optional): + One or more file extensions to search on. If None, uses + :attr:`allowed_extensions`. Otherwise, this *must* be a subset + of the allowed extensions for the selected spectrograph. + + Returns: + :obj:`list`: List of `Path`_ objects with the full path to the set of + unique raw data filenames that match the provided criteria search + strings. + """ + if extension is None: + _ext = cls.allowed_extensions + else: + _ext = [extension] if isinstance(extension, str) else extension + _ext = [e for e in _ext if e in cls.allowed_extensions] + if len(_ext) == 0: + msgs.error(f'{extension} is not or does not include allowed extensions for ' + f'{cls.name}; choose from {cls.allowed_extensions}.') + return io.files_from_extension(root, extension=_ext) def _check_extensions(self, filename): """ @@ -280,9 +320,9 @@ def _check_extensions(self, filename): """ if self.allowed_extensions is not None: _filename = Path(filename).absolute() - if _filename.suffix not in self.allowed_extensions: + if not any([_filename.name.endswith(ext) for ext in self.allowed_extensions]): msgs.error(f'The input file ({_filename.name}) does not have a recognized ' - f'extension ({_filename.suffix}). The allowed extensions for ' + f'extension. The allowed extensions for ' f'{self.name} include {",".join(self.allowed_extensions)}.') def _check_telescope(self): @@ -627,6 +667,28 @@ def list_detectors(self, mosaic=False): dets = self.allowed_mosaics if mosaic else range(1,self.ndet+1) return np.array([self.get_det_name(det) for det in dets]) + def parse_raw_files(self, fitstbl, det=1, ftype=None): + """ + Parse the list of raw files with given frame type and detector. + This is spectrograph-specific, and it is not defined for all + spectrographs. Therefore, this generic method + returns the indices of all the files in the input table. + + Args: + fitstbl (`astropy.table.Table`_): + Table with metadata of the raw files to parse. + det (:obj:`int`, optional): + 1-indexed detector number to parse. + ftype (:obj:`str`, optional): + Frame type to parse. If None, no frames are parsed + and the indices of all frames are returned. + + Returns: + `numpy.ndarray`_: The indices of the raw files in the fitstbl that are parsed. + + """ + + return np.arange(len(fitstbl)) def get_lamps(self, fitstbl): """ @@ -1596,6 +1658,73 @@ def check_frame_type(self, ftype, fitstbl, exprng=None): """ raise NotImplementedError('Frame typing not defined for {0}.'.format(self.name)) + def vet_assigned_ftypes(self, type_bits, fitstbl): + """ + NOTE: this function should only be called when running pypeit_setup, + in order to not overwrite any user-provided frame types. + + This method checks the assigned frame types for consistency. + For frames that are assigned both the science and standard types, + this method chooses the one that is most likely, by checking if the + frames are within 10 arcmin of a listed standard star. + + In addition, this method can perform other checks on the assigned frame types + that are spectrograph-specific. + + Args: + type_bits (`numpy.ndarray`_): + Array with the frame types assigned to each frame. + fitstbl (:class:`~pypeit.metadata.PypeItMetaData`): + The class holding the metadata for all the frames. + + Returns: + `numpy.ndarray`_: The updated frame types. + + """ + # For frames that are assigned both science and standard types, choose the one that is most likely + # find frames that are assigned both science and standard star types + indx = fitstbl.type_bitmask.flagged(type_bits, flag='standard') & \ + fitstbl.type_bitmask.flagged(type_bits, flag='science') + if np.any(indx): + msgs.warn('Some frames are assigned both science and standard types. Choosing the most likely type.') + if 'ra' not in fitstbl.keys() or 'dec' not in fitstbl.keys(): + msgs.warn('Sky coordinates are not available. Standard stars cannot be identified.') + # turn off the standard flag for all frames + type_bits[indx] = fitstbl.type_bitmask.turn_off(type_bits[indx], flag='standard') + return type_bits + # check if any coordinates are None + none_coords = indx & ((fitstbl['ra'] == 'None') | (fitstbl['dec'] == 'None') | + np.isnan(fitstbl['ra']) | np.isnan(fitstbl['dec'])) + if np.any(none_coords): + msgs.warn('The following frames have None coordinates. ' + 'They could be a twilight flat frame that was missed by the automatic identification') + [msgs.prindent(f) for f in fitstbl['filename'][none_coords]] + # turn off the standard star flag for these frames + type_bits[none_coords] = fitstbl.type_bitmask.turn_off(type_bits[none_coords], flag='standard') + + # If the frame is within 10 arcmin of a listed standard star, then it is probably a standard star + # Find the nearest standard star to each frame that is assigned both science and standard types + # deal with possible None coordinates + is_std = np.array([], dtype=bool) + for ra, dec in zip(fitstbl['ra'], fitstbl['dec']): + if ra == 'None' or dec == 'None' or np.isnan(ra) or np.isnan(dec): + is_std = np.append(is_std, False) + else: + is_std = np.append(is_std, flux_calib.find_standard_file(ra, dec, toler=10.*units.arcmin, check=True)) + + foundstd = indx & is_std + # turn off the science flag for frames that are found to be standard stars and + # turn off the standard flag for frames that are not + if np.any(foundstd): + type_bits[foundstd] = fitstbl.type_bitmask.turn_off(type_bits[foundstd], flag='science') + type_bits[np.logical_not(foundstd)] = \ + fitstbl.type_bitmask.turn_off(type_bits[np.logical_not(foundstd)], flag='standard') + else: + # if no standard stars are found, turn off the standard flag for all frames + type_bits[indx] = fitstbl.type_bitmask.turn_off(type_bits[indx], flag='standard') + + return type_bits + def idname(self, ftype): """ Return the ``idname`` for the selected frame type for this @@ -1794,7 +1923,6 @@ def spec1d_match_spectra(self, sobjs): def tweak_standard(self, wave_in, counts_in, counts_ivar_in, gpm_in, meta_table, log10_blaze_function=None): """ - This routine is for performing instrument/disperser specific tweaks to standard stars so that sensitivity function fits will be well behaved. For example, masking second order light. For instruments that don't require such tweaks it will just return the inputs, but for instruments that do this function is overloaded diff --git a/pypeit/spectrographs/vlt_xshooter.py b/pypeit/spectrographs/vlt_xshooter.py index 98b5cf6895..9160f834e3 100644 --- a/pypeit/spectrographs/vlt_xshooter.py +++ b/pypeit/spectrographs/vlt_xshooter.py @@ -574,8 +574,8 @@ def get_detector_par(self, det, hdu=None): # Grab the gain and read noise from the header. # If hdu not present, use typical defaults - gain = None if hdu is None else hdu[0].header['HIERARCH ESO DET OUT1 CONAD'] - ronoise = None if hdu is None else hdu[0].header['HIERARCH ESO DET OUT1 RON'] + gain = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 CONAD']) + ronoise = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 RON']) # Detector 1 detector_dict = dict( @@ -591,8 +591,8 @@ def get_detector_par(self, det, hdu=None): nonlinear = 0.86, mincounts = -1e10, numamplifiers = 1, - gain = np.atleast_1d(gain), - ronoise = np.atleast_1d(ronoise), + gain = gain, + ronoise = ronoise, datasec=np.atleast_1d('[:,11:2058]'), # FITS format is flipped: PrimaryHDU (2106, 4000) w/respect to Python oscansec=np.atleast_1d('[:,2059:2106]'), # raw unbinned images are (4000,2106) (spec, spat) ) # pre and oscan are in the spatial direction @@ -866,8 +866,8 @@ def get_detector_par(self, det, hdu=None): binning = '1,1' if hdu is None else self.get_meta_value(self.get_headarr(hdu), 'binning') # Grab the gain and read noise from the header. - gain = None if hdu is None else hdu[0].header['HIERARCH ESO DET OUT1 CONAD'] - ronoise = None if hdu is None else hdu[0].header['HIERARCH ESO DET OUT1 RON'] + gain = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 CONAD']) + ronoise = None if hdu is None else np.atleast_1d(hdu[0].header['HIERARCH ESO DET OUT1 RON']) # Detector 1 detector_dict = dict( @@ -883,8 +883,8 @@ def get_detector_par(self, det, hdu=None): nonlinear = 0.86, mincounts = -1e10, numamplifiers = 1, - gain = np.atleast_1d(gain), - ronoise = np.atleast_1d(ronoise), + gain = gain, + ronoise = ronoise, datasec = np.atleast_1d('[:,49:2096]'), # '[49:2000,1:2999]', oscansec = np.atleast_1d('[:,1:48]'), # '[1:48, 1:2999]', ) diff --git a/pypeit/spectrographs/wht_isis.py b/pypeit/spectrographs/wht_isis.py index e107264ac3..7464e459b8 100644 --- a/pypeit/spectrographs/wht_isis.py +++ b/pypeit/spectrographs/wht_isis.py @@ -125,6 +125,7 @@ class WHTISISBlueSpectrograph(WHTISISSpectrograph): name = 'wht_isis_blue' camera = 'ISISb' comment = 'Blue camera' + allowed_extensions = ['.fit', '.fit.gz'] def get_detector_par(self, det, hdu=None): """ diff --git a/pypeit/telescopes.py b/pypeit/telescopes.py index e8bc8c3d3a..32e9cbff80 100644 --- a/pypeit/telescopes.py +++ b/pypeit/telescopes.py @@ -10,6 +10,18 @@ #TODO: Remove 'Par' from class name? + +class AATTelescopePar(TelescopePar): + def __init__(self): + loc = EarthLocation.of_site('Siding Spring Observatory') + super(AATTelescopePar, self).__init__(name='AAT', + longitude=loc.lon.to(units.deg).value, + latitude=loc.lat.to(units.deg).value, + elevation=loc.height.to(units.m).value, + diameter=3.9, + eff_aperture=12.0) + + class GTCTelescopePar(TelescopePar): def __init__(self): loc = EarthLocation.of_site('Roque de los Muchachos') diff --git a/pypeit/tests/test_arc.py b/pypeit/tests/test_arc.py index fef470534e..3cde322d37 100644 --- a/pypeit/tests/test_arc.py +++ b/pypeit/tests/test_arc.py @@ -2,10 +2,12 @@ Module to run tests on ararclines """ import pytest +import numpy as np from pypeit.core import arc from pypeit import io + def test_detect_lines(): # Using Paranal night sky as an 'arc' arx_sky = io.load_sky_spectrum('paranal_sky.fits') diff --git a/pypeit/tests/test_calibrations.py b/pypeit/tests/test_calibrations.py index 3705674b59..52ac1958ac 100644 --- a/pypeit/tests/test_calibrations.py +++ b/pypeit/tests/test_calibrations.py @@ -1,13 +1,10 @@ -""" -Module to run tests on FlatField class -Requires files in Development suite and an Environmental variable -""" from pathlib import Path -import os import yaml import pytest import shutil +from IPython import embed + import numpy as np from pypeit import dataPaths @@ -16,9 +13,8 @@ from pypeit.images import buildimage from pypeit.par import pypeitpar from pypeit.spectrographs.util import load_spectrograph -from IPython import embed -from pypeit.tests.tstutils import dummy_fitstbl, data_output_path +from pypeit.tests.tstutils import data_output_path @pytest.fixture def fitstbl(): diff --git a/pypeit/tests/test_fluxspec.py b/pypeit/tests/test_fluxspec.py index fc0a73da34..a51b3aed00 100644 --- a/pypeit/tests/test_fluxspec.py +++ b/pypeit/tests/test_fluxspec.py @@ -181,7 +181,8 @@ def extinction_correction_tester(algorithm): wave = np.linspace(4000, 6000) counts = np.ones_like(wave) ivar = np.ones_like(wave) - sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, counts, ivar) + flat = np.ones_like(wave) + sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, counts, ivar, flat) sobjs = specobjs.SpecObjs([sobj]) # choice of PYP_SPEC, DISPNAME and EXPTIME are unimportant here diff --git a/pypeit/tests/test_io.py b/pypeit/tests/test_io.py index 95d03abb14..2a6f226297 100644 --- a/pypeit/tests/test_io.py +++ b/pypeit/tests/test_io.py @@ -48,5 +48,5 @@ def test_grab_rawfiles(): _raw_files = inputfiles.grab_rawfiles(raw_paths=[str(root)], extension='.fits.gz') assert len(_raw_files) == 9, 'Found the wrong number of files' - assert all([str(root / f) in _raw_files for f in tbl['filename']]), 'Missing expected files' + assert all([root / f in _raw_files for f in tbl['filename']]), 'Missing expected files' diff --git a/pypeit/tests/test_metadata.py b/pypeit/tests/test_metadata.py index 048191cc3f..467daa44bf 100644 --- a/pypeit/tests/test_metadata.py +++ b/pypeit/tests/test_metadata.py @@ -30,7 +30,7 @@ def test_read_combid(): # Generate the pypeit file with the comb_id droot = tstutils.data_output_path('b') pargs = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blue', '-c', 'all', '-b', - '--extension', 'fits.gz', '--output_path', f'{config_dir.parent}']) + '--output_path', f'{config_dir.parent}']) Setup.main(pargs) pypeit_file = config_dir / 'shane_kast_blue_A.pypeit' diff --git a/pypeit/tests/test_setups.py b/pypeit/tests/test_setups.py index e4356968e1..5759072024 100644 --- a/pypeit/tests/test_setups.py +++ b/pypeit/tests/test_setups.py @@ -63,12 +63,12 @@ def test_run_setup(): droot = tstutils.data_output_path('b') odir = Path(tstutils.data_output_path('')).absolute() / 'shane_kast_blue_A' pargs = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blue', '-c', 'all', - '--extension', 'fits.gz', '--output_path', f'{odir.parent}']) + '--output_path', f'{odir.parent}']) Setup.main(pargs) # Fails because name of spectrograph is wrong pargs2 = Setup.parse_args(['-r', droot, '-s', 'shane_kast_blu', '-c', 'all', - '--extension', 'fits.gz', '--output_path', f'{odir.parent}']) + '--output_path', f'{odir.parent}']) with pytest.raises(ValueError): Setup.main(pargs2) diff --git a/pypeit/tests/test_specobj.py b/pypeit/tests/test_specobj.py index 5e647648df..6ec2da6cfe 100644 --- a/pypeit/tests/test_specobj.py +++ b/pypeit/tests/test_specobj.py @@ -101,7 +101,8 @@ def test_from_arrays(): wave = np.linspace(5000., 6000, 1000) flux = np.ones_like(wave) ivar = 0.1*np.ones_like(wave) - sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, flux, ivar) + flat = np.ones_like(wave) + sobj = specobj.SpecObj.from_arrays('MultiSlit', wave, flux, ivar, flat) assert sobj.OPT_WAVE[0] == 5000. diff --git a/pypeit/wavecalib.py b/pypeit/wavecalib.py index 4e576ee53f..ca223a39e2 100644 --- a/pypeit/wavecalib.py +++ b/pypeit/wavecalib.py @@ -526,11 +526,11 @@ def __init__(self, msarc, slits, spectrograph, par, lamps, self.arccen = None # central arc spectrum # Get the non-linear count level - # TODO: This is currently hacked to deal with Mosaics - try: + if self.msarc.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + self.nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in self.msarc.detector.detectors]) + else: self.nonlinear_counts = self.msarc.detector.nonlinear_counts() - except: - self.nonlinear_counts = 1e10 # -------------------------------------------------------------- # TODO: Build another base class that does these things for both diff --git a/pypeit/wavetilts.py b/pypeit/wavetilts.py index cf9977744a..2b24d0005f 100644 --- a/pypeit/wavetilts.py +++ b/pypeit/wavetilts.py @@ -318,11 +318,11 @@ def __init__(self, mstilt, slits, spectrograph, par, wavepar, det=1, qa_path=Non self.spat_flexure = spat_flexure # Get the non-linear count level - # TODO: This is currently hacked to deal with Mosaics - try: + if self.mstilt.is_mosaic: + # if this is a mosaic we take the maximum value among all the detectors + self.nonlinear_counts = np.max([rawdets.nonlinear_counts() for rawdets in self.mstilt.detector.detectors]) + else: self.nonlinear_counts = self.mstilt.detector.nonlinear_counts() - except: - self.nonlinear_counts = 1e10 # Set the slitmask and slit boundary related attributes that the # code needs for execution. This also deals with arcimages that diff --git a/setup.cfg b/setup.cfg index 71821b46a4..294a1e1615 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,7 +17,6 @@ classifiers = Natural Language :: English Operating System :: OS Independent Programming Language :: Python - Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 Programming Language :: Python :: 3.12 Topic :: Documentation :: Sphinx @@ -29,28 +28,28 @@ classifiers = zip_safe = False use_2to3=False packages = find: -python_requires = >=3.10,<3.13 +python_requires = >=3.11,<3.13 setup_requires = setuptools_scm include_package_data = True install_requires = - numpy>=1.23,<2.0.0 + numpy>=1.24 astropy>=6.0 - extension-helpers>=0.1 - packaging>=0.19 - scipy>=1.7 + extension-helpers>=1.0 + packaging>=22.0 + scipy>=1.9 matplotlib>=3.7 - PyYAML>=5.1 + PyYAML>=6.0 PyERFA>=2.0.0 fast-histogram>=0.11 configobj>=5.0.6 - scikit-learn>=1.0 - IPython>=7.10.0 + scikit-learn>=1.2 + IPython>=8.0.0 ginga>=5.1.0 - linetools>=0.3.1 - qtpy>=2.0.1 + linetools>=0.3.2 + qtpy>=2.2.0 pygithub bottleneck - pyqt6<=6.7.0 + pyqt6 scripts = bin/pypeit_c_enabled bin/pypeit_chk_plugins @@ -69,7 +68,7 @@ specutils = specutils>=1.13 test = pygit2 - pytest>=6.0.0 + pytest>=7.0.0 pytest-astropy tox pytest-cov @@ -84,12 +83,12 @@ devsuite = pytest-qt dev = # scikit-image - scikit-image + scikit-image>=0.23 # specutils specutils>=1.13 # test pygit2 - pytest>=6.0.0 + pytest>=7.0.0 pytest-astropy tox pytest-cov @@ -110,6 +109,7 @@ console_scripts = pypeit_arxiv_solution = pypeit.scripts.arxiv_solution:ArxivSolution.entry_point pypeit_cache_github_data = pypeit.scripts.cache_github_data:CacheGithubData.entry_point pypeit_clean_cache = pypeit.scripts.clean_cache:CleanCache.entry_point + pypeit_chk_flexure = pypeit.scripts.chk_flexure:ChkFlexure.entry_point pypeit_chk_for_calibs = pypeit.scripts.chk_for_calibs:ChkForCalibs.entry_point pypeit_chk_noise_1dspec = pypeit.scripts.chk_noise_1dspec:ChkNoise1D.entry_point pypeit_chk_noise_2dspec = pypeit.scripts.chk_noise_2dspec:ChkNoise2D.entry_point @@ -169,6 +169,7 @@ console_scripts = pypeit_skysub_regions = pypeit.scripts.skysub_regions:SkySubRegions.entry_point pypeit_view_fits = pypeit.scripts.view_fits:ViewFits.entry_point pypeit_setup_gui = pypeit.scripts.setup_gui:SetupGUI.entry_point + pypeit_show_pixflat = pypeit.scripts.show_pixflat:ShowPixFlat.entry_point ginga.rv.plugins = SlitWavelength = pypeit.display:setup_SlitWavelength diff --git a/tox.ini b/tox.ini index 55bbe46e07..c8b7d51210 100644 --- a/tox.ini +++ b/tox.ini @@ -1,12 +1,12 @@ [tox] envlist = - {3.10,3.11,3.12}-test{,-alldeps,-shapely,-specutils}{,-cov} - {3.10,3.11,3.12}-test-numpy{123,124,125,126} - {3.10,3.11,3.12}-test-{numpy,astropy,linetools,ginga}dev + {3.11,3.12}-test{,-alldeps,-shapely,-specutils}{,-cov} + {3.11,3.12}-test-numpy{124,125,126,200,201} + {3.11,3.12}-test-{numpy,astropy,linetools,ginga}dev codestyle requires = - setuptools >= 30.3.0 - pip >= 19.3.1 + setuptools >= 65.0 + pip >= 22.0 isolated_build = true [testenv] @@ -36,19 +36,21 @@ description = devdeps: with the latest developer version of key dependencies oldestdeps: with the oldest supported version of key dependencies cov: and test coverage - numpy123: with numpy 1.23.* numpy124: with numpy 1.24.* numpy125: with numpy 1.25.* numpy126: with numpy 1.26.* + numpy200: with numpy 2.0.* + numpy201: with numpy 2.1.* # The following provides some specific pinnings for key packages deps = cov: coverage - numpy123: numpy==1.23.* numpy124: numpy==1.24.* numpy125: numpy==1.25.* numpy126: numpy==1.26.* + numpy200: numpy==2.0.* + numpy201: numpy==2.1.* numpydev: numpy>=0.0.dev0 astropydev: git+https://github.com/astropy/astropy.git#egg=astropy