Merge remote-tracking branch 'upstream/develop' into develop

.github/workflows/ci_cron.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.11', '3.12']
         toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev]
     steps:
     - name: Check out repository

.github/workflows/ci_tests.yml

@@ -17,8 +17,8 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ['3.10', '3.11', '3.12']
-        toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev]
+        python: ['3.11', '3.12']
+        toxenv: [test, test-alldeps-cov, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev]
     steps:
     - name: Check out repository
       uses: actions/checkout@v3
@@ -48,7 +48,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-latest, macos-latest]
-        python: ['3.10', '3.11', '3.12']
+        python: ['3.11', '3.12']
         toxenv: [test-alldeps]
     steps:
     - name: Check out repository
@@ -71,7 +71,7 @@ jobs:
     - name: Conda environment check
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Install base dependencies
       run: |
         python -m pip install --upgrade pip tox
@@ -86,7 +86,7 @@ jobs:
     - name: Python codestyle check
       uses: actions/setup-python@v4
       with:
-        python-version: '3.11'
+        python-version: '3.12'
     - name: Install base dependencies
       run: |
         python -m pip install --upgrade pip

deprecated/arc_old.py

@@ -855,3 +855,53 @@ def saturation_mask(a, satlevel):
 
     return mask.astype(int)
 
+def mask_around_peaks(spec, inbpm):
+    """
+    Find peaks in the input spectrum and mask pixels around them.
+
+    All pixels to the left and right of a peak is masked until
+    a pixel has a lower value than the adjacent pixel. At this
+    point, we assume that spec has reached the noise level.
+
+    Parameters
+    ----------
+    spec: `numpy.ndarray`_
+        Spectrum (1D array) in counts
+    inbpm: `numpy.ndarray`_
+        Input bad pixel mask
+
+    Returns
+    -------
+    outbpm: `numpy.ndarray`_
+        Bad pixel mask with pixels around peaks masked
+    """
+    # Find the peak locations
+    pks = detect_peaks(spec)
+
+    # Initialise some useful variables and the output bpm
+    xarray = np.arange(spec.size)
+    specdiff = np.append(np.diff(spec), 0.0)
+    outbpm = inbpm.copy()
+
+    # Loop over the peaks and mask pixels around them
+    for i in range(len(pks)):
+        # Find all pixels to the left of the peak that are above the noise level
+        wl = np.where((xarray <= pks[i]) & (specdiff > 0.0))[0]
+        ww = (pks[i]-wl)[::-1]
+        # Find the first pixel to the left of the peak that is below the noise level
+        nmask = np.where(np.diff(ww) > 1)[0]
+        if nmask.size != 0 and nmask[0] > 5:
+            # Mask all pixels to the left of the peak
+            mini = max(0,wl.size-nmask[0]-1)
+            outbpm[wl[mini]:pks[i]] = True
+        # Find all pixels to the right of the peak that are above the noise level
+        ww = np.where((xarray >= pks[i]) & (specdiff < 0.0))[0]
+        # Find the first pixel to the right of the peak that is below the noise level
+        nmask = np.where(np.diff(ww) > 1)[0]
+        if nmask.size != 0 and nmask[0] > 5:
+            # Mask all pixels to the right of the peak
+            maxi = min(nmask[0], ww.size)
+            outbpm[pks[i]:ww[maxi]+2] = True
+    # Return the output bpm
+    return outbpm
+

deprecated/sensfunc.py

@@ -0,0 +1,56 @@
+def compute_blaze(self, wave, trace_spec, trace_spat, flatfile, box_radius=10.0,
+                  min_blaze_value=1e-3, debug=False):
+    """
+    Compute the blaze function from a flat field image.
+
+    Args:
+        wave (`numpy.ndarray`_):
+            Wavelength array. Shape = (nspec, norddet)
+        trace_spec (`numpy.ndarray`_):
+            Spectral pixels for the trace of the spectrum. Shape = (nspec, norddet)
+        trace_spat (`numpy.ndarray`_):
+            Spatial pixels for the trace of the spectrum. Shape = (nspec, norddet)
+        flatfile (:obj:`str`):
+            Filename for the flat field calibration image
+        box_radius (:obj:`float`, optional):
+            Radius of the boxcar extraction region used to extract the blaze function in pixels
+        min_blaze_value (:obj:`float`, optional):
+            Minimum value of the blaze function. Values below this are clipped and set to this value. Default=1e-3
+        debug (:obj:`bool`, optional):
+            Show plots useful for debugging. Default=False
+
+    Returns:
+        `numpy.ndarray`_: The log10 blaze function. Shape = (nspec, norddet)
+        if norddet > 1, else shape = (nspec,)
+    """
+    flatImages = flatfield.FlatImages.from_file(flatfile, chk_version=self.chk_version)
+
+    pixelflat_raw = flatImages.pixelflat_raw
+    pixelflat_norm = flatImages.pixelflat_norm
+    pixelflat_proc, flat_bpm = flat.flatfield(pixelflat_raw, pixelflat_norm)
+
+    flux_box = moment1d(pixelflat_proc * np.logical_not(flat_bpm), trace_spat, 2 * box_radius, row=trace_spec)[0]
+
+    pixtot = moment1d(pixelflat_proc * 0 + 1.0, trace_spat, 2 * box_radius, row=trace_spec)[0]
+    pixmsk = moment1d(flat_bpm, trace_spat, 2 * box_radius, row=trace_spec)[0]
+
+    mask_box = (pixmsk != pixtot) & np.isfinite(wave) & (wave > 0.0)
+
+    # TODO This is ugly and redundant with spec_atleast_2d, but the order of operations compels me to do it this way
+    blaze_function = (np.clip(flux_box * mask_box, 1e-3, 1e9)).reshape(-1, 1) \
+        if flux_box.ndim == 1 else flux_box * mask_box
+    wave_debug = wave.reshape(-1, 1) if wave.ndim == 1 else wave
+    log10_blaze_function = np.zeros_like(blaze_function)
+    norddet = log10_blaze_function.shape[1]
+    for iorddet in range(norddet):
+        blaze_function_smooth = utils.fast_running_median(blaze_function[:, iorddet], 5)
+        blaze_function_norm = blaze_function_smooth / blaze_function_smooth.max()
+        log10_blaze_function[:, iorddet] = np.log10(np.clip(blaze_function_norm, min_blaze_value, None))
+        if debug:
+            plt.plot(wave_debug[:, iorddet], log10_blaze_function[:, iorddet])
+    if debug:
+        plt.show()
+
+    # TODO It would probably better to just return an array of shape (nspec, norddet) even if norddet = 1, i.e.
+    # to get rid of this .squeeze()
+    return log10_blaze_function.squeeze()

doc/api/pypeit.scripts.chk_flexure.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.chk\_flexure module
+==================================
+
+.. automodule:: pypeit.scripts.chk_flexure
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.scripts.extract_datacube.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.extract\_datacube module
+=======================================
+
+.. automodule:: pypeit.scripts.extract_datacube
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.scripts.rst

@@ -12,6 +12,7 @@ Submodules
    pypeit.scripts.chk_alignments
    pypeit.scripts.chk_edges
    pypeit.scripts.chk_flats
+   pypeit.scripts.chk_flexure
    pypeit.scripts.chk_for_calibs
    pypeit.scripts.chk_noise_1dspec
    pypeit.scripts.chk_noise_2dspec
@@ -26,6 +27,7 @@ Submodules
    pypeit.scripts.compare_sky
    pypeit.scripts.compile_wvarxiv
    pypeit.scripts.edge_inspector
+   pypeit.scripts.extract_datacube
    pypeit.scripts.flux_calib
    pypeit.scripts.flux_setup
    pypeit.scripts.identify
@@ -50,6 +52,7 @@ Submodules
    pypeit.scripts.show_1dspec
    pypeit.scripts.show_2dspec
    pypeit.scripts.show_arxiv
+   pypeit.scripts.show_pixflat
    pypeit.scripts.show_wvcalib
    pypeit.scripts.skysub_regions
    pypeit.scripts.tellfit

doc/api/pypeit.scripts.show_pixflat.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.show\_pixflat module
+===================================
+
+.. automodule:: pypeit.scripts.show_pixflat
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.spectrographs.aat_uhrf.rst

@@ -0,0 +1,8 @@
+pypeit.spectrographs.aat\_uhrf module
+=====================================
+
+.. automodule:: pypeit.spectrographs.aat_uhrf
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.spectrographs.rst

@@ -7,6 +7,7 @@ Submodules
 .. toctree::
    :maxdepth: 4
 
+   pypeit.spectrographs.aat_uhrf
    pypeit.spectrographs.bok_bc
    pypeit.spectrographs.gemini_flamingos
    pypeit.spectrographs.gemini_gmos

doc/calibrations/flat.rst

@@ -51,7 +51,7 @@ spectrograph that we have not included example
 screen-shots.
 
 Raw Flat
---------
+++++++++
 
 This is the processed and combined ``pixelflat`` image.
 Despite the name, it is not completely raw.
@@ -60,7 +60,7 @@ This image should look like any one of your input
 ``pixelflat`` images.
 
 Pixel Flat
-----------
+++++++++++
 
 This is the normalized to unity image which is used to
 correct for pixel-to-pixel variations across the detector.
@@ -77,17 +77,32 @@ true if there is limited flux at these ends (e.g. the data
 goes below the atmospheric cutoff).
 
 Illumination Flat
------------------
++++++++++++++++++
 
 This image should also have most values near unity, but
 there will be vertical coherence.  And the edges (left/right)
 may fall off well below unity.
 
 Flat Model
-----------
+++++++++++
 
 This image should largely resemble the `Raw Flat`_.
 
+pypeit_show_pixflat
+-------------------
+
+In addition to ``pypeit_chk_flats``, if a custom pixel flat is provided by the user,
+another script ``pypeit_show_pixflat`` is available to inspect it. The script is called as follows:
+
+.. code-block:: console
+
+        pypeit_show_pixflat PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz
+
+The script usage can be displayed by calling the script with the ``-h`` option:
+
+.. include:: ../help/pypeit_show_pixflat.rst
+
+
 Troubleshooting
 ===============
 

doc/calibrations/flat_fielding.rst

@@ -204,7 +204,8 @@ need to provide the matching flat field images in your
 In short, if ``use_pixelflat=True`` for *any* of your frame types,
 at least one of the data files in the
 :ref:`pypeit_file` :ref:`data_block` must
-be labelled as ``pixelflat`` (unless you `Feed a PixelFlat`_).
+be labelled as ``pixelflat``, or ``slitless_pixflat``
+(unless you `Feed a PixelFlat`_).
 
 And, if ``use_illumflat=True`` for *any* of your frame types,
 at least one of the data files in the
@@ -216,26 +217,47 @@ frames for the pixel and illumination corrections. This is
 supported, but we recommend that you set the ``trace`` frames
 to be the same as the ``illumflat`` frames.
 
+.. _generate-pixflat:
+
+Generate a Slitless PixelFlat
+-----------------------------
+
+If a set of ``slitless_pixflat`` frames are available in the
+:ref:`data_block` of the :ref:`pypeit_file`, PypeIt will generate
+a slitless pixel flat (unless you `Feed a PixelFlat`_ instead)
+during the main :ref:`run-pypeit`, and will apply it to frame
+types that have ``use_pixelflat=True``.
+The slitless pixel flat is generated separately for each detector
+(even in the case of a mosaic reduction) and it is stored in a FITS
+file in the reduction directory, with one extension per detector.
+In addition to saving the file in your reduction directory,
+the constructed pixelflat is saved to the PypeIt cache (see ref:`data_installation`).
+This allows you to use the file for both current and future reductions.
+To use this file in future reductions, the user should add the slitless
+pixel flat file name to the :ref:`pypeit_file` as shown in `Feed a PixelFlat`_.
+
+If you generate your own slitless pixel flat, and you think it is generally
+applicable for your instrument, please consider sharing it with the PypeIt Developers.
+
+
 Feed a PixelFlat
 ----------------
 
 If you have generated your own pixel flat (or were provided one)
 and it is trimmed and oriented following the expected :ref:`pypeit-orientation`,
-then you may feed this into PypeIt.  This is the recommended approach
-at present for :ref:`lrisb`.
+then you may feed this into PypeIt.
 
-And you perform this by modifying the :ref:`parameter_block`:
-
-.. TODO: IS THIS STILL THE CORRECT APPROACH?  WHAT DO PEOPLE DO IF THEY DON'T
-.. HAVE THE DEV SUITE?
+To use the available PixelFlat, you need to modify the :ref:`parameter_block` like, e.g.:
 
 .. code-block:: ini
 
-    [calibrations]
-        [[flatfield]]
-            pixelflat_file = /Users/joe/python/PypeIt-development-suite/CALIBS/PYPEIT_LRISb_pixflat_B600_2x2_17sep2009.fits.gz
+   [calibrations]
+      [[flatfield]]
+         pixelflat_file = PYPEIT_LRISb_pixflat_B600_2x2_17sep2009_specflip.fits.gz
+
+If any of the frames in the :ref:`data_block` are labelled as ``pixelflat``, or ``slitless_pixflat``,
+the provided pixel flat file will still be used instead of generating a new one.
 
-None of the frames in the :ref:`data_block` should be labelled as ``pixelflat``.
 
 Algorithms
 ----------

doc/calibrations/image_proc.rst

@@ -42,8 +42,9 @@ where:
     - the quantity :math:`C=N_{\rm frames}\ c/s\prime=c/s` is the number of electron counts excited by
       photons hitting the detector,
     - :math:`1/s=N_{\rm frames}/s\prime` is a factor that accounts for the number
-      of frames contributing to the electron counts, and the relative
-      throughput factors (see below) that can be measured from flat-field frames,
+      of frames contributing to the electron counts (`N_{\rm frames}`), and (`s\prime`) the relative
+      throughput factors (see below) that can be measured from flat-field frames plus a scaling factor
+      applied if the counts of each frame are scaled to the mean counts of all frames,
     - :math:`D` is the dark-current, i.e., the rate at which the detector
       generates thermal electrons, in e-/pixel/s,
     - :math:`N_{\rm bin}` is the number of pixels in a binned pixel,

doc/coadd1d.rst

@@ -303,3 +303,40 @@ and launches a GUI from the `linetools`_ package. e.g.:
 
     lt_xspec J1217p3905_coadd.fits
 
+UVES_popler coaddition
+======================
+
+If you prefer to use a GUI for the coaddition (to manually remove
+bad pixels, ghosts, cosmic rays etc.), then you can use the UVES_popler tool.
+This tool is developed by Michael Murphy and is available at
+`this link <https://github.com/MTMurphy77/UVES_popler/>`__.
+Here is an example of a coadded spectrum using UVES_popler:
+
+.. image:: ../figures/uves_popler.png
+   :scale: 60%
+
+UVES_popler was originally written to coadd ESO/UVES echelle spectra
+that were reduced by the ESO pipeline, and it has been recently modified
+to support the reduction of PypeIt longslit and echelle data.
+For details on how to use the tool, please refer to the
+`UVES_popler documentation <https://astronomy.swin.edu.au/~mmurphy/UVES_popler/>`__.
+To get you started with reading in PypeIt :doc:`out_spec1D` files,
+you need to generate a text file that lists the absolute paths to the
+:doc:`out_spec1D` files. Here is an example of how to generate this file:
+
+.. code-block:: console
+
+    ls -1 /path/to/your/pypeit_output/Science/spec1d/*.fits > /path/to/your/pypeit_output/pypeit_spec1d_files.txt
+
+Then you can use this file as input to UVES_popler, by using the following command:
+
+.. code-block:: console
+
+    cd /path/to/your/pypeit_output/
+    UVES_popler -disp 50 -filetype 11 pypeit_spec1d_files.txt
+
+This will launch the GUI, where you can interactively coadd your spectra. The
+``-disp 50`` option is used to set the pixel sampling of the spectra to 50 km/s,
+and the ``-filetype 11`` option is used to specify that the input files are PypeIt
+:doc:`out_spec1D` files. For more information on the options available, you can
+specify the ``-h`` option.