Merge pull request #1765 from pypeit/staged

Merges develop into release (1.15.0 tag prep; take 2)

.gitignore

@@ -52,7 +52,7 @@ tst*.fits
 # Logs and databases #
 ######################
 *.log
-!pypeit/data/spectrographs/keck_deimos/gain_ronoise/*.log
+!pypeit/data/spectrographs/keck_deimos/gain_ronoise/*/*.log
 *.sql
 *.sqlite
 

CHANGES.rst

@@ -58,8 +58,7 @@ RELEASE FILE IN doc/releases
       files that have frametype None (this prevent ``run_pypeit`` to crash)
     - Added a function ``check_spectrograph()`` (currently only defined for LRIS),
       that checks (during ``pypeit_setup``) if the selected spectrograph is the
-      corrected one for the data used.
-
+      corrected one for the data used. 
 
 1.13.0 (2 June 2023)
 --------------------

CITATION.cff

@@ -0,0 +1,9 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - family-names: Prochaska, Hennawi, Westfall, Cooke, Wang, Hsyu, Davies, Farina, Pelliccia
+    given-names: J. Xavier, Joseph, Kyle, Ryan, Feige, Tiffany, Fred, Emanuele, Debora
+title: "PypeIt"
+version: 1.14.0
+doi: 10.21105/joss.02308
+date-released: 2020-08-28

MANIFEST.in

@@ -15,7 +15,7 @@ recursive-include scripts *
 prune build
 prune docs/_build
 prune docs/api
-prune pypeit/deprecated
+prune deprecated
 
 global-exclude *.pyc *.o *.so *.DS_Store
 

pypeit/deprecated/flat.py → deprecated/flat.py

@@ -1025,3 +1025,102 @@ def norm_slits(mstrace, datasec_img, lordloc, rordloc, pixwid,
 #
 #    return slit_profiles, mstracenrm, extrap_blz
 
+# Deprecated because it is slow, inefficient, and needs more work. The idea is to use a spline fit to control points
+# along the spectral direction to construct a map between modelimg and rawimg. See core.flat.poly_map for a faster
+# approach.
+def spline_map(rawimg, rawivar, waveimg, slitmask, slitmask_trim, modelimg,
+               slit_illum_ref_idx=0, gpmask=None, thismask=None, nbins=20, debug=False):
+    """
+    Use a spline fit to control points along the spectral direction to construct a map between modelimg and
+    rawimg. Currently, this routine is only used for image slicer IFUs.
+
+    This problem needs to be recast into a smoothing problem, that can take advantage of the following:
+    https://docs.scipy.org/doc/scipy/reference/generated/scipy.interpolate.UnivariateSpline.html#scipy.interpolate.UnivariateSpline
+    where:
+    y = science/model
+    w = model/science_error
+    resid = w*(y-f(x))
+    Then, iterate over this. The reason to iterate is that there should be a mapping between science and
+    science_error. Once we have an estimate of f(x), we can do the following:
+    spl = spline(science, science_error)
+    new_science_error = spl(model*f(x))
+    Now iterate a few times until new_science_error (and the fit) is converged.
+
+    Parameters
+    ----------
+    rawimg : `numpy.ndarray`_
+        Image data that will be used to estimate the spectral relative sensitivity
+    rawivar : `numpy.ndarray`_
+        Inverse variance image of rawimg
+    waveimg : `numpy.ndarray`_
+        Wavelength image
+    slitmask : `numpy.ndarray`_
+        A 2D int mask, the same shape as rawimg, indicating which pixels are on a slit. A -1 value
+        indicates not on a slit, while any pixels on a slit should have the value of the slit spatial ID
+        number.
+    slitmask_trim : `numpy.ndarray`_
+        Same as slitmask, but the slit edges are trimmed.
+    modelimg : `numpy.ndarray`_
+        A model of the rawimg data.
+    slit_illum_ref_idx : :obj:`int`
+        Index of slit that is used as the reference.
+    gpmask : `numpy.ndarray`_, optional
+        Boolean good pixel mask (True = Good)
+    thismask : `numpy.ndarray`_, optional
+        A boolean mask (True = good) that indicates all pixels where the scaleImg should be constructed.
+        If None, the slitmask that is generated by this routine will be used.
+    nbins : :obj:`int`
+        Number of bins in the spectral direction to sample the relative spectral sensitivity
+    debug : :obj:`bool`
+        If True, some plots will be output to test if the fitting is working correctly.
+
+    Returns
+    -------
+    scale_model: `numpy.ndarray`_
+        An image containing the appropriate scaling
+    """
+    # Some variables to consider putting as function arguments
+    numiter = 4
+
+    # Start by calculating a ratio of the raming and the modelimg
+    nspec = rawimg.shape[0]
+    _ratio = rawimg * utils.inverse(modelimg)
+    _ratio_ivar = rawivar * modelimg**2
+    _fit_wghts = modelimg * np.sqrt(rawivar)
+
+    # Generate the mask
+    _thismask = thismask if (thismask is not None) else (slitmask > 0)
+    gpm = gpmask if (gpmask is not None) else np.ones_like(rawimg, dtype=bool)
+    # Extract the list of  spatial IDs from the slitmask
+    slitmask_spatid = np.unique(slitmask)
+    slitmask_spatid = np.sort(slitmask_spatid[slitmask_spatid > 0])
+
+    # Create a spline between the raw data and the error
+    # TODO :: This is slow and inefficient.
+    embed()
+    flxsrt = np.argsort(np.ravel(rawimg))
+    spl = scipy.interpolate.interp1d(np.ravel(rawimg)[flxsrt], np.ravel(rawivar)[flxsrt], kind='linear',
+                                     bounds_error=False, fill_value=0.0, assume_sorted=True)
+    modelmap = np.zeros_like(rawimg)
+    for sl, spatid in enumerate(slitmask_spatid):
+        print("sl")
+        # Prepare the masks, edges, and fitting variables
+        this_slit = (slitmask == spatid)
+        this_slit_trim = (slitmask_trim == spatid)
+        this_slit_mask = gpm & this_slit_trim
+        this_wave = waveimg[this_slit_mask]
+        this_wghts = _fit_wghts[this_slit_mask]
+        asrt = np.argsort(this_wave)
+        for ii in range(numiter):
+            # Generate the map between model and data
+            splmap = scipy.interpolate.UnivariateSpline(this_wave[asrt], _ratio[this_slit_mask][asrt], w=this_wghts[asrt],
+                                                        bbox=[None, None], k=3, s=nspec, ext=0, check_finite=False)
+            # Construct the mapping, and use this to make a model of the rawdata
+            this_modmap = splmap(this_wave[this_slit_mask])
+            this_modflx = modelimg[this_slit_mask] * this_modmap
+            # Update the fit weights
+            this_wghts = modelimg[this_slit_mask] * np.sqrt(spl(this_modflx))
+        # Produce the final model for this slit
+        modelmap[this_slit] = splmap(this_wave[this_slit])
+    return modelmap
+

doc/Makefile

@@ -56,7 +56,7 @@ clean:
 	rm -rf $(LOCALFILES)
 
 apirst:
-	SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/deprecated/* ../pypeit/version.py ../pypeit/compiler_version*
+	SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/version.py ../pypeit/compiler_version*
 	wget -O ./include/dev_suite_readme.rst https://raw.githubusercontent.com/pypeit/PypeIt-development-suite/main/README.rst
 	python ./scripts/build_datacontainer_datamodels.py
 	python ./scripts/build_dependency_rst.py

doc/api/pypeit.scripts.print_bpm.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.print\_bpm module
+================================
+
+.. automodule:: pypeit.scripts.print_bpm
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.scripts.rst

@@ -35,6 +35,7 @@ Submodules
    pypeit.scripts.multislit_flexure
    pypeit.scripts.obslog
    pypeit.scripts.parse_slits
+   pypeit.scripts.print_bpm
    pypeit.scripts.qa_html
    pypeit.scripts.ql
    pypeit.scripts.run_pypeit

doc/calibrations/wvcalib.rst

@@ -31,13 +31,13 @@ with the **pypeit_chk_wavecalib** script, e.g. :
 
     $ pypeit_chk_wavecalib Calibrations/WaveCalib_A_1_MSC03.fits
 
-     N. SpatID minWave Wave_cen maxWave dWave Nlin     IDs_Wave_range    IDs_Wave_cov(%) mesured_fwhm  RMS
-    --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------ -----
-      0     35  6422.5   7753.8  9053.2 0.325   48  6508.325 -  9047.930            96.5          3.5 0.046
-      1     93  6310.0   7641.4  8940.8 0.325   49  6336.179 -  8931.145            98.6          3.6 0.036
-      2    140  6440.8   7772.1  9071.5 0.325   47  6508.325 -  9047.930            96.5          3.6 0.049
-      3    184  6301.2   7632.6  8932.0 0.325   50  6306.533 -  8931.145            99.8          3.6 0.037
-      4    243  6257.1   7588.5  8887.9 0.325   49  6268.229 -  8821.832            97.1          3.6 0.034
+     N. SpatID minWave Wave_cen maxWave dWave Nlin     IDs_Wave_range    IDs_Wave_cov(%) measured_fwhm  RMS
+    --- ------ ------- -------- ------- ----- ---- --------------------- --------------- ------------- -----
+      0     35  6422.5   7753.8  9053.2 0.325   48  6508.325 -  9047.930            96.5           3.5 0.046
+      1     93  6310.0   7641.4  8940.8 0.325   49  6336.179 -  8931.145            98.6           3.6 0.036
+      2    140  6440.8   7772.1  9071.5 0.325   47  6508.325 -  9047.930            96.5           3.6 0.049
+      3    184  6301.2   7632.6  8932.0 0.325   50  6306.533 -  8931.145            99.8           3.6 0.037
+      4    243  6257.1   7588.5  8887.9 0.325   49  6268.229 -  8821.832            97.1           3.6 0.034
 
 - ``SpatID`` is the spatial position of the slit/order.
 
@@ -50,7 +50,7 @@ with the **pypeit_chk_wavecalib** script, e.g. :
   number, the wavelength range, and the spectral coverage of the identified and
   fitted arc lines.
 
-- ``mesured_fwhm`` is the measured arc lines FWHM (in binned pixels of the input
+- ``measured_fwhm`` is the measured arc lines FWHM (in binned pixels of the input
   arc frame), i.e, the approximate spectral resolution. Note that this not
   necessarily the ``fwhm`` used to identify the arc lines during the wavelength
   calibration, see :ref:`wvcalib-fwhm`.

doc/dev/deimosconfig.rst

@@ -48,7 +48,7 @@ as determined by
 :func:`pypeit.metadata.PypeItMetaData.unique_configurations`. The
 ``AMPMODE`` value is included, even though ``PypeIt`` (currently)
 restricts itself to only attempting to reduce frames read by the B
-amplifier; see
+and A amplifiers; see
 :func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.valid_configuration_values`.
 Additionally, ``PypeIt`` requires all frames to have ``MOSMODE ==
 'Spectral'``. Frames that do not match these header keyword
@@ -178,7 +178,7 @@ The algorithm for this test is as follows:
 
     5. Check that "cleaning" the configurations of frames that cannot
        be reduced by ``PypeIt`` (those with ``MOSMODE != 'Spectral'``
-       or ``AMPMODE != SINGLE:B``), using
+       or ``AMPMODE != SINGLE:B`` or ``AMPMODE != SINGLE:A``), using
        :func:`~pypeit.metadata.PypeItMetaData.clean_configurations`
        does not remove any file because all of the dev-suite files
        are valid.

doc/dev/deimosframes.rst

@@ -79,7 +79,7 @@ Also note that ``PypeIt`` will ignore frames observed under
 conditions that do not meet the restricted values set by
 :func:`~pypeit.spectrographs.keck_deimos.KeckDEIMOSSpectrograph.valid_configuration_values`.
 This currently requires all frames to have ``MOSMODE == 'Spectral'``
-and ``AMPMODE == 'SINGLE:B'``. Frames that do not meet these criteria
+and ``AMPMODE == 'SINGLE:B'`` or ``AMPMODE == 'SINGLE:A'``. Frames that do not meet these criteria
 will not be included in the automatically generated
 :ref:`pypeit_file` created by :ref:`pypeit_setup`.
 

doc/dev/development.rst

@@ -422,6 +422,12 @@ are as follows:
  * The docstrings for any changes to existing methods that were altered
    must have been modified so that they are up-to-date and accurate.
 
+ * The documentation must be successfully recompiled, either using the
+   ``update_docs`` scripts or but running ``make clean ; make html`` in the
+   ``doc/`` directory.  (We plan for this to be added to the dev-suite testing;
+   in the meantime, PR authors simply need to affirm that the documentation
+   builds successfully.)
+
  * Spurious commented code used for debugging or testing is fine, but
    please let us know if you want it to be kept by adding a relevant
    comment, something like ``# TODO: Keep this around for now``, at the
@@ -491,7 +497,11 @@ tagging process is as follows:
         # Push the new tag
         git push --tags
 
- * The tag is released for `pip`_ installation.
+   Similarly, a matching tag is executed for the dev-suite code (these tags only
+   exist for versions 1.15 and later).
+
+ * The tag of the ``pypeit`` code-base (not the dev-suite) is released for
+   `pip`_ installation.
 
     .. code-block:: bash
 

doc/help/pypeit_chk_alignments.rst

@@ -1,7 +1,7 @@
 .. code-block:: console
 
     $ pypeit_chk_alignments -h
-    usage: pypeit_chk_alignments [-h] [--chname CHNAME] file
+    usage: pypeit_chk_alignments [-h] [--chname CHNAME] [--try_old] file
     
     Display Alignment image and the trace data
     
@@ -11,4 +11,6 @@
     options:
       -h, --help       show this help message and exit
       --chname CHNAME  Channel name for image in Ginga (default: Alignments)
+      --try_old        Attempt to load old datamodel versions. A crash may ensue..
+                       (default: False)
     

doc/help/pypeit_chk_noise_1dspec.rst

@@ -6,6 +6,7 @@
                                    [--z [Z ...]] [--maskdef_objname MASKDEF_OBJNAME]
                                    [--pypeit_name PYPEIT_NAME] [--wavemin WAVEMIN]
                                    [--wavemax WAVEMAX] [--plot_or_save PLOT_OR_SAVE]
+                                   [--try_old]
                                    [files ...]
     
     Examine the noise in a PypeIt spectrum
@@ -41,4 +42,6 @@
                             window. If you choose save, a folder called
                             spec1d*_noisecheck will be created and all the relevant
                             plot will be placed there. (default: plot)
+      --try_old             Attempt to load old datamodel versions. A crash may
+                            ensue.. (default: False)
     

doc/help/pypeit_chk_noise_2dspec.rst

@@ -5,7 +5,7 @@
                                    [--maskdef_id MASKDEF_ID] [--pypeit_id PYPEIT_ID]
                                    [--pad PAD] [--aspect_ratio ASPECT_RATIO]
                                    [--wavemin WAVEMIN] [--wavemax WAVEMAX]
-                                   [--mode MODE] [--list]
+                                   [--mode MODE] [--list] [--try_old]
                                    [files ...]
     
     Examine the noise in a PypeIt slit/order
@@ -43,4 +43,6 @@
                             placed. "print" will cause the check noise values to be
                             printed in the terminal. (default: plot)
       --list                List the extensions only? (default: False)
+      --try_old             Attempt to load old datamodel versions. A crash may
+                            ensue.. (default: False)
     

doc/help/pypeit_chk_scattlight.rst

@@ -2,6 +2,7 @@
 
     $ pypeit_chk_scattlight -h
     usage: pypeit_chk_scattlight [-h] [--spec2d SPEC2D] [--det DET] [--mask MASK]
+                                 [--try_old]
                                  file slits
     
     Display the scattered light image in a Ginga viewer
@@ -21,4 +22,6 @@
       --mask MASK      If True, the detector pixels that are considered on the slit
                        will be masked to highlight the scattered light regions
                        (default: False)
+      --try_old        Attempt to load old datamodel versions. A crash may ensue..
+                       (default: False)
     

doc/help/pypeit_chk_tilts.rst

@@ -17,5 +17,5 @@
                      Ginga). If not set, only the fitted, masked and rejected in the
                      fit tilts are shown. (default: False)
       --try_old      Attempt to load old datamodel versions. A crash may ensue..
-                     (default: True)
+                     (default: False)
     

doc/help/pypeit_chk_wavecalib.rst

@@ -1,7 +1,7 @@
 .. code-block:: console
 
     $ pypeit_chk_wavecalib -h
-    usage: pypeit_chk_wavecalib [-h] input_file [input_file ...]
+    usage: pypeit_chk_wavecalib [-h] [--try_old] input_file [input_file ...]
     
     Print QA on Wavelength Calib to the screen
     
@@ -11,4 +11,6 @@
     
     options:
       -h, --help  show this help message and exit
+      --try_old   Attempt to load old datamodel versions. A crash may ensue..
+                  (default: False)
     

doc/help/pypeit_collate_1d.rst

@@ -36,8 +36,6 @@
                                                     value are skipped, else all wavelength rms values are accepted.
                               refframe              Perform reference frame correction prior to coadding.
                                                     Options are ['observed', 'heliocentric', 'barycentric']. Defaults to None.
-                              chk_version           If true, spec1ds and archival sensfuncs must match the currently
-                                                    supported versions. If false (the default) version numbers are not checked.
                              
                             spec1d read
                             <path to spec1d files, wildcards allowed>
@@ -88,8 +86,14 @@
       --refframe {observed,heliocentric,barycentric}
                             Perform reference frame correction prior to coadding.
                             Options are: observed, heliocentric, barycentric
-      --chk_version         Whether to check the data model versions of spec1d files
-                            and sensfunc files.
+      --chk_version         If True enforce strict PypeIt version checking to ensure
+                            that all files were created with the current version of
+                            PypeIt. If set to False, the code will attempt to read
+                            out-of-date files and keep going. Beware (!!) that this
+                            can lead to unforeseen bugs that either cause the code
+                            to crash or lead to erroneous results. I.e., you really
+                            need to know what you are doing if you set this to
+                            False!
       -v VERBOSITY, --verbosity VERBOSITY
                             Verbosity level between 0 [none] and 2 [all]. Default:
                             1. Level 2 writes a log with filename

doc/help/pypeit_edge_inspector.rst

@@ -1,7 +1,7 @@
 .. code-block:: console
 
     $ pypeit_edge_inspector -h
-    usage: pypeit_edge_inspector [-h] trace_file
+    usage: pypeit_edge_inspector [-h] [--try_old] trace_file
     
     Interactively inspect/edit slit edge traces
     
@@ -10,4 +10,6 @@
     
     options:
       -h, --help  show this help message and exit
+      --try_old   Attempt to load old datamodel versions. A crash may ensue..
+                  (default: False)