Merge pull request #1603 from pypeit/staged

Merges develop into release (1.13.0 tag prep)

.github/workflows/ci_cron.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ['3.9', '3.10']
+        python: ['3.9', '3.10', '3.11']
         toxenv: [test-alldeps, test-numpydev, test-linetoolsdev, test-gingadev, test-astropydev, conda]
     steps:
     - name: Check out repository

.github/workflows/ci_tests.yml

@@ -17,7 +17,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        python: ['3.9', '3.10']
+        python: ['3.9', '3.10', '3.11']
         toxenv: [test, test-alldeps-cov, test-linetoolsdev, test-gingadev, test-astropydev, conda]
     steps:
     - name: Check out repository
@@ -48,7 +48,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [windows-latest, macos-latest]
-        python: ['3.9', '3.10']
+        python: ['3.9', '3.10', '3.11']
         toxenv: [test-alldeps]
     steps:
     - name: Check out repository
@@ -64,54 +64,14 @@ jobs:
       run: |
         tox -e ${{ matrix.toxenv }}
 
-  centos:
-    # Some PypeIt users have to run under CentOS and have run into unique issues. GitHub Actions does not
-    # provide CentOS as a runtime environment like it does Ubuntu. However, it can be used by running
-    # it as a container within a provided Ubuntu environment.
-    name: CentOS ${{ matrix.centos_ver }} ${{ matrix.toxenv }}
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        # Test CentOS 7
-        centos_ver: [7]
-        # Test both pip and conda for installing dependencies
-        toxenv: [test-alldeps, conda]
-
-    container:
-      image: "centos:${{ matrix.centos_ver }}"
-
-    steps:
-    - name: Install base CentOS dependencies
-      run: |
-        yum install -y https://packages.endpointdev.com/rhel/7/os/x86_64/endpoint-repo.x86_64.rpm
-        yum update -y && yum install -y wget git gcc libgomp
-    - name: Check out repository
-      uses: actions/checkout@v3
-      with:
-        fetch-depth: 0
-    - name: Install and configure miniconda
-      # Restrict to 3.9 until we migrate to 3.10
-      run: |
-        wget https://repo.anaconda.com/miniconda/Miniconda3-py39_22.11.1-1-Linux-x86_64.sh -O miniconda.sh
-        bash ./miniconda.sh -b -p /conda
-        /conda/bin/conda update -y --all
-        /conda/bin/conda init
-    - name: Install base dependencies
-      run: |
-        source ~/.bashrc && python -m pip install --upgrade pip tox setuptools_scm && python --version
-    - name: Test with tox
-      run: |
-        source ~/.bashrc && tox -e ${{ matrix.toxenv }}
-
   codestyle:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v3
     - name: Python codestyle check
       uses: actions/setup-python@v4
       with:
-        python-version: '3.10'
+        python-version: '3.11'
     - name: Install base dependencies
       run: |
         python -m pip install --upgrade pip

.gitignore

@@ -9,7 +9,7 @@ doc/build_/*
 bin/bin/*
 # Test folders that crop up
 keck_lris_blue_A/
-Masters/
+Calibrations/
 Science/
 QA/
 
@@ -79,7 +79,7 @@ Thumbs.db
 *.swp
 pypeit/_compiler.c
 pypeit/data/telluric/atm_grids/*.fits
-pypeit/data/QL_MASTERS
+pypeit/data/QL_CALIB
 
 *.pckl
 

CHANGES.rst

@@ -1,8 +1,37 @@
 
-1.12.3dev
----------
+1.13.0 (2 June 2023)
+--------------------
 
+- Implemented a resample algorithm when generating datacubes
 - Hotfix to docs to ensure pypeit_loaders api doc is generated
+- Allow user control of the local sky subtraction window
+- Deprecate use of python 3.8 with PypeIt, allow python 3.11
+- Make pypeit_show_2dspec (somewhat) backwards compatible.
+- Added the option to disable strict version checking for 1d coadds.
+- Hotfix for KCWI when using alignment (aka ContBars) frames for the astrometric correction.
+- Sensitivity function masking and output updates
+- Fixed a bug in the `variance_model` calculation for combined images.
+- Added the possibility to use dither offsets saved in the header of the science frames for
+  coadding 2D spectra (``dithoff`` must be part of the spectrograph metadata).
+- Calibration group numbers can now be anything, as long as there are no more
+  than 63 unique integers.
+- Removed use of the term "master", renamed to calibration frames/files.
+  Default output directory for calibration frames is now ``Calibrations``.
+  Calibration frames renamed; e.g., ``MasterArc`` is now ``Arc``.
+- Calibration frame naming now done via ``calibframe.CalibFrame`` class.
+- Start to deprecate use of ``os.path`` in favor of ``pathlib``
+- Deprecated ``pypeit_parse_calib_id`` script, but improved the ``.calib`` file
+  provided by ``pypeit_setup``.  The ``.calib`` file is now always written, and
+  provides a direct association between input raw files and output calibration
+  files.  Discussed in new docs.
+- The ``'calib'`` column is now always added to the pypeit file, regardless of
+  whether or not you also request the ``'comb_id'`` and ``'bkg_id'`` columns.
+- Names of associated calibration frames now written to ``spec2d`` file headers.
+- Major quicklook updates.  ql_multislit.py deprecated.
+- Improve speed in ginga visualization of traces and added
+  `pypeit_chk_tilts`. Note that this script uses an update
+  of the tilts datamodel, so it will not work on older reductions.
+- Updates to reduction parameters for LDT/DeVeny
 
 1.12.2 (29 Mar 2023)
 --------------------
@@ -16,6 +45,8 @@
 - Added new specutils interface
 - Fixed bugs when only performing calibrations and (1) calib groups are all set
   to 'all' or (2) anything other than '0'.
+- Added `MASKDEF_OBJMAG` and `MASKDEF_OBJMAG_BAND` in spec1d datamodel.
+- Improved NIRES dither pattern parsing and automatic assignment of `comb_id` and `bkg_id`.
 
 1.12.1 (21 Feb 2023)
 --------------------

bin/pypeit_clean

@@ -21,7 +21,7 @@ def delete_if_exists(fname, f_or_dir):
             shutil.rmtree(fname)
 
 if __name__ == '__main__':
-    flist = ['fitstbl.fits', 'MasterArc_A_01_22.fits', 'tmp.fits', 'tmp_detector.fits', 'tst_pypeitimage.fits', 'shane_kast_blue.calib', 'shane_kast_blue.sorted', 'sensfunc.fits', 'datacube.fits', 'tst_coadd_files-SPAT0175-DET01.fits']
+    flist = ['fitstbl.fits', 'Arc_A_01_22.fits', 'tmp.fits', 'tmp_detector.fits', 'tst_pypeitimage.fits', 'shane_kast_blue.calib', 'shane_kast_blue.sorted', 'sensfunc.fits', 'datacube.fits', 'tst_coadd_files-SPAT0175-DET01.fits']
     try:
         from pypeit.tests.test_fluxspec import kast_blue_files
         f = os.path.basename(kast_blue_files()[1])

checkout_current_tag

@@ -5,7 +5,7 @@ echo "Checkout out tag ${tag}"
 git checkout ${tag}
 echo "To reset:"
 echo ""
-echo "  git checkout master"
+echo "  git checkout release"
 echo ""
 
 exit 0

doc/Makefile

@@ -57,7 +57,7 @@ clean:
 
 apirst:
 	SPHINX_APIDOC_OPTIONS=$(SPHINXAPIOPT) $(SPHINXAPI) --separate -o ./api ../pypeit ../pypeit/tests/* ../pypeit/deprecated/* ../pypeit/version.py ../pypeit/compiler_version*
-	wget -O ./include/dev_suite_readme.rst https://raw.githubusercontent.com/pypeit/PypeIt-development-suite/master/README.rst
+	wget -O ./include/dev_suite_readme.rst https://raw.githubusercontent.com/pypeit/PypeIt-development-suite/main/README.rst
 	python ./scripts/build_datacontainer_datamodels.py
 	python ./scripts/build_dependency_rst.py
 	python ./scripts/build_detector_table.py

doc/api/pypeit.calibframe.rst

@@ -0,0 +1,8 @@
+pypeit.calibframe module
+========================
+
+.. automodule:: pypeit.calibframe
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.rst

@@ -26,8 +26,8 @@ Submodules
 
    pypeit.alignframe
    pypeit.archive
-   pypeit.biasframe
    pypeit.bitmask
+   pypeit.calibframe
    pypeit.calibrations
    pypeit.coadd1d
    pypeit.coadd2d
@@ -41,7 +41,6 @@ Submodules
    pypeit.inputfiles
    pypeit.io
    pypeit.manual_extract
-   pypeit.masterframe
    pypeit.metadata
    pypeit.onespec
    pypeit.pypeit

doc/api/pypeit.scripts.chk_tilts.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.chk\_tilts module
+================================
+
+.. automodule:: pypeit.scripts.chk_tilts
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.scripts.install_ql_calibs.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.install\_ql\_calibs module
+=========================================
+
+.. automodule:: pypeit.scripts.install_ql_calibs
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/api/pypeit.scripts.rst

@@ -14,6 +14,7 @@ Submodules
    pypeit.scripts.chk_for_calibs
    pypeit.scripts.chk_noise_1dspec
    pypeit.scripts.chk_noise_2dspec
+   pypeit.scripts.chk_tilts
    pypeit.scripts.chk_wavecalib
    pypeit.scripts.coadd_1dspec
    pypeit.scripts.coadd_2dspec
@@ -25,12 +26,11 @@ Submodules
    pypeit.scripts.identify
    pypeit.scripts.install_extinctfile
    pypeit.scripts.install_linelist
-   pypeit.scripts.install_ql_masters
+   pypeit.scripts.install_ql_calibs
    pypeit.scripts.install_telluric
    pypeit.scripts.lowrdx_skyspec
    pypeit.scripts.multislit_flexure
    pypeit.scripts.obslog
-   pypeit.scripts.parse_calib_id
    pypeit.scripts.parse_slits
    pypeit.scripts.qa_html
    pypeit.scripts.ql
@@ -39,6 +39,7 @@ Submodules
    pypeit.scripts.scriptbase
    pypeit.scripts.sensfunc
    pypeit.scripts.setup
+   pypeit.scripts.setup_coadd2d
    pypeit.scripts.show_1dspec
    pypeit.scripts.show_2dspec
    pypeit.scripts.show_arxiv

doc/api/pypeit.scripts.setup_coadd2d.rst

@@ -0,0 +1,8 @@
+pypeit.scripts.setup\_coadd2d module
+====================================
+
+.. automodule:: pypeit.scripts.setup_coadd2d
+   :members:
+   :private-members:
+   :undoc-members:
+   :show-inheritance:

doc/calibrations/master_align.rst → doc/calibrations/align.rst

@@ -1,21 +1,21 @@
 
 .. include:: ../include/links.rst
 
-===============
-MasterAlignment
-===============
+=========
+Alignment
+=========
 
 Overview
 ========
 
-This file describes the data model for the ``MasterAlignment`` image.
+This file describes the data model for the ``Alignment`` image.
 It is generally a simple combination of all input alignment frames,
 which are used to trace constant spatial position on a slit. These
 frames are useful in slit-based IFUs (e.g. ``cbar`` frames for KCWI).
 
 The image is written to disk as a multi-extension FITS file
-prefixed by ``MasterAlignment`` in the ``Masters/`` folder.
-See :ref:`master-naming` for the naming convention.
+prefixed by ``Alignment`` in the ``Calibrations/`` folder.
+See :ref:`calib-naming` for the naming convention.
 
 
 Inspecting
@@ -28,7 +28,7 @@ You can view it with any standard image viewer, e.g.:
 
 .. code-block:: console
 
-    ginga Masters/MasterAlignment_A_1_DET01.fits
+    ginga Calibrations/Alignment_A_1_DET01.fits
 
 The image will also be a trimmed portion of the raw image and also re-oriented
 so that vertical is the spectral dimension with blue at the bottom, following

doc/calibrations/alignment.rst

@@ -6,7 +6,7 @@ Spatial Alignment
 Alignment frames are observed to provide a spatial reference frame for the
 observations.  Currently these are only used for Keck/KCWI reductions.
 
-See :doc:`master_align`.
+See :doc:`align`.
 
 .. TODO: Need input from Ryan
 

doc/calibrations/master_arc.rst → doc/calibrations/arc.rst

@@ -1,21 +1,21 @@
 
 .. include:: ../include/links.rst
 
-.. _masterarc:
+.. _arc:
 
-=========
-MasterArc
-=========
+===
+Arc
+===
 
 Overview
 ========
 
-This file describes the data model for the ``MasterArc`` image.
+This file describes the data model for the ``Arc`` image.
 It is generally a simple combination of all input arc frames.
 
 The image is written to disk as a multi-extension FITS file
-prefixed by ``MasterArc`` in the ``Masters/`` folder.
-See :ref:`master-naming` for the naming convention.
+prefixed by ``Arc`` in the ``Calibrations/`` folder.
+See :ref:`calib-naming` for the naming convention.
 
 
 Inspecting
@@ -26,7 +26,7 @@ You can view it with any standard image viewer, e.g.:
 
 .. code-block:: console
 
-    ginga Masters/MasterArc_A_1_01.fits
+    ginga Calibrations/Arc_A_1_01.fits
 
 Most often you use only one arc frame and this appears
 very similar to the raw image.  If you do stack several,