Skip to content

Commit

Permalink
Merge pull request #1813 from pypeit/combine_refactor
Browse files Browse the repository at this point in the history 
Combine refactor
  • Loading branch information
@kbwestfall
kbwestfall authored Jul 22, 2024
2 parents 9c7bb98 + 66d5784 commit 9e19d4c
Show file tree
Hide file tree
Showing 7 changed files with 170 additions and 141 deletions.
81 changes: 81 additions & 0 deletions pypeit/calibrations.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
.. include common links, assuming primary doc root is up one directory
.. include:: ../include/links.rst
"""
import os
from pathlib import Path
from datetime import datetime
from copy import deepcopy
Expand Down Expand Up @@ -209,6 +210,46 @@ def __init__(self, fitstbl, par, spectrograph, caldir, qadir=None,
self.success = False
self.failed_step = None

def check_calibrations(self, file_list, check_lamps=True):
"""
Check if the input calibration files are consistent with each other.
This step is usually needed when combining calibration frames of a given type.
This routine currently only prints out warning messages if the calibration files are not consistent.
Note: The exposure times are currently checked in the combine step, so they are not checked here.
Parameters
----------
file_list : list
List of calibration files to check
check_lamps : bool, optional
Check if the lamp status is the same for all the files. Default is True.
"""

lampstat = [None] * len(file_list)
# Loop on the files
for ii, ifile in enumerate(file_list):
# Save the lamp status
headarr = deepcopy(self.spectrograph.get_headarr(ifile))
lampstat[ii] = self.spectrograph.get_lamps_status(headarr)

# Check that the lamps being combined are all the same
if check_lamps:
if not lampstat[1:] == lampstat[:-1]:
msgs.warn("The following files contain different lamp status")
# Get the longest strings
maxlen = max([len("Filename")] + [len(os.path.split(x)[1]) for x in file_list])
maxlmp = max([len("Lamp status")] + [len(x) for x in lampstat])
strout = "{0:" + str(maxlen) + "} {1:s}"
# Print the messages
print(msgs.indent() + '-' * maxlen + " " + '-' * maxlmp)
print(msgs.indent() + strout.format("Filename", "Lamp status"))
print(msgs.indent() + '-' * maxlen + " " + '-' * maxlmp)
for ff, file in enumerate(file_list):
print(msgs.indent()
+ strout.format(os.path.split(file)[1], " ".join(lampstat[ff].split("_"))))
print(msgs.indent() + '-' * maxlen + " " + '-' * maxlmp)

def find_calibrations(self, frametype, frameclass):
"""
Find calibration files and identifiers.
Expand Down Expand Up @@ -334,6 +375,9 @@ def get_arc(self):
# Reset the BPM
self.get_bpm(frame=raw_files[0])

# Perform a check on the files
self.check_calibrations(raw_files)

# Otherwise, create the processed file.
msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
self.msarc = buildimage.buildimage_fromlist(self.spectrograph, self.det,
Expand Down Expand Up @@ -377,6 +421,9 @@ def get_tiltimg(self):
# Reset the BPM
self.get_bpm(frame=raw_files[0])

# Perform a check on the files
self.check_calibrations(raw_files)

# Otherwise, create the processed file.
msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
self.mstilt = buildimage.buildimage_fromlist(self.spectrograph, self.det,
Expand Down Expand Up @@ -426,6 +473,9 @@ def get_align(self):
# Reset the BPM
self.get_bpm(frame=raw_files[0])

# Perform a check on the files
self.check_calibrations(raw_files)

# Otherwise, create the processed file.
msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
msalign = buildimage.buildimage_fromlist(self.spectrograph, self.det,
Expand Down Expand Up @@ -473,6 +523,9 @@ def get_bias(self):
self.msbias = frame['class'].from_file(cal_file, chk_version=self.chk_version)
return self.msbias

# Perform a check on the files
self.check_calibrations(raw_files)

# Otherwise, create the processed file.
msgs.info(f'Preparing a {frame["class"].calib_type} calibration frame.')
self.msbias = buildimage.buildimage_fromlist(self.spectrograph, self.det,
Expand Down Expand Up @@ -523,6 +576,9 @@ def get_dark(self):
# there any reason why creation of the bpm should come after the dark,
# or can we change the order?

# Perform a check on the files
self.check_calibrations(raw_files)

# Otherwise, create the processed file.
self.msdark = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['darkframe'], raw_files,
Expand Down Expand Up @@ -597,6 +653,9 @@ def get_scattlight(self):
# Reset the BPM
self.get_bpm(frame=raw_scattlight_files[0])

# Perform a check on the files
self.check_calibrations(raw_scattlight_files)

binning = self.fitstbl[scatt_idx[0]]['binning']
dispname = self.fitstbl[scatt_idx[0]]['dispname']
scattlightImage = buildimage.buildimage_fromlist(self.spectrograph, self.det,
Expand Down Expand Up @@ -725,6 +784,10 @@ def get_flats(self):
if len(raw_pixel_files) > 0:
# Reset the BPM
self.get_bpm(frame=raw_pixel_files[0])

# Perform a check on the files
self.check_calibrations(raw_pixel_files)

msgs.info('Creating pixel-flat calibration frame using files: ')
for f in raw_pixel_files:
msgs.prindent(f'{Path(f).name}')
Expand All @@ -737,6 +800,10 @@ def get_flats(self):
if len(raw_lampoff_files) > 0:
# Reset the BPM
self.get_bpm(frame=raw_lampoff_files[0])

# Perform a check on the files
self.check_calibrations(raw_lampoff_files)

msgs.info('Subtracting lamp off flats using files: ')
for f in raw_lampoff_files:
msgs.prindent(f'{Path(f).name}')
Expand All @@ -763,6 +830,10 @@ def get_flats(self):
if not pix_is_illum and len(raw_illum_files) > 0:
# Reset the BPM
self.get_bpm(frame=raw_illum_files[0])

# Perform a check on the files
self.check_calibrations(raw_illum_files)

msgs.info('Creating slit-illumination flat calibration frame using files: ')
for f in raw_illum_files:
msgs.prindent(f'{Path(f).name}')
Expand All @@ -775,6 +846,10 @@ def get_flats(self):
for f in raw_lampoff_files:
msgs.prindent(f'{Path(f).name}')
if lampoff_flat is None:
# Perform a check on the files
self.check_calibrations(raw_lampoff_files)

# Build the image
lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['lampoffflatsframe'],
raw_lampoff_files,
Expand Down Expand Up @@ -889,6 +964,9 @@ def get_slits(self):
# Reset the BPM
self.get_bpm(frame=raw_trace_files[0])

# Perform a check on the files
self.check_calibrations(raw_trace_files)

traceImage = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['traceframe'], raw_trace_files,
bias=self.msbias, bpm=self.msbpm,
Expand All @@ -903,6 +981,9 @@ def get_slits(self):
# Reset the BPM
self.get_bpm(frame=raw_trace_files[0])

# Perform a check on the files
self.check_calibrations(raw_lampoff_files)

lampoff_flat = buildimage.buildimage_fromlist(self.spectrograph, self.det,
self.par['lampoffflatsframe'],
raw_lampoff_files, dark=self.msdark,
Expand Down
27 changes: 18 additions & 9 deletions pypeit/images/buildimage.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@

from pypeit import msgs
from pypeit.par import pypeitpar
from pypeit.images import rawimage
from pypeit.images import combineimage
from pypeit.images import pypeitimage
from pypeit.core.framematch import valid_frametype



class ArcImage(pypeitimage.PypeItCalibrationImage):
"""
Simple DataContainer for the Arc Image
Expand Down Expand Up @@ -160,7 +162,10 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
scattlight=None, flatimages=None, maxiters=5, ignore_saturation=True,
slits=None, mosaic=None, calib_dir=None, setup=None, calib_id=None):
"""
Perform basic image processing on a list of images and combine the results.
Perform basic image processing on a list of images and combine the results. All
core processing steps for each image are handled by :class:`~pypeit.images.rawimage.RawImage` and
image combination is handled by :class:`~pypeit.images.combineimage.CombineImage`.
This function can be used to process both single images, lists of images, and detector mosaics.
.. warning::
Expand Down Expand Up @@ -246,16 +251,20 @@ def buildimage_fromlist(spectrograph, det, frame_par, file_list, bias=None, bpm=
# Should the detectors be reformatted into a single image mosaic?
if mosaic is None:
mosaic = isinstance(det, tuple) and frame_par['frametype'] not in ['bias', 'dark']

rawImage_list = []
# Loop on the files
for ifile in file_list:
# Load raw image
rawImage = rawimage.RawImage(ifile, spectrograph, det)
# Process
rawImage_list.append(rawImage.process(
frame_par['process'], scattlight=scattlight, bias=bias,
bpm=bpm, dark=dark, flatimages=flatimages, slits=slits, mosaic=mosaic))

# Do it
combineImage = combineimage.CombineImage(spectrograph, det, frame_par['process'], file_list)
pypeitImage = combineImage.run(bias=bias, bpm=bpm, dark=dark, flatimages=flatimages,
scattlight=scattlight, sigma_clip=frame_par['process']['clip'],
sigrej=frame_par['process']['comb_sigrej'],
maxiters=maxiters, ignore_saturation=ignore_saturation,
slits=slits, combine_method=frame_par['process']['combine'],
mosaic=mosaic)

combineImage = combineimage.CombineImage(rawImage_list, frame_par['process'])
pypeitImage = combineImage.run(maxiters=maxiters, ignore_saturation=ignore_saturation)
# Return class type, if returning any of the frame_image_classes
cls = frame_image_classes[frame_par['frametype']] \
if frame_par['frametype'] in frame_image_classes.keys() else None
Expand Down
Loading

0 comments on commit 9e19d4c

Please sign in to comment.