From 9452ddc12fa287eb8696dbc621e8f623c7a23530 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:44:28 -0700 Subject: [PATCH 01/86] introduce new variables for paths --- mirar/paths.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirar/paths.py b/mirar/paths.py index 2c8dedaf8..6c2db804a 100644 --- a/mirar/paths.py +++ b/mirar/paths.py @@ -252,6 +252,7 @@ def get_astrometry_keys() -> list: GAIN_KEY = "GAIN" EXPTIME_KEY = "AEXPTIME" ZP_KEY = "ZP" +SATURATE_KEY = "SATURATE" PSF_FLUX_KEY = "psf_flux" PSF_FLUXUNC_KEY = "psf_fluxunc" MAG_PSF_KEY = "magpsf" @@ -261,13 +262,14 @@ def get_astrometry_keys() -> list: CAND_NAME_KEY = "objectId" CAND_RA_KEY = "ra" CAND_DEC_KEY = "dec" +CSV_MASK_KEY = "MASKCSV" sextractor_checkimg_keys = { "BACKGROUND": "BKGPT", "BACKGROUND_RMS": "BKGRMS", "MINIBACKGROUND": "MINIBKG", "MINIBACK_RMS": "MINIBGRM", } - +STACKED_COMPONENT_IMAGES_KEY = "COMPENTS" APFLUX_PREFIX_KEY = "fluxap" APFLUXUNC_PREFIX_KEY = "fluxucap" APMAG_PREFIX_KEY = "magap" From f78da22fd6bd3b1be2b29bc1338b77d6887301b1 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:45:08 -0700 Subject: [PATCH 02/86] new range of masking processors --- mirar/processors/mask.py | 208 +++++++++++++++++++++++++++++++++++---- 1 file changed, 189 insertions(+), 19 deletions(-) diff --git a/mirar/processors/mask.py b/mirar/processors/mask.py index decbe86a4..c20f21bf9 100644 --- a/mirar/processors/mask.py +++ b/mirar/processors/mask.py @@ -5,9 +5,13 @@ from pathlib import Path import numpy as np +import pandas as pd from mirar.data import ImageBatch from mirar.processors.base_processor import BaseImageProcessor +from astropy.wcs import WCS +from astropy.coordinates import SkyCoord +from mirar.paths import BASE_NAME_KEY, get_output_dir, CSV_MASK_KEY logger = logging.getLogger(__name__) @@ -15,40 +19,206 @@ MASK_VALUE = np.nan -class MaskPixels(BaseImageProcessor): +class BaseMask(BaseImageProcessor): + """ + Base class for masking processors + """ + + def __init__(self, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + *args, **kwargs): + super().__init__(*args, **kwargs) + self.write_masked_pixels_to_file = write_masked_pixels_to_file + self.output_dir = output_dir + + def get_mask(self, image) -> np.ndarray: + raise NotImplementedError + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + for image in batch: + data = image.get_data() + mask = self.get_mask(image) + data[mask] = MASK_VALUE + logger.info(f"Masked {np.sum(mask)} pixels in {image[BASE_NAME_KEY]}") + image.set_data(data) + + if self.write_masked_pixels_to_file: + wcs = WCS(image.get_header()) + masked_pixels_indices = np.where(mask) + masked_pixels_wcs = wcs.pixel_to_world(masked_pixels_indices[1], + masked_pixels_indices[0]) + mask_directory = get_output_dir(self.output_dir, self.night_sub_dir) + if not mask_directory.exists(): + mask_directory.mkdir(parents=True) + mask_file_path = mask_directory / f"{image[BASE_NAME_KEY]}" \ + f"_masked_pixels_wcs.csv" + with open(mask_file_path, "w") as f: + f.write("ra,dec\n") + for ra, dec in zip(masked_pixels_wcs.ra, masked_pixels_wcs.dec): + f.write(f"{ra.deg},{dec.deg}\n") + + image[CSV_MASK_KEY] = mask_file_path.as_posix() + return batch + + +class MaskPixelsFromPath(BaseMask): """ Processor to apply bias calibration """ - base_key = "mask" + base_key = "maskfrompath" - def __init__(self, mask_path: str | Path): - super().__init__() + def __init__(self, mask_path: str | Path = None, + mask_path_key: str = None, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask"): + super().__init__(write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir) self.mask = None - self.mask_path = Path(mask_path) + self.mask_path = mask_path + self.mask_path_key = mask_path_key + if mask_path is None and mask_path_key is None: + raise ValueError("Must specify either mask_path or mask_path_key") + if mask_path is not None and mask_path_key is not None: + raise ValueError("Must specify either mask_path or mask_path_key, " + "not both") def __str__(self) -> str: return f"Processor to mask bad pixels using a pre-defined map: {self.mask_path}" - def get_mask(self): + def get_mask(self, image) -> np.ndarray: """ loads mask if needed, and returns it :return: mask """ if self.mask is None: - self.mask = self.open_fits(self.mask_path) - return self.mask + if self.mask_path is not None: + self.mask = self.open_fits(self.mask_path) + elif self.mask_path_key is not None: + self.mask = self.open_fits(image[self.mask_path_key]) + mask = self.mask.get_data() + mask = mask != 0 + return mask - def _apply_to_images( - self, - batch: ImageBatch, - ) -> ImageBatch: - for image in batch: - data = image.get_data() - mask = self.get_mask().get_data() - mask = mask != 0 - data[mask] = MASK_VALUE - image.set_data(data) - return batch +class MaskAboveThreshold(BaseMask): + """ + Processor to mask pixels above a threshold + """ + + base_key = "maskthresh" + + def __init__(self, threshold: float = None, + threshold_key: str = None, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask"): + """ + :param threshold: threshold to mask above + :param threshold_key: key to use to get threshold from image header + """ + super().__init__(write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir) + self.threshold = threshold + self.threshold_key = threshold_key + self.write_masked_pixels_to_file = write_masked_pixels_to_file + if threshold is None and threshold_key is None: + raise ValueError("Must specify either threshold or threshold_key") + if threshold is not None and threshold_key is not None: + raise ValueError("Must specify either threshold or threshold_key, " + "not both") + + def __str__(self) -> str: + return f"Processor to mask pixels above a threshold: {self.threshold}" + + def get_mask(self, image) -> np.ndarray: + """ + Returns a mask for pixels above a threshold + + :return: mask + """ + if self.threshold is None: + self.threshold = image.get_header()[self.threshold_key] + mask = image.get_data() > self.threshold + return mask + + +class MaskPixelsFromWCS(BaseMask): + """ + Processor to mask pixels from a file where WCS coordinates of masked pixels are + given + """ + + base_key = "maskwcs" + + def __init__(self, + mask_pixels_ra: float | list[float] = None, + mask_pixels_dec: float | list[float] = None, + mask_file_key: str = CSV_MASK_KEY, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask"): + super().__init__(write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir) + self.mask_pixels_ra = mask_pixels_ra + self.mask_pixels_dec = mask_pixels_dec + self.mask_file_key = mask_file_key + + if self.mask_pixels_ra is not None: + self.mask_file_key = None + + def __str__(self) -> str: + return f"Processor to mask pixels using a list of RA/Dec: " + + def get_mask(self, image) -> np.ndarray: + """ + loads mask if needed, and returns it + + :return: mask + """ + wcs = WCS(image.get_header()) + if self.mask_file_key is not None: + mask_file_path = image.get_header()[self.mask_file_key] + mask_pixels = pd.read_csv(mask_file_path) + mask_pixels_ra = mask_pixels["ra"].to_numpy() + mask_pixels_dec = mask_pixels["dec"].to_numpy() + else: + mask_pixels_ra = self.mask_pixels_ra + mask_pixels_dec = self.mask_pixels_dec + logger.debug(f"Masking {mask_pixels_ra} ras and {mask_pixels_dec} decs") + mask_pixel_coords = SkyCoord(mask_pixels_ra, mask_pixels_dec, unit="deg") + mask_pixels_x, mask_pixels_y = wcs.world_to_pixel(mask_pixel_coords) + mask_pixels_x = mask_pixels_x.astype(int) + mask_pixels_y = mask_pixels_y.astype(int) + mask = np.zeros(image.get_data().shape, dtype=bool) + mask_in_image = np.logical_and(mask_pixels_x >= 0, + mask_pixels_x < mask.shape[1]) \ + & np.logical_and(mask_pixels_y >= 0, + mask_pixels_y < mask.shape[0]) + logger + mask_pixels_x = mask_pixels_x[mask_in_image] + mask_pixels_y = mask_pixels_y[mask_in_image] + mask[mask_pixels_y, mask_pixels_x] = True + return mask + + +class WriteMaskedCoordsToFile(BaseMask): + base_key = "writemaskedcoords" + + def __init__(self, output_dir: str | Path = "mask"): + super().__init__(write_masked_pixels_to_file=True, + output_dir=output_dir) + + def get_mask(self, image) -> np.ndarray: + mask = np.zeros(image.get_data().shape, dtype=bool) + + # For some reason, MASK_VALUE == np.nan returns False. Issue/Feature of numpy? + # This is a workaround + if np.isnan(MASK_VALUE): + mask[np.isnan(image.get_data())] = True + else: + mask[image.get_data() == MASK_VALUE] = True + return mask From ceb5966e8015e2b38dec8509c31f037a0f50edf4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:50:21 -0700 Subject: [PATCH 03/86] lint --- mirar/processors/mask.py | 107 +++++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 44 deletions(-) diff --git a/mirar/processors/mask.py b/mirar/processors/mask.py index c20f21bf9..f0ad84389 100644 --- a/mirar/processors/mask.py +++ b/mirar/processors/mask.py @@ -6,12 +6,12 @@ import numpy as np import pandas as pd +from astropy.coordinates import SkyCoord +from astropy.wcs import WCS from mirar.data import ImageBatch +from mirar.paths import BASE_NAME_KEY, CSV_MASK_KEY, get_output_dir from mirar.processors.base_processor import BaseImageProcessor -from astropy.wcs import WCS -from astropy.coordinates import SkyCoord -from mirar.paths import BASE_NAME_KEY, get_output_dir, CSV_MASK_KEY logger = logging.getLogger(__name__) @@ -24,15 +24,19 @@ class BaseMask(BaseImageProcessor): Base class for masking processors """ - def __init__(self, - write_masked_pixels_to_file: bool = False, - output_dir: str | Path = "mask", - *args, **kwargs): - super().__init__(*args, **kwargs) + def __init__( + self, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + ): + super().__init__() self.write_masked_pixels_to_file = write_masked_pixels_to_file self.output_dir = output_dir def get_mask(self, image) -> np.ndarray: + """ + Function to get the mask for a given image + """ raise NotImplementedError def _apply_to_images( @@ -49,13 +53,15 @@ def _apply_to_images( if self.write_masked_pixels_to_file: wcs = WCS(image.get_header()) masked_pixels_indices = np.where(mask) - masked_pixels_wcs = wcs.pixel_to_world(masked_pixels_indices[1], - masked_pixels_indices[0]) + masked_pixels_wcs = wcs.pixel_to_world( + masked_pixels_indices[1], masked_pixels_indices[0] + ) mask_directory = get_output_dir(self.output_dir, self.night_sub_dir) if not mask_directory.exists(): mask_directory.mkdir(parents=True) - mask_file_path = mask_directory / f"{image[BASE_NAME_KEY]}" \ - f"_masked_pixels_wcs.csv" + mask_file_path = ( + mask_directory / f"{image[BASE_NAME_KEY]}" f"_masked_pixels_wcs.csv" + ) with open(mask_file_path, "w") as f: f.write("ra,dec\n") for ra, dec in zip(masked_pixels_wcs.ra, masked_pixels_wcs.dec): @@ -72,20 +78,24 @@ class MaskPixelsFromPath(BaseMask): base_key = "maskfrompath" - def __init__(self, mask_path: str | Path = None, - mask_path_key: str = None, - write_masked_pixels_to_file: bool = False, - output_dir: str | Path = "mask"): - super().__init__(write_masked_pixels_to_file=write_masked_pixels_to_file, - output_dir=output_dir) + def __init__( + self, + mask_path: str | Path = None, + mask_path_key: str = None, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + ): + super().__init__( + write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir, + ) self.mask = None self.mask_path = mask_path self.mask_path_key = mask_path_key if mask_path is None and mask_path_key is None: raise ValueError("Must specify either mask_path or mask_path_key") if mask_path is not None and mask_path_key is not None: - raise ValueError("Must specify either mask_path or mask_path_key, " - "not both") + raise ValueError("Must specify either mask_path or mask_path_key, not both") def __str__(self) -> str: return f"Processor to mask bad pixels using a pre-defined map: {self.mask_path}" @@ -113,24 +123,28 @@ class MaskAboveThreshold(BaseMask): base_key = "maskthresh" - def __init__(self, threshold: float = None, - threshold_key: str = None, - write_masked_pixels_to_file: bool = False, - output_dir: str | Path = "mask"): + def __init__( + self, + threshold: float = None, + threshold_key: str = None, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + ): """ :param threshold: threshold to mask above :param threshold_key: key to use to get threshold from image header """ - super().__init__(write_masked_pixels_to_file=write_masked_pixels_to_file, - output_dir=output_dir) + super().__init__( + write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir, + ) self.threshold = threshold self.threshold_key = threshold_key self.write_masked_pixels_to_file = write_masked_pixels_to_file if threshold is None and threshold_key is None: raise ValueError("Must specify either threshold or threshold_key") if threshold is not None and threshold_key is not None: - raise ValueError("Must specify either threshold or threshold_key, " - "not both") + raise ValueError("Must specify either threshold or threshold_key, not both") def __str__(self) -> str: return f"Processor to mask pixels above a threshold: {self.threshold}" @@ -155,14 +169,18 @@ class MaskPixelsFromWCS(BaseMask): base_key = "maskwcs" - def __init__(self, - mask_pixels_ra: float | list[float] = None, - mask_pixels_dec: float | list[float] = None, - mask_file_key: str = CSV_MASK_KEY, - write_masked_pixels_to_file: bool = False, - output_dir: str | Path = "mask"): - super().__init__(write_masked_pixels_to_file=write_masked_pixels_to_file, - output_dir=output_dir) + def __init__( + self, + mask_pixels_ra: float | list[float] = None, + mask_pixels_dec: float | list[float] = None, + mask_file_key: str = CSV_MASK_KEY, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + ): + super().__init__( + write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir, + ) self.mask_pixels_ra = mask_pixels_ra self.mask_pixels_dec = mask_pixels_dec self.mask_file_key = mask_file_key @@ -171,7 +189,7 @@ def __init__(self, self.mask_file_key = None def __str__(self) -> str: - return f"Processor to mask pixels using a list of RA/Dec: " + return "Processor to mask pixels using a list of RA/Dec." def get_mask(self, image) -> np.ndarray: """ @@ -194,11 +212,9 @@ def get_mask(self, image) -> np.ndarray: mask_pixels_x = mask_pixels_x.astype(int) mask_pixels_y = mask_pixels_y.astype(int) mask = np.zeros(image.get_data().shape, dtype=bool) - mask_in_image = np.logical_and(mask_pixels_x >= 0, - mask_pixels_x < mask.shape[1]) \ - & np.logical_and(mask_pixels_y >= 0, - mask_pixels_y < mask.shape[0]) - logger + mask_in_image = np.logical_and( + mask_pixels_x >= 0, mask_pixels_x < mask.shape[1] + ) & np.logical_and(mask_pixels_y >= 0, mask_pixels_y < mask.shape[0]) mask_pixels_x = mask_pixels_x[mask_in_image] mask_pixels_y = mask_pixels_y[mask_in_image] mask[mask_pixels_y, mask_pixels_x] = True @@ -206,11 +222,14 @@ def get_mask(self, image) -> np.ndarray: class WriteMaskedCoordsToFile(BaseMask): + """ + Processor to write masked coordinates to a file + """ + base_key = "writemaskedcoords" def __init__(self, output_dir: str | Path = "mask"): - super().__init__(write_masked_pixels_to_file=True, - output_dir=output_dir) + super().__init__(write_masked_pixels_to_file=True, output_dir=output_dir) def get_mask(self, image) -> np.ndarray: mask = np.zeros(image.get_data().shape, dtype=bool) From eb9334a7bcae52a737f1cbcd80d3cc460efe3f6f Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:52:09 -0700 Subject: [PATCH 04/86] update to new mask name --- mirar/pipelines/sedmv2/blocks.py | 4 ++-- mirar/pipelines/summer/blocks.py | 6 +++--- mirar/pipelines/winter/blocks.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/mirar/pipelines/sedmv2/blocks.py b/mirar/pipelines/sedmv2/blocks.py index 6cd106120..34c2a3543 100644 --- a/mirar/pipelines/sedmv2/blocks.py +++ b/mirar/pipelines/sedmv2/blocks.py @@ -26,7 +26,7 @@ from mirar.processors.anet import AstrometryNet from mirar.processors.astromatic import PSFex, Sextractor, Swarp from mirar.processors.csvlog import CSVLog -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import ( CandidateAperturePhotometry, @@ -79,7 +79,7 @@ ] # pylint: disable=duplicate-code reduce = [ - MaskPixels(mask_path=sedmv2_mask_path), + MaskPixelsFromPath(mask_path=sedmv2_mask_path), BiasCalibrator(), ImageSelector(("OBSTYPE", ["FLAT", "SCIENCE"])), ImageBatcher(split_key="filter"), diff --git a/mirar/pipelines/summer/blocks.py b/mirar/pipelines/summer/blocks.py index cc892b6fe..bed3edcbc 100644 --- a/mirar/pipelines/summer/blocks.py +++ b/mirar/pipelines/summer/blocks.py @@ -44,7 +44,7 @@ DatabaseImageExporter as PSQLDatabaseImageExporter, ) from mirar.processors.database.database_modifier import ModifyImageDatabaseSeq -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import CandidateAperturePhotometry from mirar.processors.photometry.psf_photometry import CandidatePSFPhotometry @@ -123,7 +123,7 @@ duplicate_protocol="replace", q3c_bool=False, ), - MaskPixels(mask_path=summer_mask_path), + MaskPixelsFromPath(mask_path=summer_mask_path), DatabaseImageExporter(db_table=Raw, duplicate_protocol="replace", q3c_bool=False), ImageSelector(("OBSTYPE", ["BIAS", "FLAT", "SCIENCE"])), ] @@ -133,7 +133,7 @@ ] test_cr = [ - MaskPixels(mask_path=summer_mask_path), + MaskPixelsFromPath(mask_path=summer_mask_path), BiasCalibrator(), ImageSelector(("OBSTYPE", ["FLAT", "SCIENCE"])), ImageBatcher(split_key="filter"), diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 0550925b3..b71066acf 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -1,6 +1,15 @@ -from mirar.pipelines.winter.generator import winter_reference_generator +from mirar.pipelines.winter.generator import ( + winter_mask_path, + winter_reference_generator, +) +from mirar.processors.csvlog import CSVLog +from mirar.processors.dark import DarkCalibrator +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.reference import GetReferenceImage +from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator +from mirar.processors.split import SplitImage from mirar.processors.utils import ImageDebatcher, ImageSaver +from mirar.processors.utils.multi_ext_parser import MultiExtParser refbuild = [ ImageDebatcher(), @@ -9,3 +18,23 @@ ), ImageSaver(output_dir_name="stacked_ref"), ] + +commissioning = [ + CSVLog( + export_keys=[ + "OBJECT", + "FILTER", + "UTSHUT", + "EXPTIME", + "COADDS", + "OBSTYPE", + "OBSCLASS", + ] + ), + MultiExtParser(input_sub_dir="raw/mef/"), + SplitImage(), + MaskPixelsFromPath(mask_path=winter_mask_path), + DarkCalibrator(), + SkyFlatCalibrator(), + NightSkyMedianCalibrator(), +] From d4b7e4adee07d20ccf063849066c0645816af7e5 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:55:01 -0700 Subject: [PATCH 05/86] first draft at winter pipeline --- mirar/pipelines/winter/generator.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index c52048ebc..4de91e93a 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -23,6 +23,7 @@ winter_dir = os.path.dirname(__file__) astromatic_config_dir = os.path.join(winter_dir, "config/") swarp_config_path = os.path.join(astromatic_config_dir, "config.swarp") +winter_mask_path = os.path.join(winter_dir, "winter_mask.fits") def winter_reference_generator(image: Image, db_table: Type[BaseDB] = RefStacks): From b17006b484c1c2acac899d76d78ce6bf3271fef4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:56:19 -0700 Subject: [PATCH 06/86] lint --- mirar/processors/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/mirar/processors/__init__.py b/mirar/processors/__init__.py index 9cfd025f1..60653314d 100644 --- a/mirar/processors/__init__.py +++ b/mirar/processors/__init__.py @@ -3,7 +3,7 @@ from mirar.processors.bias import BiasCalibrator from mirar.processors.dark import DarkCalibrator from mirar.processors.flat import FlatCalibrator, SkyFlatCalibrator -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.utils.image_saver import ImageSaver # @@ -16,7 +16,8 @@ # processor = BaseProcessor.subclasses[processor_name] # except KeyError: # err = f"Processor type '{processor_name}' not recognised. " \ -# f"The following processors are available: {BaseProcessor.subclasses.keys()}" +# f"The following processors are available: +# {BaseProcessor.subclasses.keys()}" # logger.error(err) # raise KeyError(err) # From a82366486e56dc8734ad154a3a598dd7d6dbd5b9 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 13:58:51 -0700 Subject: [PATCH 07/86] saturate keyword --- mirar/processors/astromatic/sextractor/settings.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/mirar/processors/astromatic/sextractor/settings.py b/mirar/processors/astromatic/sextractor/settings.py index c53c8aa62..2c9530134 100644 --- a/mirar/processors/astromatic/sextractor/settings.py +++ b/mirar/processors/astromatic/sextractor/settings.py @@ -12,6 +12,9 @@ def write_param_file(param_path: str = default_param_path): + """ + Write a default parameter file for sextractor + """ params = """X_IMAGE Y_IMAGE ALPHA_J2000 @@ -26,6 +29,9 @@ def write_param_file(param_path: str = default_param_path): def write_conv_file(conv_path: str = default_conv_path): + """ + Write a default convolution file for sextractor + """ convol = """CONV NORM # 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. 1 2 1 @@ -42,11 +48,14 @@ def write_config_file( config_path: str = default_config_path, saturation: float = 55000.0, ): + """ + Write a default configuration file for sextractor + """ configs = f""" #-------------------------------- Catalog ------------------------------------ CATALOG_NAME temp.cat # name of the output catalog -CATALOG_TYPE ASCII_HEAD # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, +CATALOG_TYPE ASCII_HEAD # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC PARAMETERS_NAME {param_path} # name of the file containing catalog contents @@ -114,6 +123,7 @@ def write_config_file( VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL WRITE_XML N # Write XML file (Y/N)? XML_NAME sex.xml # Filename for XML output +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) SATUR_LEVEL {saturation} # level (in ADUs) at which arises saturation """ with open(config_path, "w") as pf: From 5c2baae3bc49d655a3ff4a78515a269927796037 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:01:59 -0700 Subject: [PATCH 08/86] set aturation to None by default --- .../astromatic/sextractor/sextractor.py | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mirar/processors/astromatic/sextractor/sextractor.py b/mirar/processors/astromatic/sextractor/sextractor.py index 136ad0ed9..16274342b 100644 --- a/mirar/processors/astromatic/sextractor/sextractor.py +++ b/mirar/processors/astromatic/sextractor/sextractor.py @@ -18,7 +18,6 @@ get_temp_path, ) from mirar.processors.astromatic.sextractor.sourceextractor import ( - default_saturation, parse_checkimage, run_sextractor_single, ) @@ -35,6 +34,7 @@ "BACKGROUND_RMS": "BKGRMS", "MINIBACKGROUND": "MINIBKG", "MINIBACK_RMS": "MINIBGRM", + "SEGMENTATION": "SEGMAP", } @@ -53,7 +53,7 @@ def __init__( parameter_path: str, filter_path: str, starnnw_path: str, - saturation: float = default_saturation, + saturation: float = None, verbose_type: str = "QUIET", checkimage_name: Optional[str | list] = None, checkimage_type: Optional[str | list] = None, @@ -63,6 +63,24 @@ def __init__( mag_zp: Optional[float] = None, write_regions_bool: bool = False, ): + """ + :param output_sub_dir: subdirectory to output sextractor files + :param config_path: path to sextractor config file + :param parameter_path: path to sextractor parameter file + :param filter_path: path to sextractor filter file + :param starnnw_path: path to sextractor starnnw file + :param saturation: saturation level for sextractor. Leave to None if not known, + no saturation will be applied + :param verbose_type: verbose type for sextractor + :param checkimage_type: type of checkimage to output + :param checkimage_name: name of checkimage to output. Leave to None to use + pipeline defaults in sextractor_checkimage_map for output name (recommended). + :param gain: gain for sextractor. Leave to None if not known. + :param dual: whether to run sextractor in dual mode + :param cache: whether to cache sextractor output + :param mag_zp: magnitude zero point for sextractor. Leave to None if not known. + :param write_regions_bool: whether to write regions file for ds9 + """ # pylint: disable=too-many-arguments super().__init__() self.output_sub_dir = output_sub_dir @@ -87,7 +105,7 @@ def __str__(self) -> str: f"and save detected sources to the '{self.output_sub_dir}' directory." ) - def get_sextractor_output_dir(self) -> str: + def get_sextractor_output_dir(self) -> Path: """ Get the directory to output From ec60321c4580f3b54da7bb9c9459aaf64c9e37ed Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:20:05 -0700 Subject: [PATCH 09/86] more removing saturations, ad regions file writer --- .../astromatic/sextractor/sourceextractor.py | 43 ++++++++++++++----- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/mirar/processors/astromatic/sextractor/sourceextractor.py b/mirar/processors/astromatic/sextractor/sourceextractor.py index 98b9c1db6..43721ea6d 100644 --- a/mirar/processors/astromatic/sextractor/sourceextractor.py +++ b/mirar/processors/astromatic/sextractor/sourceextractor.py @@ -7,13 +7,15 @@ from typing import Optional from mirar.processors.astromatic.config import astromatic_config_dir +from mirar.processors.candidates.utils.regions_writer import write_regions_file from mirar.utils import ExecutionError, execute +from mirar.utils.ldac_tools import get_table_from_ldac logger = logging.getLogger(__name__) # sextractor_cmd = os.getenv("SEXTRACTOR_CMD") -default_saturation = 1.0e10 +default_saturation = None default_config_path = os.path.join(astromatic_config_dir, "astrom.sex") default_param_path = os.path.join(astromatic_config_dir, "astrom.param") default_filter_name = os.path.join(astromatic_config_dir, "default.conv") @@ -79,8 +81,8 @@ def parse_checkimage( ) logger.error(err) raise ValueError(err) - else: - cmd += f"-CHECKIMAGE_NAME {','.join(checkimage_name)}" + + cmd += f"-CHECKIMAGE_NAME {','.join(checkimage_name)}" else: if image is not None: @@ -135,20 +137,21 @@ def run_sextractor_single( parameters_name: str = default_param_path, filter_name: str = default_filter_name, starnnw_name: str = default_starnnw_path, - saturation: float = default_saturation, + saturation: float = None, weight_image: Optional[str] = None, verbose_type: str = "QUIET", checkimage_name: Optional[str | list] = None, checkimage_type: Optional[str | list] = None, gain: Optional[float] = None, mag_zp: Optional[float] = None, + write_regions: bool = False, ): """ Function to run sextractor in single mode Args: - img: - output_dir: - catalog_name: + img: The image to run sextractor on + output_dir: The directory to output the catalog to + catalog_name: The name of the catalog to output. config: parameters_name: filter_name: @@ -158,9 +161,9 @@ def run_sextractor_single( verbose_type: checkimage_name: checkimage_type: - gain: - mag_zp: - + gain: The gain to use for the catalog + mag_zp: The magnitude zero point to use for the catalog + write_regions: Whether to write ds9 regions for the objects in the catalog Returns: """ @@ -207,6 +210,21 @@ def run_sextractor_single( except ExecutionError as e: raise SextractorError(e) + if write_regions: + output_catalog = get_table_from_ldac(catalog_name) + + x_coords = output_catalog["X_IMAGE"] + y_coords = output_catalog["Y_IMAGE"] + + regions_path = catalog_name.as_posix() + ".reg" + + write_regions_file( + regions_path=regions_path, + x_coords=x_coords, + y_coords=y_coords, + system="image", + region_radius=5, + ) return catalog_name, checkimage_name @@ -219,7 +237,7 @@ def run_sextractor_dual( parameters_name: str = default_param_path, filter_name: str = default_filter_name, starnnw_name: str = default_starnnw_path, - saturation: float = default_saturation, + saturation: float = None, weight_image: Optional[str] = None, verbose_type: str = "QUIET", checkimage_name: Optional[str | list] = None, @@ -275,6 +293,9 @@ def run_sextractor_dual( if starnnw_name is not None: cmd += f"-STARNNW_NAME {starnnw_name} " + if mag_zp is not None: + cmd += f" -MAG_ZEROPOINT {mag_zp}" + checkimage_cmd, checkimage_name = parse_checkimage( checkimage_type=checkimage_type, checkimage_name=checkimage_name, From eda2150a1004906048ee594a88140a2db245fb99 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:25:09 -0700 Subject: [PATCH 10/86] more desaturation --- mirar/processors/autoastrometry/autoastrometry.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mirar/processors/autoastrometry/autoastrometry.py b/mirar/processors/autoastrometry/autoastrometry.py index a9ed6c8e4..b6e4b8ebc 100644 --- a/mirar/processors/autoastrometry/autoastrometry.py +++ b/mirar/processors/autoastrometry/autoastrometry.py @@ -45,6 +45,8 @@ from mirar.processors.autoastrometry.errors import ( AstrometryCrossmatchError, AstrometrySourceError, + AstrometryURLError, + AstrometryReferenceError ) from mirar.processors.autoastrometry.io import ( export_src_lists, @@ -83,11 +85,11 @@ def autoastrometry( outfile: str = "", output_dir: str = base_output_dir, temp_file: Optional[str] = None, - saturation: float = default_saturation, + saturation: float = None, no_rot: bool = False, min_fwhm: float = DEFAULT_MIN_FWHM, max_fwhm: float = DEFAULT_MAX_FWHM, - write_crosscheck_files: bool = False, + write_crosscheck_files: bool = True, ): """ @@ -244,7 +246,8 @@ def autoastrometry( center_dec=center_dec, box_size_arcsec=box_size_arcsec, ) - except TimeoutError: + except (TimeoutError, AstrometryURLError, AstrometrySourceError, + AstrometryReferenceError): ref_src_list, n_ref, ref_density = get_ref_sources_from_catalog_astroquery( catalog=catalog, center_ra=center_ra, @@ -550,7 +553,7 @@ def run_autoastrometry_single( overwrite: bool = False, outfile: Optional[str] = None, output_dir: str = base_output_dir, - saturation: float = default_saturation, + saturation: float = None, no_rot: bool = False, write_crosscheck_files: bool = False, ): @@ -661,7 +664,7 @@ def run_autoastrometry_batch( overwrite: bool = False, outfile: Optional[str] = None, output_dir: str = base_output_dir, - saturation: float = default_saturation, + saturation: float = None, no_rot: bool = False, write_crosscheck_files: bool = False, ): From 1ad9b4910afdb16c16413d1c110086aad38547a9 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:36:19 -0700 Subject: [PATCH 11/86] more error handling --- mirar/processors/autoastrometry/autoastrometry.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mirar/processors/autoastrometry/autoastrometry.py b/mirar/processors/autoastrometry/autoastrometry.py index b6e4b8ebc..a787e6dc4 100644 --- a/mirar/processors/autoastrometry/autoastrometry.py +++ b/mirar/processors/autoastrometry/autoastrometry.py @@ -32,7 +32,6 @@ write_config_file, write_param_file, ) -from mirar.processors.astromatic.sextractor.sourceextractor import default_saturation from mirar.processors.autoastrometry.crossmatch import ( crosscheck_source_lists, distance_match, @@ -44,9 +43,9 @@ ) from mirar.processors.autoastrometry.errors import ( AstrometryCrossmatchError, + AstrometryReferenceError, AstrometrySourceError, AstrometryURLError, - AstrometryReferenceError ) from mirar.processors.autoastrometry.io import ( export_src_lists, @@ -246,8 +245,12 @@ def autoastrometry( center_dec=center_dec, box_size_arcsec=box_size_arcsec, ) - except (TimeoutError, AstrometryURLError, AstrometrySourceError, - AstrometryReferenceError): + except ( + TimeoutError, + AstrometryURLError, + AstrometrySourceError, + AstrometryReferenceError, + ): ref_src_list, n_ref, ref_density = get_ref_sources_from_catalog_astroquery( catalog=catalog, center_ra=center_ra, From e47832ebb598cb01c02540779f6965e4d1c5b831 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:37:11 -0700 Subject: [PATCH 12/86] more sats --- mirar/processors/autoastrometry/autoastrometry_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/processors/autoastrometry/autoastrometry_processor.py b/mirar/processors/autoastrometry/autoastrometry_processor.py index 87da6b9e1..d32fbd904 100644 --- a/mirar/processors/autoastrometry/autoastrometry_processor.py +++ b/mirar/processors/autoastrometry/autoastrometry_processor.py @@ -20,7 +20,7 @@ class AutoAstrometry(BaseImageProcessor): def __init__( self, temp_output_sub_dir: str = "autoastrometry", - write_crosscheck_files: bool = False, + write_crosscheck_files: bool = True, catalog: Optional[str] = None, pixel_scale: Optional[float] = None, inv: bool = False, From afc9575244a0438b9572b3acdad3040c92e6b3cb Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:39:24 -0700 Subject: [PATCH 13/86] some logging --- mirar/processors/autoastrometry/reference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/processors/autoastrometry/reference.py b/mirar/processors/autoastrometry/reference.py index e784117db..670139801 100644 --- a/mirar/processors/autoastrometry/reference.py +++ b/mirar/processors/autoastrometry/reference.py @@ -334,7 +334,7 @@ def get_ref_sources_from_catalog( f"scat?catalog={trycat}&ra={center_ra}" f"&dec={center_dec}&system=J2000&rad=-90" ) - + logger.debug(f"Trying {testqueryurl}") with urllib.request.urlopen(testqueryurl, timeout=30) as check: checklines = check.readlines() logger.debug(f"Found {len(checklines)}") From 60230baed5f40f01a9939c9c13e5141aae644274 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:40:51 -0700 Subject: [PATCH 14/86] fix bug --- mirar/processors/autoastrometry/io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/processors/autoastrometry/io.py b/mirar/processors/autoastrometry/io.py index 3501735ea..a9085eca2 100644 --- a/mirar/processors/autoastrometry/io.py +++ b/mirar/processors/autoastrometry/io.py @@ -349,7 +349,7 @@ def write_region_file( out.write("image\n") for i, src in enumerate(src_list): out.write( - f"point({src.ra_deg:.3f},{src.dec_deg:.3f}) " + f"point({src.x:.3f},{src.y:.3f}) " f"# point=boxcircle text={{{i + 1}}}\n" ) From 3439c516774c8b700f35b732f6875ca837687082 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:42:30 -0700 Subject: [PATCH 15/86] calculate saturation values --- mirar/processors/bias.py | 5 +++-- mirar/processors/dark.py | 5 +++++ mirar/processors/sky.py | 4 ++++ 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/mirar/processors/bias.py b/mirar/processors/bias.py index f94fca6c5..ecf7d5c27 100644 --- a/mirar/processors/bias.py +++ b/mirar/processors/bias.py @@ -8,7 +8,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError -from mirar.paths import BIAS_FRAME_KEY, LATEST_SAVE_KEY +from mirar.paths import BIAS_FRAME_KEY, LATEST_SAVE_KEY, SATURATE_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -57,7 +57,8 @@ def _apply_to_images( data = data - master_bias.get_data() image.set_data(data) image[BIAS_FRAME_KEY] = master_bias[LATEST_SAVE_KEY] - + if SATURATE_KEY in image.header: + image[SATURATE_KEY] -= np.nanmedian(master_bias.get_data()) return batch def make_image( diff --git a/mirar/processors/dark.py b/mirar/processors/dark.py index f266701a1..15f86edf8 100644 --- a/mirar/processors/dark.py +++ b/mirar/processors/dark.py @@ -8,6 +8,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError +from mirar.paths import SATURATE_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -60,6 +61,10 @@ def _apply_to_images( data = data - (master_dark.get_data() * image["EXPTIME"]) image.set_data(data) + if SATURATE_KEY in image.header: + image[SATURATE_KEY] -= ( + np.nanmedian(master_dark.get_data()) * image["EXPTIME"] + ) return batch def make_image( diff --git a/mirar/processors/sky.py b/mirar/processors/sky.py index ff452ca8b..cd63be513 100644 --- a/mirar/processors/sky.py +++ b/mirar/processors/sky.py @@ -6,6 +6,7 @@ import numpy as np from mirar.data import ImageBatch +from mirar.paths import SATURATE_KEY from mirar.processors.base_processor import ProcessorPremadeCache from mirar.processors.flat import SkyFlatCalibrator @@ -40,6 +41,9 @@ def _apply_to_images( header.append( ("SKMEDSUB", subtract_median, "Median sky level subtracted"), end=True ) + if SATURATE_KEY in image.header: + # image[SATURATE_KEY] -= subtract_median + image[SATURATE_KEY] = 25000 image.set_data(data) image.set_header(header) From 218656f823781eed51aa1e9173c30dae4fa07884 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:45:45 -0700 Subject: [PATCH 16/86] make possible to retrieve components of a stack --- mirar/processors/astromatic/swarp/swarp.py | 73 ++++++++++++++++++++-- 1 file changed, 67 insertions(+), 6 deletions(-) diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index 7e7d217d0..c4e23ec00 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -3,18 +3,23 @@ """ import logging import os +from collections.abc import Callable from pathlib import Path from typing import Optional +import astropy import numpy as np from astropy.wcs import WCS -from mirar.data import ImageBatch +from mirar.data import Image, ImageBatch from mirar.errors import ProcessorError +from mirar.io import open_fits from mirar.paths import ( BASE_NAME_KEY, + LATEST_SAVE_KEY, LATEST_WEIGHT_SAVE_KEY, RAW_IMG_KEY, + STACKED_COMPONENT_IMAGES_KEY, SWARP_FLUX_SCALING_KEY, all_astrometric_keywords, copy_temp_file, @@ -180,18 +185,23 @@ def __init__( temp_output_sub_dir: str output sub-directory pixscale: float - Pixel scale in degrees + Pixel scale in degrees. If None, set as median of the pixel scales + of input images. x_imgpixsize: float - X-dimension in pixels + X-dimension in pixels. If None, set as max x-size of input images. If + you want a stacked image covering all input images, set + calculate_dims_in_swarp to True instead. y_imgpixsize: float - Y-dimension in pixels + Y-dimension in pixels. If None, set as max y-size of input images. propogate_headerlist: list List of header keywords to propagate. Recommended to leave None, the processor will take care of it. center_ra: float - Desired central RA of output image + Desired central RA of output image. If None, set as the median of the + input images. center_dec: - Desired central Dec of output image + Desired central Dec of output image. If None, set as the median of the + input images. gain: float Gain include_scamp: bool @@ -460,6 +470,9 @@ def _apply_to_images( new_image["COADDS"] = np.sum([x["COADDS"] for x in batch]) new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) + new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join( + [x[LATEST_SAVE_KEY] for x in batch] + ) new_image[BASE_NAME_KEY] = output_image_path.name new_image[LATEST_WEIGHT_SAVE_KEY] = output_image_weight_path.as_posix() self.save_fits(new_image, output_image_path) @@ -471,3 +484,51 @@ def _apply_to_images( logger.debug(f"Deleted temporary file {temp_file}") return ImageBatch([new_image]) + + +class GetSwarpComponentImages(BaseImageProcessor): + base_key = "swarp_component_images" + + def __init__( + self, + load_image: Callable[[str], [np.ndarray, astropy.io.fits.Header]] = open_fits, + header_key=STACKED_COMPONENT_IMAGES_KEY, + copy_header_keys: str | list[str] = None, + ): + super().__init__() + self.load_image = load_image + self.header_key = header_key + self.copy_header_keys = copy_header_keys + if isinstance(copy_header_keys, str): + self.copy_header_keys = [copy_header_keys] + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + if len(batch) > 1: + raise NotImplementedError( + "GetSwarpComponentImages only works on a batch containing a " + "single images. Consider adding an ImageDebatcher before " + "this processor." + ) + component_batch = ImageBatch() + image = batch[0] + component_images_list = image[self.header_key].split(",") + + for component_image_path in component_images_list: + if not Path(component_image_path).exists(): + raise FileNotFoundError( + f"Component image {component_image_path} not found. " + f"Are you sure it was saved using ImageSaver to this path just " + f"before the Swarp processor that stacked it?" + ) + component_data, component_header = self.load_image(component_image_path) + component_image = Image(component_data, component_header) + if self.copy_header_keys is not None: + for key in self.copy_header_keys: + if key in image.keys(): + component_image[key] = image[key] + component_batch.append(component_image) + logger.info(f"Loaded {len(component_batch)} component images") + return component_batch From 855dcd9cbf7a0011a6852a5e6ce439d359bfa799 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:57:02 -0700 Subject: [PATCH 17/86] make more compatible with reading processed imags --- mirar/pipelines/wirc/load_wirc_image.py | 17 +++++++++++++---- tests/test_wirc_pipeline.py | 4 ++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/mirar/pipelines/wirc/load_wirc_image.py b/mirar/pipelines/wirc/load_wirc_image.py index 1f83745a6..73e83cd0d 100644 --- a/mirar/pipelines/wirc/load_wirc_image.py +++ b/mirar/pipelines/wirc/load_wirc_image.py @@ -16,10 +16,13 @@ PROC_FAIL_KEY, PROC_HISTORY_KEY, RAW_IMG_KEY, + SATURATE_KEY, ) logger = logging.getLogger(__name__) +wirc_nonlinear_level = 30000 + def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: """ @@ -34,17 +37,23 @@ def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Hea header = img[0].header # pylint: enable=E1101 header["FILTER"] = header["AFT"].split("__")[0] - + header["DETCOADD"] = header["COADDS"] + if SATURATE_KEY not in header: + header[SATURATE_KEY] = wirc_nonlinear_level * header["DETCOADD"] if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: header["OBSTYPE"] = "calibration" header["OBSCLASS"] = ["calibration", "science"][header["OBSTYPE"] == "object"] header[BASE_NAME_KEY] = os.path.basename(path) - header[RAW_IMG_KEY] = path + if RAW_IMG_KEY not in header.keys(): + header[RAW_IMG_KEY] = path header["TARGET"] = header["OBJECT"].lower() - header["UTCTIME"] = header["UTSHUT"] - header["MJD-OBS"] = Time(header["UTSHUT"]).mjd + if "MJD-OBS" in header.keys(): + header["UTCTIME"] = Time(header["MJD-OBS"], format="mjd").isot + else: + header["UTCTIME"] = header["UTSHUT"] + header["MJD-OBS"] = Time(header["UTSHUT"]).mjd if COADD_KEY not in header.keys(): logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") header[COADD_KEY] = 1 diff --git a/tests/test_wirc_pipeline.py b/tests/test_wirc_pipeline.py index 7a1fcd811..a88af1948 100644 --- a/tests/test_wirc_pipeline.py +++ b/tests/test_wirc_pipeline.py @@ -23,7 +23,7 @@ from mirar.processors.csvlog import CSVLog from mirar.processors.dark import MasterDarkCalibrator from mirar.processors.flat import MasterFlatCalibrator -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.photcal import PhotCalibrator from mirar.processors.sky import MasterSkyCalibrator from mirar.processors.utils import ImageSaver @@ -88,7 +88,7 @@ def get_cal_path(name: str) -> str: "OBSCLASS", ], ), - MaskPixels(mask_path=wirc_mask_path), + MaskPixelsFromPath(mask_path=wirc_mask_path), ImageSelector(("exptime", "45.0")), MasterDarkCalibrator(get_cal_path("dark")), ImageDebatcher(), From 5a2c37e53c808f1e5264b485459de0ef9eaa97e1 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 14:59:52 -0700 Subject: [PATCH 18/86] reduce xmatch radii --- mirar/pipelines/wirc/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirar/pipelines/wirc/generator.py b/mirar/pipelines/wirc/generator.py index 5ab3fc33f..24b6d8a65 100644 --- a/mirar/pipelines/wirc/generator.py +++ b/mirar/pipelines/wirc/generator.py @@ -24,7 +24,7 @@ def wirc_astrometric_catalog_generator(_) -> Gaia2Mass: :return: catalogue """ - return Gaia2Mass(min_mag=10, max_mag=20, search_radius_arcmin=30) + return Gaia2Mass(min_mag=10, max_mag=20, search_radius_arcmin=10) def wirc_photometric_catalog_generator(image: Image) -> Gaia2Mass: @@ -36,7 +36,7 @@ def wirc_photometric_catalog_generator(image: Image) -> Gaia2Mass: """ filter_name = image["FILTER"] return Gaia2Mass( - min_mag=10, max_mag=20, search_radius_arcmin=30, filter_name=filter_name + min_mag=10, max_mag=20, search_radius_arcmin=10, filter_name=filter_name ) From 7c7789a56870f6c32c55fd81e5396697be578d3a Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 15:03:12 -0700 Subject: [PATCH 19/86] more processors for second pass processing --- mirar/pipelines/wirc/blocks.py | 73 ++++++++++++++++++++++++---------- 1 file changed, 53 insertions(+), 20 deletions(-) diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index 602429b35..1f7d7526e 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -24,6 +24,8 @@ ) from mirar.processors.alerts import AvroPacketMaker, SendToFritz from mirar.processors.astromatic import Scamp, Sextractor, Swarp +from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map +from mirar.processors.astromatic.swarp.swarp import GetSwarpComponentImages from mirar.processors.astromatic.psfex import PSFex from mirar.processors.autoastrometry import AutoAstrometry from mirar.processors.candidates.candidate_detector import DetectCandidates @@ -37,7 +39,8 @@ from mirar.processors.database.database_exporter import DatabaseDataframeExporter from mirar.processors.database.database_importer import DatabaseHistoryImporter from mirar.processors.flat import SkyFlatCalibrator -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath, MaskPixelsFromWCS, \ + MaskAboveThreshold, WriteMaskedCoordsToFile from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import ( CandidateAperturePhotometry, @@ -57,27 +60,56 @@ ) from mirar.processors.xmatch import XMatch from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare +from mirar.paths import SATURATE_KEY, RAW_IMG_KEY, CSV_MASK_KEY -load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)] +# load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)] +load_raw = [ImageLoader(input_sub_dir="firstpassstack", load_image=load_raw_wirc_image)] reduce = [ - CSVLog( - export_keys=[ - "OBJECT", - "FILTER", - "UTSHUT", - "EXPTIME", - "COADDS", - "OBSTYPE", - "OBSCLASS", - ] - ), - MaskPixels(mask_path=wirc_mask_path), - ImageSelector(("exptime", "45.0")), - DarkCalibrator(), - ImageDebatcher(), - ImageSelector(("obsclass", "science")), - ImageBatcher(split_key="filter"), + # CSVLog( + # export_keys=[ + # "OBJECT", + # "FILTER", + # "UTSHUT", + # "EXPTIME", + # "COADDS", + # "OBSTYPE", + # "OBSCLASS", + # ] + # ), + # MaskPixelsFromPath(mask_path=wirc_mask_path), + # ImageSelector(("exptime", "45.0")), + # DarkCalibrator(), + # ImageDebatcher(), + # ImageSelector(("obsclass", "science")), + # ImageSelector(("object", "ZTF18aavqmki")), + # ImageBatcher(split_key=["filter", "object"]), + # SkyFlatCalibrator(), + # NightSkyMedianCalibrator(), + # AutoAstrometry(catalog="tmc"), + # Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), + # ImageSaver(output_dir_name="firstpass"), + # Scamp( + # ref_catalog_generator=wirc_astrometric_catalog_generator, + # scamp_config_path=scamp_fp_path, + # ), + # Swarp(swarp_config_path=swarp_sp_path, + # calculate_dims_in_swarp=True), + # ImageSaver(output_dir_name="firstpassstack"), + Sextractor(output_sub_dir="firstpasssextractor", **sextractor_astrometry_config, + checkimage_type="SEGMENTATION", + cache=True, + ), + MaskPixelsFromPath(mask_path_key=sextractor_checkimg_map["SEGMENTATION"]), + ImageSaver(output_dir_name="mask1", write_mask=True), + MaskAboveThreshold(threshold_key=SATURATE_KEY), + ImageSaver(output_dir_name="mask2", write_mask=True), + WriteMaskedCoordsToFile(output_dir="mask"), + GetSwarpComponentImages(header_key=RAW_IMG_KEY, + load_image=load_raw_wirc_image, + copy_header_keys=CSV_MASK_KEY), + MaskPixelsFromWCS(), + ImageSaver(output_dir_name="firstpassmasked", write_mask=True), SkyFlatCalibrator(), NightSkyMedianCalibrator(), AutoAstrometry(catalog="tmc"), @@ -86,7 +118,8 @@ ref_catalog_generator=wirc_astrometric_catalog_generator, scamp_config_path=scamp_fp_path, ), - Swarp(swarp_config_path=swarp_sp_path), + Swarp(swarp_config_path=swarp_sp_path, + calculate_dims_in_swarp=True), Sextractor(output_sub_dir="final_sextractor", **sextractor_astrometry_config), PhotCalibrator(ref_catalog_generator=wirc_photometric_catalog_generator), ImageSaver(output_dir_name="final"), From fa18d2e8972ef7bcab7e11ce33d027897b400407 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 8 Jun 2023 15:05:25 -0700 Subject: [PATCH 20/86] more processors for second pass processing --- mirar/pipelines/wirc/blocks.py | 33 ++++++++++++++++++++------------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index 1f7d7526e..edb9cc5d7 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -2,6 +2,7 @@ Module containing standard processing blocks for WIRC """ from mirar.catalog.kowalski import PS1, TMASS +from mirar.paths import CSV_MASK_KEY, RAW_IMG_KEY, SATURATE_KEY from mirar.pipelines.wirc.generator import ( wirc_astrometric_catalog_generator, wirc_photometric_catalog_generator, @@ -24,9 +25,9 @@ ) from mirar.processors.alerts import AvroPacketMaker, SendToFritz from mirar.processors.astromatic import Scamp, Sextractor, Swarp +from mirar.processors.astromatic.psfex import PSFex from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map from mirar.processors.astromatic.swarp.swarp import GetSwarpComponentImages -from mirar.processors.astromatic.psfex import PSFex from mirar.processors.autoastrometry import AutoAstrometry from mirar.processors.candidates.candidate_detector import DetectCandidates from mirar.processors.candidates.candidate_extractor import ( @@ -39,8 +40,12 @@ from mirar.processors.database.database_exporter import DatabaseDataframeExporter from mirar.processors.database.database_importer import DatabaseHistoryImporter from mirar.processors.flat import SkyFlatCalibrator -from mirar.processors.mask import MaskPixelsFromPath, MaskPixelsFromWCS, \ - MaskAboveThreshold, WriteMaskedCoordsToFile +from mirar.processors.mask import ( + MaskAboveThreshold, + MaskPixelsFromPath, + MaskPixelsFromWCS, + WriteMaskedCoordsToFile, +) from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import ( CandidateAperturePhotometry, @@ -60,7 +65,6 @@ ) from mirar.processors.xmatch import XMatch from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare -from mirar.paths import SATURATE_KEY, RAW_IMG_KEY, CSV_MASK_KEY # load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)] load_raw = [ImageLoader(input_sub_dir="firstpassstack", load_image=load_raw_wirc_image)] @@ -96,18 +100,22 @@ # Swarp(swarp_config_path=swarp_sp_path, # calculate_dims_in_swarp=True), # ImageSaver(output_dir_name="firstpassstack"), - Sextractor(output_sub_dir="firstpasssextractor", **sextractor_astrometry_config, - checkimage_type="SEGMENTATION", - cache=True, - ), + Sextractor( + output_sub_dir="firstpasssextractor", + **sextractor_astrometry_config, + checkimage_type="SEGMENTATION", + cache=True, + ), MaskPixelsFromPath(mask_path_key=sextractor_checkimg_map["SEGMENTATION"]), ImageSaver(output_dir_name="mask1", write_mask=True), MaskAboveThreshold(threshold_key=SATURATE_KEY), ImageSaver(output_dir_name="mask2", write_mask=True), WriteMaskedCoordsToFile(output_dir="mask"), - GetSwarpComponentImages(header_key=RAW_IMG_KEY, - load_image=load_raw_wirc_image, - copy_header_keys=CSV_MASK_KEY), + GetSwarpComponentImages( + header_key=RAW_IMG_KEY, + load_image=load_raw_wirc_image, + copy_header_keys=CSV_MASK_KEY, + ), MaskPixelsFromWCS(), ImageSaver(output_dir_name="firstpassmasked", write_mask=True), SkyFlatCalibrator(), @@ -118,8 +126,7 @@ ref_catalog_generator=wirc_astrometric_catalog_generator, scamp_config_path=scamp_fp_path, ), - Swarp(swarp_config_path=swarp_sp_path, - calculate_dims_in_swarp=True), + Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True), Sextractor(output_sub_dir="final_sextractor", **sextractor_astrometry_config), PhotCalibrator(ref_catalog_generator=wirc_photometric_catalog_generator), ImageSaver(output_dir_name="final"), From ac32267f4652bfd492969a15f361b2306d1ba221 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:27:18 -0700 Subject: [PATCH 21/86] separate directory for astrometry processors --- mirar/processors/anet/__init__.py | 6 -- mirar/processors/astrometry/__init__.py | 0 mirar/processors/astrometry/anet/__init__.py | 6 ++ .../processors/{ => astrometry}/anet/anet.py | 0 .../{ => astrometry}/anet/anet_processor.py | 2 +- .../autoastrometry/__init__.py | 5 +- .../autoastrometry/autoastrometry.py | 12 ++-- .../autoastrometry_processor.py | 4 +- .../autoastrometry/crossmatch.py | 4 +- .../{ => astrometry}/autoastrometry/detect.py | 9 ++- .../{ => astrometry}/autoastrometry/errors.py | 0 .../{ => astrometry}/autoastrometry/io.py | 7 +- .../autoastrometry/reference.py | 6 +- .../autoastrometry/sources.py | 0 .../{ => astrometry}/autoastrometry/utils.py | 0 mirar/processors/astrometry/utils.py | 70 +++++++++++++++++++ 16 files changed, 106 insertions(+), 25 deletions(-) delete mode 100644 mirar/processors/anet/__init__.py create mode 100644 mirar/processors/astrometry/__init__.py create mode 100644 mirar/processors/astrometry/anet/__init__.py rename mirar/processors/{ => astrometry}/anet/anet.py (100%) rename mirar/processors/{ => astrometry}/anet/anet_processor.py (97%) rename mirar/processors/{ => astrometry}/autoastrometry/__init__.py (58%) rename mirar/processors/{ => astrometry}/autoastrometry/autoastrometry.py (98%) rename mirar/processors/{ => astrometry}/autoastrometry/autoastrometry_processor.py (95%) rename mirar/processors/{ => astrometry}/autoastrometry/crossmatch.py (99%) rename mirar/processors/{ => astrometry}/autoastrometry/detect.py (96%) rename mirar/processors/{ => astrometry}/autoastrometry/errors.py (100%) rename mirar/processors/{ => astrometry}/autoastrometry/io.py (98%) rename mirar/processors/{ => astrometry}/autoastrometry/reference.py (97%) rename mirar/processors/{ => astrometry}/autoastrometry/sources.py (100%) rename mirar/processors/{ => astrometry}/autoastrometry/utils.py (100%) create mode 100644 mirar/processors/astrometry/utils.py diff --git a/mirar/processors/anet/__init__.py b/mirar/processors/anet/__init__.py deleted file mode 100644 index 9f13a24c2..000000000 --- a/mirar/processors/anet/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Central module for running anet.net solve -""" - -from mirar.processors.anet.anet import run_astrometry_net -from mirar.processors.anet.anet_processor import AstrometryNet diff --git a/mirar/processors/astrometry/__init__.py b/mirar/processors/astrometry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/processors/astrometry/anet/__init__.py b/mirar/processors/astrometry/anet/__init__.py new file mode 100644 index 000000000..fdef8e9df --- /dev/null +++ b/mirar/processors/astrometry/anet/__init__.py @@ -0,0 +1,6 @@ +""" +Central module for running anet.net solve +""" + +from mirar.processors.astrometry.anet.anet import run_astrometry_net +from mirar.processors.astrometry.anet.anet_processor import AstrometryNet diff --git a/mirar/processors/anet/anet.py b/mirar/processors/astrometry/anet/anet.py similarity index 100% rename from mirar/processors/anet/anet.py rename to mirar/processors/astrometry/anet/anet.py diff --git a/mirar/processors/anet/anet_processor.py b/mirar/processors/astrometry/anet/anet_processor.py similarity index 97% rename from mirar/processors/anet/anet_processor.py rename to mirar/processors/astrometry/anet/anet_processor.py index 1f6b463ad..2094291af 100644 --- a/mirar/processors/anet/anet_processor.py +++ b/mirar/processors/astrometry/anet/anet_processor.py @@ -10,7 +10,7 @@ from mirar.data import ImageBatch from mirar.paths import BASE_NAME_KEY, get_output_dir, get_temp_path -from mirar.processors.anet.anet import run_astrometry_net_single +from mirar.processors.astrometry.anet.anet import run_astrometry_net_single from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/__init__.py b/mirar/processors/astrometry/autoastrometry/__init__.py similarity index 58% rename from mirar/processors/autoastrometry/__init__.py rename to mirar/processors/astrometry/autoastrometry/__init__.py index bdb973b44..e60d22d9f 100644 --- a/mirar/processors/autoastrometry/__init__.py +++ b/mirar/processors/astrometry/autoastrometry/__init__.py @@ -4,4 +4,7 @@ author: Daniel Perley (dperley@astro.caltech.edu) last significant modifications 2012-04-23 """ -from mirar.processors.autoastrometry.autoastrometry_processor import AutoAstrometry + +from mirar.processors.astrometry.autoastrometry.autoastrometry_processor import ( + AutoAstrometry, +) diff --git a/mirar/processors/autoastrometry/autoastrometry.py b/mirar/processors/astrometry/autoastrometry/autoastrometry.py similarity index 98% rename from mirar/processors/autoastrometry/autoastrometry.py rename to mirar/processors/astrometry/autoastrometry/autoastrometry.py index a787e6dc4..042133455 100644 --- a/mirar/processors/autoastrometry/autoastrometry.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry.py @@ -32,32 +32,32 @@ write_config_file, write_param_file, ) -from mirar.processors.autoastrometry.crossmatch import ( +from mirar.processors.astrometry.autoastrometry.crossmatch import ( crosscheck_source_lists, distance_match, ) -from mirar.processors.autoastrometry.detect import ( +from mirar.processors.astrometry.autoastrometry.detect import ( DEFAULT_MAX_FWHM, DEFAULT_MIN_FWHM, get_img_src_list, ) -from mirar.processors.autoastrometry.errors import ( +from mirar.processors.astrometry.autoastrometry.errors import ( AstrometryCrossmatchError, AstrometryReferenceError, AstrometrySourceError, AstrometryURLError, ) -from mirar.processors.autoastrometry.io import ( +from mirar.processors.astrometry.autoastrometry.io import ( export_src_lists, parse_header, write_region_file, write_text_file, ) -from mirar.processors.autoastrometry.reference import ( +from mirar.processors.astrometry.autoastrometry.reference import ( get_ref_sources_from_catalog, get_ref_sources_from_catalog_astroquery, ) -from mirar.processors.autoastrometry.utils import median, stdev +from mirar.processors.astrometry.autoastrometry.utils import median, stdev logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/autoastrometry_processor.py b/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py similarity index 95% rename from mirar/processors/autoastrometry/autoastrometry_processor.py rename to mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py index d32fbd904..cc2f00125 100644 --- a/mirar/processors/autoastrometry/autoastrometry_processor.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py @@ -6,7 +6,9 @@ from mirar.data import ImageBatch from mirar.paths import BASE_NAME_KEY, get_output_dir -from mirar.processors.autoastrometry.autoastrometry import run_autoastrometry_single +from mirar.processors.astrometry.autoastrometry.autoastrometry import ( + run_autoastrometry_single, +) from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/crossmatch.py b/mirar/processors/astrometry/autoastrometry/crossmatch.py similarity index 99% rename from mirar/processors/autoastrometry/crossmatch.py rename to mirar/processors/astrometry/autoastrometry/crossmatch.py index ff9e94fc8..37462c0de 100644 --- a/mirar/processors/autoastrometry/crossmatch.py +++ b/mirar/processors/astrometry/autoastrometry/crossmatch.py @@ -8,7 +8,7 @@ import numpy as np -from mirar.processors.autoastrometry.sources import ( +from mirar.processors.astrometry.autoastrometry.sources import ( BaseSource, SextractorSource, distance, @@ -16,7 +16,7 @@ position_angle, quickdistance, ) -from mirar.processors.autoastrometry.utils import median, mode, stdev, unique +from mirar.processors.astrometry.autoastrometry.utils import median, mode, stdev, unique logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/detect.py b/mirar/processors/astrometry/autoastrometry/detect.py similarity index 96% rename from mirar/processors/autoastrometry/detect.py rename to mirar/processors/astrometry/autoastrometry/detect.py index 4d3f3cbcf..21f0f7c6a 100644 --- a/mirar/processors/autoastrometry/detect.py +++ b/mirar/processors/astrometry/autoastrometry/detect.py @@ -16,9 +16,12 @@ default_saturation, run_sextractor_single, ) -from mirar.processors.autoastrometry.errors import AstrometrySourceError -from mirar.processors.autoastrometry.sources import SextractorSource, compare_mag -from mirar.processors.autoastrometry.utils import median, mode +from mirar.processors.astrometry.autoastrometry.errors import AstrometrySourceError +from mirar.processors.astrometry.autoastrometry.sources import ( + SextractorSource, + compare_mag, +) +from mirar.processors.astrometry.autoastrometry.utils import median, mode logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/errors.py b/mirar/processors/astrometry/autoastrometry/errors.py similarity index 100% rename from mirar/processors/autoastrometry/errors.py rename to mirar/processors/astrometry/autoastrometry/errors.py diff --git a/mirar/processors/autoastrometry/io.py b/mirar/processors/astrometry/autoastrometry/io.py similarity index 98% rename from mirar/processors/autoastrometry/io.py rename to mirar/processors/astrometry/autoastrometry/io.py index a9085eca2..ae10d2df1 100644 --- a/mirar/processors/autoastrometry/io.py +++ b/mirar/processors/astrometry/autoastrometry/io.py @@ -10,8 +10,11 @@ import numpy as np from astropy.io import fits -from mirar.processors.autoastrometry.sources import BaseSource, SextractorSource -from mirar.processors.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg +from mirar.processors.astrometry.autoastrometry.sources import ( + BaseSource, + SextractorSource, +) +from mirar.processors.astrometry.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/reference.py b/mirar/processors/astrometry/autoastrometry/reference.py similarity index 97% rename from mirar/processors/autoastrometry/reference.py rename to mirar/processors/astrometry/autoastrometry/reference.py index 670139801..43ebf2875 100644 --- a/mirar/processors/autoastrometry/reference.py +++ b/mirar/processors/astrometry/autoastrometry/reference.py @@ -12,12 +12,12 @@ from astropy.coordinates import SkyCoord from astroquery.vizier import Vizier -from mirar.processors.autoastrometry.errors import ( +from mirar.processors.astrometry.autoastrometry.errors import ( AstrometryReferenceError, AstrometryURLError, ) -from mirar.processors.autoastrometry.sources import BaseSource, compare_mag -from mirar.processors.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg +from mirar.processors.astrometry.autoastrometry.sources import BaseSource, compare_mag +from mirar.processors.astrometry.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/sources.py b/mirar/processors/astrometry/autoastrometry/sources.py similarity index 100% rename from mirar/processors/autoastrometry/sources.py rename to mirar/processors/astrometry/autoastrometry/sources.py diff --git a/mirar/processors/autoastrometry/utils.py b/mirar/processors/astrometry/autoastrometry/utils.py similarity index 100% rename from mirar/processors/autoastrometry/utils.py rename to mirar/processors/astrometry/autoastrometry/utils.py diff --git a/mirar/processors/astrometry/utils.py b/mirar/processors/astrometry/utils.py new file mode 100644 index 000000000..308489a39 --- /dev/null +++ b/mirar/processors/astrometry/utils.py @@ -0,0 +1,70 @@ +""" +Module containing processors to add astrometry headers to images +""" +import logging +import os + +from astropy.io import fits + +from mirar.data import ImageBatch +from mirar.paths import ASTROMETRY_FILE_KEY, BASE_NAME_KEY, get_astrometry_keys +from mirar.processors.base_processor import BaseImageProcessor + +logger = logging.getLogger(__name__) + + +class AstrometryFromFile(BaseImageProcessor): + """ + Processor to add astrometry headers to images from file. + """ + + base_key = "astrometry_from_file" + + def __init__(self, astrometry_file_key: str = ASTROMETRY_FILE_KEY): + super().__init__() + self.astrometry_file_key = astrometry_file_key + + def __str__(self) -> str: + return "Processor to add astrometry headers to images from file." + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + astrometry_keys = get_astrometry_keys() + new_batch = ImageBatch() + for image in batch: + astrometry_file = image[self.astrometry_file_key] + + if not os.path.exists(astrometry_file): + raise FileNotFoundError( + f"Could not find astrometry file " + f"{astrometry_file}. Are you for sure running " + f"scamp with cache=True?" + ) + for key in astrometry_keys: + if key in image.header.keys(): + logger.debug(f"Removing {key} from {image[BASE_NAME_KEY]}") + del image.header[key] + + logger.info( + f"Adding astrometry headers from {astrometry_file} " + f"to {image[BASE_NAME_KEY]}" + ) + + with open(astrometry_file, "r") as f: + header_data = f.read() + # Scamp v 2.10.0 writes this annoying character in a comment + header_data = header_data.replace("é", "e") + + astrometry_header = fits.Header.fromstring(header_data, sep="\n") + for k in astrometry_header: + if (k == "HISTORY") | (k == "COMMENT"): + continue + + logger.debug(f"Adding {k} to {astrometry_header[k]}") + image.header.append( + (k, astrometry_header[k], astrometry_header.comments[k]) + ) + new_batch.append(image) + return new_batch From a16f5561095fbc64e279be98dea56488721564ce Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:28:42 -0700 Subject: [PATCH 22/86] more default keywords --- mirar/paths.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirar/paths.py b/mirar/paths.py index 6c2db804a..005908f47 100644 --- a/mirar/paths.py +++ b/mirar/paths.py @@ -238,6 +238,7 @@ def get_astrometry_keys() -> list: UNC_IMG_KEY = "UNCPATH" PROC_HISTORY_KEY = "CALSTEPS" PROC_FAIL_KEY = "PROCFAIL" +ASTROMETRY_FILE_KEY = "ASTRFILE" LATEST_SAVE_KEY = "SAVEPATH" LATEST_WEIGHT_SAVE_KEY = "WGHTPATH" SEXTRACTOR_HEADER_KEY = "SRCCAT" @@ -262,7 +263,7 @@ def get_astrometry_keys() -> list: CAND_NAME_KEY = "objectId" CAND_RA_KEY = "ra" CAND_DEC_KEY = "dec" -CSV_MASK_KEY = "MASKCSV" +FITS_MASK_KEY = "MASKFITS" sextractor_checkimg_keys = { "BACKGROUND": "BKGPT", "BACKGROUND_RMS": "BKGRMS", From 800e8e1019003b0c2dce184827da3bd74a249c92 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:37:23 -0700 Subject: [PATCH 23/86] make photometric qual cut default --- mirar/catalog/gaia.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirar/catalog/gaia.py b/mirar/catalog/gaia.py index f7db5fd17..ac3f99a67 100644 --- a/mirar/catalog/gaia.py +++ b/mirar/catalog/gaia.py @@ -26,7 +26,7 @@ def __init__( self, *args, filter_name: str = "j", - ph_qual_cut: bool = False, + ph_qual_cut: bool = True, snr_threshold: float = 5, trim: bool = False, image_catalog_path: Optional[str] = None, @@ -66,7 +66,7 @@ def get_catalog( ) if self.ph_qual_cut: - cmd += "AND tmass.ph_qual='AAA';" + cmd += " AND tmass.ph_qual='AAA';" else: cmd += ";" From 1c376db8e7b5243a79c472297360f5ccfd6d55c7 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:43:15 -0700 Subject: [PATCH 24/86] make sure non-temporary images are being propagated to headers --- mirar/processors/astromatic/swarp/swarp.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index c4e23ec00..f63ea7361 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -279,6 +279,7 @@ def _apply_to_images( ) logger.debug(f"Writing file list to {swarp_image_list_path}") + component_images_list = [x[LATEST_SAVE_KEY] for x in batch] temp_files = [swarp_image_list_path, swarp_weight_list_path] # If swarp is run with combine -N option, @@ -470,9 +471,7 @@ def _apply_to_images( new_image["COADDS"] = np.sum([x["COADDS"] for x in batch]) new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) - new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join( - [x[LATEST_SAVE_KEY] for x in batch] - ) + new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join(component_images_list) new_image[BASE_NAME_KEY] = output_image_path.name new_image[LATEST_WEIGHT_SAVE_KEY] = output_image_weight_path.as_posix() self.save_fits(new_image, output_image_path) @@ -487,6 +486,10 @@ def _apply_to_images( class GetSwarpComponentImages(BaseImageProcessor): + """ + Get the component images used to make a swarp stack + """ + base_key = "swarp_component_images" def __init__( From 2fff54bf7a77e7ec4175df8e85571a87e33cbe9a Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:47:36 -0700 Subject: [PATCH 25/86] refactor somewhat, introduce custom catalog purifier object --- mirar/processors/photcal.py | 223 ++++++++++++++++++------------------ 1 file changed, 112 insertions(+), 111 deletions(-) diff --git a/mirar/processors/photcal.py b/mirar/processors/photcal.py index ac40fde75..bd4643ad9 100644 --- a/mirar/processors/photcal.py +++ b/mirar/processors/photcal.py @@ -11,11 +11,12 @@ from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.stats import sigma_clip, sigma_clipped_stats +from astropy.table import Table from mirar.catalog.base_catalog import BaseCatalog from mirar.data import Image, ImageBatch from mirar.errors import ProcessorError -from mirar.paths import copy_temp_file, get_output_dir, get_output_path +from mirar.paths import BASE_NAME_KEY, copy_temp_file, get_output_dir, get_output_path from mirar.processors.astromatic.sextractor.sextractor import ( SEXTRACTOR_HEADER_KEY, Sextractor, @@ -60,6 +61,29 @@ class PhotometryCalculationError(PhotometryError): """Error related to the photometric calibration""" +def default_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Table: + """ + Default function to purify the photometric image catalog + """ + edge_width_pixels = 100 + fwhm_threshold_arcsec = 4.0 + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + class PhotCalibrator(BaseImageProcessor): """ Photometric calibrator processor @@ -72,11 +96,9 @@ def __init__( ref_catalog_generator: Callable[[Image], BaseCatalog], temp_output_sub_dir: str = "phot", redo: bool = True, - x_lower_limit: float = 100, - x_upper_limit: float = 2800, # Are these floats or ints? - y_lower_limit: float = 100, - y_upper_limit: float = 2800, - fwhm_threshold_arcsec: float = 4.0, + image_photometric_catalog_purifier: Callable[ + [Table, Image], Table + ] = default_photometric_img_catalog_purifier, num_matches_threshold: int = 5, write_regions: bool = False, cache: bool = False, @@ -85,15 +107,9 @@ def __init__( self.redo = redo # What is this for? self.ref_catalog_generator = ref_catalog_generator self.temp_output_sub_dir = temp_output_sub_dir - self.x_lower_limit = x_lower_limit - self.x_upper_limit = x_upper_limit - self.y_lower_limit = y_lower_limit - self.y_upper_limit = y_upper_limit + self.image_photometric_catalog_purifier = image_photometric_catalog_purifier self.cache = cache - # Why is this here not in catalog? - self.fwhm_threshold_arcsec = fwhm_threshold_arcsec - self.num_matches_threshold = num_matches_threshold self.write_regions = write_regions @@ -108,60 +124,25 @@ def get_phot_output_dir(self): return get_output_dir(self.temp_output_sub_dir, self.night_sub_dir) def calculate_zeropoint( - self, ref_cat_path: Path, img_cat_path: Path, img_filt + self, + ref_cat: Table, + clean_img_cat: Table, ) -> list[dict]: """ Function to calculate zero point from two catalogs Args: - ref_cat_path: Path to reference ldac catalog - img_cat_path: Path to image ldac catalog + ref_cat: Reference catalog table + clean_img_cat: Catalog of sources from image to xmatch with ref_cat Returns: """ - ref_cat_with_flagged = get_table_from_ldac(ref_cat_path) - img_cat = get_table_from_ldac(img_cat_path) - - if len(ref_cat_with_flagged) == 0: - err = "No sources found in reference catalog" - logger.error(err) - raise PhotometryReferenceError(err) - - if str(ref_cat_path).split(".")[-2] == "ps1": - # this reference catalog is from ps1 - # remove sources with SATURATED flag - ref_cat = self.remove_sat_ps1(ref_cat_with_flagged, img_filt) - - else: - # reference not ps1, no flags to check - ref_cat = ref_cat_with_flagged - ref_coords = SkyCoord(ra=ref_cat["ra"], dec=ref_cat["dec"], unit=(u.deg, u.deg)) - clean_mask = ( - (img_cat["FLAGS"] == 0) - & (img_cat["FWHM_WORLD"] < self.fwhm_threshold_arcsec / 3600.0) - & (img_cat["X_IMAGE"] > self.x_lower_limit) - & (img_cat["X_IMAGE"] < self.x_upper_limit) - & (img_cat["Y_IMAGE"] > self.y_lower_limit) - & (img_cat["Y_IMAGE"] < self.y_upper_limit) - ) - img_coords = SkyCoord( - ra=img_cat["ALPHAWIN_J2000"], - dec=img_cat["DELTAWIN_J2000"], - unit=(u.deg, u.deg), - ) - clean_img_cat = img_cat[clean_mask] - logger.debug(f"Found {len(clean_img_cat)} clean sources in image.") clean_img_coords = SkyCoord( ra=clean_img_cat["ALPHAWIN_J2000"], dec=clean_img_cat["DELTAWIN_J2000"], unit=(u.deg, u.deg), ) - if len(clean_img_coords) == 0: - err = "No clean sources found in image" - logger.error(err) - raise PhotometrySourceError(err) - idx, d2d, _ = ref_coords.match_to_catalog_sky(clean_img_coords) match_mask = d2d < 1.0 * u.arcsec matched_ref_cat = ref_cat[match_mask] @@ -170,45 +151,6 @@ def calculate_zeropoint( f"Cross-matched {len(matched_img_cat)} sources from catalog to the image." ) - if self.write_regions: - ref_regions_path = get_output_path( - base_name="ref.reg", - dir_root=self.temp_output_sub_dir, - sub_dir=self.night_sub_dir, - ) - cleaned_img_regions_path = get_output_path( - base_name="cleaned_img.reg", - dir_root=self.temp_output_sub_dir, - sub_dir=self.night_sub_dir, - ) - img_regions_path = get_output_path( - base_name="img.reg", - dir_root=self.temp_output_sub_dir, - sub_dir=self.night_sub_dir, - ) - - write_regions_file( - regions_path=ref_regions_path, - x_coords=ref_coords.ra.deg, - y_coords=ref_coords.dec.deg, - system="wcs", - region_radius=2.0 / 3600, - ) - write_regions_file( - regions_path=cleaned_img_regions_path, - x_coords=clean_img_coords.ra.deg, - y_coords=clean_img_coords.dec.deg, - system="wcs", - region_radius=2.0 / 3600, - ) - write_regions_file( - regions_path=img_regions_path, - x_coords=img_coords.ra.deg, - y_coords=img_coords.dec.deg, - system="wcs", - region_radius=2.0 / 3600, - ) - if len(matched_img_cat) < self.num_matches_threshold: err = ( "Not enough cross-matched sources " @@ -296,9 +238,79 @@ def _apply_to_images( image["FWHM_MED"] = fwhm_med image["FWHM_STD"] = fwhm_std - zp_dicts = self.calculate_zeropoint( - ref_cat_path, temp_cat_path, image.header["FILTER"] - ) + ref_cat = get_table_from_ldac(ref_cat_path) + img_cat = get_table_from_ldac(temp_cat_path) + + if len(ref_cat) == 0: + err = "No sources found in reference catalog" + logger.error(err) + raise PhotometryReferenceError(err) + + clean_img_cat = self.image_photometric_catalog_purifier(img_cat, image) + logger.debug(f"Found {len(clean_img_cat)} clean sources in image.") + + if len(clean_img_cat) == 0: + err = "No clean sources found in image" + logger.error(err) + raise PhotometrySourceError(err) + + if self.write_regions: + ref_coords = SkyCoord( + ra=ref_cat["ra"], dec=ref_cat["dec"], unit=(u.deg, u.deg) + ) + + img_coords = SkyCoord( + ra=img_cat["ALPHAWIN_J2000"], + dec=img_cat["DELTAWIN_J2000"], + unit=(u.deg, u.deg), + ) + + clean_img_coords = SkyCoord( + ra=clean_img_cat["ALPHAWIN_J2000"], + dec=clean_img_cat["DELTAWIN_J2000"], + unit=(u.deg, u.deg), + ) + + ref_regions_path = get_output_path( + base_name=image.header[BASE_NAME_KEY] + "ref.reg", + dir_root=self.temp_output_sub_dir, + sub_dir=self.night_sub_dir, + ) + cleaned_img_regions_path = get_output_path( + base_name=image.header[BASE_NAME_KEY] + "cleaned_img.reg", + dir_root=self.temp_output_sub_dir, + sub_dir=self.night_sub_dir, + ) + img_regions_path = get_output_path( + base_name=image.header[BASE_NAME_KEY] + "img.reg", + dir_root=self.temp_output_sub_dir, + sub_dir=self.night_sub_dir, + ) + + write_regions_file( + regions_path=ref_regions_path, + x_coords=ref_coords.ra.deg, + y_coords=ref_coords.dec.deg, + system="wcs", + region_radius=2.0 / 3600, + ) + write_regions_file( + regions_path=cleaned_img_regions_path, + x_coords=clean_img_coords.ra.deg, + y_coords=clean_img_coords.dec.deg, + system="wcs", + region_radius=2.0 / 3600, + ) + write_regions_file( + regions_path=img_regions_path, + x_coords=img_coords.ra.deg, + y_coords=img_coords.dec.deg, + system="wcs", + region_radius=2.0 / 3600, + ) + + zp_dicts = self.calculate_zeropoint(ref_cat, clean_img_cat) + aperture_diameters = [] zp_values = [] for zpvals in zp_dicts: @@ -314,7 +326,8 @@ def _apply_to_images( aperture_diameters.append(med_fwhm_pix) zp_values.append(image["ZP_AUTO"]) - if sextractor_checkimg_map["BACKGROUND_RMS"] in image: + if sextractor_checkimg_map["BACKGROUND_RMS"] in image.header.keys(): + logger.info("Calculating limiting magnitudes from background RMS file") limmags = self.get_maglim( image[sextractor_checkimg_map["BACKGROUND_RMS"]], zp_values, @@ -389,6 +402,9 @@ def get_maglim( return maglim def get_sextractor_module(self) -> Sextractor: + """ + Get the Sextractor module from the preceding steps + """ mask = [isinstance(x, Sextractor) for x in self.preceding_steps] return np.array(self.preceding_steps)[mask][-1] @@ -452,18 +468,3 @@ def get_sextractor_apertures(self) -> list[float]: line = aperture_lines[0].replace("PHOT_APERTURES", " ").split("#")[0] return [float(x) for x in line.split(",") if x not in [""]] - - def remove_sat_ps1(self, catalog, filt: str): - """ - remove ps1 sources flagged as "SATURATED" - """ - logger.info(f"original ps1 table length: {len(catalog)}") - logger.info("removing ps1 sources with SATURATED flag...") - sat_flag = 4096 # SATURATED value - column = catalog[str(filt) + "Flags"] - check = (column & sat_flag) / sat_flag - # check != 0 means this flag is there - # check == 0 means this flag is not there - clean_cat = catalog[np.where(check == 0)[0]] - logger.info(f"found {len(clean_cat)} columns without this flag \n") - return clean_cat From 4f2c3562ba8fe9267dea0a01750da8c1487c086b Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:48:42 -0700 Subject: [PATCH 26/86] make possible to mask flats --- mirar/processors/flat.py | 38 ++++++++++++++++++++++++++++++++------ 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index 7b18eab2d..172c5b603 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -2,14 +2,15 @@ Module containing processors for flat calibration """ import logging +import os.path import sys from collections.abc import Callable - +from astropy.io import fits import numpy as np from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError -from mirar.paths import FLAT_FRAME_KEY, LATEST_SAVE_KEY +from mirar.paths import FLAT_FRAME_KEY, LATEST_SAVE_KEY, BASE_NAME_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -44,6 +45,7 @@ def __init__( y_max: int = sys.maxsize, flat_nan_threshold: float = 0.0, select_flat_images: Callable[[ImageBatch], ImageBatch] = default_select_flat, + flat_mask_key: str = None, **kwargs, ): super().__init__(*args, **kwargs) @@ -53,6 +55,7 @@ def __init__( self.y_max = y_max self.flat_nan_threshold = flat_nan_threshold self.select_cache_images = select_flat_images + self.flat_mask_key = flat_mask_key def __str__(self) -> str: return "Creates a flat image, divides other images by this image." @@ -94,10 +97,32 @@ def make_image( flats = np.zeros((nx, ny, n_frames)) for i, img in enumerate(images): + data = img.get_data() + + if self.flat_mask_key is not None: + if self.flat_mask_key not in img.header.keys(): + err = f"Image {img} does not have a mask with key " \ + f"{self.flat_mask_key}" + logger.error(err) + raise KeyError(err) + + mask_file = img[self.flat_mask_key] + logger.info(f"Masking flat {img[BASE_NAME_KEY]} with mask {mask_file}") + if not os.path.exists(mask_file): + err = f"Mask file {mask_file} does not exist" + logger.error(err) + raise FileNotFoundError(err) + with fits.open(mask_file) as mask_img: + mask = mask_img[0].data + mask = (mask > 0) + logger.info(f"Masking {np.sum(mask)} pixels in flat " + f"{img[BASE_NAME_KEY]}") + data[mask] = np.nan + median = np.nanmedian( - img.get_data()[self.x_min : self.x_max, self.y_min : self.y_max] + data[self.x_min : self.x_max, self.y_min : self.y_max] ) - flats[:, :, i] = img.get_data() / median + flats[:, :, i] = data / median logger.info(f"Median combining {n_frames} flats") master_flat = np.nanmedian(flats, axis=2) @@ -110,8 +135,9 @@ class SkyFlatCalibrator(FlatCalibrator): Processor to do flat calibration using sky flats """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, select_flat_images=self.select_sky_flat) + def __init__(self, flat_mask_key=None, *args, **kwargs): + super().__init__(*args, **kwargs, select_flat_images=self.select_sky_flat, + flat_mask_key=flat_mask_key) @staticmethod def select_sky_flat( From 080c11f9961dc051ada9c782db7ad38e99bcab1d Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:49:18 -0700 Subject: [PATCH 27/86] make possible to mask flats --- mirar/processors/flat.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index 172c5b603..3ef236fc4 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -5,12 +5,13 @@ import os.path import sys from collections.abc import Callable -from astropy.io import fits + import numpy as np +from astropy.io import fits from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError -from mirar.paths import FLAT_FRAME_KEY, LATEST_SAVE_KEY, BASE_NAME_KEY +from mirar.paths import BASE_NAME_KEY, FLAT_FRAME_KEY, LATEST_SAVE_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -101,8 +102,10 @@ def make_image( if self.flat_mask_key is not None: if self.flat_mask_key not in img.header.keys(): - err = f"Image {img} does not have a mask with key " \ - f"{self.flat_mask_key}" + err = ( + f"Image {img} does not have a mask with key " + f"{self.flat_mask_key}" + ) logger.error(err) raise KeyError(err) @@ -114,9 +117,11 @@ def make_image( raise FileNotFoundError(err) with fits.open(mask_file) as mask_img: mask = mask_img[0].data - mask = (mask > 0) - logger.info(f"Masking {np.sum(mask)} pixels in flat " - f"{img[BASE_NAME_KEY]}") + mask = mask > 0 + logger.info( + f"Masking {np.sum(mask)} pixels in flat " + f"{img[BASE_NAME_KEY]}" + ) data[mask] = np.nan median = np.nanmedian( @@ -136,8 +141,12 @@ class SkyFlatCalibrator(FlatCalibrator): """ def __init__(self, flat_mask_key=None, *args, **kwargs): - super().__init__(*args, **kwargs, select_flat_images=self.select_sky_flat, - flat_mask_key=flat_mask_key) + super().__init__( + *args, + **kwargs, + select_flat_images=self.select_sky_flat, + flat_mask_key=flat_mask_key, + ) @staticmethod def select_sky_flat( From 0218eaf9f3827dcce3b84b3da909ff4450cf0a5a Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:51:22 -0700 Subject: [PATCH 28/86] write regions, make possible to only write the mask file without changing image data values --- mirar/processors/mask.py | 63 ++++++++++++++++++++++++---------------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/mirar/processors/mask.py b/mirar/processors/mask.py index f0ad84389..943ade078 100644 --- a/mirar/processors/mask.py +++ b/mirar/processors/mask.py @@ -5,12 +5,12 @@ from pathlib import Path import numpy as np -import pandas as pd from astropy.coordinates import SkyCoord +from astropy.io import fits from astropy.wcs import WCS -from mirar.data import ImageBatch -from mirar.paths import BASE_NAME_KEY, CSV_MASK_KEY, get_output_dir +from mirar.data import Image, ImageBatch +from mirar.paths import BASE_NAME_KEY, FITS_MASK_KEY, get_output_dir from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) @@ -28,10 +28,12 @@ def __init__( self, write_masked_pixels_to_file: bool = False, output_dir: str | Path = "mask", + only_write_mask: bool = False, ): super().__init__() self.write_masked_pixels_to_file = write_masked_pixels_to_file self.output_dir = output_dir + self.only_write_mask = only_write_mask def get_mask(self, image) -> np.ndarray: """ @@ -46,28 +48,23 @@ def _apply_to_images( for image in batch: data = image.get_data() mask = self.get_mask(image) - data[mask] = MASK_VALUE + + if not self.only_write_mask: + data[mask] = MASK_VALUE + image.set_data(data) + logger.info(f"Masked {np.sum(mask)} pixels in {image[BASE_NAME_KEY]}") - image.set_data(data) if self.write_masked_pixels_to_file: - wcs = WCS(image.get_header()) - masked_pixels_indices = np.where(mask) - masked_pixels_wcs = wcs.pixel_to_world( - masked_pixels_indices[1], masked_pixels_indices[0] - ) mask_directory = get_output_dir(self.output_dir, self.night_sub_dir) if not mask_directory.exists(): mask_directory.mkdir(parents=True) - mask_file_path = ( - mask_directory / f"{image[BASE_NAME_KEY]}" f"_masked_pixels_wcs.csv" - ) - with open(mask_file_path, "w") as f: - f.write("ra,dec\n") - for ra, dec in zip(masked_pixels_wcs.ra, masked_pixels_wcs.dec): - f.write(f"{ra.deg},{dec.deg}\n") - - image[CSV_MASK_KEY] = mask_file_path.as_posix() + mask_file_path = mask_directory / f"{image[BASE_NAME_KEY]}_mask.fits" + + mask_image = Image(data=mask.astype(int), header=image.get_header()) + self.save_fits(mask_image, mask_file_path) + + image[FITS_MASK_KEY] = mask_file_path.as_posix() return batch @@ -84,10 +81,12 @@ def __init__( mask_path_key: str = None, write_masked_pixels_to_file: bool = False, output_dir: str | Path = "mask", + only_write_mask: bool = False, ): super().__init__( write_masked_pixels_to_file=write_masked_pixels_to_file, output_dir=output_dir, + only_write_mask=only_write_mask, ) self.mask = None self.mask_path = mask_path @@ -129,6 +128,7 @@ def __init__( threshold_key: str = None, write_masked_pixels_to_file: bool = False, output_dir: str | Path = "mask", + only_write_mask: bool = False, ): """ :param threshold: threshold to mask above @@ -137,6 +137,7 @@ def __init__( super().__init__( write_masked_pixels_to_file=write_masked_pixels_to_file, output_dir=output_dir, + only_write_mask=only_write_mask, ) self.threshold = threshold self.threshold_key = threshold_key @@ -173,13 +174,15 @@ def __init__( self, mask_pixels_ra: float | list[float] = None, mask_pixels_dec: float | list[float] = None, - mask_file_key: str = CSV_MASK_KEY, + mask_file_key: str = FITS_MASK_KEY, write_masked_pixels_to_file: bool = False, output_dir: str | Path = "mask", + only_write_mask: bool = False, ): super().__init__( write_masked_pixels_to_file=write_masked_pixels_to_file, output_dir=output_dir, + only_write_mask=only_write_mask, ) self.mask_pixels_ra = mask_pixels_ra self.mask_pixels_dec = mask_pixels_dec @@ -200,13 +203,19 @@ def get_mask(self, image) -> np.ndarray: wcs = WCS(image.get_header()) if self.mask_file_key is not None: mask_file_path = image.get_header()[self.mask_file_key] - mask_pixels = pd.read_csv(mask_file_path) - mask_pixels_ra = mask_pixels["ra"].to_numpy() - mask_pixels_dec = mask_pixels["dec"].to_numpy() + with fits.open(mask_file_path) as mask_image: + mask = mask_image[0].data + mask_wcs = WCS(mask_image[0].header) + + masked_pixel_x, masked_pixel_y = np.where(mask) + mask_pixel_coords = mask_wcs.pixel_to_world(masked_pixel_y, masked_pixel_x) + mask_pixels_ra = mask_pixel_coords.ra.deg + mask_pixels_dec = mask_pixel_coords.dec.deg else: mask_pixels_ra = self.mask_pixels_ra mask_pixels_dec = self.mask_pixels_dec logger.debug(f"Masking {mask_pixels_ra} ras and {mask_pixels_dec} decs") + mask_pixel_coords = SkyCoord(mask_pixels_ra, mask_pixels_dec, unit="deg") mask_pixels_x, mask_pixels_y = wcs.world_to_pixel(mask_pixel_coords) mask_pixels_x = mask_pixels_x.astype(int) @@ -228,8 +237,12 @@ class WriteMaskedCoordsToFile(BaseMask): base_key = "writemaskedcoords" - def __init__(self, output_dir: str | Path = "mask"): - super().__init__(write_masked_pixels_to_file=True, output_dir=output_dir) + def __init__(self, output_dir: str | Path = "mask", only_write_mask: bool = False): + super().__init__( + write_masked_pixels_to_file=True, + output_dir=output_dir, + only_write_mask=only_write_mask, + ) def get_mask(self, image) -> np.ndarray: mask = np.zeros(image.get_data().shape, dtype=bool) From 158be96ed5c7c865e738bec31c637922230ca22a Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:54:22 -0700 Subject: [PATCH 29/86] reformat --- mirar/processors/utils/image_loader.py | 42 +++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/mirar/processors/utils/image_loader.py b/mirar/processors/utils/image_loader.py index 530716522..d0ce156d1 100644 --- a/mirar/processors/utils/image_loader.py +++ b/mirar/processors/utils/image_loader.py @@ -13,7 +13,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError from mirar.io import open_fits -from mirar.paths import RAW_IMG_SUB_DIR, base_raw_dir, core_fields +from mirar.paths import RAW_IMG_KEY, RAW_IMG_SUB_DIR, base_raw_dir, core_fields from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) @@ -110,3 +110,43 @@ def unzip(zipped_list: list[str]) -> list[str]: os.rename(file, unzipped_list[i]) return unzipped_list + + +class LoadImageFromHeader(BaseImageProcessor): + """ + Class to load images from header information + """ + + base_key = "load_from_header" + + def __init__( + self, + header_key: str = RAW_IMG_KEY, + copy_header_keys: str | list[str] = None, + load_image: Callable[[str], [np.ndarray, astropy.io.fits.Header]] = open_fits, + ): + super().__init__() + self.header_key = header_key + self.copy_header_keys = copy_header_keys + self.load_image = load_image + if isinstance(self.copy_header_keys, str): + self.copy_header_keys = [self.copy_header_keys] + + def __str__(self): + return f"Processor to load images from header key {self.header_key}" + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + new_batch = ImageBatch() + for image in batch: + new_image_file = image.header[self.header_key] + new_image_data, new_header = self.load_image(new_image_file) + new_image = Image(new_image_data, new_header) + if self.copy_header_keys is not None: + for key in self.copy_header_keys: + new_image.header[key] = image.header[key] + new_batch.append(new_image) + + return new_batch From 169ea67106292a747eab680f357dd7b58c3a4598 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:56:23 -0700 Subject: [PATCH 30/86] update to reflect astrometry changes --- mirar/pipelines/sedmv2/blocks.py | 2 +- mirar/pipelines/summer/blocks.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mirar/pipelines/sedmv2/blocks.py b/mirar/pipelines/sedmv2/blocks.py index 34c2a3543..c5b3712fc 100644 --- a/mirar/pipelines/sedmv2/blocks.py +++ b/mirar/pipelines/sedmv2/blocks.py @@ -23,8 +23,8 @@ ) from mirar.pipelines.sedmv2.load_sedmv2_image import load_raw_sedmv2_image from mirar.processors import BiasCalibrator, FlatCalibrator -from mirar.processors.anet import AstrometryNet from mirar.processors.astromatic import PSFex, Sextractor, Swarp +from mirar.processors.astrometry.anet import AstrometryNet from mirar.processors.csvlog import CSVLog from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.photcal import PhotCalibrator diff --git a/mirar/pipelines/summer/blocks.py b/mirar/pipelines/summer/blocks.py index bed3edcbc..3345e77b0 100644 --- a/mirar/pipelines/summer/blocks.py +++ b/mirar/pipelines/summer/blocks.py @@ -35,7 +35,7 @@ from mirar.pipelines.summer.models import Exposure, Proc, Raw from mirar.processors import BiasCalibrator, FlatCalibrator from mirar.processors.astromatic import PSFex, Scamp, Sextractor, Swarp -from mirar.processors.autoastrometry import AutoAstrometry +from mirar.processors.astrometry.autoastrometry import AutoAstrometry from mirar.processors.candidates.candidate_detector import DetectCandidates from mirar.processors.candidates.utils import DataframeWriter, RegionsWriter from mirar.processors.cosmic_rays import LACosmicCleaner From 38234e863538d2f8c7aded447e66604aafdc2969 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:57:27 -0700 Subject: [PATCH 31/86] make more firendly to PROC images --- mirar/pipelines/wirc/load_wirc_image.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mirar/pipelines/wirc/load_wirc_image.py b/mirar/pipelines/wirc/load_wirc_image.py index 73e83cd0d..9ee26088e 100644 --- a/mirar/pipelines/wirc/load_wirc_image.py +++ b/mirar/pipelines/wirc/load_wirc_image.py @@ -37,7 +37,8 @@ def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Hea header = img[0].header # pylint: enable=E1101 header["FILTER"] = header["AFT"].split("__")[0] - header["DETCOADD"] = header["COADDS"] + if "COADDS" in header.keys(): + header["DETCOADD"] = header["COADDS"] if SATURATE_KEY not in header: header[SATURATE_KEY] = wirc_nonlinear_level * header["DETCOADD"] if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: From 720794010d9f195389c5d21ca5f142b8df38e716 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 12:58:23 -0700 Subject: [PATCH 32/86] sextractor photometry --- mirar/pipelines/wirc/wirc_files/__init__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/mirar/pipelines/wirc/wirc_files/__init__.py b/mirar/pipelines/wirc/wirc_files/__init__.py index b78bff7ac..8db669ff5 100644 --- a/mirar/pipelines/wirc/wirc_files/__init__.py +++ b/mirar/pipelines/wirc/wirc_files/__init__.py @@ -17,6 +17,13 @@ "starnnw_path": wirc_file_dir.joinpath("default.nnw"), } +sextractor_photometry_config = { + "config_path": wirc_file_dir.joinpath("matchcat.sex"), + "filter_path": wirc_file_dir.joinpath("default.conv"), + "parameter_path": wirc_file_dir.joinpath("astrom.param"), + "starnnw_path": wirc_file_dir.joinpath("default.nnw"), +} + scamp_fp_path = wirc_file_dir.joinpath("scamp_fp.conf") swarp_sp_path = wirc_file_dir.joinpath("second_pass.swarp") From a764a11749ae5ea4d9a9c3cdeff5d2ed85f8a705 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 13:01:07 -0700 Subject: [PATCH 33/86] add photometry purifier --- mirar/pipelines/wirc/generator.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/mirar/pipelines/wirc/generator.py b/mirar/pipelines/wirc/generator.py index 24b6d8a65..c29a886aa 100644 --- a/mirar/pipelines/wirc/generator.py +++ b/mirar/pipelines/wirc/generator.py @@ -18,6 +18,24 @@ logger = logging.getLogger(__name__) +def wirc_photometric_img_catalog_purifier(catalog, image): + """ + Function to purify the photometric catalog + + :return: purified catalog + """ + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < 4.0 / 3600.0) + & (catalog["X_IMAGE"] > 200) + & (catalog["X_IMAGE"] < 1800) + & (catalog["Y_IMAGE"] > 200) + & (catalog["Y_IMAGE"] < 1800) + ) + + return catalog[clean_mask] + + def wirc_astrometric_catalog_generator(_) -> Gaia2Mass: """ Function to crossmatch WIRC to GAIA/2mass for astrometry @@ -36,7 +54,10 @@ def wirc_photometric_catalog_generator(image: Image) -> Gaia2Mass: """ filter_name = image["FILTER"] return Gaia2Mass( - min_mag=10, max_mag=20, search_radius_arcmin=10, filter_name=filter_name + min_mag=10, + max_mag=20, + search_radius_arcmin=10, + filter_name=filter_name, ) From c3d653d2d08b3b476cca572b8ff16406c320b64d Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 13:04:12 -0700 Subject: [PATCH 34/86] update for astrometry folder changes --- tests/test_wirc_pipeline.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_wirc_pipeline.py b/tests/test_wirc_pipeline.py index a88af1948..0456c9d9d 100644 --- a/tests/test_wirc_pipeline.py +++ b/tests/test_wirc_pipeline.py @@ -19,7 +19,7 @@ ) from mirar.pipelines.wirc.wirc_pipeline import WircPipeline from mirar.processors.astromatic import Scamp, Sextractor, Swarp -from mirar.processors.autoastrometry import AutoAstrometry +from mirar.processors.astrometry.autoastrometry import AutoAstrometry from mirar.processors.csvlog import CSVLog from mirar.processors.dark import MasterDarkCalibrator from mirar.processors.flat import MasterFlatCalibrator From 9231c7febc24ca4b6baffd8561b12fbf35bd6ab4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 13:05:25 -0700 Subject: [PATCH 35/86] works now? --- mirar/pipelines/wirc/blocks.py | 125 ++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 50 deletions(-) diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index edb9cc5d7..561fb2685 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -2,10 +2,11 @@ Module containing standard processing blocks for WIRC """ from mirar.catalog.kowalski import PS1, TMASS -from mirar.paths import CSV_MASK_KEY, RAW_IMG_KEY, SATURATE_KEY +from mirar.paths import FITS_MASK_KEY, LATEST_SAVE_KEY, RAW_IMG_KEY, SATURATE_KEY from mirar.pipelines.wirc.generator import ( wirc_astrometric_catalog_generator, wirc_photometric_catalog_generator, + wirc_photometric_img_catalog_purifier, wirc_reference_image_generator, wirc_reference_image_resampler, wirc_reference_psfex, @@ -18,6 +19,7 @@ scamp_fp_path, sextractor_astrometry_config, sextractor_candidate_config, + sextractor_photometry_config, sextractor_reference_config, swarp_sp_path, wirc_candidate_schema_path, @@ -26,9 +28,11 @@ from mirar.processors.alerts import AvroPacketMaker, SendToFritz from mirar.processors.astromatic import Scamp, Sextractor, Swarp from mirar.processors.astromatic.psfex import PSFex +from mirar.processors.astromatic.scamp.scamp import scamp_header_key from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map from mirar.processors.astromatic.swarp.swarp import GetSwarpComponentImages -from mirar.processors.autoastrometry import AutoAstrometry +from mirar.processors.astrometry.autoastrometry import AutoAstrometry +from mirar.processors.astrometry.utils import AstrometryFromFile from mirar.processors.candidates.candidate_detector import DetectCandidates from mirar.processors.candidates.candidate_extractor import ( ForcedPhotometryCandidateTable, @@ -57,78 +61,99 @@ ) from mirar.processors.reference import ProcessReference from mirar.processors.sky import NightSkyMedianCalibrator -from mirar.processors.utils import ImageLoader, ImageSaver -from mirar.processors.utils.image_selector import ( +from mirar.processors.utils import ( + HeaderAnnotator, ImageBatcher, ImageDebatcher, + ImageLoader, + ImageSaver, ImageSelector, ) +from mirar.processors.utils.image_loader import LoadImageFromHeader from mirar.processors.xmatch import XMatch from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare -# load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)] -load_raw = [ImageLoader(input_sub_dir="firstpassstack", load_image=load_raw_wirc_image)] +load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)] +# load_raw = [ImageLoader(input_sub_dir="firstpassstack", +# load_image=load_raw_wirc_image)] reduce = [ - # CSVLog( - # export_keys=[ - # "OBJECT", - # "FILTER", - # "UTSHUT", - # "EXPTIME", - # "COADDS", - # "OBSTYPE", - # "OBSCLASS", - # ] - # ), - # MaskPixelsFromPath(mask_path=wirc_mask_path), - # ImageSelector(("exptime", "45.0")), - # DarkCalibrator(), - # ImageDebatcher(), - # ImageSelector(("obsclass", "science")), - # ImageSelector(("object", "ZTF18aavqmki")), - # ImageBatcher(split_key=["filter", "object"]), - # SkyFlatCalibrator(), - # NightSkyMedianCalibrator(), - # AutoAstrometry(catalog="tmc"), - # Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), - # ImageSaver(output_dir_name="firstpass"), - # Scamp( - # ref_catalog_generator=wirc_astrometric_catalog_generator, - # scamp_config_path=scamp_fp_path, - # ), - # Swarp(swarp_config_path=swarp_sp_path, - # calculate_dims_in_swarp=True), - # ImageSaver(output_dir_name="firstpassstack"), + CSVLog( + export_keys=[ + "OBJECT", + "FILTER", + "UTSHUT", + "EXPTIME", + "COADDS", + "OBSTYPE", + "OBSCLASS", + ] + ), + MaskPixelsFromPath(mask_path=wirc_mask_path), + ImageSelector(("exptime", "45.0")), + DarkCalibrator(), + ImageSaver(output_dir_name="darkcal"), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), + ImageDebatcher(), + ImageSelector(("obsclass", "science")), + ImageSelector(("object", "SN2018hna")), + ImageBatcher(split_key=["filter", "object"]), + SkyFlatCalibrator(), + NightSkyMedianCalibrator(), + AutoAstrometry(catalog="tmc"), + Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), + Scamp( + ref_catalog_generator=wirc_astrometric_catalog_generator, + scamp_config_path=scamp_fp_path, + cache=True, + ), + ImageSaver(output_dir_name="firstpass"), + Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True), + ImageSaver(output_dir_name="firstpassstack"), + # ImageSelector(("BASENAME", "image0125.fits_stack.fits")), Sextractor( output_sub_dir="firstpasssextractor", **sextractor_astrometry_config, checkimage_type="SEGMENTATION", cache=True, ), - MaskPixelsFromPath(mask_path_key=sextractor_checkimg_map["SEGMENTATION"]), + MaskPixelsFromPath( + mask_path_key=sextractor_checkimg_map["SEGMENTATION"], + write_masked_pixels_to_file=True, + output_dir="mask1", + ), ImageSaver(output_dir_name="mask1", write_mask=True), - MaskAboveThreshold(threshold_key=SATURATE_KEY), + MaskAboveThreshold( + threshold_key=SATURATE_KEY, write_masked_pixels_to_file=True, output_dir="mask2" + ), ImageSaver(output_dir_name="mask2", write_mask=True), - WriteMaskedCoordsToFile(output_dir="mask"), + WriteMaskedCoordsToFile(output_dir="mask_stack"), GetSwarpComponentImages( + load_image=load_raw_wirc_image, + copy_header_keys=FITS_MASK_KEY, + ), + LoadImageFromHeader( header_key=RAW_IMG_KEY, + copy_header_keys=[scamp_header_key, FITS_MASK_KEY], load_image=load_raw_wirc_image, - copy_header_keys=CSV_MASK_KEY, ), - MaskPixelsFromWCS(), + AstrometryFromFile(astrometry_file_key=scamp_header_key), + ImageSaver(output_dir_name="firstpassastrom", write_mask=True), + MaskPixelsFromWCS( + write_masked_pixels_to_file=True, + output_dir="mask_secondpass", + only_write_mask=True, + ), ImageSaver(output_dir_name="firstpassmasked", write_mask=True), - SkyFlatCalibrator(), - NightSkyMedianCalibrator(), - AutoAstrometry(catalog="tmc"), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY), + NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), - Scamp( - ref_catalog_generator=wirc_astrometric_catalog_generator, - scamp_config_path=scamp_fp_path, - ), Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True), - Sextractor(output_sub_dir="final_sextractor", **sextractor_astrometry_config), - PhotCalibrator(ref_catalog_generator=wirc_photometric_catalog_generator), + Sextractor(output_sub_dir="final_sextractor", **sextractor_photometry_config), + PhotCalibrator( + ref_catalog_generator=wirc_photometric_catalog_generator, + image_photometric_catalog_purifier=wirc_photometric_img_catalog_purifier, + ), ImageSaver(output_dir_name="final"), ] From a4aff4d5896c3b777440b7ba32d6e2535c2ffbc4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 21:21:39 -0700 Subject: [PATCH 36/86] sextractor file for photcal --- .../wirc/wirc_files/files/matchcat.sex | 107 ++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100644 mirar/pipelines/wirc/wirc_files/files/matchcat.sex diff --git a/mirar/pipelines/wirc/wirc_files/files/matchcat.sex b/mirar/pipelines/wirc/wirc_files/files/matchcat.sex new file mode 100644 index 000000000..a4877668b --- /dev/null +++ b/mirar/pipelines/wirc/wirc_files/files/matchcat.sex @@ -0,0 +1,107 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME matchCat.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 10 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 2 # or , in mag.arcsec-2 +ANALYSIS_THRESH 2 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 6.0,10.0,14.0,18.0 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +PHOT_AUTOPARAMS 2.5,3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0,3.5 # MAG_PETRO parameters: , + # + # + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +#PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 5.467 # detector gain in e-/ADU + +PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.7 # stellar FWHM in arcsec +STARNNW_NAME default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------- ASSOCiation --------------------------------- + +#ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +#ASSOC_DATA 5,6,7,8,9,10 # columns of the data to replicate (0=all) +#ASSOC_PARAMS 3,4 # columns of xpos,ypos[,mag] +#ASSOC_RADIUS 3.0 # cross-matching radius (pixels) +#ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +#ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE BACKGROUND, BACKGROUND_RMS # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME check.fits # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output From a31f6576718f33d664157c26759310fd8caf316b Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 21:24:41 -0700 Subject: [PATCH 37/86] better purifier --- mirar/pipelines/wirc/generator.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/mirar/pipelines/wirc/generator.py b/mirar/pipelines/wirc/generator.py index c29a886aa..2c00ed3bd 100644 --- a/mirar/pipelines/wirc/generator.py +++ b/mirar/pipelines/wirc/generator.py @@ -24,13 +24,21 @@ def wirc_photometric_img_catalog_purifier(catalog, image): :return: purified catalog """ + edge_width_pixels = 300 + fwhm_threshold_arcsec = 4.0 + + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + clean_mask = ( (catalog["FLAGS"] == 0) - & (catalog["FWHM_WORLD"] < 4.0 / 3600.0) - & (catalog["X_IMAGE"] > 200) - & (catalog["X_IMAGE"] < 1800) - & (catalog["Y_IMAGE"] > 200) - & (catalog["Y_IMAGE"] < 1800) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) ) return catalog[clean_mask] @@ -58,6 +66,8 @@ def wirc_photometric_catalog_generator(image: Image) -> Gaia2Mass: max_mag=20, search_radius_arcmin=10, filter_name=filter_name, + acceptable_h_ph_quals=["A"], + acceptable_k_ph_quals=["A"], ) From 4c2a99ef1b20155c5f3d5ac0ba6dad26a2abd4bc Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 21:27:29 -0700 Subject: [PATCH 38/86] cut on photometric quality of 2mass --- mirar/catalog/gaia.py | 45 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/mirar/catalog/gaia.py b/mirar/catalog/gaia.py index ac3f99a67..969133d37 100644 --- a/mirar/catalog/gaia.py +++ b/mirar/catalog/gaia.py @@ -6,6 +6,7 @@ import astropy.table import astropy.units as u +import numpy as np from astropy.coordinates import SkyCoord from astroquery.gaia import Gaia @@ -26,18 +27,40 @@ def __init__( self, *args, filter_name: str = "j", - ph_qual_cut: bool = True, snr_threshold: float = 5, trim: bool = False, image_catalog_path: Optional[str] = None, + acceptable_j_ph_quals: str | list[str] = None, + acceptable_h_ph_quals: str | list[str] = None, + acceptable_k_ph_quals: str | list[str] = None, **kwargs, ): super().__init__(*args, filter_name=filter_name, **kwargs) - self.ph_qual_cut = ph_qual_cut + self.trim = trim self.image_catalog_path = image_catalog_path self.snr_threshold = snr_threshold + if isinstance(acceptable_j_ph_quals, str): + acceptable_j_ph_quals = [acceptable_j_ph_quals] + if isinstance(acceptable_h_ph_quals, str): + acceptable_h_ph_quals = [acceptable_h_ph_quals] + if isinstance(acceptable_k_ph_quals, str): + acceptable_k_ph_quals = [acceptable_k_ph_quals] + + self.acceptable_ph_quals = { + "j": acceptable_j_ph_quals, + "h": acceptable_h_ph_quals, + "k": acceptable_k_ph_quals, + } + + if self.acceptable_ph_quals[self.filter_name.lower()] is None: + self.acceptable_ph_quals[self.filter_name.lower()] = ["A"] + + for filt in self.acceptable_ph_quals.items(): + if self.acceptable_ph_quals[filt] is None: + self.acceptable_ph_quals[filt] = ["A", "B", "C"] + logger.debug(f"Sextractor catalog path is {self.image_catalog_path}") def get_catalog( @@ -62,14 +85,9 @@ def get_catalog( f"AND tmass.{self.filter_name}_m > {self.min_mag:.2f} " f"AND tmass.{self.filter_name}_m < {self.max_mag:.2f} " f"AND tbest.number_of_mates=0 " - f"AND tbest.number_of_neighbours=1" + f"AND tbest.number_of_neighbours=1;" ) - if self.ph_qual_cut: - cmd += " AND tmass.ph_qual='AAA';" - else: - cmd += ";" - job = Gaia.launch_job_async(cmd, dump_to_file=False) src_list = job.get_results() src_list["ph_qual"] = src_list["ph_qual"].astype(str) @@ -80,6 +98,17 @@ def get_catalog( src_list["magnitude_err"] = src_list[f"{self.filter_name.lower()}_msigcom"] logger.info(f"Found {len(src_list)} sources in Gaia") + j_phquals = [x[0] for x in src_list["ph_qual"]] + h_phquals = [x[0] for x in src_list["ph_qual"]] + k_phquals = [x[0] for x in src_list["ph_qual"]] + + j_phmask = np.array([x in self.acceptable_ph_quals["j"] for x in j_phquals]) + h_phmask = np.array([x in self.acceptable_ph_quals["h"] for x in h_phquals]) + k_phmask = np.array([x in self.acceptable_ph_quals["k"] for x in k_phquals]) + + phmask = j_phmask & h_phmask & k_phmask + + src_list = src_list[phmask] src_list = src_list[src_list["magnitude_err"] < 1.086 / self.snr_threshold] if self.trim: if self.image_catalog_path is None: From a8bc5091eac4cc80e49c4e27d6ab4bcfe3dec354 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 21:35:14 -0700 Subject: [PATCH 39/86] refactor blocks, have test use actual wirc pipeline --- mirar/pipelines/wirc/blocks.py | 19 ++++++--- tests/test_wirc_pipeline.py | 73 +++++++--------------------------- 2 files changed, 28 insertions(+), 64 deletions(-) diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index 561fb2685..7bca2f60c 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -77,7 +77,7 @@ # load_raw = [ImageLoader(input_sub_dir="firstpassstack", # load_image=load_raw_wirc_image)] -reduce = [ +log = [ CSVLog( export_keys=[ "OBJECT", @@ -88,15 +88,19 @@ "OBSTYPE", "OBSCLASS", ] - ), - MaskPixelsFromPath(mask_path=wirc_mask_path), - ImageSelector(("exptime", "45.0")), - DarkCalibrator(), + ) +] + +masking = [MaskPixelsFromPath(mask_path=wirc_mask_path)] + +dark_calibration = [ImageSelector(("exptime", "45.0")), DarkCalibrator()] + +reduction = [ ImageSaver(output_dir_name="darkcal"), HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), ImageDebatcher(), ImageSelector(("obsclass", "science")), - ImageSelector(("object", "SN2018hna")), + ImageSelector(("object", "ZTF18aavqmki")), ImageBatcher(split_key=["filter", "object"]), SkyFlatCalibrator(), NightSkyMedianCalibrator(), @@ -153,10 +157,13 @@ PhotCalibrator( ref_catalog_generator=wirc_photometric_catalog_generator, image_photometric_catalog_purifier=wirc_photometric_img_catalog_purifier, + write_regions=True, ), ImageSaver(output_dir_name="final"), ] +reduce = log + masking + dark_calibration + reduction + reference = [ ProcessReference( ref_image_generator=wirc_reference_image_generator, diff --git a/tests/test_wirc_pipeline.py b/tests/test_wirc_pipeline.py index 0456c9d9d..c7a437de8 100644 --- a/tests/test_wirc_pipeline.py +++ b/tests/test_wirc_pipeline.py @@ -6,33 +6,12 @@ from mirar.data import Dataset, ImageBatch from mirar.downloader.get_test_data import get_test_data_dir -from mirar.pipelines.wirc.generator import ( - wirc_astrometric_catalog_generator, - wirc_photometric_catalog_generator, -) +from mirar.pipelines.wirc.blocks import log, masking, reduction from mirar.pipelines.wirc.load_wirc_image import load_raw_wirc_image -from mirar.pipelines.wirc.wirc_files import ( - scamp_fp_path, - sextractor_astrometry_config, - swarp_sp_path, - wirc_mask_path, -) from mirar.pipelines.wirc.wirc_pipeline import WircPipeline -from mirar.processors.astromatic import Scamp, Sextractor, Swarp -from mirar.processors.astrometry.autoastrometry import AutoAstrometry -from mirar.processors.csvlog import CSVLog from mirar.processors.dark import MasterDarkCalibrator -from mirar.processors.flat import MasterFlatCalibrator -from mirar.processors.mask import MaskPixelsFromPath -from mirar.processors.photcal import PhotCalibrator -from mirar.processors.sky import MasterSkyCalibrator -from mirar.processors.utils import ImageSaver from mirar.processors.utils.image_loader import ImageLoader -from mirar.processors.utils.image_selector import ( - ImageBatcher, - ImageDebatcher, - ImageSelector, -) +from mirar.processors.utils.image_selector import ImageSelector from mirar.testing import BaseTestCase logger = logging.getLogger(__name__) @@ -73,41 +52,19 @@ def get_cal_path(name: str) -> str: return os.path.join(test_data_dir, f"wirc/cals/test_{name}.fits") -test_configuration = [ - ImageLoader( - input_img_dir=test_data_dir, input_sub_dir="raw", load_image=load_raw_wirc_image - ), - CSVLog( - export_keys=[ - "OBJECT", - "FILTER", - "UTSHUT", - "EXPTIME", - "COADDS", - "OBSTYPE", - "OBSCLASS", - ], - ), - MaskPixelsFromPath(mask_path=wirc_mask_path), - ImageSelector(("exptime", "45.0")), - MasterDarkCalibrator(get_cal_path("dark")), - ImageDebatcher(), - ImageSelector(("obsclass", "science")), - ImageBatcher(split_key="filter"), - MasterFlatCalibrator(get_cal_path("flat")), - MasterSkyCalibrator(get_cal_path("sky")), - ImageSelector(("object", "ZTF21aagppzg"), ("filter", "J")), - AutoAstrometry(catalog="tmc"), - Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), - Scamp( - ref_catalog_generator=wirc_astrometric_catalog_generator, - scamp_config_path=scamp_fp_path, - ), - Swarp(swarp_config_path=swarp_sp_path), - Sextractor(output_sub_dir="final_sextractor", **sextractor_astrometry_config), - PhotCalibrator(ref_catalog_generator=wirc_photometric_catalog_generator), - ImageSaver(output_dir_name="final"), -] +test_configuration = ( + [ + ImageLoader( + input_img_dir=test_data_dir, + input_sub_dir="raw", + load_image=load_raw_wirc_image, + ), + ] + + log + + masking + + [ImageSelector(("exptime", "45.0")), MasterDarkCalibrator(get_cal_path("dark"))] + + reduction +) pipeline = WircPipeline(night="20210330", selected_configurations="test") pipeline.add_configuration(configuration_name="test", configuration=test_configuration) From 0b75c2fa21560fa1850cd2a62e97598a3c18c682 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 21:56:33 -0700 Subject: [PATCH 40/86] fix bug --- mirar/catalog/gaia.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/catalog/gaia.py b/mirar/catalog/gaia.py index 969133d37..c414d1f76 100644 --- a/mirar/catalog/gaia.py +++ b/mirar/catalog/gaia.py @@ -57,7 +57,7 @@ def __init__( if self.acceptable_ph_quals[self.filter_name.lower()] is None: self.acceptable_ph_quals[self.filter_name.lower()] = ["A"] - for filt in self.acceptable_ph_quals.items(): + for filt in self.acceptable_ph_quals.keys(): if self.acceptable_ph_quals[filt] is None: self.acceptable_ph_quals[filt] = ["A", "B", "C"] From 5746127bbe5136d35dda0a5c5d20342cc8779269 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 22:08:05 -0700 Subject: [PATCH 41/86] new edgewidth --- mirar/pipelines/wirc/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/wirc/generator.py b/mirar/pipelines/wirc/generator.py index 2c00ed3bd..b4a76898b 100644 --- a/mirar/pipelines/wirc/generator.py +++ b/mirar/pipelines/wirc/generator.py @@ -24,7 +24,7 @@ def wirc_photometric_img_catalog_purifier(catalog, image): :return: purified catalog """ - edge_width_pixels = 300 + edge_width_pixels = 200 fwhm_threshold_arcsec = 4.0 x_lower_limit = edge_width_pixels From 4ba67b931204244e061b714ae6d8c2ca0a7ddea3 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 22:09:05 -0700 Subject: [PATCH 42/86] remove imageselector --- mirar/pipelines/wirc/blocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index 7bca2f60c..f8367557f 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -100,7 +100,7 @@ HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), ImageDebatcher(), ImageSelector(("obsclass", "science")), - ImageSelector(("object", "ZTF18aavqmki")), + # ImageSelector(("object", "ZTF18aavqmki")), ImageBatcher(split_key=["filter", "object"]), SkyFlatCalibrator(), NightSkyMedianCalibrator(), From d0b8e32e501e86d6faf33fd2a8d9d68304c16392 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Fri, 9 Jun 2023 23:18:09 -0700 Subject: [PATCH 43/86] lint tint patint --- mirar/catalog/gaia.py | 2 +- mirar/processors/astromatic/config/__init__.py | 3 +++ mirar/processors/astromatic/psfex/__init__.py | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/mirar/catalog/gaia.py b/mirar/catalog/gaia.py index c414d1f76..af175f8fb 100644 --- a/mirar/catalog/gaia.py +++ b/mirar/catalog/gaia.py @@ -57,7 +57,7 @@ def __init__( if self.acceptable_ph_quals[self.filter_name.lower()] is None: self.acceptable_ph_quals[self.filter_name.lower()] = ["A"] - for filt in self.acceptable_ph_quals.keys(): + for filt in self.acceptable_ph_quals: if self.acceptable_ph_quals[filt] is None: self.acceptable_ph_quals[filt] = ["A", "B", "C"] diff --git a/mirar/processors/astromatic/config/__init__.py b/mirar/processors/astromatic/config/__init__.py index e775cdf8d..ab2590fbb 100644 --- a/mirar/processors/astromatic/config/__init__.py +++ b/mirar/processors/astromatic/config/__init__.py @@ -1,3 +1,6 @@ +""" +Module for configs of astrometry +""" import os astromatic_config_dir = os.path.dirname(__file__) diff --git a/mirar/processors/astromatic/psfex/__init__.py b/mirar/processors/astromatic/psfex/__init__.py index 0592e9eaf..eca8f375c 100644 --- a/mirar/processors/astromatic/psfex/__init__.py +++ b/mirar/processors/astromatic/psfex/__init__.py @@ -1 +1,4 @@ +""" +Module for running PSFex +""" from mirar.processors.astromatic.psfex.psfex import PSFex From f3fb4fdd8cd67f3063d4edaf0f3af9b94f1b1c49 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 10:32:04 -0500 Subject: [PATCH 44/86] make mefparser more generic --- mirar/pipelines/winter/config/__init__.py | 0 .../pipelines/winter/config/files/Scorr.param | 134 +++++++++++++++++ .../winter/config/files/astrom.param | 28 ++++ .../pipelines/winter/config/files/astrom.sex | 111 ++++++++++++++ .../winter/config/files/astrom_anet.sex | 0 .../winter/config/files/config.swarp | 102 +++++++++++++ .../winter/config/files/default.conv | 5 + .../pipelines/winter/config/files/default.nnw | 28 ++++ .../winter/config/files/photom.param | 28 ++++ .../winter/config/files/photom.psfex | 58 +++++++ .../winter/config/files/photomCat.sex | 97 ++++++++++++ .../pipelines/winter/config/files/scamp.conf | 142 ++++++++++++++++++ .../pipelines/winter/config/files/sex.config | 0 mirar/pipelines/winter/config/files/sex.conv | 5 + .../pipelines/winter/config/files/temp.param | 9 ++ mirar/pipelines/winter/fix_headers.py | 0 mirar/pipelines/winter/load_winter_image.py | 0 .../wirc/wirc_files/files/focusloop.sex | 0 mirar/processors/utils/multi_ext_parser.py | 57 ++++++- 19 files changed, 797 insertions(+), 7 deletions(-) create mode 100644 mirar/pipelines/winter/config/__init__.py create mode 100644 mirar/pipelines/winter/config/files/Scorr.param create mode 100644 mirar/pipelines/winter/config/files/astrom.param create mode 100644 mirar/pipelines/winter/config/files/astrom.sex create mode 100644 mirar/pipelines/winter/config/files/astrom_anet.sex create mode 100644 mirar/pipelines/winter/config/files/config.swarp create mode 100644 mirar/pipelines/winter/config/files/default.conv create mode 100644 mirar/pipelines/winter/config/files/default.nnw create mode 100644 mirar/pipelines/winter/config/files/photom.param create mode 100644 mirar/pipelines/winter/config/files/photom.psfex create mode 100644 mirar/pipelines/winter/config/files/photomCat.sex create mode 100644 mirar/pipelines/winter/config/files/scamp.conf create mode 100644 mirar/pipelines/winter/config/files/sex.config create mode 100644 mirar/pipelines/winter/config/files/sex.conv create mode 100644 mirar/pipelines/winter/config/files/temp.param create mode 100644 mirar/pipelines/winter/fix_headers.py create mode 100644 mirar/pipelines/winter/load_winter_image.py create mode 100644 mirar/pipelines/wirc/wirc_files/files/focusloop.sex diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/pipelines/winter/config/files/Scorr.param b/mirar/pipelines/winter/config/files/Scorr.param new file mode 100644 index 000000000..d116900f0 --- /dev/null +++ b/mirar/pipelines/winter/config/files/Scorr.param @@ -0,0 +1,134 @@ +NUMBER + +#FLUX_ISO +#FLUXERR_ISO +#MAG_ISO +#MAGERR_ISO + +#FLUX_ISOCOR +#FLUXERR_ISOCOR +#MAG_ISOCOR +#MAGERR_ISOCOR + +FLUX_APER(1) +FLUXERR_APER(1) +#MAG_APER(1) +#MAGERR_APER(1) + +FLUX_AUTO +FLUXERR_AUTO +#MAG_AUTO +#MAGERR_AUTO + +#FLUX_BEST +#FLUXERR_BEST +#MAG_BEST +#MAGERR_BEST + +#KRON_RADIUS +FLUX_RADIUS +#BACKGROUND + +#THRESHOLD +#MU_THRESHOLD +FLUX_MAX +#MU_MAX +#ISOAREA_IMAGE +#ISOAREA_WORLD + +#XMIN_IMAGE +#YMIN_IMAGE +#XMAX_IMAGE +#YMAX_IMAGE + +X_IMAGE +Y_IMAGE +XPEAK_IMAGE +YPEAK_IMAGE +#XWIN_IMAGE +#YWIN_IMAGE +ERRX2_IMAGE +ERRY2_IMAGE +#ERRAWIN_IMAGE +#ERRBWIN_IMAGE +#ERRTHETAWIN_IMAGE +#X_WORLD +#Y_WORLD +#ALPHA_SKY +#DELTA_SKY +ALPHA_J2000 +DELTA_J2000 +#ALPHA_B1950 +#DELTA_B1950 + +#X2_IMAGE +#Y2_IMAGE +#XY_IMAGE +#X2_WORLD +#Y2_WORLD +#XY_WORLD + +#CXX_IMAGE +#CYY_IMAGE +#CXY_IMAGE +#CXX_WORLD +#CYY_WORLD +#CXY_WORLD + +A_IMAGE +B_IMAGE +#A_WORLD +#B_WORLD + +#THETA_IMAGE +#THETA_WORLD +#THETA_SKY +#THETA_J2000 +#THETA_B1950 + +ELONGATION +#ELLIPTICITY + +#ERRX2_IMAGE +#ERRY2_IMAGE +#ERRXY_IMAGE +#ERRX2_WORLD +#ERRY2_WORLD +#ERRXY_WORLD + +#ERRCXX_IMAGE +#ERRCYY_IMAGE +#ERRCXY_IMAGE +#ERRCXX_WORLD +#ERRCYY_WORLD +#ERRCXY_WORLD + +#ERRA_IMAGE +#ERRB_IMAGE +#ERRA_WORLD +#ERRB_WORLD + +#ERRTHETA_IMAGE +#ERRTHETA_WORLD +#ERRTHETA_SKY +#ERRTHETA_J2000 +#ERRTHETA_B1950 + +FWHM_IMAGE +FWHM_WORLD + +#ISO0 +#ISO1 +#ISO2 +#ISO3 +#ISO4 +#ISO5 +#ISO6 +#ISO7 + +FLAGS +#IMAFLAGS_ISO +#NIMAFLAGS_ISO(1) + +#CLASS_STAR +#VIGNET(5,5) diff --git a/mirar/pipelines/winter/config/files/astrom.param b/mirar/pipelines/winter/config/files/astrom.param new file mode 100644 index 000000000..3def7c01b --- /dev/null +++ b/mirar/pipelines/winter/config/files/astrom.param @@ -0,0 +1,28 @@ +#VECTOR_ASSOC(10) +ALPHAWIN_J2000 +DELTAWIN_J2000 +X_IMAGE +Y_IMAGE +ELONGATION +ELLIPTICITY +XWIN_IMAGE +YWIN_IMAGE +ERRAWIN_IMAGE +ERRBWIN_IMAGE +FLUX_RADIUS +FWHM_WORLD +FWHM_IMAGE +FLUX_AUTO +FLUXERR_AUTO +FLUX_MAX +MAG_AUTO +MAGERR_AUTO +FLAGS +BACKGROUND +CLASS_STAR +FLUX_APER(5) +FLUXERR_APER(5) +MAG_APER(5) +MAGERR_APER(5) +VIGNET(41,41) +SNR_WIN diff --git a/mirar/pipelines/winter/config/files/astrom.sex b/mirar/pipelines/winter/config/files/astrom.sex new file mode 100644 index 000000000..b5aca201c --- /dev/null +++ b/mirar/pipelines/winter/config/files/astrom.sex @@ -0,0 +1,111 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /home/viraj/rts2/python/img_subtraction/astrom.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 10 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /home/viraj/rts2/python/img_subtraction/default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 16 # Number of deblending sub-thresholds +DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + +#------------------------------ Photometry ----------------------------------- + +#PHOT_APERTURES 6.0,10.0,14.0,18.0,22.0 # MAG_APER aperture diameter(s) in pixels +PHOT_APERTURES 4.0, 6.0, 8.0, 10.0 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +#PHOT_AUTOPARAMS 2.5,3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0,3.5 # MAG_PETRO parameters: , + # + # +PHOT_AUTOPARAMS 1.0,2.0 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: , + + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +#PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 1.6 # detector gain in e-/ADU + +PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 2 # stellar FWHM in arcsec +STARNNW_NAME /home/viraj/rts2/python/img_subtraction/default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 256 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------- ASSOCiation --------------------------------- + +#ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +#ASSOC_DATA 5,6,7,8,9,10,11,12,13,14 # columns of the data to replicate (0=all) +#ASSOC_PARAMS 3,4 # columns of xpos,ypos[,mag] +#ASSOC_RADIUS 3.0 # cross-matching radius (pixels) +#ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, +# # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +#ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/pipelines/winter/config/files/astrom_anet.sex b/mirar/pipelines/winter/config/files/astrom_anet.sex new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/pipelines/winter/config/files/config.swarp b/mirar/pipelines/winter/config/files/config.swarp new file mode 100644 index 000000000..91c9265b1 --- /dev/null +++ b/mirar/pipelines/winter/config/files/config.swarp @@ -0,0 +1,102 @@ +# Default configuration file for SWarp 2.19.1 +# EB 2011-05-22 +# +#----------------------------------- Output ----------------------------------- +IMAGEOUT_NAME coadd.fits # Output filename +WEIGHTOUT_NAME coadd.weight.fits # Output weight-map filename + +HEADER_ONLY N # Only a header as an output file (Y/N)? +HEADER_SUFFIX .head # Filename extension for additional headers + +#------------------------------- Input Weights -------------------------------- + +WEIGHT_TYPE NONE # BACKGROUND,MAP_RMS,MAP_VARIANCE + +#RESCALE_WEIGHTS N # or MAP_WEIGHT +WEIGHT_SUFFIX .weight.fits # Suffix to use for weight-maps +WEIGHT_IMAGE # Weightmap filename if suffix not used + # (all or for each weight-map) + +#------------------------------- Co-addition ---------------------------------- + +COMBINE N # Combine resampled images (Y/N)? +COMBINE_TYPE MEDIAN # MEDIAN,AVERAGE,MIN,MAX,WEIGHTED,CHI2 + # or SUM +#CLIP_AMPFRAC 0.3 # +#CLIP_SIGMA 4.0 +#CLIP_WRITELOG N +#CLIP_LOGNAME clipped.log +#BLANK_BADPIXELS N + +#-------------------------------- Astrometry ---------------------------------- + +CELESTIAL_TYPE NATIVE # NATIVE, PIXEL, EQUATORIAL, + # GALACTIC,ECLIPTIC, or SUPERGALACTIC +PROJECTION_TYPE TAN # Any WCS projection code or NONE +PROJECTION_ERR 0.001 # Maximum projection error (in output + # pixels), or 0 for no approximation +CENTER_TYPE MANUAL # MANUAL, ALL or MOST +CENTER 00:00:00.0, +00:00:00.0 # Coordinates of the image center +PIXELSCALE_TYPE MANUAL # MANUAL,FIT,MIN,MAX or MEDIAN +PIXEL_SCALE 0.0 # Pixel scale +IMAGE_SIZE 0 # Image size (0 = AUTOMATIC) + +#-------------------------------- Resampling ---------------------------------- + +RESAMPLE Y # Resample input images (Y/N)? +RESAMPLE_DIR . # Directory path for resampled images +RESAMPLE_SUFFIX .resamp.fits # filename extension for resampled images + +RESAMPLING_TYPE LANCZOS3 # NEAREST,BILINEAR,LANCZOS2,LANCZOS3 + # or LANCZOS4 (1 per axis) +OVERSAMPLING 0 # Oversampling in each dimension + # (0 = automatic) +INTERPOLATE N # Interpolate bad input pixels (Y/N)? + # (all or for each image) + +FSCALASTRO_TYPE FIXED # NONE,FIXED, or VARIABLE +FSCALE_KEYWORD FLXSCALE # FITS keyword for the multiplicative + # factor applied to each input image +FSCALE_DEFAULT 1.0 # Default FSCALE value if not in header + +GAIN_KEYWORD GAIN # FITS keyword for effect. gain (e-/ADU) +GAIN_DEFAULT 0.0 # Default gain if no FITS keyword found +SATLEV_KEYWORD SATURATE +SATLEV_DEFAULT 40000 + +#--------------------------- Background subtraction --------------------------- + +SUBTRACT_BACK Y # Subtraction sky background (Y/N)? + # (all or for each image) + +BACK_TYPE AUTO # AUTO or MANUAL + # (all or for each image) +BACK_DEFAULT 0.0 # Default background value in MANUAL + # (all or for each image) +BACK_SIZE 128 # Background mesh size (pixels) + # (all or for each image) +BACK_FILTERSIZE 6 # Background map filter range (meshes) + # (all or for each image) + +#------------------------------ Memory management ----------------------------- + +VMEM_DIR . # Directory path for swap files +VMEM_MAX 2047 # Maximum amount of virtual memory (MB) +MEM_MAX 256 # Maximum amount of usable RAM (MB) +COMBINE_BUFSIZE 256 # RAM dedicated to co-addition(MB) + +#------------------------------ Miscellaneous --------------------------------- + +DELETE_TMPFILES Y # Delete temporary resampled FITS files + # (Y/N)? +COPY_KEYWORDS OBJECT # List of FITS keywords to propagate + # from the input to the output headers +WRITE_FILEINFO N # Write information about each input + # file in the output image header? +WRITE_XML N # Write XML file (Y/N)? +XML_NAME swarp.xml # Filename for XML output +VERBOSE_TYPE NORMAL # QUIET,NORMAL or FULL + +NTHREADS 0 # Number of simultaneous threads for + # the SMP version of SWarp + # 0 = automatic diff --git a/mirar/pipelines/winter/config/files/default.conv b/mirar/pipelines/winter/config/files/default.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/mirar/pipelines/winter/config/files/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/mirar/pipelines/winter/config/files/default.nnw b/mirar/pipelines/winter/config/files/default.nnw new file mode 100644 index 000000000..4b521bd43 --- /dev/null +++ b/mirar/pipelines/winter/config/files/default.nnw @@ -0,0 +1,28 @@ +NNW +# Neural Network Weights for the SExtractor star/galaxy classifier (V1.3) +# inputs: 9 for profile parameters + 1 for seeing. +# outputs: ``Stellarity index'' (0.0 to 1.0) +# Seeing FWHM range: from 0.025 to 5.5'' (images must have 1.5 < FWHM < 5 pixels) +# Optimized for Moffat profiles with 2<= beta <= 4. + + 3 10 10 1 + +-1.56604e+00 -2.48265e+00 -1.44564e+00 -1.24675e+00 -9.44913e-01 -5.22453e-01 4.61342e-02 8.31957e-01 2.15505e+00 2.64769e-01 + 3.03477e+00 2.69561e+00 3.16188e+00 3.34497e+00 3.51885e+00 3.65570e+00 3.74856e+00 3.84541e+00 4.22811e+00 3.27734e+00 + +-3.22480e-01 -2.12804e+00 6.50750e-01 -1.11242e+00 -1.40683e+00 -1.55944e+00 -1.84558e+00 -1.18946e-01 5.52395e-01 -4.36564e-01 -5.30052e+00 + 4.62594e-01 -3.29127e+00 1.10950e+00 -6.01857e-01 1.29492e-01 1.42290e+00 2.90741e+00 2.44058e+00 -9.19118e-01 8.42851e-01 -4.69824e+00 +-2.57424e+00 8.96469e-01 8.34775e-01 2.18845e+00 2.46526e+00 8.60878e-02 -6.88080e-01 -1.33623e-02 9.30403e-02 1.64942e+00 -1.01231e+00 + 4.81041e+00 1.53747e+00 -1.12216e+00 -3.16008e+00 -1.67404e+00 -1.75767e+00 -1.29310e+00 5.59549e-01 8.08468e-01 -1.01592e-02 -7.54052e+00 + 1.01933e+01 -2.09484e+01 -1.07426e+00 9.87912e-01 6.05210e-01 -6.04535e-02 -5.87826e-01 -7.94117e-01 -4.89190e-01 -8.12710e-02 -2.07067e+01 +-5.31793e+00 7.94240e+00 -4.64165e+00 -4.37436e+00 -1.55417e+00 7.54368e-01 1.09608e+00 1.45967e+00 1.62946e+00 -1.01301e+00 1.13514e-01 + 2.20336e-01 1.70056e+00 -5.20105e-01 -4.28330e-01 1.57258e-03 -3.36502e-01 -8.18568e-02 -7.16163e+00 8.23195e+00 -1.71561e-02 -1.13749e+01 + 3.75075e+00 7.25399e+00 -1.75325e+00 -2.68814e+00 -3.71128e+00 -4.62933e+00 -2.13747e+00 -1.89186e-01 1.29122e+00 -7.49380e-01 6.71712e-01 +-8.41923e-01 4.64997e+00 5.65808e-01 -3.08277e-01 -1.01687e+00 1.73127e-01 -8.92130e-01 1.89044e+00 -2.75543e-01 -7.72828e-01 5.36745e-01 +-3.65598e+00 7.56997e+00 -3.76373e+00 -1.74542e+00 -1.37540e-01 -5.55400e-01 -1.59195e-01 1.27910e-01 1.91906e+00 1.42119e+00 -4.35502e+00 + +-1.70059e+00 -3.65695e+00 1.22367e+00 -5.74367e-01 -3.29571e+00 2.46316e+00 5.22353e+00 2.42038e+00 1.22919e+00 -9.22250e-01 -2.32028e+00 + + + 0.00000e+00 + 1.00000e+00 diff --git a/mirar/pipelines/winter/config/files/photom.param b/mirar/pipelines/winter/config/files/photom.param new file mode 100644 index 000000000..2e275fa46 --- /dev/null +++ b/mirar/pipelines/winter/config/files/photom.param @@ -0,0 +1,28 @@ +#VECTOR_ASSOC(10) +ALPHAWIN_J2000 +DELTAWIN_J2000 +X_IMAGE +Y_IMAGE +ELONGATION +ELLIPTICITY +XWIN_IMAGE +YWIN_IMAGE +ERRAWIN_IMAGE +ERRBWIN_IMAGE +FLUX_RADIUS +FWHM_WORLD +FWHM_IMAGE +FLUX_AUTO +FLUXERR_AUTO +FLUX_MAX +MAG_AUTO +MAGERR_AUTO +FLAGS +BACKGROUND +CLASS_STAR +FLUX_APER(7) +FLUXERR_APER(7) +MAG_APER(7) +MAGERR_APER(7) +VIGNET(41,41) +SNR_WIN diff --git a/mirar/pipelines/winter/config/files/photom.psfex b/mirar/pipelines/winter/config/files/photom.psfex new file mode 100644 index 000000000..7f1f88c11 --- /dev/null +++ b/mirar/pipelines/winter/config/files/photom.psfex @@ -0,0 +1,58 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2014-09-19 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL_AUTO # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +PSF_SAMPLING 1.0 # Sampling step in pixel units (0.0 = auto) +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 41, 41 # Image size of the PSF model + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS X_IMAGE,Y_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_APER(3) # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_APER(3) # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS X_IMAGE,Y_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 3 # Polynom degree for each group + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT Y # Automatically select the FWHM (Y/N) ? +SAMPLEVAR_TYPE SEEING # File-to-file PSF variability: NONE or SEEING +SAMPLE_FWHMRANGE 1.5, 14 # Allowed FWHM range +SAMPLE_VARIABILITY 0.5 # Allowed FWHM variability (1.0 = 100%) +SAMPLE_MINSN 40 # Minimum S/N for a source to be used #was 20 +SAMPLE_MAXELLIP 0.5 # Maximum (A-B)/(A+B) for a source to be used + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV PNG # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_TYPE NONE #FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME NONE #fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +CHECKIMAGE_TYPE NONE #CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames + +#----------------------------- Miscellaneous --------------------------------- + +PSF_DIR # Where to write PSFs (empty=same as input) +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME psfex.xml # Filename for XML output +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic + diff --git a/mirar/pipelines/winter/config/files/photomCat.sex b/mirar/pipelines/winter/config/files/photomCat.sex new file mode 100644 index 000000000..fb391e219 --- /dev/null +++ b/mirar/pipelines/winter/config/files/photomCat.sex @@ -0,0 +1,97 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /home/kde/reduxScripts/gattini/drp_commissioning/astrometry/photomCat.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 10 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /home/kde/reduxScripts/gattini/drp_commissioning/astrometry/default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN N # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 2, 3, 4, 5, 6, 7, 8 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +#PHOT_AUTOPARAMS 1.0,2.0 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: , + # + # + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 25000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 1 # detector gain in e-/ADU +PIXEL_SCALE 0.466 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 2 # stellar FWHM in arcsec +STARNNW_NAME /home/kde/reduxScripts/gattini/drp_commissioning/astrometry/default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 256 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE BACKGROUND,BACKGROUND_RMS # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME sources.fits # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +NTHREADS 1 #Number of threads to use +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/pipelines/winter/config/files/scamp.conf b/mirar/pipelines/winter/config/files/scamp.conf new file mode 100644 index 000000000..df968f20a --- /dev/null +++ b/mirar/pipelines/winter/config/files/scamp.conf @@ -0,0 +1,142 @@ +# Default configuration file for SCAMP 2.0.4 +# EB 2018-01-23 +# + +#----------------------------- Field grouping --------------------------------- + +#FGROUP_RADIUS 0.3 # Max dist (deg) between field groups + +#---------------------------- Reference catalogs ------------------------------ + +REF_SERVER cocat1.u-strasbg.fr # Internet addresses of catalog servers +REF_PORT 80 # Ports to connect to catalog servers +CDSCLIENT_EXEC aclient_cgi # CDSclient executable +ASTREF_CATALOG FILE # NONE, FILE, USNO-A1,USNO-A2,USNO-B1, + # GSC-1.3,GSC-2.2,GSC-2.3, + # TYCHO-2, UCAC-1,UCAC-2,UCAC-3,UCAC-4, + # NOMAD-1, PPMX, CMC-14, 2MASS, DENIS-3, + # SDSS-R3,SDSS-R5,SDSS-R6,SDSS-R7, + # SDSS-R8, SDSS-R9 +ASTREF_BAND DEFAULT # Photom. band for astr.ref.magnitudes + # or DEFAULT, BLUEST, or REDDEST +ASTREFCAT_NAME astrefcat.cat # Local astrometric reference catalogs +ASTREFCENT_KEYS ra, dec # Local ref.cat. centroid parameters +ASTREFERR_KEYS ra_errdeg, dec_errdeg #e_RAJ2000, e_DEJ2000 + # Local ref.cat. err. ellipse params +ASTREFMAG_KEY phot_rp_mean_mag # Local ref.cat. magnitude parameter +ASTREFMAGERR_KEY phot_rp_mean_mag_error # Local ref.cat. mag. error parameter +ASTREFOBSDATE_KEY ref_epoch # Local ref.cat. obs. date parameter +ASTREFMAG_LIMITS -99.0,99.0 # Select magnitude range in ASTREF_BAND +SAVE_REFCATALOG N # Save ref catalogs in FITS-LDAC format? +REFOUT_CATPATH . # Save path for reference catalogs + +#--------------------------- Merged output catalogs --------------------------- + +MERGEDOUTCAT_TYPE NONE # NONE, ASCII_HEAD, ASCII, FITS_LDAC +MERGEDOUTCAT_NAME merged.cat # Merged output catalog filename + +#--------------------------- Full output catalogs --------------------------- + +FULLOUTCAT_TYPE NONE # NONE, ASCII_HEAD, ASCII, FITS_LDAC +FULLOUTCAT_NAME full.cat # Full output catalog filename + +#----------------------------- Pattern matching ------------------------------- + +MATCH Y # Do pattern-matching (Y/N) ? +MATCH_NMAX 0 # Max.number of detections for MATCHing + # (0=auto) +PIXSCALE_MAXERR 1.2 # Max scale-factor uncertainty +POSANGLE_MAXERR 1.0 # Max position-angle uncertainty (deg) +POSITION_MAXERR 0.15 # Max positional uncertainty (arcmin) +MATCH_RESOL 0 # Matching resolution (arcsec); 0=auto +MATCH_FLIPPED N # Allow matching with flipped axes? +MOSAIC_TYPE UNCHANGED # UNCHANGED, SAME_CRVAL, SHARE_PROJAXIS, + # FIX_FOCALPLANE or LOOSE +FIXFOCALPLANE_NMIN 1 # Min number of dets for FIX_FOCALPLANE + +#---------------------------- Cross-identification ---------------------------- + +CROSSID_RADIUS 1.0 # Cross-id initial radius (arcsec) + +#---------------------------- Astrometric solution ---------------------------- + +SOLVE_ASTROM Y # Compute astrometric solution (Y/N) ? +PROJECTION_TYPE SAME # SAME, TPV or TAN +ASTRINSTRU_KEY # FITS keyword(s) defining the astrom +STABILITY_TYPE INSTRUMENT # EXPOSURE, PRE-DISTORTED or INSTRUMENT +CENTROID_KEYS XWIN_IMAGE,YWIN_IMAGE # Cat. parameters for centroiding +CENTROIDERR_KEYS ERRAWIN_IMAGE,ERRBWIN_IMAGE,ERRTHETAWIN_IMAGE + # Cat. params for centroid err ellipse +DISTORT_KEYS XWIN_IMAGE,YWIN_IMAGE # Cat. parameters or FITS keywords +DISTORT_GROUPS 1,1 # Polynom group for each context key +DISTORT_DEGREES 3 # Polynom degree for each group +FOCDISTORT_DEGREE 3 # Polynom degree for focal plane coords +#ASTREF_WEIGHT 1.0 # Relative weight of ref.astrom.cat. +#ASTRACCURACY_TYPE SIGMA-PIXEL # SIGMA-PIXEL, SIGMA-ARCSEC, + # or TURBULENCE-ARCSEC +#ASTRACCURACY_KEY ASTRACCU # FITS keyword for ASTR_ACCURACY param. +#ASTR_ACCURACY 0.01 # Astrom. uncertainty floor parameter +#ASTRCLIP_NSIGMA 3.0 # Astrom. clipping threshold in sigmas +#COMPUTE_PARALLAXES N # Compute trigonom. parallaxes (Y/N)? +#COMPUTE_PROPERMOTIONS Y # Compute proper motions (Y/N)? +#CORRECT_COLOURSHIFTS N # Correct for colour shifts (Y/N)? +#INCLUDE_ASTREFCATALOG Y # Include ref.cat in prop.motions (Y/N)? +#ASTR_FLAGSMASK 0x00fc # Astrometry rejection mask on SEx FLAGS +#ASTR_IMAFLAGSMASK 0x0 # Astrometry rejection mask on IMAFLAGS + +#---------------------------- Photometric solution ---------------------------- + +SOLVE_PHOTOM Y # Compute photometric solution (Y/N) ? +MAGZERO_OUT 0.0 # Magnitude zero-point(s) in output +MAGZERO_INTERR 0.01 # Internal mag.zero-point accuracy +MAGZERO_REFERR 0.03 # Photom.field mag.zero-point accuracy +PHOTINSTRU_KEY # FITS keyword(s) defining the photom. +MAGZERO_KEY PHOT_C # FITS keyword for the mag zero-point +EXPOTIME_KEY EXPTIME # FITS keyword for the exposure time (s) +AIRMASS_KEY AIRMASS # FITS keyword for the airmass +EXTINCT_KEY PHOT_K # FITS keyword for the extinction coeff +PHOTOMFLAG_KEY PHOTFLAG # FITS keyword for the photometry flag +PHOTFLUX_KEY FLUX_AUTO # Catalog param. for the flux measurement +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalog parameter for the flux error +PHOTCLIP_NSIGMA 5.0 # Photom.clipping threshold in sigmas +PHOT_ACCURACY 1e-3 # Photometric uncertainty floor (frac.) +PHOT_FLAGSMASK 0x00fc # Photometry rejection mask on SEx FLAGS +PHOT_IMAFLAGSMASK 0x0 # Photometry rejection mask on IMAFLAGS + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_CKEY SCAMPCOL # FITS keyword for PLPLOT field colour +CHECKPLOT_DEV PSC # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE FGROUPS,DISTORTION,ASTR_INTERROR2D,ASTR_INTERROR1D,ASTR_REFERROR2D,ASTR_REFERROR1D,ASTR_CHI2,PHOT_ERROR +CHECKPLOT_NAME fgroups,distort,astr_interror2d,astr_interror1d,astr_referror2d,astr_referror1d,astr_chi2,psphot_error # Check-plot filename(s) + +#------------------------------- Check-images --------------------------------- + +CHECKIMAGE_TYPE NONE # NONE, AS_PAIR, AS_REFPAIR, or AS_XCORR +CHECKIMAGE_NAME check.fits # Check-image filename(s) + +#------------------------------ Miscellaneous --------------------------------- + +SN_THRESHOLDS 10.0,100.0 # S/N thresholds (in sigmas) for all and + # high-SN sample +FWHM_THRESHOLDS 1.0,10.0 # FWHM thresholds (in pixels) for sources +ELLIPTICITY_MAX 0.5 # Max. source ellipticity +FLAGS_MASK 0x00f0 # Global rejection mask on SEx FLAGS +WEIGHTFLAGS_MASK 0x00ff # Global rejec. mask on SEx FLAGS_WEIGHT +IMAFLAGS_MASK 0x0 # Global rejec. mask on SEx IMAFLAGS_ISO +AHEADER_GLOBAL scamp.ahead # Filename of the global INPUT header +AHEADER_SUFFIX .ahead # Filename extension for additional + # INPUT headers +HEADER_SUFFIX .head # Filename extension for OUTPUT headers +HEADER_TYPE NORMAL # NORMAL or FOCAL_PLANE +VERBOSE_TYPE NORMAL # QUIET, NORMAL, LOG or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME scamp.xml # Filename for XML output +#XSL_URL file:///scr2/kde/scamp/share/scamp/scamp.xsl + # Filename for XSL style-sheet +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of SCAMP + # 0 = automatic diff --git a/mirar/pipelines/winter/config/files/sex.config b/mirar/pipelines/winter/config/files/sex.config new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/pipelines/winter/config/files/sex.conv b/mirar/pipelines/winter/config/files/sex.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/mirar/pipelines/winter/config/files/sex.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/mirar/pipelines/winter/config/files/temp.param b/mirar/pipelines/winter/config/files/temp.param new file mode 100644 index 000000000..7a5bccb76 --- /dev/null +++ b/mirar/pipelines/winter/config/files/temp.param @@ -0,0 +1,9 @@ +X_IMAGE +Y_IMAGE +ALPHA_J2000 +DELTA_J2000 +MAG_AUTO +MAGERR_AUTO +ELLIPTICITY +FWHM_IMAGE +FLAGS diff --git a/mirar/pipelines/winter/fix_headers.py b/mirar/pipelines/winter/fix_headers.py new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/pipelines/wirc/wirc_files/files/focusloop.sex b/mirar/pipelines/wirc/wirc_files/files/focusloop.sex new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/processors/utils/multi_ext_parser.py b/mirar/processors/utils/multi_ext_parser.py index b3dc7dd51..4775ef98f 100644 --- a/mirar/processors/utils/multi_ext_parser.py +++ b/mirar/processors/utils/multi_ext_parser.py @@ -13,7 +13,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError from mirar.io import open_fits -from mirar.paths import RAW_IMG_SUB_DIR, base_raw_dir +from mirar.paths import RAW_IMG_SUB_DIR, base_raw_dir, get_output_dir, get_output_path from mirar.processors.base_processor import BaseImageProcessor from mirar.processors.utils.image_loader import unzip @@ -30,15 +30,33 @@ class MultiExtParser(BaseImageProcessor): def __init__( self, input_sub_dir: str = RAW_IMG_SUB_DIR, + output_sub_dir: str = "raw_split", input_img_dir: str = base_raw_dir, load_image: Callable[[str], [np.ndarray, astropy.io.fits.Header]] = open_fits, skip_first: bool = False, + extension_num_header_key: str = None, + only_extract_num: int = None, ): + """ + :param input_sub_dir: subdirectory to look for images + :param output_sub_dir: subdirectory to save split single extenion images + :param input_img_dir: parent directory of input_sub_dir + :param load_image: function to load image + :param extension_num_header_key: If provided, will use the corresponding value + in the header to identify an extension_number for every image and save the file + as <>_{extension_number}.fits. If None, will serially number the extensions. + :param only_extract_num: If provided, will only extract the extension with this + number. If None, will extract all extensions. extension_number is calculated + as described in extension_num_header_key. + """ super().__init__() self.input_sub_dir = input_sub_dir self.input_img_dir = input_img_dir self.load_image = load_image self.skip_first = skip_first + self.output_sub_dir = output_sub_dir + self.extension_num_header_key = extension_num_header_key + self.only_extract_num = only_extract_num def __str__(self): return ( @@ -56,10 +74,16 @@ def parse(self, path: str) -> list: ex: /[instrument]/[night]/raw/mef/ """ + output_dir = get_output_dir( + dir_root=self.output_sub_dir, sub_dir=self.night_sub_dir + ) + if not output_dir.exists(): + output_dir.mkdir(parents=True) + new_paths = [] with astropy.io.fits.open(path) as hdu: num_ext = len(hdu) - logger.info(f"This file has {num_ext} extensions.") + logger.info(f"This file - {path} - has {num_ext} extensions.") hdr0 = hdu[0].header # pylint: disable=no-member # zip hdr0's values and comments @@ -74,17 +98,36 @@ def parse(self, path: str) -> list: data = hdu[ext].data hdrext = hdu[ext].header + extension_num_str = str(ext) + if self.extension_num_header_key is not None: + extension_num_str = hdrext[self.extension_num_header_key] + + if self.only_extract_num is not None: + if int(extension_num_str) != self.only_extract_num: + continue + # append hdr0 to hdrext for count, key in enumerate(list(hdr0.keys())): hdrext.append((key, zipped[count][0], zipped[count][1])) # save to new file with 1 extension - notmefpath = path.split("/mef/")[0] + path.split("/mef")[1] - newpath = notmefpath.split(".fits")[0] + "_" + str(ext) + ".fits" + # notmefpath = path.split("/mef/")[0] + path.split("/mef")[1] + + splitfile_basename = ( + f"{os.path.basename(path).split('.fits')[0]}_" + f"{extension_num_str}.fits" + ) + + splitfile_path = get_output_path( + base_name=splitfile_basename, + dir_root=self.output_sub_dir, + sub_dir=self.night_sub_dir, + ) + # newpath = notmefpath.split(".fits")[0] + "_" + str(ext) + ".fits" astropy.io.fits.writeto( - newpath, data, hdrext, overwrite=True + splitfile_path, data, hdrext, overwrite=True ) # pylint: disable=no-member - new_paths.append(newpath) + new_paths.append(splitfile_path) return new_paths @@ -96,7 +139,7 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: def load_from_dir( - input_dir: str | Path, parse_f: Callable[[str | Path], Image] + input_dir: str | Path, parse_f: Callable[[list[str | Path]], Image] ) -> ImageBatch: """ Function to parse all MEF images in a directory From fb9adc18a8245d094299e91fa8c4c7fa795c649d Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 10:39:37 -0500 Subject: [PATCH 45/86] propagate mask --- mirar/processors/sky.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mirar/processors/sky.py b/mirar/processors/sky.py index cd63be513..fce32cfb9 100644 --- a/mirar/processors/sky.py +++ b/mirar/processors/sky.py @@ -29,7 +29,9 @@ def _apply_to_images( mask = master_sky.get_data() <= self.flat_nan_threshold if np.sum(mask) > 0: - master_sky[mask] = np.nan + data = master_sky.get_data() + data[mask] = np.nan + master_sky.set_data(data) for image in batch: data = image.get_data() From 42fb2f304634c7a767a0fb6ce65420d81a87a5f4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 10:44:13 -0500 Subject: [PATCH 46/86] some text for debug --- mirar/processors/flat.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index 3ef236fc4..13c8fdec1 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -87,6 +87,8 @@ def make_image( ) -> Image: images = self.select_cache_images(images) + logger.debug(f"Found {len(images)} suitable flats in batch") + n_frames = len(images) if n_frames == 0: err = f"Found {n_frames} suitable flats in batch" @@ -130,6 +132,7 @@ def make_image( flats[:, :, i] = data / median logger.info(f"Median combining {n_frames} flats") + master_flat = np.nanmedian(flats, axis=2) return Image(master_flat, header=images[0].get_header()) From b677f307e7d9955ed38d720325c6071d1e415071 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 10:45:10 -0500 Subject: [PATCH 47/86] propagate correct header keys --- mirar/processors/dark.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/mirar/processors/dark.py b/mirar/processors/dark.py index 15f86edf8..777286b13 100644 --- a/mirar/processors/dark.py +++ b/mirar/processors/dark.py @@ -83,12 +83,18 @@ def make_image( darks = np.zeros((nx, ny, n_frames)) + individual_dark_exptimes = [] for i, img in enumerate(images): dark_exptime = img["EXPTIME"] darks[:, :, i] = img.get_data() / dark_exptime + individual_dark_exptimes.append(str(dark_exptime)) logger.info(f"Median combining {n_frames} darks") - master_dark = Image(np.nanmedian(darks, axis=2), header=images[0].get_header()) + master_dark_header = images[0].get_header() + master_dark_header["EXPTIME"] = 1.0 + master_dark_header["NCOMBINE"] = n_frames + master_dark_header["INDIVEXP"] = ",".join(individual_dark_exptimes) + master_dark = Image(np.nanmedian(darks, axis=2), header=master_dark_header) return master_dark From 33cc70b4b58125825017d9def903308ddaa8c25e Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 10:48:13 -0500 Subject: [PATCH 48/86] fix typo/bug? --- mirar/processors/mask.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/mirar/processors/mask.py b/mirar/processors/mask.py index 943ade078..838688544 100644 --- a/mirar/processors/mask.py +++ b/mirar/processors/mask.py @@ -47,6 +47,7 @@ def _apply_to_images( ) -> ImageBatch: for image in batch: data = image.get_data() + logger.debug(f"Masking {image[BASE_NAME_KEY]}") mask = self.get_mask(image) if not self.only_write_mask: @@ -105,11 +106,12 @@ def get_mask(self, image) -> np.ndarray: :return: mask """ - if self.mask is None: - if self.mask_path is not None: - self.mask = self.open_fits(self.mask_path) - elif self.mask_path_key is not None: - self.mask = self.open_fits(image[self.mask_path_key]) + # if self.mask is None: # why is this needed? + if self.mask_path is not None: + self.mask = self.open_fits(self.mask_path) + elif self.mask_path_key is not None: + logger.debug(f"Loading mask from {image[self.mask_path_key]}") + self.mask = self.open_fits(image[self.mask_path_key]) mask = self.mask.get_data() mask = mask != 0 return mask From 76f173ae415ca3429d0ae7215c2eb832f55f36fe Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 11:00:00 -0500 Subject: [PATCH 49/86] use existing sextractor catalog --- .../astrometry/autoastrometry/detect.py | 40 ++++++++++++++----- 1 file changed, 30 insertions(+), 10 deletions(-) diff --git a/mirar/processors/astrometry/autoastrometry/detect.py b/mirar/processors/astrometry/autoastrometry/detect.py index 21f0f7c6a..baee8844c 100644 --- a/mirar/processors/astrometry/autoastrometry/detect.py +++ b/mirar/processors/astrometry/autoastrometry/detect.py @@ -6,6 +6,9 @@ from pathlib import Path from typing import Optional +from astropy.io import fits + +from mirar.paths import SEXTRACTOR_HEADER_KEY from mirar.processors.astromatic.sextractor.settings import ( default_config_path, default_conv_path, @@ -22,6 +25,7 @@ compare_mag, ) from mirar.processors.astrometry.autoastrometry.utils import median, mode +from mirar.utils.ldac_tools import get_table_from_ldac logger = logging.getLogger(__name__) @@ -71,16 +75,28 @@ def get_img_src_list( except FileNotFoundError: pass - run_sextractor_single( - img=img_path, - output_dir=os.path.dirname(output_catalog), - config=config_path, - saturation=saturation, - catalog_name=output_catalog, - parameters_name=default_param_path, - filter_name=default_conv_path, - starnnw_name=default_starnnw_path, - ) + sextractor_catalog_path = fits.getval(img_path, SEXTRACTOR_HEADER_KEY) + if sextractor_catalog_path is not None: + logger.info("Using existing sextractor catalog") + output_catalog = sextractor_catalog_path + output_catalog_table = get_table_from_ldac(output_catalog) + output_catalog_table.write( + output_catalog.replace(".cat", ".ascii.cat"), + format="ascii.ecsv", + overwrite=True, + ) + output_catalog = output_catalog.replace(".cat", ".ascii.cat") + else: + run_sextractor_single( + img=img_path, + output_dir=os.path.dirname(output_catalog), + config=config_path, + saturation=saturation, + catalog_name=output_catalog, + parameters_name=default_param_path, + filter_name=default_conv_path, + starnnw_name=default_starnnw_path, + ) # Read in the sextractor catalog with open(output_catalog, "rb") as cat: @@ -109,6 +125,10 @@ def get_img_src_list( if line[0] == "#": continue + # TODO : make more generic to skip column names + if line[0] == "X": + continue + src = SextractorSource(line) # process the line into an object n_src_init += 1 From f6806091f4000ff9a30fccb49df697c31759f7ec Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 11:08:10 -0500 Subject: [PATCH 50/86] more user spec. options for anet --- mirar/processors/astrometry/anet/anet.py | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/mirar/processors/astrometry/anet/anet.py b/mirar/processors/astrometry/anet/anet.py index 384a82771..98abb304a 100644 --- a/mirar/processors/astrometry/anet/anet.py +++ b/mirar/processors/astrometry/anet/anet.py @@ -42,6 +42,14 @@ def run_astrometry_net_single( scale_units: Optional[str] = None, # scale units ('degw', 'amw') downsample: Optional[float | int] = None, # downsample by factor of __ timeout: Optional[float] = None, # astrometry cmd execute timeout, in seconds + use_sextractor: bool = False, + sextractor_path: str = "sex", + search_radius_deg: float = 5, + parity: str = None, + sextractor_config_path: str = None, + x_image_key: str = "X_IMAGE", + y_image_key: str = "Y_IMAGE", + sort_key_name: str = "FLUX_AUTO", ): """ function to run astrometry.net locally on one image, with options to adjust settings @@ -74,8 +82,20 @@ def run_astrometry_net_single( # cmd with a ra, dec first guess (speeds up solution) header = fits.open(img)[0].header # pylint: disable=no-member ra_req, dec_req = header["RA"], header["DEC"] # requested ra, dec + if use_sextractor: + cmd += f"--use-source-extractor --source-extractor-path '{sextractor_path}' " + + if sextractor_config_path is not None: + cmd += f"--source-extractor-config {sextractor_config_path} " + + cmd += f"-X {x_image_key} -Y {y_image_key} -s {sort_key_name} " + + if parity is not None: + assert parity in ["pos", "neg"] + cmd += f"--parity {parity} " + cmd_loc = ( - cmd + f"--ra {ra_req}, --dec {dec_req} --radius 5 " + cmd + f"--ra {ra_req}, --dec {dec_req} --radius {search_radius_deg} " ) # radius takes on units of ra, dec try: From 2db148c113ba84207671213c293047628a5be3b4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 11:10:24 -0500 Subject: [PATCH 51/86] user-defined sextractor config file for anet --- .../astrometry/anet/anet_processor.py | 44 +++++++++++++++++-- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/mirar/processors/astrometry/anet/anet_processor.py b/mirar/processors/astrometry/anet/anet_processor.py index 2094291af..ff225a2b3 100644 --- a/mirar/processors/astrometry/anet/anet_processor.py +++ b/mirar/processors/astrometry/anet/anet_processor.py @@ -9,13 +9,17 @@ from astropy.io import fits from mirar.data import ImageBatch -from mirar.paths import BASE_NAME_KEY, get_output_dir, get_temp_path +from mirar.paths import ( + BASE_NAME_KEY, + LATEST_WEIGHT_SAVE_KEY, + get_output_dir, + get_temp_path, +) from mirar.processors.astrometry.anet.anet import run_astrometry_net_single from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) - ASTROMETRY_TIMEOUT = 900 # astrometry cmd execute timeout, in seconds @@ -27,12 +31,21 @@ class AstrometryNet(BaseImageProcessor): def __init__( self, output_sub_dir: str, # = "a-net" - scale_bounds: Optional[tuple | list] = None, # limits on scale (lower, upper) + scale_bounds: Optional[tuple | list] = None, + # limits on scale (lower, upper) scale_units: Optional[str] = None, # scale units ('degw', 'amw') downsample: Optional[float | int] = None, timeout: Optional[ float ] = ASTROMETRY_TIMEOUT, # astrometry cmd execute timeout, in seconds + use_sextractor: bool = False, + sextractor_path: str = "sex", + search_radius_deg: float = 5.0, + parity: str = None, + sextractor_config_path: str = None, + x_image_key: str = "X_IMAGE", + y_image_key: str = "Y_IMAGE", + sort_key_name: str = "FLUX_AUTO", ): super().__init__() @@ -41,6 +54,16 @@ def __init__( self.scale_units = scale_units self.downsample = downsample self.timeout = timeout + self.use_sextractor = use_sextractor + self.sextractor_path = sextractor_path + self.search_radius_deg = search_radius_deg + self.parity = parity + + self.x_image_key = x_image_key + self.y_image_key = y_image_key + self.sort_key_name = sort_key_name + + self.sextractor_config_path = sextractor_config_path def __str__(self) -> str: return "Processor to perform astrometric calibration via astrometry.net." @@ -67,6 +90,13 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: self.save_fits(image, temp_path) temp_files = [temp_path] + sextractor_path = None + if self.use_sextractor: + weight_image = image[LATEST_WEIGHT_SAVE_KEY] + sextractor_path = ( + f"{self.sextractor_path} -WEIGHT_TYPE MAP_WEIGHT" + + f" -WEIGHT_IMAGE {weight_image}" + ) run_astrometry_net_single( img=temp_path, @@ -75,6 +105,14 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: scale_units=self.scale_units, downsample=self.downsample, timeout=self.timeout, + use_sextractor=self.use_sextractor, + sextractor_path=sextractor_path, + sextractor_config_path=self.sextractor_config_path, + search_radius_deg=self.search_radius_deg, + parity=self.parity, + x_image_key=self.x_image_key, + y_image_key=self.y_image_key, + sort_key_name=self.sort_key_name, ) newname = anet_out_dir.joinpath(Path(str(temp_path).split("temp_")[1])) From bb48e195f10af1825c67f44e30aefb22b260d00e Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 11:14:37 -0500 Subject: [PATCH 52/86] docstrings! --- .../astrometry/anet/anet_processor.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/mirar/processors/astrometry/anet/anet_processor.py b/mirar/processors/astrometry/anet/anet_processor.py index ff225a2b3..63fa5d9f8 100644 --- a/mirar/processors/astrometry/anet/anet_processor.py +++ b/mirar/processors/astrometry/anet/anet_processor.py @@ -47,6 +47,23 @@ def __init__( y_image_key: str = "Y_IMAGE", sort_key_name: str = "FLUX_AUTO", ): + """ + :param output_sub_dir: subdirectory to output astrometry.net results + :param scale_bounds: limits on scale (lower, upper) + :param scale_units: scale units ('degw', 'amw') + :param downsample: downsample by factor of __ + :param timeout: astrometry cmd execute timeout, in seconds + :param use_sextractor: use sextractor to find sources + :param sextractor_path: path to sextractor executable (e.g. sex) + :param search_radius_deg: search radius in degrees + :param parity: parity of the image, if known (e.g. "odd" or "even") + :param sextractor_config_path: path to sextractor config file, NOTE that you + cannot specify other config files (param, conv, nnw, etc.)to astrometry-net. + Make sure to set the config file to use the correct filter, etc. + :param x_image_key: key for x-image coordinate in sextractor catalog + :param y_image_key: key for y-image coordinate in sextractor catalog + :param sort_key_name: key for sorting sextractor catalog + """ super().__init__() self.output_sub_dir = output_sub_dir From 93f56c888a29c6751dc00fbc2839c1391f889e5f Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:44:47 -0500 Subject: [PATCH 53/86] make using existing weight optional --- mirar/processors/astromatic/swarp/swarp.py | 21 ++++++++++++++++++++- mirar/processors/utils/image_saver.py | 14 ++++++++++++-- 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index f63ea7361..9d1bccbf2 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -54,6 +54,7 @@ def run_swarp( subtract_bkg: bool = False, flux_scaling_keyword: str = None, cache: bool = False, + center_type: str = None ): """ Wrapper to resample and stack images with swarp @@ -133,6 +134,9 @@ def run_swarp( if np.logical_and(center_ra is not None, center_dec is not None): swarp_command += f" -CENTER_TYPE MANUAL -CENTER {center_ra},{center_dec}" + else: + swarp_command += f" -CENTER_TYPE {center_type}" + if x_imgpixsize is not None: swarp_command += f" -IMAGE_SIZE {x_imgpixsize}" if y_imgpixsize is not None: @@ -167,6 +171,7 @@ def __init__( x_imgpixsize: Optional[float] = None, y_imgpixsize: Optional[float] = None, propogate_headerlist: Optional[list] = None, + center_type: Optional[str] = None, center_ra: Optional[float] = None, center_dec: Optional[float] = None, gain: Optional[float] = None, @@ -251,6 +256,9 @@ def __init__( self.subtract_bkg = subtract_bkg self.flux_scaling_factor = flux_scaling_factor self.calculate_dims_in_swarp = calculate_dims_in_swarp + self.center_type = center_type + if self.center_type is not None: + assert self.center_type in ['MOST', 'ALL', 'MANUAL'] def __str__(self) -> str: return "Processor to apply swarp to images, stacking them together." @@ -409,6 +417,13 @@ def _apply_to_images( x_imgpixsize_to_use = None y_imgpixsize_to_use = None + if self.center_type is None: + self.center_type = 'MANUAL' + + if self.center_type != 'MANUAL': + center_dec_to_use = None + center_ra_to_use = None + output_image_weight_path = output_image_path.with_suffix(".weight.fits") run_swarp( @@ -421,6 +436,7 @@ def _apply_to_images( x_imgpixsize=x_imgpixsize_to_use, y_imgpixsize=y_imgpixsize_to_use, propogate_headerlist=self.propogate_headerlist, + center_type=self.center_type, center_ra=center_ra_to_use, center_dec=center_dec_to_use, combine=self.combine, @@ -467,7 +483,10 @@ def _apply_to_images( key.strip() not in all_astrometric_keywords, ): if key not in new_image.keys(): - new_image[key] = batch[0][key] + try: + new_image[key] = batch[0][key] + except ValueError: + continue new_image["COADDS"] = np.sum([x["COADDS"] for x in batch]) new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) diff --git a/mirar/processors/utils/image_saver.py b/mirar/processors/utils/image_saver.py index 1666cddcb..5c8abc041 100644 --- a/mirar/processors/utils/image_saver.py +++ b/mirar/processors/utils/image_saver.py @@ -1,6 +1,7 @@ """ Module for saving images """ +import logging import shutil from pathlib import Path @@ -14,6 +15,8 @@ ) from mirar.processors.base_processor import BaseImageProcessor +logger = logging.getLogger(__name__) + class ImageSaver(BaseImageProcessor): """ @@ -27,11 +30,13 @@ def __init__( output_dir_name: str, write_mask: bool = True, output_dir: str | Path = base_output_dir, + use_existing_weight: bool = True, ): super().__init__() self.output_dir_name = output_dir_name self.write_mask = write_mask self.output_dir = Path(output_dir) + self.use_existing_weight = use_existing_weight def __str__(self): return f"Processor to save images to the '{self.output_dir_name}' subdirectory" @@ -53,7 +58,10 @@ def _apply_to_images( image[LATEST_SAVE_KEY] = str(path) if self.write_mask: weight_image_found, mask_path = False, "" - if LATEST_WEIGHT_SAVE_KEY in image.header.keys(): + if self.use_existing_weight & ( + LATEST_WEIGHT_SAVE_KEY in image.header.keys() + ): + logger.debug(f"Searching for existing weight image") existing_weightpath = Path(image[LATEST_WEIGHT_SAVE_KEY]) if existing_weightpath.exists(): weight_image_found = True @@ -67,7 +75,9 @@ def _apply_to_images( shutil.copy(existing_weightpath, mask_path) if not weight_image_found: - mask_path = self.save_weight_image(image, img_path=path) + mask_path = self.save_weight_image( + image, img_path=path, use_existing=self.use_existing_weight + ) image[LATEST_WEIGHT_SAVE_KEY] = str(mask_path) self.save_fits(image, path) From 16ef05b15de1d60e98a61f3e40348c8d519e79ed Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:45:51 -0500 Subject: [PATCH 54/86] optional existing weight --- mirar/processors/base_processor.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/mirar/processors/base_processor.py b/mirar/processors/base_processor.py index e2702aa44..45921453e 100644 --- a/mirar/processors/base_processor.py +++ b/mirar/processors/base_processor.py @@ -305,19 +305,22 @@ def save_fits( logger.info(f"Saving to {path}") save_to_path(data, header, path) - def save_weight_image(self, image: Image, img_path: Path) -> Path: + def save_weight_image( + self, image: Image, img_path: Path, use_existing: bool = True + ) -> Path: """ Saves a weight image :param image: Weight image :param img_path: Path of parent image + :param use_existing: If True, will use existing weight image if it exists :return: Path of weight image """ weight_path = get_weight_path(img_path) header = image.get_header() weight_found = False - if LATEST_WEIGHT_SAVE_KEY in header.keys(): + if use_existing & (LATEST_WEIGHT_SAVE_KEY in header.keys()): existing_weightpath = Path(image[LATEST_WEIGHT_SAVE_KEY]) logger.info(f"WGHTPATH {existing_weightpath}") if existing_weightpath.exists(): From 2f96b5d732ebc68a23f87bc224d14cbbe30ebea3 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:46:44 -0500 Subject: [PATCH 55/86] error handling, use weight optional --- .../astrometry/anet/anet_processor.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mirar/processors/astrometry/anet/anet_processor.py b/mirar/processors/astrometry/anet/anet_processor.py index 63fa5d9f8..2eb09e953 100644 --- a/mirar/processors/astrometry/anet/anet_processor.py +++ b/mirar/processors/astrometry/anet/anet_processor.py @@ -9,6 +9,7 @@ from astropy.io import fits from mirar.data import ImageBatch +from mirar.errors import ProcessorError from mirar.paths import ( BASE_NAME_KEY, LATEST_WEIGHT_SAVE_KEY, @@ -23,6 +24,12 @@ ASTROMETRY_TIMEOUT = 900 # astrometry cmd execute timeout, in seconds +class AstrometryNetError(ProcessorError): + """ + Class for errors in astrometry.net + """ + + class AstrometryNet(BaseImageProcessor): """Processor to run astrometry.net""" @@ -46,6 +53,7 @@ def __init__( x_image_key: str = "X_IMAGE", y_image_key: str = "Y_IMAGE", sort_key_name: str = "FLUX_AUTO", + use_weight: bool = True, ): """ :param output_sub_dir: subdirectory to output astrometry.net results @@ -79,7 +87,7 @@ def __init__( self.x_image_key = x_image_key self.y_image_key = y_image_key self.sort_key_name = sort_key_name - + self.use_weight = use_weight self.sextractor_config_path = sextractor_config_path def __str__(self) -> str: @@ -107,8 +115,8 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: self.save_fits(image, temp_path) temp_files = [temp_path] - sextractor_path = None - if self.use_sextractor: + sextractor_path = f"{self.sextractor_path}" + if self.use_sextractor & self.use_weight: weight_image = image[LATEST_WEIGHT_SAVE_KEY] sextractor_path = ( f"{self.sextractor_path} -WEIGHT_TYPE MAP_WEIGHT" @@ -133,6 +141,11 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: ) newname = anet_out_dir.joinpath(Path(str(temp_path).split("temp_")[1])) + if not newname.exists(): + raise AstrometryNetError( + f"AstrometryNet did not run successfully - no output " + f"file {newname} found." + ) solved = fits.open(newname) hdr = solved[0].header # pylint: disable=no-member From 142aaf468a73484bb9915be4e33479b5b265c2f3 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:47:33 -0500 Subject: [PATCH 56/86] center type specification --- mirar/processors/astromatic/swarp/swarp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index 9d1bccbf2..3818f66b4 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -54,7 +54,7 @@ def run_swarp( subtract_bkg: bool = False, flux_scaling_keyword: str = None, cache: bool = False, - center_type: str = None + center_type: str = None, ): """ Wrapper to resample and stack images with swarp @@ -258,7 +258,7 @@ def __init__( self.calculate_dims_in_swarp = calculate_dims_in_swarp self.center_type = center_type if self.center_type is not None: - assert self.center_type in ['MOST', 'ALL', 'MANUAL'] + assert self.center_type in ["MOST", "ALL", "MANUAL"] def __str__(self) -> str: return "Processor to apply swarp to images, stacking them together." @@ -418,9 +418,9 @@ def _apply_to_images( y_imgpixsize_to_use = None if self.center_type is None: - self.center_type = 'MANUAL' + self.center_type = "MANUAL" - if self.center_type != 'MANUAL': + if self.center_type != "MANUAL": center_dec_to_use = None center_ra_to_use = None From 6c8db99bcd309cf009538ac04d05e676ae2be21d Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:48:58 -0500 Subject: [PATCH 57/86] fix bug in checkimage --- mirar/processors/astromatic/sextractor/sextractor.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/mirar/processors/astromatic/sextractor/sextractor.py b/mirar/processors/astromatic/sextractor/sextractor.py index 16274342b..53ccfb786 100644 --- a/mirar/processors/astromatic/sextractor/sextractor.py +++ b/mirar/processors/astromatic/sextractor/sextractor.py @@ -156,12 +156,14 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: sextractor_out_dir, image[BASE_NAME_KEY].replace(".fits", ".cat") ) - _, self.checkimage_name = parse_checkimage( - checkimage_name=self.checkimage_name, + _, checkimage_name = parse_checkimage( + checkimage_name=None, checkimage_type=self.checkimage_type, image=os.path.join(sextractor_out_dir, image[BASE_NAME_KEY]), ) + logger.debug(f"Sextractor checkimage name is {checkimage_name}") + output_cat, checkimage_name = run_sextractor_single( img=temp_path, config=self.config, @@ -172,7 +174,7 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: saturation=self.saturation, weight_image=mask_path, verbose_type=self.verbose_type, - checkimage_name=self.checkimage_name, + checkimage_name=checkimage_name, checkimage_type=self.checkimage_type, gain=self.gain, catalog_name=output_cat, From 74fb6b5789b15c3702e09a21f160638c662b77b8 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:49:38 -0500 Subject: [PATCH 58/86] config files for winter --- mirar/pipelines/winter/config/__init__.py | 37 ++++++ .../winter/config/files/astrom.param | 2 + .../pipelines/winter/config/files/astrom.sex | 6 +- .../winter/config/files/astrom_anet.sex | 111 ++++++++++++++++++ .../pipelines/winter/config/files/sex.config | 74 ++++++++++++ 5 files changed, 227 insertions(+), 3 deletions(-) diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py index e69de29bb..4ae05a6dc 100644 --- a/mirar/pipelines/winter/config/__init__.py +++ b/mirar/pipelines/winter/config/__init__.py @@ -0,0 +1,37 @@ +from pathlib import Path + +PIPELINE_NAME = "winter" + +winter_file_dir = Path(__file__).parent.joinpath("files") + +sextractor_astrometry_config = { + "config_path": winter_file_dir.joinpath("photomCat.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("astrom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + + +sextractor_photometry_config = { + "config_path": winter_file_dir.joinpath("photomCat.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("photom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + + +sextractor_anet_config = { + "config_path": winter_file_dir.joinpath("astrom_anet.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("astrom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + +sextractor_autoastrometry_config = { + "config_path": winter_file_dir.joinpath("astrom.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("astrom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + +swarp_config_path = winter_file_dir.joinpath("swarp.config") diff --git a/mirar/pipelines/winter/config/files/astrom.param b/mirar/pipelines/winter/config/files/astrom.param index 3def7c01b..5f72f7204 100644 --- a/mirar/pipelines/winter/config/files/astrom.param +++ b/mirar/pipelines/winter/config/files/astrom.param @@ -26,3 +26,5 @@ MAG_APER(5) MAGERR_APER(5) VIGNET(41,41) SNR_WIN +X +Y diff --git a/mirar/pipelines/winter/config/files/astrom.sex b/mirar/pipelines/winter/config/files/astrom.sex index b5aca201c..605f80cc8 100644 --- a/mirar/pipelines/winter/config/files/astrom.sex +++ b/mirar/pipelines/winter/config/files/astrom.sex @@ -7,7 +7,7 @@ CATALOG_NAME test.cat # name of the output catalog CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC -PARAMETERS_NAME /home/viraj/rts2/python/img_subtraction/astrom.param # name of the file containing catalog contents +PARAMETERS_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/astrom.param # name of the file containing catalog contents #------------------------------- Extraction ---------------------------------- @@ -19,7 +19,7 @@ DETECT_THRESH 5 # or , in mag.arcsec-2 ANALYSIS_THRESH 5 # or , in mag.arcsec-2 FILTER Y # apply filter for detection (Y or N)? -FILTER_NAME /home/viraj/rts2/python/img_subtraction/default.conv # name of the file containing the filter +FILTER_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.conv # name of the file containing the filter DEBLEND_NTHRESH 16 # Number of deblending sub-thresholds DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending @@ -69,7 +69,7 @@ PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) #------------------------- Star/Galaxy Separation ---------------------------- SEEING_FWHM 2 # stellar FWHM in arcsec -STARNNW_NAME /home/viraj/rts2/python/img_subtraction/default.nnw # Neural-Network_Weight table filename +STARNNW_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.nnw # Neural-Network_Weight table filename #------------------------------ Background ----------------------------------- diff --git a/mirar/pipelines/winter/config/files/astrom_anet.sex b/mirar/pipelines/winter/config/files/astrom_anet.sex index e69de29bb..071a1ac8c 100644 --- a/mirar/pipelines/winter/config/files/astrom_anet.sex +++ b/mirar/pipelines/winter/config/files/astrom_anet.sex @@ -0,0 +1,111 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE ASCII_HEAD # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/astrom.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 3 # or , in mag.arcsec-2 +ANALYSIS_THRESH 3 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE NONE # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + +#------------------------------ Photometry ----------------------------------- + +#PHOT_APERTURES 6.0,10.0,14.0,18.0,22.0 # MAG_APER aperture diameter(s) in pixels +PHOT_APERTURES 4.0, 6.0, 8.0, 10.0 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +#PHOT_AUTOPARAMS 2.5,3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0,3.5 # MAG_PETRO parameters: , + # + # +PHOT_AUTOPARAMS 1.0,2.0 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: , + + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +#PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 1.6 # detector gain in e-/ADU + +PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 2 # stellar FWHM in arcsec +STARNNW_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 256 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------- ASSOCiation --------------------------------- + +#ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +#ASSOC_DATA 5,6,7,8,9,10,11,12,13,14 # columns of the data to replicate (0=all) +#ASSOC_PARAMS 3,4 # columns of xpos,ypos[,mag] +#ASSOC_RADIUS 3.0 # cross-matching radius (pixels) +#ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, +# # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +#ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/pipelines/winter/config/files/sex.config b/mirar/pipelines/winter/config/files/sex.config index e69de29bb..42ce7fc99 100644 --- a/mirar/pipelines/winter/config/files/sex.config +++ b/mirar/pipelines/winter/config/files/sex.config @@ -0,0 +1,74 @@ + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME temp.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /Users/viraj/winter_telescope/mirar/mirar/processors/astromatic/config/temp.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # minimum number of pixels above threshold +DETECT_THRESH 3 # or , in mag.arcsec-2 +ANALYSIS_THRESH 3 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /Users/viraj/winter_telescope/mirar/mirar/processors/astromatic/config/sex.conv # name of the file containing the filter + +DEBLEND_NTHRESH 16 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.02 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + + + +MAG_ZEROPOINT 0.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 0.0 # detector gain in e-/ADU +PIXEL_SCALE 1.0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 1.2 # stellar FWHM in arcsec +STARNNW_NAME default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE NONE # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME check.fits # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation From 13cbb349833e9dffa93e063a6ffd001e9f46eda2 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 16:56:49 -0500 Subject: [PATCH 59/86] generators --- mirar/pipelines/winter/generator.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 4de91e93a..8eaaedc64 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -24,6 +24,7 @@ astromatic_config_dir = os.path.join(winter_dir, "config/") swarp_config_path = os.path.join(astromatic_config_dir, "config.swarp") winter_mask_path = os.path.join(winter_dir, "winter_mask.fits") +scamp_config_path = os.path.join(winter_dir, "scamp.conf") def winter_reference_generator(image: Image, db_table: Type[BaseDB] = RefStacks): @@ -115,17 +116,13 @@ def winter_reference_phot_calibrator(image: Image, **kwargs) -> PhotCalibrator: :param kwargs: kwargs :return: Swarp processor """ - x_lower_limit = 0 - y_lower_limit = 0 - x_upper_limit = image.header["NAXIS1"] - y_upper_limit = image.header["NAXIS2"] + # x_lower_limit = 0 + # y_lower_limit = 0 + # x_upper_limit = image.header["NAXIS1"] + # y_upper_limit = image.header["NAXIS2"] return PhotCalibrator( ref_catalog_generator=winter_photometric_catalog_generator, - x_lower_limit=x_lower_limit, - x_upper_limit=x_upper_limit, - y_lower_limit=y_lower_limit, - y_upper_limit=y_upper_limit, write_regions=True, **kwargs, ) @@ -162,5 +159,14 @@ def ref_phot_calibrator(image: Image): return PhotCalibrator( ref_catalog_generator=winter_photometric_catalog_generator, write_regions=True, - fwhm_threshold_arcsec=3, + # fwhm_threshold_arcsec=3, ) + + +def winter_astrometric_catalog_generator(_) -> Gaia2Mass: + """ + Function to crossmatch WIRC to GAIA/2mass for astrometry + + :return: catalogue + """ + return Gaia2Mass(min_mag=10, max_mag=20, search_radius_arcmin=15) From 616d75ba346eebca5cfcf7ca086216a566299df4 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 17:05:18 -0500 Subject: [PATCH 60/86] load image --- mirar/pipelines/winter/load_winter_image.py | 243 ++++++++++++++++++++ 1 file changed, 243 insertions(+) diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py index e69de29bb..b4dd37f92 100644 --- a/mirar/pipelines/winter/load_winter_image.py +++ b/mirar/pipelines/winter/load_winter_image.py @@ -0,0 +1,243 @@ +""" +Module for loading raw WINTER images and ensuring they have the correct format +""" +import logging +import os +from pathlib import Path + +import astropy +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.time import Time + +from mirar.paths import ( + BASE_NAME_KEY, + COADD_KEY, + PROC_FAIL_KEY, + PROC_HISTORY_KEY, + RAW_IMG_KEY, +) + +logger = logging.getLogger(__name__) + + +def mask_datasec(data: np.ndarray, header: astropy.io.fits.Header) -> np.ndarray: + """ + Function to mask the data section of an image + """ + datasec = header["DATASEC"].replace("[", "").replace("]", "").split(",") + datasec_xmin = int(datasec[0].split(":")[0]) + datasec_xmax = int(datasec[0].split(":")[1]) + datasec_ymin = int(datasec[1].split(":")[0]) + datasec_ymax = int(datasec[1].split(":")[1]) + + data[:, :datasec_xmin] = np.nan + data[:, datasec_xmax:] = np.nan + data[:datasec_ymin, :] = np.nan + data[datasec_ymax:, :] = np.nan + return data + + +def load_raw_winter_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: + """ + Function to load a raw WIRC image + + :param path: path of file + :return: data and header of image + """ + logger.info(f"Loading {path}") + with fits.open(path) as img: + # pylint: disable=E1101 + data = img[0].data + header = img[0].header + + header["UNIQTYPE"] = f"{header['OBSTYPE']}_{header['BOARD_ID']}" + + # if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: + # header["OBSTYPE"] = "calibration" + + # if '065456' in path or '064257' in path: + # if '073730' in path or '075817' in path or '080024' in path: + # if '_082' in path: + basename = os.path.basename(path) + timestamp = basename.split(".fits")[0].split("_")[1] + date = timestamp.split("-")[0] + time = timestamp.split("-")[1] + + if "RADEG" not in header.keys(): + header["RADEG"] = header["RA"] + header["DECDEG"] = header["DEC"] + + header["UTCTIME"] = ( + f"{date[:4]}-{date[4:6]}-{date[6:]}T" f"{time[:2]}:{time[2:4]}:{time[4:]}" + ) + logger.info(header["UTCTIME"]) + header["MJD-OBS"] = Time(header["UTCTIME"]).mjd + + # if '085739' in path or '-09' in path: + # if Time("2023-06-13T09:12:01") >= Time(header["UTCTIME"]) \ + # >= Time("2023-06-13T08:57:39"): + # if header['TARGNAME'] == 'm16': + # header["OBSTYPE"] = "SCIENCE" + # header['TARGNAME'] = 'INTERESTING' + # elif header["OBSTYPE"] != "DARK": + # header["OBSTYPE"] = "OTHER" + header["OBSCLASS"] = ["science", "calibration"][ + header["OBSTYPE"] in ["DARK", "FLAT"] + ] + # if header["OBSTYPE"] == "TEST" and ("_mef" not in path): + # header["OBSTYPE"] = "FLAT" + header["EXPTIME"] = np.rint(header["EXPTIME"]) + header[BASE_NAME_KEY] = os.path.basename(path) + if RAW_IMG_KEY not in header.keys(): + header[RAW_IMG_KEY] = path + header["TARGET"] = header["OBSTYPE"].lower() + + if (header["FILTERID"] == "dark") & (header["OBSTYPE"] != "BIAS"): + header["OBSTYPE"] = "DARK" + header["TARGET"] = "dark" + + if ".weight" in path: + header["OBSTYPE"] = "WEIGHT" + header["RA"] = header["RADEG"] + header["DEC"] = header["DECDEG"] + # elif '053618' in path: + # header["FILTER"] = "Hs" + # elif '053936' in path: + # header['FILTER'] = 'Y' + # elif Time(header["UTCTIME"]) >= Time("2023-06-10T06:50:29"): + # header["FILTER"] = "Hs" + # else: + # header["FILTER"] = "J" + + if COADD_KEY not in header.keys(): + logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") + header[COADD_KEY] = 1 + + header[PROC_HISTORY_KEY] = "" + header[PROC_FAIL_KEY] = "" + + filter_dict = {"J": 1, "H": 2, "Ks": 3} + + if "FILTERID" not in header.keys(): + header["FILTERID"] = filter_dict[header["FILTER"]] + if "FIELDID" not in header.keys(): + header["FIELDID"] = 99999 + if "PROGPI" not in header.keys(): + header["PROGPI"] = "Kasliwal" + if "PROGID" not in header.keys(): + header["PROGID"] = 0 + + if "CTYPE1" not in header: + header["CTYPE1"] = "RA---TAN" + if "CTYPE2" not in header: + header["CTYPE2"] = "DEC--TAN" + # if 'RADEG' in header.keys(): + # header['CRVAL1'] = header['RADEG'] + # if 'DECDEG' in header.keys(): + # header['CRVAL2'] = header['DECDEG'] + data = data.astype(float) + # data[data > 40000.0] = np.nan + # data[:, :250] = np.nan + # data[:, 1800:] = np.nan + # data[:20, :] = np.nan + # data[1060:, :] = np.nan + + header["FILTER"] = header["FILTERID"] + if "DATASEC" in header.keys(): + data = mask_datasec(data, header) + del header["DATASEC"] + # TODO: check if this is necessary for short exposures + # (Non-linearity/1700 counts) + data[data > 40000] = np.nan + if header["BOARD_ID"] == 0: + # data[:500, 700:1500] = np.nan + data[1075:, :] = np.nan + data[:, 1950:] = np.nan + data[:20, :] = np.nan + + if header["BOARD_ID"] == 1: + pass + + if header["BOARD_ID"] == 2: + data[1085:, :] = np.nan + data[:, 1970:] = np.nan + data[:55, :] = np.nan + data[:, :20] = np.nan + + if header["BOARD_ID"] == 3: + data[1085:, :] = np.nan + data[:, 1970:] = np.nan + data[:55, :] = np.nan + data[:, :20] = np.nan + + if header["BOARD_ID"] == 4: + # data[610:, :280] = np.nan + data[:, 1948:] = np.nan + data[:, :61] = np.nan + data[:20, :] = np.nan + data[1060:, :] = np.nan + data[:, 999:1002] = np.nan + + if header["BOARD_ID"] == 5: + # data[740:, 1270: 1850] = np.nan + data[1072:, :] = np.nan + data[:, 1940:] = np.nan + data[:15, :] = np.nan + data[680:, 1180:] = np.nan + # data[data > 25000] = np.nan + + _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5) + header["MEDCOUNT"] = med + header["STDDEV"] = std + # if header['RADEG']>300: + # header['TARGNAME'] = 'other' + + if "weight" in path: + header["OBSTYPE"] = "weight" + + return data, header + + +def load_proc_winter_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: + """ + Load proc image + """ + logger.info(f"Loading {path}") + with fits.open(path) as img: + data = img[0].data + header = img[0].header + if "weight" in path: + header["OBSTYPE"] = "weight" + + header["FILTER"] = header["FILTERID"] + + return data, header + + +def load_stacked_winter_image( + path: str | Path, +) -> tuple[np.array, astropy.io.fits.Header]: + """ + Load proc image + """ + logger.info(f"Loading {path}") + with fits.open(path) as img: + data = img[0].data + header = img[0].header + if "weight" in path: + header["OBSTYPE"] = "weight" + + header["OBSCLASS"] = "weight" + header["COADDS"] = 1 + header["TARGET"] = "science" + header["CALSTEPS"] = "" + header["PROCFAIL"] = 1 + header["RAWPATH"] = "" + header["BASENAME"] = os.path.basename(path) + header["TARGNAME"] = "weight" + if "UTCTIME" not in header.keys(): + header["UTCTIME"] = "2023-06-14T00:00:00" + + return data, header From 526c90340d70c3c9ce0ab62ee1bcbc4bef17e5a1 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 17:06:36 -0500 Subject: [PATCH 61/86] blocks --- mirar/pipelines/winter/blocks.py | 234 ++++++++++++++++++++++++++++--- 1 file changed, 216 insertions(+), 18 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index b71066acf..7139892fd 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -1,15 +1,32 @@ from mirar.pipelines.winter.generator import ( winter_mask_path, winter_reference_generator, + winter_photometric_catalog_generator, + winter_astrometric_catalog_generator, + scamp_config_path ) +from mirar.processors.mask import WriteMaskedCoordsToFile +from mirar.pipelines.winter.load_winter_image import load_raw_winter_image, \ + load_proc_winter_image, load_stacked_winter_image from mirar.processors.csvlog import CSVLog from mirar.processors.dark import DarkCalibrator from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.reference import GetReferenceImage from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator +from mirar.processors.flat import FlatCalibrator from mirar.processors.split import SplitImage -from mirar.processors.utils import ImageDebatcher, ImageSaver +from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map +from mirar.processors.utils import ImageDebatcher, ImageSaver, ImageLoader, \ + ImageSelector, ImageBatcher from mirar.processors.utils.multi_ext_parser import MultiExtParser +from mirar.paths import FITS_MASK_KEY +from mirar.processors.astromatic.sextractor.sextractor import Sextractor +from mirar.pipelines.winter.config import sextractor_astrometry_config, \ + sextractor_autoastrometry_config, swarp_config_path, sextractor_photometry_config +from mirar.processors.astromatic.swarp.swarp import Swarp +from mirar.processors.photcal import PhotCalibrator +from mirar.processors.astrometry.anet.anet_processor import AstrometryNet +from mirar.processors.astromatic import Scamp refbuild = [ ImageDebatcher(), @@ -19,22 +36,203 @@ ImageSaver(output_dir_name="stacked_ref"), ] -commissioning = [ - CSVLog( - export_keys=[ - "OBJECT", - "FILTER", - "UTSHUT", - "EXPTIME", - "COADDS", - "OBSTYPE", - "OBSCLASS", - ] +board_id = 5 +target_name = "M101" +split = [MultiExtParser(input_sub_dir="raw/", + extension_num_header_key="BOARD_ID", + only_extract_num=board_id, + output_sub_dir=f"raw_split_{board_id}")] +load = [ImageLoader(input_sub_dir=f"raw_split_{board_id}", + load_image=load_raw_winter_image), + ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), ] + +load_proc = [ImageLoader(input_sub_dir=f"skysub_{board_id}", + load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), + ] + +load_dark = [ImageLoader(input_sub_dir=f"darkcal_{board_id}", + load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"))] + +load_anet = [ImageLoader(input_sub_dir=f"anet_{board_id}", + load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE"))] + +load_stack = [ImageLoader(input_sub_dir=f"anet_{board_id}", + load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), + ImageBatcher("EXPTIME")] + +load_multiboard_stack = [ImageLoader(input_sub_dir=f"stack_all_{target_name}", + load_image=load_stacked_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), + ("OBSTYPE", "SCIENCE"), + ),] + # ImageBatcher("COADDS")] +log = split + load + \ + [CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC" + ] + )] + +dark_cal = [ + ImageSelector(("BOARD_ID", f"{board_id}")), + ImageBatcher(["BOARD_ID", "EXPTIME"]), + WriteMaskedCoordsToFile(output_dir="mask_raw"), + DarkCalibrator(cache_sub_dir=f"calibration_{board_id}"), + ImageSaver(output_dir_name=f"darkcal_{board_id}"), + ImageDebatcher()] + +flat_cal = [ + # ImageSelector(("OBSTYPE", ["FOCUS", "SCIENCE", "FLAT"])), + # ImageSelector(("TARGNAME", ["INTERESTING"])), + # ImageBatcher(["BOARD_ID", "FILTER"]), + # FlatCalibrator(flat_mask_key=FITS_MASK_KEY, + # cache_sub_dir=f"calibration_{board_id}" + # ), + ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{target_name}")), + ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, + cache_sub_dir=f'skycals_{board_id}'), + ImageSelector(("OBSTYPE", ["SCIENCE"])), + ImageSaver(output_dir_name=f"skyflatcal_{board_id}"), + # ImageSelector(("OBSTYPE", ["SCIENCE"])), + # Sextractor(**sextractor_astrometry_config, + # write_regions_bool=True, + # output_sub_dir="sextractor", + # cache=True), + # ImageSelector(("TARGNAME", [""])), + # ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), + NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), + ImageSaver(output_dir_name=f"skysub_{board_id}"), + # Sextractor(**sextractor_astrometry_config, + # write_regions_bool=True, + # output_sub_dir="sextractor", + # cache=True), + # AutoAstrometry(catalog="tmc", pixel_scale=1.0, pa=0, inv=True, + # write_crosscheck_files=True), + # ImageSaver(output_dir_name="skysub"), + # Sextractor(**sextractor_astrometry_config, + # write_regions_bool=True, + # output_sub_dir="sextractor"), + + # MultiExtParser(input_sub_dir="raw/mef/"), + # SplitImage(), + # MaskPixelsFromPath(mask_path=winter_mask_path), + # DarkCalibrator(), + # SkyFlatCalibrator(), + # NightSkyMedianCalibrator(), +] + +process = dark_cal + flat_cal +process_proc = [ImageDebatcher(), + AstrometryNet(output_sub_dir=f'anet_{board_id}', + scale_bounds=[25, 40], + scale_units='amw', + use_sextractor=True, + parity='neg', + search_radius_deg=1.0, + # sextractor_config_path=sextractor_autoastrometry_config[ + # 'config_path'], + use_weight=False + ), + ImageSaver(output_dir_name=f"anet_{board_id}", + use_existing_weight=False), + Sextractor(**sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="scamp", + ), + Scamp( + temp_output_sub_dir="scamp", + ref_catalog_generator=winter_astrometric_catalog_generator, + scamp_config_path=scamp_config_path, + cache=False, + ), + ImageDebatcher(), + Swarp(swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, + include_scamp=True, + subtract_bkg=False, + cache=True, + center_type="ALL", + temp_output_sub_dir=f"stack_all_{target_name}")] +process_noise = [ + Sextractor(**sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="mask_sextractor", + checkimage_type="SEGMENTATION", + cache=True, + verbose_type="FULL"), + MaskPixelsFromPath( + mask_path_key=sextractor_checkimg_map["SEGMENTATION"], + write_masked_pixels_to_file=True, + output_dir="mask1", ), - MultiExtParser(input_sub_dir="raw/mef/"), - SplitImage(), - MaskPixelsFromPath(mask_path=winter_mask_path), - DarkCalibrator(), - SkyFlatCalibrator(), - NightSkyMedianCalibrator(), + ImageSaver(output_dir_name=f"noisemask_{board_id}"), ] +# process_proc = [ImageDebatcher(), +# ImageSelector(("OBSTYPE", ["SCIENCE"])), +# Sextractor(**sextractor_autoastrometry_config, +# write_regions_bool=True, +# output_sub_dir="sextractor", +# cache=False), +# AutoAstrometry(catalog="tmc", pixel_scale=1.07, +# write_crosscheck_files=True, +# inv=False, +# ), ] + +stack_proc = [ImageBatcher(["TARGNAME", "FILTER"]), + Swarp(swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=True, + cache=True, + center_type="MANUAL"), + ImageSaver(output_dir_name=f"stack_{target_name}"), + Sextractor(**sextractor_photometry_config, + output_sub_dir=f"phot_{board_id}_{target_name}", + checkimage_type="BACKGROUND_RMS"), + PhotCalibrator(ref_catalog_generator=winter_photometric_catalog_generator, + temp_output_sub_dir=f"phot_{board_id}_{target_name}", + write_regions=True), + ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}"), + ] + +stack_multiboard = [Swarp(swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=False, + cache=True, + temp_output_sub_dir=f"multiboard_stack_{target_name}", + center_type="MANUAL", + + )] +# commissioning = \ +# log + process_proc + +commissioning = log + process + +commissioning_dark = log + dark_cal +commissioning_proc = load_proc + process_proc +commissioning_flat = load_dark + flat_cal +commissioning_reduce = log + dark_cal + flat_cal +commissioning_stack = load_stack + stack_proc +commissioning_multiboard_stack = load_multiboard_stack + stack_multiboard +commissioning_noise = load_anet + process_noise + +full_commissioning = log + process + process_proc # + stack_proc From 252fc8f648d6175a2d03a369a4e857b123c57af8 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 17:06:56 -0500 Subject: [PATCH 62/86] blocks --- mirar/pipelines/winter/blocks.py | 314 ++++++++++++++++++------------- 1 file changed, 183 insertions(+), 131 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 7139892fd..13cab4d10 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -1,32 +1,48 @@ +""" +Module for WINTER data reduction +""" +from mirar.paths import FITS_MASK_KEY +from mirar.pipelines.winter.config import ( + sextractor_astrometry_config, + sextractor_autoastrometry_config, + sextractor_photometry_config, + swarp_config_path, +) from mirar.pipelines.winter.generator import ( + scamp_config_path, + winter_astrometric_catalog_generator, winter_mask_path, - winter_reference_generator, winter_photometric_catalog_generator, - winter_astrometric_catalog_generator, - scamp_config_path + winter_reference_generator, ) -from mirar.processors.mask import WriteMaskedCoordsToFile -from mirar.pipelines.winter.load_winter_image import load_raw_winter_image, \ - load_proc_winter_image, load_stacked_winter_image +from mirar.pipelines.winter.load_winter_image import ( + load_proc_winter_image, + load_raw_winter_image, + load_stacked_winter_image, +) +from mirar.processors.astromatic import Scamp +from mirar.processors.astromatic.sextractor.sextractor import ( + Sextractor, + sextractor_checkimg_map, +) +from mirar.processors.astromatic.swarp.swarp import Swarp +from mirar.processors.astrometry.anet.anet_processor import AstrometryNet from mirar.processors.csvlog import CSVLog from mirar.processors.dark import DarkCalibrator -from mirar.processors.mask import MaskPixelsFromPath +from mirar.processors.flat import FlatCalibrator +from mirar.processors.mask import MaskPixelsFromPath, WriteMaskedCoordsToFile +from mirar.processors.photcal import PhotCalibrator from mirar.processors.reference import GetReferenceImage from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator -from mirar.processors.flat import FlatCalibrator from mirar.processors.split import SplitImage -from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map -from mirar.processors.utils import ImageDebatcher, ImageSaver, ImageLoader, \ - ImageSelector, ImageBatcher +from mirar.processors.utils import ( + ImageBatcher, + ImageDebatcher, + ImageLoader, + ImageSaver, + ImageSelector, +) from mirar.processors.utils.multi_ext_parser import MultiExtParser -from mirar.paths import FITS_MASK_KEY -from mirar.processors.astromatic.sextractor.sextractor import Sextractor -from mirar.pipelines.winter.config import sextractor_astrometry_config, \ - sextractor_autoastrometry_config, swarp_config_path, sextractor_photometry_config -from mirar.processors.astromatic.swarp.swarp import Swarp -from mirar.processors.photcal import PhotCalibrator -from mirar.processors.astrometry.anet.anet_processor import AstrometryNet -from mirar.processors.astromatic import Scamp refbuild = [ ImageDebatcher(), @@ -38,59 +54,78 @@ board_id = 5 target_name = "M101" -split = [MultiExtParser(input_sub_dir="raw/", - extension_num_header_key="BOARD_ID", - only_extract_num=board_id, - output_sub_dir=f"raw_split_{board_id}")] -load = [ImageLoader(input_sub_dir=f"raw_split_{board_id}", - load_image=load_raw_winter_image), - ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), ] +split = [ + MultiExtParser( + input_sub_dir="raw/", + extension_num_header_key="BOARD_ID", + only_extract_num=board_id, + output_sub_dir=f"raw_split_{board_id}", + ) +] +load = [ + ImageLoader( + input_sub_dir=f"raw_split_{board_id}", load_image=load_raw_winter_image + ), + ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), +] -load_proc = [ImageLoader(input_sub_dir=f"skysub_{board_id}", - load_image=load_raw_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), - ] +load_proc = [ + ImageLoader(input_sub_dir=f"skysub_{board_id}", load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), +] -load_dark = [ImageLoader(input_sub_dir=f"darkcal_{board_id}", - load_image=load_raw_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"))] +load_dark = [ + ImageLoader(input_sub_dir=f"darkcal_{board_id}", load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{target_name}")), +] -load_anet = [ImageLoader(input_sub_dir=f"anet_{board_id}", - load_image=load_proc_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE"))] +load_anet = [ + ImageLoader(input_sub_dir=f"anet_{board_id}", load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), +] -load_stack = [ImageLoader(input_sub_dir=f"anet_{board_id}", - load_image=load_proc_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), - ImageBatcher("EXPTIME")] +load_stack = [ + ImageLoader(input_sub_dir=f"anet_{board_id}", load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), + ImageBatcher("EXPTIME"), +] -load_multiboard_stack = [ImageLoader(input_sub_dir=f"stack_all_{target_name}", - load_image=load_stacked_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), - ("OBSTYPE", "SCIENCE"), - ),] - # ImageBatcher("COADDS")] -log = split + load + \ - [CSVLog( - export_keys=[ - "FILTER", - "UTCTIME", - "EXPTIME", - "OBSTYPE", - "UNIQTYPE", - "BOARD_ID", - "OBSCLASS", - "TARGET", - "FILTER", - "BASENAME", - "TARGNAME", - "RADEG", - "DECDEG", - "MEDCOUNT", - "STDDEV", - "T_ROIC" - ] - )] +load_multiboard_stack = [ + ImageLoader( + input_sub_dir=f"stack_all_{target_name}", load_image=load_stacked_winter_image + ), + ImageSelector( + ("TARGNAME", f"{target_name}"), + ("OBSTYPE", "SCIENCE"), + ), +] +# ImageBatcher("COADDS")] +log = ( + split + + load + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] +) dark_cal = [ ImageSelector(("BOARD_ID", f"{board_id}")), @@ -98,7 +133,8 @@ WriteMaskedCoordsToFile(output_dir="mask_raw"), DarkCalibrator(cache_sub_dir=f"calibration_{board_id}"), ImageSaver(output_dir_name=f"darkcal_{board_id}"), - ImageDebatcher()] + ImageDebatcher(), +] flat_cal = [ # ImageSelector(("OBSTYPE", ["FOCUS", "SCIENCE", "FLAT"])), @@ -109,8 +145,7 @@ # ), ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{target_name}")), ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), - SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, - cache_sub_dir=f'skycals_{board_id}'), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir=f"skycals_{board_id}"), ImageSelector(("OBSTYPE", ["SCIENCE"])), ImageSaver(output_dir_name=f"skyflatcal_{board_id}"), # ImageSelector(("OBSTYPE", ["SCIENCE"])), @@ -132,7 +167,6 @@ # Sextractor(**sextractor_astrometry_config, # write_regions_bool=True, # output_sub_dir="sextractor"), - # MultiExtParser(input_sub_dir="raw/mef/"), # SplitImage(), # MaskPixelsFromPath(mask_path=winter_mask_path), @@ -142,43 +176,51 @@ ] process = dark_cal + flat_cal -process_proc = [ImageDebatcher(), - AstrometryNet(output_sub_dir=f'anet_{board_id}', - scale_bounds=[25, 40], - scale_units='amw', - use_sextractor=True, - parity='neg', - search_radius_deg=1.0, - # sextractor_config_path=sextractor_autoastrometry_config[ - # 'config_path'], - use_weight=False - ), - ImageSaver(output_dir_name=f"anet_{board_id}", - use_existing_weight=False), - Sextractor(**sextractor_autoastrometry_config, - write_regions_bool=True, - output_sub_dir="scamp", - ), - Scamp( - temp_output_sub_dir="scamp", - ref_catalog_generator=winter_astrometric_catalog_generator, - scamp_config_path=scamp_config_path, - cache=False, - ), - ImageDebatcher(), - Swarp(swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, - include_scamp=True, - subtract_bkg=False, - cache=True, - center_type="ALL", - temp_output_sub_dir=f"stack_all_{target_name}")] +process_proc = [ + ImageDebatcher(), + AstrometryNet( + output_sub_dir=f"anet_{board_id}", + scale_bounds=[25, 40], + scale_units="amw", + use_sextractor=True, + parity="neg", + search_radius_deg=1.0, + # sextractor_config_path=sextractor_autoastrometry_config[ + # 'config_path'], + use_weight=False, + ), + ImageSaver(output_dir_name=f"anet_{board_id}", use_existing_weight=False), + Sextractor( + **sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="scamp", + ), + Scamp( + temp_output_sub_dir="scamp", + ref_catalog_generator=winter_astrometric_catalog_generator, + scamp_config_path=scamp_config_path, + cache=False, + ), + ImageDebatcher(), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=True, + subtract_bkg=False, + cache=True, + center_type="ALL", + temp_output_sub_dir=f"stack_all_{target_name}", + ), +] process_noise = [ - Sextractor(**sextractor_autoastrometry_config, - write_regions_bool=True, - output_sub_dir="mask_sextractor", - checkimage_type="SEGMENTATION", - cache=True, - verbose_type="FULL"), + Sextractor( + **sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="mask_sextractor", + checkimage_type="SEGMENTATION", + cache=True, + verbose_type="FULL", + ), MaskPixelsFromPath( mask_path_key=sextractor_checkimg_map["SEGMENTATION"], write_masked_pixels_to_file=True, @@ -197,31 +239,41 @@ # inv=False, # ), ] -stack_proc = [ImageBatcher(["TARGNAME", "FILTER"]), - Swarp(swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, - include_scamp=False, - subtract_bkg=True, - cache=True, - center_type="MANUAL"), - ImageSaver(output_dir_name=f"stack_{target_name}"), - Sextractor(**sextractor_photometry_config, - output_sub_dir=f"phot_{board_id}_{target_name}", - checkimage_type="BACKGROUND_RMS"), - PhotCalibrator(ref_catalog_generator=winter_photometric_catalog_generator, - temp_output_sub_dir=f"phot_{board_id}_{target_name}", - write_regions=True), - ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}"), - ] - -stack_multiboard = [Swarp(swarp_config_path=swarp_config_path, - calculate_dims_in_swarp=True, - include_scamp=False, - subtract_bkg=False, - cache=True, - temp_output_sub_dir=f"multiboard_stack_{target_name}", - center_type="MANUAL", +stack_proc = [ + ImageBatcher(["TARGNAME", "FILTER"]), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=True, + cache=True, + center_type="MANUAL", + ), + ImageSaver(output_dir_name=f"stack_{target_name}"), + Sextractor( + **sextractor_photometry_config, + output_sub_dir=f"phot_{board_id}_{target_name}", + checkimage_type="BACKGROUND_RMS", + ), + PhotCalibrator( + ref_catalog_generator=winter_photometric_catalog_generator, + temp_output_sub_dir=f"phot_{board_id}_{target_name}", + write_regions=True, + ), + ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}"), +] - )] +stack_multiboard = [ + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=False, + cache=True, + temp_output_sub_dir=f"multiboard_stack_{target_name}", + center_type="MANUAL", + ) +] # commissioning = \ # log + process_proc From 6fc5574a7eebe07b72675bf544bd9be29c2f4715 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 17:10:38 -0500 Subject: [PATCH 63/86] winter pipeline --- mirar/pipelines/winter/winter_pipeline.py | 40 ++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index 308f9f17d..b72b1bea5 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -3,20 +3,46 @@ """ import logging +from mirar.downloader.caltech import download_via_ssh from mirar.pipelines.base_pipeline import Pipeline -from mirar.pipelines.winter.blocks import refbuild +from mirar.pipelines.winter.blocks import ( + commissioning, + commissioning_dark, + commissioning_flat, + commissioning_multiboard_stack, + commissioning_noise, + commissioning_proc, + commissioning_reduce, + commissioning_stack, + full_commissioning, + log, + refbuild, +) +from mirar.pipelines.winter.config import PIPELINE_NAME logger = logging.getLogger(__name__) class WINTERPipeline(Pipeline): """ - Pipeline for building reference images in the IR from WFAU + Pipeline for processing WINTER data """ name = "winter" - all_pipeline_configurations = {"refbuild": refbuild} + all_pipeline_configurations = { + "refbuild": refbuild, + "commissioning": commissioning, + "log": log, + "commissioning_dark": commissioning_dark, + "commissioning_stack": commissioning_stack, + "commissioning_flat": commissioning_flat, + "commissioning_reduce": commissioning_reduce, + "commissioning_proc": commissioning_proc, + "commissioning_multiboard_stack": commissioning_multiboard_stack, + "full_commissioning": full_commissioning, + "commissioning_noise": commissioning_noise, + } gain = 1.0 non_linear_level = 65535 @@ -27,4 +53,10 @@ def _load_raw_image(path: str): @staticmethod def download_raw_images_for_night(night: str): - pass + download_via_ssh( + server="winter.caltech.edu", + base_dir="/data/loki/raw_data/winter", + night=night, + pipeline=PIPELINE_NAME, + server_sub_dir="raw", + ) From 359dee50590035fba80fb5f981565b6209d153f9 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 17:21:33 -0500 Subject: [PATCH 64/86] script to fix headers --- mirar/pipelines/winter/fix_headers.py | 39 +++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/mirar/pipelines/winter/fix_headers.py b/mirar/pipelines/winter/fix_headers.py index e69de29bb..77c3fc44a 100644 --- a/mirar/pipelines/winter/fix_headers.py +++ b/mirar/pipelines/winter/fix_headers.py @@ -0,0 +1,39 @@ +""" +Script to fix headers of images based on a log file +""" +import pandas as pd +from astropy.io import fits +from mirar.paths import get_output_path +import argparse + + +def fix_headers(logfile, night, keyword_list, sub_dir='raw'): + """ + Function to fix headers of images based on a log file + """ + obslog = pd.read_csv(logfile) + for ind, image_basename in enumerate(obslog['BASENAME']): + for i in range(5): + if f'_{i}.fits' in image_basename: + image_basename = image_basename.replace(f'_{i}.fits', '.fits') + image_filename = get_output_path(base_name=image_basename, + dir_root=sub_dir, + sub_dir='winter/'+night) + img_hdulist = fits.open(image_filename, 'update') + img_header = img_hdulist[0].header + for key in keyword_list: + print(key) + img_header[key] = obslog.loc[ind][key] + img_hdulist.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Fix headers of images') + parser.add_argument('logfile', type=str, help='CSV file with image metadata') + parser.add_argument('night', type=str, help='Night of observation') + parser.add_argument('--sub_dir', type=str, default='raw', + help='Subdirectory of images') + parser.add_argument('-k', '--keywords', type=str, nargs='+') + args = parser.parse_args() + + fix_headers(args.logfile, args.night, args.sub_dir, args.keywords) From 43eab2de32d17785dccadd047ad4690cd420e608 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Tue, 27 Jun 2023 17:21:38 -0500 Subject: [PATCH 65/86] script to fix headers --- mirar/pipelines/winter/fix_headers.py | 35 +++++++++++++++------------ 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/mirar/pipelines/winter/fix_headers.py b/mirar/pipelines/winter/fix_headers.py index 77c3fc44a..6acc018a8 100644 --- a/mirar/pipelines/winter/fix_headers.py +++ b/mirar/pipelines/winter/fix_headers.py @@ -1,25 +1,27 @@ """ Script to fix headers of images based on a log file """ +import argparse + import pandas as pd from astropy.io import fits + from mirar.paths import get_output_path -import argparse -def fix_headers(logfile, night, keyword_list, sub_dir='raw'): +def fix_headers(logfile, night, keyword_list, sub_dir="raw"): """ Function to fix headers of images based on a log file """ obslog = pd.read_csv(logfile) - for ind, image_basename in enumerate(obslog['BASENAME']): + for ind, image_basename in enumerate(obslog["BASENAME"]): for i in range(5): - if f'_{i}.fits' in image_basename: - image_basename = image_basename.replace(f'_{i}.fits', '.fits') - image_filename = get_output_path(base_name=image_basename, - dir_root=sub_dir, - sub_dir='winter/'+night) - img_hdulist = fits.open(image_filename, 'update') + if f"_{i}.fits" in image_basename: + image_basename = image_basename.replace(f"_{i}.fits", ".fits") + image_filename = get_output_path( + base_name=image_basename, dir_root=sub_dir, sub_dir="winter/" + night + ) + img_hdulist = fits.open(image_filename, "update") img_header = img_hdulist[0].header for key in keyword_list: print(key) @@ -27,13 +29,14 @@ def fix_headers(logfile, night, keyword_list, sub_dir='raw'): img_hdulist.close() -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Fix headers of images') - parser.add_argument('logfile', type=str, help='CSV file with image metadata') - parser.add_argument('night', type=str, help='Night of observation') - parser.add_argument('--sub_dir', type=str, default='raw', - help='Subdirectory of images') - parser.add_argument('-k', '--keywords', type=str, nargs='+') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fix headers of images") + parser.add_argument("logfile", type=str, help="CSV file with image metadata") + parser.add_argument("night", type=str, help="Night of observation") + parser.add_argument( + "--sub_dir", type=str, default="raw", help="Subdirectory of images" + ) + parser.add_argument("-k", "--keywords", type=str, nargs="+") args = parser.parse_args() fix_headers(args.logfile, args.night, args.sub_dir, args.keywords) From 6ad06c12c7057a8f083381a0e96709d02089e863 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Wed, 28 Jun 2023 18:06:57 -0500 Subject: [PATCH 66/86] fix radius for xmatch --- mirar/pipelines/winter/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 8eaaedc64..9846f9b05 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -99,7 +99,7 @@ def winter_photometric_catalog_generator(image: Image) -> Gaia2Mass: filter_name = image["FILTER"] search_radius_arcmin = ( np.max([image["NAXIS1"], image["NAXIS2"]]) * np.abs(image["CD1_1"]) * 60 - ) + ) / 2 return Gaia2Mass( min_mag=10, max_mag=20, From 6b1cf9f9e87c9f662c040315859707b3b53a6070 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Wed, 28 Jun 2023 18:07:48 -0500 Subject: [PATCH 67/86] working --- mirar/pipelines/winter/fix_headers.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/mirar/pipelines/winter/fix_headers.py b/mirar/pipelines/winter/fix_headers.py index 6acc018a8..979364c0d 100644 --- a/mirar/pipelines/winter/fix_headers.py +++ b/mirar/pipelines/winter/fix_headers.py @@ -3,6 +3,7 @@ """ import argparse +import numpy as np import pandas as pd from astropy.io import fits @@ -24,8 +25,11 @@ def fix_headers(logfile, night, keyword_list, sub_dir="raw"): img_hdulist = fits.open(image_filename, "update") img_header = img_hdulist[0].header for key in keyword_list: - print(key) - img_header[key] = obslog.loc[ind][key] + if obslog.loc[ind][key] is np.nan: + img_header[key] = "" + else: + img_header[key] = obslog.loc[ind][key] + img_hdulist.close() @@ -39,4 +43,4 @@ def fix_headers(logfile, night, keyword_list, sub_dir="raw"): parser.add_argument("-k", "--keywords", type=str, nargs="+") args = parser.parse_args() - fix_headers(args.logfile, args.night, args.sub_dir, args.keywords) + fix_headers(args.logfile, args.night, args.keywords, args.sub_dir) From f354fc4c695739628dd3e269a573915ad5f9508b Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 02:25:15 -0500 Subject: [PATCH 68/86] fix bug --- mirar/pipelines/winter/blocks.py | 185 +++++++++++++++--- mirar/pipelines/winter/winter_pipeline.py | 6 + .../astrometry/autoastrometry/detect.py | 6 +- 3 files changed, 167 insertions(+), 30 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 13cab4d10..b48906f7a 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -52,8 +52,8 @@ ImageSaver(output_dir_name="stacked_ref"), ] -board_id = 5 -target_name = "M101" +board_id = 4 +target_name = "EMGW_gal1" split = [ MultiExtParser( input_sub_dir="raw/", @@ -62,6 +62,14 @@ output_sub_dir=f"raw_split_{board_id}", ) ] + +split_all_boards = [ + MultiExtParser( + input_sub_dir="raw/", + extension_num_header_key="BOARD_ID", + output_sub_dir="raw_split", + ) +] load = [ ImageLoader( input_sub_dir=f"raw_split_{board_id}", load_image=load_raw_winter_image @@ -69,6 +77,13 @@ ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), ] +load_all_boards = [ + ImageLoader( + input_sub_dir="raw_split", load_image=load_raw_winter_image + ), + ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), +] + load_proc = [ ImageLoader(input_sub_dir=f"skysub_{board_id}", load_image=load_raw_winter_image), ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), @@ -101,30 +116,57 @@ ] # ImageBatcher("COADDS")] log = ( - split - + load - + [ - CSVLog( - export_keys=[ - "FILTER", - "UTCTIME", - "EXPTIME", - "OBSTYPE", - "UNIQTYPE", - "BOARD_ID", - "OBSCLASS", - "TARGET", - "FILTER", - "BASENAME", - "TARGNAME", - "RADEG", - "DECDEG", - "MEDCOUNT", - "STDDEV", - "T_ROIC", - ] - ) - ] + split + + load + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] +) + +log_all_boards = ( + split_all_boards + + load_all_boards + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] ) dark_cal = [ @@ -136,6 +178,14 @@ ImageDebatcher(), ] +dark_cal_all_boards = [ + ImageBatcher(["BOARD_ID", "EXPTIME"]), + WriteMaskedCoordsToFile(output_dir="mask_raw"), + DarkCalibrator(cache_sub_dir="calibration"), + ImageSaver(output_dir_name="darkcal"), + ImageDebatcher(), +] + flat_cal = [ # ImageSelector(("OBSTYPE", ["FOCUS", "SCIENCE", "FLAT"])), # ImageSelector(("TARGNAME", ["INTERESTING"])), @@ -175,6 +225,16 @@ # NightSkyMedianCalibrator(), ] +flat_cal_all_boards = [ + ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{target_name}")), + ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir="skycals"), + ImageSelector(("OBSTYPE", ["SCIENCE"])), + ImageSaver(output_dir_name="skyflatcal"), + NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), + ImageSaver(output_dir_name="skysub"), +] + process = dark_cal + flat_cal process_proc = [ ImageDebatcher(), @@ -212,6 +272,45 @@ temp_output_sub_dir=f"stack_all_{target_name}", ), ] + +process_proc_all_boards = [ + ImageDebatcher(), + AstrometryNet( + output_sub_dir="anet", + scale_bounds=[25, 40], + scale_units="amw", + use_sextractor=True, + parity="neg", + search_radius_deg=1.0, + # sextractor_config_path=sextractor_autoastrometry_config[ + # 'config_path'], + use_weight=False, + ), + ImageSaver(output_dir_name=f"anet", use_existing_weight=False), + Sextractor( + **sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="scamp", + ), + Scamp( + temp_output_sub_dir="scamp", + ref_catalog_generator=winter_astrometric_catalog_generator, + scamp_config_path=scamp_config_path, + cache=False, + ), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME"]), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=True, + subtract_bkg=False, + cache=False, + center_type="ALL", + temp_output_sub_dir=f"stack_all_{target_name}", + ), +] + process_noise = [ Sextractor( **sextractor_autoastrometry_config, @@ -250,6 +349,12 @@ center_type="MANUAL", ), ImageSaver(output_dir_name=f"stack_{target_name}"), +] + +photcal = [ + # ImageSelector(("BOARD_ID", board_id)), + ImageDebatcher(), + ImageBatcher(["BOARD_ID"]), Sextractor( **sextractor_photometry_config, output_sub_dir=f"phot_{board_id}_{target_name}", @@ -259,9 +364,28 @@ ref_catalog_generator=winter_photometric_catalog_generator, temp_output_sub_dir=f"phot_{board_id}_{target_name}", write_regions=True, + cache=True, ), - ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}"), -] + # ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}") + ImageSaver(output_dir_name=f"phot_{target_name}") + ] + +photcal_indiv = [ + ImageSelector(("BOARD_ID", board_id)), + ImageDebatcher(), + ImageBatcher(["UTCTIME"]), + Sextractor( + **sextractor_photometry_config, + output_sub_dir=f"phot_{board_id}_{target_name}", + checkimage_type="BACKGROUND_RMS", + ), + PhotCalibrator( + ref_catalog_generator=winter_photometric_catalog_generator, + temp_output_sub_dir=f"phot_{board_id}_{target_name}", + write_regions=True, + cache=True, + ), + ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}")] stack_multiboard = [ Swarp( @@ -286,5 +410,8 @@ commissioning_stack = load_stack + stack_proc commissioning_multiboard_stack = load_multiboard_stack + stack_multiboard commissioning_noise = load_anet + process_noise - +commissioning_photcal = load_multiboard_stack + photcal +commissioning_photcal_indiv = load_anet + photcal_indiv full_commissioning = log + process + process_proc # + stack_proc +full_commissioning_all_boards = log_all_boards + dark_cal_all_boards + flat_cal_all_boards + \ + process_proc_all_boards diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index b72b1bea5..ccf6ded33 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -17,6 +17,9 @@ full_commissioning, log, refbuild, + commissioning_photcal, + commissioning_photcal_indiv, + full_commissioning_all_boards ) from mirar.pipelines.winter.config import PIPELINE_NAME @@ -42,6 +45,9 @@ class WINTERPipeline(Pipeline): "commissioning_multiboard_stack": commissioning_multiboard_stack, "full_commissioning": full_commissioning, "commissioning_noise": commissioning_noise, + "commissioning_photcal": commissioning_photcal, + "commissioning_photcal_indiv": commissioning_photcal_indiv, + "full_commissioning_all_boards": full_commissioning_all_boards } gain = 1.0 diff --git a/mirar/processors/astrometry/autoastrometry/detect.py b/mirar/processors/astrometry/autoastrometry/detect.py index baee8844c..a8def018e 100644 --- a/mirar/processors/astrometry/autoastrometry/detect.py +++ b/mirar/processors/astrometry/autoastrometry/detect.py @@ -75,7 +75,11 @@ def get_img_src_list( except FileNotFoundError: pass - sextractor_catalog_path = fits.getval(img_path, SEXTRACTOR_HEADER_KEY) + header = fits.getheader(img_path) + sextractor_catalog_path = None + if SEXTRACTOR_HEADER_KEY in header.keys(): + sextractor_catalog_path = fits.getval(img_path, SEXTRACTOR_HEADER_KEY) + if sextractor_catalog_path is not None: logger.info("Using existing sextractor catalog") output_catalog = sextractor_catalog_path From e838b8ef16ec782077261b8905687a78da910bec Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 02:28:44 -0500 Subject: [PATCH 69/86] catalog purifier --- mirar/pipelines/winter/generator.py | 42 +++++++++++-------- .../autoastrometry/autoastrometry.py | 9 ++-- .../autoastrometry_processor.py | 2 +- 3 files changed, 31 insertions(+), 22 deletions(-) diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 9846f9b05..37040ef52 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -6,6 +6,7 @@ from typing import Type import numpy as np +from astropy.table import Table from mirar.catalog import Gaia2Mass from mirar.data import Image @@ -109,6 +110,29 @@ def winter_photometric_catalog_generator(image: Image) -> Gaia2Mass: ) +def winter_ref_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Table: + """ + Default function to purify the photometric image catalog + """ + edge_width_pixels = 0 + fwhm_threshold_arcsec = 4.0 + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + def winter_reference_phot_calibrator(image: Image, **kwargs) -> PhotCalibrator: """ Generates a resampler for reference images @@ -124,6 +148,7 @@ def winter_reference_phot_calibrator(image: Image, **kwargs) -> PhotCalibrator: return PhotCalibrator( ref_catalog_generator=winter_photometric_catalog_generator, write_regions=True, + image_photometric_catalog_purifier=winter_ref_photometric_img_catalog_purifier, **kwargs, ) @@ -146,23 +171,6 @@ def ref_sextractor(image: Image): ) -def ref_phot_calibrator(image: Image): - """ - Generates a photcalibrator instance for reference images to get photometry - Args: - image: - - Returns: - - """ - logger.debug(image) - return PhotCalibrator( - ref_catalog_generator=winter_photometric_catalog_generator, - write_regions=True, - # fwhm_threshold_arcsec=3, - ) - - def winter_astrometric_catalog_generator(_) -> Gaia2Mass: """ Function to crossmatch WIRC to GAIA/2mass for astrometry diff --git a/mirar/processors/astrometry/autoastrometry/autoastrometry.py b/mirar/processors/astrometry/autoastrometry/autoastrometry.py index 042133455..9a29d901a 100644 --- a/mirar/processors/astrometry/autoastrometry/autoastrometry.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry.py @@ -32,6 +32,7 @@ write_config_file, write_param_file, ) +from mirar.processors.astromatic.sextractor.sourceextractor import default_saturation from mirar.processors.astrometry.autoastrometry.crossmatch import ( crosscheck_source_lists, distance_match, @@ -84,11 +85,11 @@ def autoastrometry( outfile: str = "", output_dir: str = base_output_dir, temp_file: Optional[str] = None, - saturation: float = None, + saturation: float = default_saturation, no_rot: bool = False, min_fwhm: float = DEFAULT_MIN_FWHM, max_fwhm: float = DEFAULT_MAX_FWHM, - write_crosscheck_files: bool = True, + write_crosscheck_files: bool = False, ): """ @@ -556,7 +557,7 @@ def run_autoastrometry_single( overwrite: bool = False, outfile: Optional[str] = None, output_dir: str = base_output_dir, - saturation: float = None, + saturation: float = default_saturation, no_rot: bool = False, write_crosscheck_files: bool = False, ): @@ -667,7 +668,7 @@ def run_autoastrometry_batch( overwrite: bool = False, outfile: Optional[str] = None, output_dir: str = base_output_dir, - saturation: float = None, + saturation: float = default_saturation, no_rot: bool = False, write_crosscheck_files: bool = False, ): diff --git a/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py b/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py index cc2f00125..21b9ee202 100644 --- a/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py @@ -22,7 +22,7 @@ class AutoAstrometry(BaseImageProcessor): def __init__( self, temp_output_sub_dir: str = "autoastrometry", - write_crosscheck_files: bool = True, + write_crosscheck_files: bool = False, catalog: Optional[str] = None, pixel_scale: Optional[float] = None, inv: bool = False, From f21542e5a8e5e26ac78836163ab28ee9a75b9b22 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 02:29:07 -0500 Subject: [PATCH 70/86] updated blocks --- mirar/pipelines/winter/blocks.py | 120 ++++++++++++++++--------------- 1 file changed, 61 insertions(+), 59 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index b48906f7a..d0caaaaf0 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -3,7 +3,7 @@ """ from mirar.paths import FITS_MASK_KEY from mirar.pipelines.winter.config import ( - sextractor_astrometry_config, + sextractor_anet_config, sextractor_autoastrometry_config, sextractor_photometry_config, swarp_config_path, @@ -78,9 +78,7 @@ ] load_all_boards = [ - ImageLoader( - input_sub_dir="raw_split", load_image=load_raw_winter_image - ), + ImageLoader(input_sub_dir="raw_split", load_image=load_raw_winter_image), ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), ] @@ -116,57 +114,57 @@ ] # ImageBatcher("COADDS")] log = ( - split - + load - + [ - CSVLog( - export_keys=[ - "FILTER", - "UTCTIME", - "EXPTIME", - "OBSTYPE", - "UNIQTYPE", - "BOARD_ID", - "OBSCLASS", - "TARGET", - "FILTER", - "BASENAME", - "TARGNAME", - "RADEG", - "DECDEG", - "MEDCOUNT", - "STDDEV", - "T_ROIC", - ] - ) - ] + split + + load + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] ) log_all_boards = ( - split_all_boards - + load_all_boards - + [ - CSVLog( - export_keys=[ - "FILTER", - "UTCTIME", - "EXPTIME", - "OBSTYPE", - "UNIQTYPE", - "BOARD_ID", - "OBSCLASS", - "TARGET", - "FILTER", - "BASENAME", - "TARGNAME", - "RADEG", - "DECDEG", - "MEDCOUNT", - "STDDEV", - "T_ROIC", - ] - ) - ] + split_all_boards + + load_all_boards + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] ) dark_cal = [ @@ -275,6 +273,7 @@ process_proc_all_boards = [ ImageDebatcher(), + ImageBatcher(["UTCTIME", "BOARD_ID"]), AstrometryNet( output_sub_dir="anet", scale_bounds=[25, 40], @@ -282,8 +281,8 @@ use_sextractor=True, parity="neg", search_radius_deg=1.0, - # sextractor_config_path=sextractor_autoastrometry_config[ - # 'config_path'], + sextractor_config_path=sextractor_anet_config[ + 'config_path'], use_weight=False, ), ImageSaver(output_dir_name=f"anet", use_existing_weight=False), @@ -300,6 +299,7 @@ ), ImageDebatcher(), ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME"]), + # ImageSaver(output_dir_name="pre-swarp"), Swarp( swarp_config_path=swarp_config_path, calculate_dims_in_swarp=True, @@ -367,8 +367,8 @@ cache=True, ), # ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}") - ImageSaver(output_dir_name=f"phot_{target_name}") - ] + ImageSaver(output_dir_name=f"phot_{target_name}"), +] photcal_indiv = [ ImageSelector(("BOARD_ID", board_id)), @@ -385,7 +385,8 @@ write_regions=True, cache=True, ), - ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}")] + ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}"), +] stack_multiboard = [ Swarp( @@ -413,5 +414,6 @@ commissioning_photcal = load_multiboard_stack + photcal commissioning_photcal_indiv = load_anet + photcal_indiv full_commissioning = log + process + process_proc # + stack_proc -full_commissioning_all_boards = log_all_boards + dark_cal_all_boards + flat_cal_all_boards + \ - process_proc_all_boards +full_commissioning_all_boards = ( + log_all_boards + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards +) From f1b9584c59a7fad1602ad10ca7d71f61fb42da2d Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 02:29:28 -0500 Subject: [PATCH 71/86] new configs --- mirar/pipelines/winter/winter_pipeline.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index ccf6ded33..aba4a2175 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -11,15 +11,15 @@ commissioning_flat, commissioning_multiboard_stack, commissioning_noise, + commissioning_photcal, + commissioning_photcal_indiv, commissioning_proc, commissioning_reduce, commissioning_stack, full_commissioning, + full_commissioning_all_boards, log, refbuild, - commissioning_photcal, - commissioning_photcal_indiv, - full_commissioning_all_boards ) from mirar.pipelines.winter.config import PIPELINE_NAME @@ -47,7 +47,7 @@ class WINTERPipeline(Pipeline): "commissioning_noise": commissioning_noise, "commissioning_photcal": commissioning_photcal, "commissioning_photcal_indiv": commissioning_photcal_indiv, - "full_commissioning_all_boards": full_commissioning_all_boards + "full_commissioning_all_boards": full_commissioning_all_boards, } gain = 1.0 From d0fdb2fea645e290b70d438687953bd24b3059b7 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 12:08:02 -0500 Subject: [PATCH 72/86] remove satur_level keyword from sextractor file --- mirar/processors/astromatic/sextractor/settings.py | 5 ++--- mirar/processors/astromatic/sextractor/sourceextractor.py | 2 +- mirar/processors/astrometry/autoastrometry/autoastrometry.py | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/mirar/processors/astromatic/sextractor/settings.py b/mirar/processors/astromatic/sextractor/settings.py index 2c9530134..050c9e978 100644 --- a/mirar/processors/astromatic/sextractor/settings.py +++ b/mirar/processors/astromatic/sextractor/settings.py @@ -46,7 +46,7 @@ def write_config_file( param_path: str = default_param_path, conv_path: str = default_conv_path, config_path: str = default_config_path, - saturation: float = 55000.0, + saturation_key: str = "SATURATE", ): """ Write a default configuration file for sextractor @@ -123,8 +123,7 @@ def write_config_file( VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL WRITE_XML N # Write XML file (Y/N)? XML_NAME sex.xml # Filename for XML output -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) -SATUR_LEVEL {saturation} # level (in ADUs) at which arises saturation +SATUR_KEY {saturation_key} # keyword for saturation level (in ADUs) """ with open(config_path, "w") as pf: pf.write(configs) diff --git a/mirar/processors/astromatic/sextractor/sourceextractor.py b/mirar/processors/astromatic/sextractor/sourceextractor.py index 43721ea6d..eadb04482 100644 --- a/mirar/processors/astromatic/sextractor/sourceextractor.py +++ b/mirar/processors/astromatic/sextractor/sourceextractor.py @@ -15,7 +15,7 @@ # sextractor_cmd = os.getenv("SEXTRACTOR_CMD") -default_saturation = None +default_saturation = 10000000000.0 default_config_path = os.path.join(astromatic_config_dir, "astrom.sex") default_param_path = os.path.join(astromatic_config_dir, "astrom.param") default_filter_name = os.path.join(astromatic_config_dir, "default.conv") diff --git a/mirar/processors/astrometry/autoastrometry/autoastrometry.py b/mirar/processors/astrometry/autoastrometry/autoastrometry.py index 9a29d901a..d288dff04 100644 --- a/mirar/processors/astrometry/autoastrometry/autoastrometry.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry.py @@ -621,7 +621,7 @@ def run_autoastrometry_single( write_param_file() - write_config_file(saturation=saturation) + write_config_file() logger.debug(f"Outfile is {outfile}") fit_info = autoastrometry( filename=img_path, From c66891d2e874f95a0cea0cf3fbb5d162520291ca Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 29 Jun 2023 12:58:21 -0700 Subject: [PATCH 73/86] Ensure paths are in image --- mirar/io.py | 11 +++ mirar/pipelines/wirc/load_wirc_image.py | 97 +++++++++++-------------- 2 files changed, 52 insertions(+), 56 deletions(-) diff --git a/mirar/io.py b/mirar/io.py index 22fb8e71f..0b110a281 100644 --- a/mirar/io.py +++ b/mirar/io.py @@ -11,6 +11,8 @@ from astropy.io import fits from astropy.utils.exceptions import AstropyUserWarning +from mirar.paths import BASE_NAME_KEY, LATEST_SAVE_KEY, RAW_IMG_KEY + def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU: """ @@ -70,6 +72,15 @@ def open_fits(path: str | Path) -> tuple[np.ndarray, fits.Header]: data = hdu.data header = hdu.header + if BASE_NAME_KEY not in header: + header[BASE_NAME_KEY] = Path(path).name + + if LATEST_SAVE_KEY not in header: + header[LATEST_SAVE_KEY] = path + + if RAW_IMG_KEY not in header.keys(): + header[RAW_IMG_KEY] = path + return data, header diff --git a/mirar/pipelines/wirc/load_wirc_image.py b/mirar/pipelines/wirc/load_wirc_image.py index 9ee26088e..5e507a894 100644 --- a/mirar/pipelines/wirc/load_wirc_image.py +++ b/mirar/pipelines/wirc/load_wirc_image.py @@ -2,26 +2,18 @@ Module for loading raw WIRC images and ensuring they have the correct format """ import logging -import os from pathlib import Path import astropy import numpy as np -from astropy.io import fits from astropy.time import Time -from mirar.paths import ( - BASE_NAME_KEY, - COADD_KEY, - PROC_FAIL_KEY, - PROC_HISTORY_KEY, - RAW_IMG_KEY, - SATURATE_KEY, -) +from mirar.io import open_fits +from mirar.paths import COADD_KEY, PROC_FAIL_KEY, PROC_HISTORY_KEY, SATURATE_KEY logger = logging.getLogger(__name__) -wirc_nonlinear_level = 30000 +WIRC_NONLINEAR_LEVEL = 30000 def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: @@ -31,54 +23,47 @@ def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Hea :param path: path of file :return: data and header of image """ - with fits.open(path) as img: - # pylint: disable=E1101 - data = img[0].data - header = img[0].header - # pylint: enable=E1101 - header["FILTER"] = header["AFT"].split("__")[0] - if "COADDS" in header.keys(): - header["DETCOADD"] = header["COADDS"] - if SATURATE_KEY not in header: - header[SATURATE_KEY] = wirc_nonlinear_level * header["DETCOADD"] - if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: - header["OBSTYPE"] = "calibration" + data, header = open_fits(path) + header["FILTER"] = header["AFT"].split("__")[0] + if "COADDS" in header.keys(): + header["DETCOADD"] = header["COADDS"] + if SATURATE_KEY not in header: + header[SATURATE_KEY] = WIRC_NONLINEAR_LEVEL * header["DETCOADD"] + if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: + header["OBSTYPE"] = "calibration" - header["OBSCLASS"] = ["calibration", "science"][header["OBSTYPE"] == "object"] + header["OBSCLASS"] = ["calibration", "science"][header["OBSTYPE"] == "object"] - header[BASE_NAME_KEY] = os.path.basename(path) - if RAW_IMG_KEY not in header.keys(): - header[RAW_IMG_KEY] = path - header["TARGET"] = header["OBJECT"].lower() - if "MJD-OBS" in header.keys(): - header["UTCTIME"] = Time(header["MJD-OBS"], format="mjd").isot - else: - header["UTCTIME"] = header["UTSHUT"] - header["MJD-OBS"] = Time(header["UTSHUT"]).mjd - if COADD_KEY not in header.keys(): - logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") - header[COADD_KEY] = 1 + header["TARGET"] = header["OBJECT"].lower() + if "MJD-OBS" in header.keys(): + header["UTCTIME"] = Time(header["MJD-OBS"], format="mjd").isot + else: + header["UTCTIME"] = header["UTSHUT"] + header["MJD-OBS"] = Time(header["UTSHUT"]).mjd + if COADD_KEY not in header.keys(): + logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") + header[COADD_KEY] = 1 - header[PROC_HISTORY_KEY] = "" - header[PROC_FAIL_KEY] = "" + header[PROC_HISTORY_KEY] = "" + header[PROC_FAIL_KEY] = "" - filter_dict = {"J": 1, "H": 2, "Ks": 3} + filter_dict = {"J": 1, "H": 2, "Ks": 3} - if "FILTERID" not in header.keys(): - header["FILTERID"] = filter_dict[header["FILTER"]] - if "FIELDID" not in header.keys(): - header["FIELDID"] = 99999 - if "PROGPI" not in header.keys(): - header["PROGPI"] = "Kasliwal" - if "PROGID" not in header.keys(): - header["PROGID"] = 0 - if "ZP" not in header.keys(): - if "TMC_ZP" in header.keys(): - header["ZP"] = header["TMC_ZP"] - header["ZP_std"] = header["TMC_ZPSD"] - if "ZP_AUTO" in header.keys(): - header["ZP"] = header["ZP_AUTO"] - header["ZP_std"] = header["ZP_AUTO_std"] - data = data.astype(float) - data[data == 0.0] = np.nan + if "FILTERID" not in header.keys(): + header["FILTERID"] = filter_dict[header["FILTER"]] + if "FIELDID" not in header.keys(): + header["FIELDID"] = 99999 + if "PROGPI" not in header.keys(): + header["PROGPI"] = "Kasliwal" + if "PROGID" not in header.keys(): + header["PROGID"] = 0 + if "ZP" not in header.keys(): + if "TMC_ZP" in header.keys(): + header["ZP"] = header["TMC_ZP"] + header["ZP_std"] = header["TMC_ZPSD"] + if "ZP_AUTO" in header.keys(): + header["ZP"] = header["ZP_AUTO"] + header["ZP_std"] = header["ZP_AUTO_std"] + data = data.astype(float) + data[data == 0.0] = np.nan return data, header From 16b931fa9d9825708743c4d29ccf5f576bf6400a Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 29 Jun 2023 13:42:12 -0700 Subject: [PATCH 74/86] Add prerequisite check --- mirar/io.py | 5 +-- mirar/processors/astromatic/swarp/swarp.py | 44 +++++++++++++++++++--- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/mirar/io.py b/mirar/io.py index 0b110a281..c03f57b75 100644 --- a/mirar/io.py +++ b/mirar/io.py @@ -11,7 +11,7 @@ from astropy.io import fits from astropy.utils.exceptions import AstropyUserWarning -from mirar.paths import BASE_NAME_KEY, LATEST_SAVE_KEY, RAW_IMG_KEY +from mirar.paths import BASE_NAME_KEY, RAW_IMG_KEY def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU: @@ -75,9 +75,6 @@ def open_fits(path: str | Path) -> tuple[np.ndarray, fits.Header]: if BASE_NAME_KEY not in header: header[BASE_NAME_KEY] = Path(path).name - if LATEST_SAVE_KEY not in header: - header[LATEST_SAVE_KEY] = path - if RAW_IMG_KEY not in header.keys(): header[RAW_IMG_KEY] = path diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index 3818f66b4..a47845f7d 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -7,8 +7,8 @@ from pathlib import Path from typing import Optional -import astropy import numpy as np +from astropy.io.fits import Header from astropy.wcs import WCS from mirar.data import Image, ImageBatch @@ -28,6 +28,7 @@ ) from mirar.processors.astromatic.scamp.scamp import scamp_header_key from mirar.processors.base_processor import BaseImageProcessor +from mirar.processors.utils.image_saver import ImageSaver from mirar.utils import execute logger = logging.getLogger(__name__) @@ -287,7 +288,6 @@ def _apply_to_images( ) logger.debug(f"Writing file list to {swarp_image_list_path}") - component_images_list = [x[LATEST_SAVE_KEY] for x in batch] temp_files = [swarp_image_list_path, swarp_weight_list_path] # If swarp is run with combine -N option, @@ -490,7 +490,13 @@ def _apply_to_images( new_image["COADDS"] = np.sum([x["COADDS"] for x in batch]) new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) - new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join(component_images_list) + + try: + component_images_list = [x[LATEST_SAVE_KEY] for x in batch] + new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join(component_images_list) + except KeyError: + pass + new_image[BASE_NAME_KEY] = output_image_path.name new_image[LATEST_WEIGHT_SAVE_KEY] = output_image_weight_path.as_posix() self.save_fits(new_image, output_image_path) @@ -513,16 +519,17 @@ class GetSwarpComponentImages(BaseImageProcessor): def __init__( self, - load_image: Callable[[str], [np.ndarray, astropy.io.fits.Header]] = open_fits, + load_image: Callable[[str], [np.ndarray, Header]] = open_fits, header_key=STACKED_COMPONENT_IMAGES_KEY, copy_header_keys: str | list[str] = None, ): super().__init__() self.load_image = load_image self.header_key = header_key - self.copy_header_keys = copy_header_keys + if isinstance(copy_header_keys, str): - self.copy_header_keys = [copy_header_keys] + copy_header_keys = [copy_header_keys] + self.copy_header_keys = copy_header_keys def _apply_to_images( self, @@ -554,3 +561,28 @@ def _apply_to_images( component_batch.append(component_image) logger.info(f"Loaded {len(component_batch)} component images") return component_batch + + def check_prerequisites( + self, + ): + mask = np.array([isinstance(x, Swarp) for x in self.preceding_steps]) + if np.sum(mask) == 0: + err = ( + f"{self.__module__} requires {Swarp} as a prerequisite. " + f"However, the following steps were found: {self.preceding_steps}." + ) + logger.error(err) + raise ValueError(err) + + index = np.argmax(mask) + + preceding_step = self.preceding_steps[index - 1] + + if not isinstance(preceding_step, ImageSaver): + err = ( + f"{self.__module__} requires an {ImageSaver} to be used to save the " + f"component images immediately before {Swarp} is run. " + f"However, the following steps were found: {self.preceding_steps}." + ) + logger.error(err) + raise ValueError(err) From 3ae7ed94f1bec67151062a894b007ce4b2e4c288 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 13:00:00 -0500 Subject: [PATCH 75/86] phot purifier --- mirar/pipelines/summer/blocks.py | 6 ++++- .../summer/config/files/photomCat.sex | 4 +-- mirar/pipelines/summer/generator.py | 25 +++++++++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/mirar/pipelines/summer/blocks.py b/mirar/pipelines/summer/blocks.py index 3345e77b0..228be078d 100644 --- a/mirar/pipelines/summer/blocks.py +++ b/mirar/pipelines/summer/blocks.py @@ -23,6 +23,7 @@ from mirar.pipelines.summer.generator import ( summer_astrometric_catalog_generator, summer_photometric_catalog_generator, + summer_photometric_img_catalog_purifier, summer_reference_image_generator, summer_reference_image_resampler, summer_reference_psfex, @@ -179,7 +180,10 @@ checkimage_type="BACKGROUND_RMS", **sextractor_photometry_config ), - PhotCalibrator(ref_catalog_generator=summer_photometric_catalog_generator), + PhotCalibrator( + ref_catalog_generator=summer_photometric_catalog_generator, + image_photometric_catalog_purifier=summer_photometric_img_catalog_purifier, + ), ImageSaver( output_dir_name="processed", # TODO: work out why this was ever here... diff --git a/mirar/pipelines/summer/config/files/photomCat.sex b/mirar/pipelines/summer/config/files/photomCat.sex index fb391e219..d6e901398 100644 --- a/mirar/pipelines/summer/config/files/photomCat.sex +++ b/mirar/pipelines/summer/config/files/photomCat.sex @@ -54,8 +54,8 @@ PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , # -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) -SATUR_LEVEL 25000 # level (in ADUs) at which arises saturation +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 1000000000.0 # level (in ADUs) at which arises saturation MAG_ZEROPOINT 0 # magnitude zero-point MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) diff --git a/mirar/pipelines/summer/generator.py b/mirar/pipelines/summer/generator.py index 82eeacf32..10e1eb953 100644 --- a/mirar/pipelines/summer/generator.py +++ b/mirar/pipelines/summer/generator.py @@ -4,6 +4,8 @@ """ import logging +from astropy.table import Table + from mirar.catalog import BaseCatalog, Gaia2Mass from mirar.catalog.vizier import PS1, SkyMapper from mirar.catalog.vizier.sdss import SDSS, NotInSDSSError, in_sdss @@ -40,6 +42,29 @@ def summer_astrometric_catalog_generator(image: Image) -> Gaia2Mass: return cat +def summer_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Table: + """ + Default function to purify the photometric image catalog + """ + edge_width_pixels = 100 + fwhm_threshold_arcsec = 4.0 + x_lower_limit = 0 # edge_width_pixels + x_upper_limit = 2800 # image.get_data().shape[1] - edge_width_pixels + y_lower_limit = 0 # edge_width_pixels + y_upper_limit = 2800 # image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + def summer_photometric_catalog_generator(image: Image) -> BaseCatalog: """ Generate a photometric calibration catalog for SUMMER images From e97c4e3c2cba152ca1e01091d3679ec6abd08ab2 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 15:44:48 -0500 Subject: [PATCH 76/86] remove SATUR_LEVEL keyword --- mirar/pipelines/summer/config/files/astrom.sex | 1 - 1 file changed, 1 deletion(-) diff --git a/mirar/pipelines/summer/config/files/astrom.sex b/mirar/pipelines/summer/config/files/astrom.sex index b5aca201c..231837b55 100644 --- a/mirar/pipelines/summer/config/files/astrom.sex +++ b/mirar/pipelines/summer/config/files/astrom.sex @@ -58,7 +58,6 @@ PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: SATUR_KEY SATURATE # keyword for saturation level (in ADUs) -SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation MAG_ZEROPOINT 0 # magnitude zero-point MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) From c8905d59de94adf6c381d6d9bcf1aabe90eae152 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 15:49:36 -0500 Subject: [PATCH 77/86] add catalog purifier --- mirar/pipelines/summer/generator.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mirar/pipelines/summer/generator.py b/mirar/pipelines/summer/generator.py index 10e1eb953..8efe76fef 100644 --- a/mirar/pipelines/summer/generator.py +++ b/mirar/pipelines/summer/generator.py @@ -38,6 +38,7 @@ def summer_astrometric_catalog_generator(image: Image) -> Gaia2Mass: trim=True, image_catalog_path=temp_cat_path, filter_name="j", + acceptable_j_ph_quals=["A", "B", "C"], ) return cat @@ -48,10 +49,10 @@ def summer_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Tab """ edge_width_pixels = 100 fwhm_threshold_arcsec = 4.0 - x_lower_limit = 0 # edge_width_pixels - x_upper_limit = 2800 # image.get_data().shape[1] - edge_width_pixels - y_lower_limit = 0 # edge_width_pixels - y_upper_limit = 2800 # image.get_data().shape[0] - edge_width_pixels + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels clean_mask = ( (catalog["FLAGS"] == 0) From c91fec5b55b1e977ce383ecd43bc56d8aa4236d0 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 15:50:24 -0500 Subject: [PATCH 78/86] update test --- tests/test_summer_pipeline.py | 40 +++++++++++++++++++++++------------ 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/tests/test_summer_pipeline.py b/tests/test_summer_pipeline.py index 6fa341b99..683845528 100644 --- a/tests/test_summer_pipeline.py +++ b/tests/test_summer_pipeline.py @@ -10,27 +10,30 @@ logger = logging.getLogger(__name__) expected_zp = { - "ZP_2.0": 24.379148900858567, - "ZP_2.0_std": 0.07320184249312667, + "ZP_2.0": 24.376073571395878, + "ZP_2.0_std": 0.07935580417736623, "ZP_2.0_nstars": 30, - "ZP_3.0": 25.072568754704793, - "ZP_3.0_std": 0.06529106709538676, + "ZP_3.0": 25.075430540593466, + "ZP_3.0_std": 0.06447897389004999, "ZP_3.0_nstars": 30, - "ZP_4.0": 25.449809682718914, - "ZP_4.0_std": 0.06168474184669056, + "ZP_4.0": 25.449352268727623, + "ZP_4.0_std": 0.061933883191802645, "ZP_4.0_nstars": 30, - "ZP_5.0": 25.66054034665426, - "ZP_5.0_std": 0.06357885150281098, + "ZP_5.0": 25.662642403793335, + "ZP_5.0_std": 0.06341347711205561, "ZP_5.0_nstars": 30, - "ZP_6.0": 25.779508960596722, - "ZP_6.0_std": 0.06472718057714481, + "ZP_6.0": 25.781888934326172, + "ZP_6.0_std": 0.0643293713949796, "ZP_6.0_nstars": 30, - "ZP_7.0": 25.851914108149213, - "ZP_7.0_std": 0.06594640100455139, + "ZP_7.0": 25.854481351725262, + "ZP_7.0_std": 0.06547451826311122, "ZP_7.0_nstars": 30, - "ZP_8.0": 25.89904772872925, - "ZP_8.0_std": 0.06650479236590319, + "ZP_8.0": 25.902204645029705, + "ZP_8.0_std": 0.06612088851303272, "ZP_8.0_nstars": 30, + "ZP_AUTO": 25.96681058044434, + "ZP_AUTO_std": 0.07490968980093107, + "ZP_AUTO_nstars": 30, } pipeline = get_pipeline( @@ -83,3 +86,12 @@ def test_pipeline(self): new_res, new_errorstack = pipeline.reduce_images( dataset=Dataset(ImageBatch()), catch_all_errors=False ) + + new_header = new_res[0][0].get_header() + + new_exp = "expected_zp = { \n" + for header_key in new_header.keys(): + if "ZP_" in header_key: + new_exp += f' "{header_key}": {new_header[header_key]}, \n' + new_exp += "}" + print(new_exp) From 6c51fa6021bb65e0f7b5fbb54c4e260b82b2e205 Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 15:56:01 -0500 Subject: [PATCH 79/86] black --- mirar/pipelines/winter/blocks.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index d0caaaaf0..f4beae78e 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -281,8 +281,7 @@ use_sextractor=True, parity="neg", search_radius_deg=1.0, - sextractor_config_path=sextractor_anet_config[ - 'config_path'], + sextractor_config_path=sextractor_anet_config["config_path"], use_weight=False, ), ImageSaver(output_dir_name=f"anet", use_existing_weight=False), From f852f1eb266a9295067364a43098df00ef76afcb Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 16:24:55 -0500 Subject: [PATCH 80/86] add edge pixels --- mirar/pipelines/winter/generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index 37040ef52..6ff3365f1 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -100,7 +100,7 @@ def winter_photometric_catalog_generator(image: Image) -> Gaia2Mass: filter_name = image["FILTER"] search_radius_arcmin = ( np.max([image["NAXIS1"], image["NAXIS2"]]) * np.abs(image["CD1_1"]) * 60 - ) / 2 + ) / 2.0 return Gaia2Mass( min_mag=10, max_mag=20, @@ -114,7 +114,7 @@ def winter_ref_photometric_img_catalog_purifier(catalog: Table, image: Image) -> """ Default function to purify the photometric image catalog """ - edge_width_pixels = 0 + edge_width_pixels = 100 fwhm_threshold_arcsec = 4.0 x_lower_limit = edge_width_pixels x_upper_limit = image.get_data().shape[1] - edge_width_pixels From 3a013fd3620561111416c714b85ad88a4734fa1f Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 16:26:20 -0500 Subject: [PATCH 81/86] update tests --- tests/test_wfau_references.py | 42 +++++++++++++++++------------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/tests/test_wfau_references.py b/tests/test_wfau_references.py index 10b8a768a..cfea35445 100644 --- a/tests/test_wfau_references.py +++ b/tests/test_wfau_references.py @@ -11,29 +11,29 @@ TEST_WINTER_FIELD_ID = 5842 expected_header = { - "ZP_2.0": 24.52299118041992, - "ZP_2.0_std": 0.0678536668419838, - "ZP_2.0_nstars": 617, - "ZP_4.0": 25.24053764343262, - "ZP_4.0_std": 0.04157085344195366, - "ZP_4.0_nstars": 616, - "ZP_5.0": 25.32686233520508, - "ZP_5.0_std": 0.03873402997851372, - "ZP_5.0_nstars": 615, - "ZP_8.0": 25.4289379119873, - "ZP_8.0_std": 0.03831901773810387, - "ZP_8.0_nstars": 616, - "ZP_10.0": 25.45285606384277, - "ZP_10.0_std": 0.03835428133606911, - "ZP_10.0_nstars": 615, + "ZP_2.0": 24.51775550842285, + "ZP_2.0_std": 0.06673479825258255, + "ZP_2.0_nstars": 522, + "ZP_4.0": 25.237825393676758, + "ZP_4.0_std": 0.041513219475746155, + "ZP_4.0_nstars": 522, + "ZP_5.0": 25.324222564697266, + "ZP_5.0_std": 0.03828829526901245, + "ZP_5.0_nstars": 520, + "ZP_8.0": 25.427032470703125, + "ZP_8.0_std": 0.038392938673496246, + "ZP_8.0_nstars": 521, + "ZP_10.0": 25.450834274291992, + "ZP_10.0_std": 0.0386686846613884, + "ZP_10.0_nstars": 521, "COADDS": 4, - "RA0_0": 282.0557287433173, - "DEC0_0": 45.35327585919234, + "RA0_0": 282.05572874331733, + "DEC0_0": 45.353275859192344, "RA1_0": 282.0582425176955, "DEC1_0": 45.82890665679109, - "RA0_1": 281.3771483296388, - "DEC0_1": 45.35301704115277, - "RA1_1": 281.3738911418988, + "RA0_1": 281.37714832963877, + "DEC0_1": 45.353017041152775, + "RA1_1": 281.37389114189875, "DEC1_1": 45.82864350802002, "FIELDID": 5842, "SUBDETID": 0, @@ -94,7 +94,7 @@ def test_pipeline(self): ) header = new_res[0][0].get_header() - print("{") + print("expected_header = {") for key, value in expected_header.items(): print(f""""{key}": {header[key]},""") print("}") From 98b0744c885c5b1e9eaac0754a2b448353cb71ff Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 17:59:28 -0500 Subject: [PATCH 82/86] reduce edge pixel --- mirar/pipelines/wirc/generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/wirc/generator.py b/mirar/pipelines/wirc/generator.py index b4a76898b..01a8991f8 100644 --- a/mirar/pipelines/wirc/generator.py +++ b/mirar/pipelines/wirc/generator.py @@ -24,7 +24,7 @@ def wirc_photometric_img_catalog_purifier(catalog, image): :return: purified catalog """ - edge_width_pixels = 200 + edge_width_pixels = 100 fwhm_threshold_arcsec = 4.0 x_lower_limit = edge_width_pixels From 7cf58ea8eac9012cf450731a1b3eac493b7af23f Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 18:01:50 -0500 Subject: [PATCH 83/86] read info before swarp overwrites it to temporary files --- mirar/processors/astromatic/swarp/swarp.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index a47845f7d..2c766b6f4 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -320,6 +320,11 @@ def _apply_to_images( ) logger.debug(f"Saving to {output_image_path}") + try: + component_images_list = [x[LATEST_SAVE_KEY] for x in batch] + except KeyError: + component_images_list = None + all_pixscales = [] all_imgpixsizes = [] all_ras = [] @@ -491,11 +496,8 @@ def _apply_to_images( new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) - try: - component_images_list = [x[LATEST_SAVE_KEY] for x in batch] + if component_images_list is not None: new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join(component_images_list) - except KeyError: - pass new_image[BASE_NAME_KEY] = output_image_path.name new_image[LATEST_WEIGHT_SAVE_KEY] = output_image_weight_path.as_posix() From d4a8dd34caf1440f1feb04e7a177fa67343677af Mon Sep 17 00:00:00 2001 From: viraj21197 Date: Thu, 29 Jun 2023 18:02:25 -0500 Subject: [PATCH 84/86] update test --- tests/test_wirc_pipeline.py | 31 ++++++++++++++----------------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/tests/test_wirc_pipeline.py b/tests/test_wirc_pipeline.py index c7a437de8..1164c00c9 100644 --- a/tests/test_wirc_pipeline.py +++ b/tests/test_wirc_pipeline.py @@ -19,23 +19,20 @@ test_data_dir = get_test_data_dir() expected_zp = { - "ZP_2.0": 25.908477783203125, - "ZP_2.0_std": 0.11761965602636337, - "ZP_2.0_nstars": 12, - "ZP_4.0": 27.235090255737305, - "ZP_4.0_std": 0.1022496446967125, - "ZP_4.0_nstars": 12, - "ZP_5.0": 27.5942440032959, - "ZP_5.0_std": 0.09162138402462006, - "ZP_5.0_nstars": 12, - "ZP_8.0": 28.16297149658203, - "ZP_8.0_std": 0.07523621618747711, - "ZP_8.0_nstars": 12, - "ZP_10.0": 28.324859619140625, - "ZP_10.0_std": 0.07975760102272034, - "ZP_10.0_nstars": 12, - "ZP_AUTO": 28.434249877929688, - "ZP_AUTO_std": 0.14610908925533295, + "ZP_6.0": 27.760366439819336, + "ZP_6.0_std": 0.2747618854045868, + "ZP_6.0_nstars": 13, + "ZP_10.0": 28.341157913208008, + "ZP_10.0_std": 0.09141725301742554, + "ZP_10.0_nstars": 11, + "ZP_14.0": 28.417837142944336, + "ZP_14.0_std": 0.13541148602962494, + "ZP_14.0_nstars": 13, + "ZP_18.0": 28.475311279296875, + "ZP_18.0_std": 0.10811392217874527, + "ZP_18.0_nstars": 13, + "ZP_AUTO": 28.48914909362793, + "ZP_AUTO_std": 0.09673704952001572, "ZP_AUTO_nstars": 12, } From be2f02b800223e378a1e8956c124c0fb9cfb8451 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 29 Jun 2023 21:00:26 -0700 Subject: [PATCH 85/86] Lintify --- mirar/pipelines/__init__.py | 22 +++++- mirar/pipelines/summer/__init__.py | 3 + mirar/pipelines/winter/__init__.py | 3 + mirar/pipelines/winter/blocks.py | 79 +++++++++---------- mirar/pipelines/winter/models/_fields.py | 2 - mirar/pipelines/winter/models/_proc.py | 10 +-- .../winter/models/_ref_components.py | 2 +- mirar/pipelines/winter/models/base_model.py | 4 +- mirar/processors/__init__.py | 3 + mirar/references/ukirt.py | 3 +- 10 files changed, 71 insertions(+), 60 deletions(-) diff --git a/mirar/pipelines/__init__.py b/mirar/pipelines/__init__.py index 422887fec..4730ba594 100644 --- a/mirar/pipelines/__init__.py +++ b/mirar/pipelines/__init__.py @@ -1,3 +1,6 @@ +""" +Central location for all pipelines. This is where you should add new pipelines. +""" import logging from mirar.errors import ProcessorError @@ -14,19 +17,30 @@ class PipelineConfigError(ProcessorError, KeyError): - pass + """ + Error raised when a pipeline is not found + """ -def get_pipeline(instrument, selected_configurations=None, *args, **kwargs): +def get_pipeline(instrument, selected_configurations=None, *args, **kwargs) -> Pipeline: + """ + Function to get pipeline + + :param instrument: Name of instrument + :param selected_configurations: Configurations to use + :param args: args + :param kwargs: kwargs + :return: pipeline + """ try: pipeline = Pipeline.pipelines[instrument.lower()] logger.info(f"Found {instrument} pipeline") - except KeyError: + except KeyError as exc: err = ( f"Unrecognised pipeline {instrument}. " f"Available pipelines are: {Pipeline.pipelines.keys()}" ) logger.error(err) - raise PipelineConfigError(err) + raise PipelineConfigError(err) from exc return pipeline(selected_configurations=selected_configurations, *args, **kwargs) diff --git a/mirar/pipelines/summer/__init__.py b/mirar/pipelines/summer/__init__.py index 66eb04b02..daa168a95 100644 --- a/mirar/pipelines/summer/__init__.py +++ b/mirar/pipelines/summer/__init__.py @@ -1 +1,4 @@ +""" +Pipline for SUMMER data +""" from mirar.pipelines.summer.summer_pipeline import SummerPipeline diff --git a/mirar/pipelines/winter/__init__.py b/mirar/pipelines/winter/__init__.py index 5dfbb5a05..43782b5c2 100644 --- a/mirar/pipelines/winter/__init__.py +++ b/mirar/pipelines/winter/__init__.py @@ -1 +1,4 @@ +""" +Pipeline for WINTER data +""" from mirar.pipelines.winter.winter_pipeline import WINTERPipeline diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index f4beae78e..a70f3629a 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -11,7 +11,6 @@ from mirar.pipelines.winter.generator import ( scamp_config_path, winter_astrometric_catalog_generator, - winter_mask_path, winter_photometric_catalog_generator, winter_reference_generator, ) @@ -29,12 +28,10 @@ from mirar.processors.astrometry.anet.anet_processor import AstrometryNet from mirar.processors.csvlog import CSVLog from mirar.processors.dark import DarkCalibrator -from mirar.processors.flat import FlatCalibrator from mirar.processors.mask import MaskPixelsFromPath, WriteMaskedCoordsToFile from mirar.processors.photcal import PhotCalibrator from mirar.processors.reference import GetReferenceImage from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator -from mirar.processors.split import SplitImage from mirar.processors.utils import ( ImageBatcher, ImageDebatcher, @@ -52,14 +49,14 @@ ImageSaver(output_dir_name="stacked_ref"), ] -board_id = 4 -target_name = "EMGW_gal1" +BOARD_ID = 4 +TARGET_NAME = "EMGW_gal1" split = [ MultiExtParser( input_sub_dir="raw/", extension_num_header_key="BOARD_ID", - only_extract_num=board_id, - output_sub_dir=f"raw_split_{board_id}", + only_extract_num=BOARD_ID, + output_sub_dir=f"raw_split_{BOARD_ID}", ) ] @@ -72,7 +69,7 @@ ] load = [ ImageLoader( - input_sub_dir=f"raw_split_{board_id}", load_image=load_raw_winter_image + input_sub_dir=f"raw_split_{BOARD_ID}", load_image=load_raw_winter_image ), ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), ] @@ -83,32 +80,32 @@ ] load_proc = [ - ImageLoader(input_sub_dir=f"skysub_{board_id}", load_image=load_raw_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), + ImageLoader(input_sub_dir=f"skysub_{BOARD_ID}", load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE")), ] load_dark = [ - ImageLoader(input_sub_dir=f"darkcal_{board_id}", load_image=load_raw_winter_image), - ImageSelector(("TARGNAME", f"{target_name}")), + ImageLoader(input_sub_dir=f"darkcal_{BOARD_ID}", load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}")), ] load_anet = [ - ImageLoader(input_sub_dir=f"anet_{board_id}", load_image=load_proc_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), + ImageLoader(input_sub_dir=f"anet_{BOARD_ID}", load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE")), ] load_stack = [ - ImageLoader(input_sub_dir=f"anet_{board_id}", load_image=load_proc_winter_image), - ImageSelector(("TARGNAME", f"{target_name}"), ("OBSTYPE", "SCIENCE")), + ImageLoader(input_sub_dir=f"anet_{BOARD_ID}", load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE")), ImageBatcher("EXPTIME"), ] load_multiboard_stack = [ ImageLoader( - input_sub_dir=f"stack_all_{target_name}", load_image=load_stacked_winter_image + input_sub_dir=f"stack_all_{TARGET_NAME}", load_image=load_stacked_winter_image ), ImageSelector( - ("TARGNAME", f"{target_name}"), + ("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE"), ), ] @@ -168,11 +165,11 @@ ) dark_cal = [ - ImageSelector(("BOARD_ID", f"{board_id}")), + ImageSelector(("BOARD_ID", f"{BOARD_ID}")), ImageBatcher(["BOARD_ID", "EXPTIME"]), WriteMaskedCoordsToFile(output_dir="mask_raw"), - DarkCalibrator(cache_sub_dir=f"calibration_{board_id}"), - ImageSaver(output_dir_name=f"darkcal_{board_id}"), + DarkCalibrator(cache_sub_dir=f"calibration_{BOARD_ID}"), + ImageSaver(output_dir_name=f"darkcal_{BOARD_ID}"), ImageDebatcher(), ] @@ -191,11 +188,11 @@ # FlatCalibrator(flat_mask_key=FITS_MASK_KEY, # cache_sub_dir=f"calibration_{board_id}" # ), - ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{target_name}")), + ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{TARGET_NAME}")), ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), - SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir=f"skycals_{board_id}"), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir=f"skycals_{BOARD_ID}"), ImageSelector(("OBSTYPE", ["SCIENCE"])), - ImageSaver(output_dir_name=f"skyflatcal_{board_id}"), + ImageSaver(output_dir_name=f"skyflatcal_{BOARD_ID}"), # ImageSelector(("OBSTYPE", ["SCIENCE"])), # Sextractor(**sextractor_astrometry_config, # write_regions_bool=True, @@ -204,7 +201,7 @@ # ImageSelector(("TARGNAME", [""])), # ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), - ImageSaver(output_dir_name=f"skysub_{board_id}"), + ImageSaver(output_dir_name=f"skysub_{BOARD_ID}"), # Sextractor(**sextractor_astrometry_config, # write_regions_bool=True, # output_sub_dir="sextractor", @@ -224,7 +221,7 @@ ] flat_cal_all_boards = [ - ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{target_name}")), + ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{TARGET_NAME}")), ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir="skycals"), ImageSelector(("OBSTYPE", ["SCIENCE"])), @@ -237,7 +234,7 @@ process_proc = [ ImageDebatcher(), AstrometryNet( - output_sub_dir=f"anet_{board_id}", + output_sub_dir=f"anet_{BOARD_ID}", scale_bounds=[25, 40], scale_units="amw", use_sextractor=True, @@ -247,7 +244,7 @@ # 'config_path'], use_weight=False, ), - ImageSaver(output_dir_name=f"anet_{board_id}", use_existing_weight=False), + ImageSaver(output_dir_name=f"anet_{BOARD_ID}", use_existing_weight=False), Sextractor( **sextractor_autoastrometry_config, write_regions_bool=True, @@ -267,7 +264,7 @@ subtract_bkg=False, cache=True, center_type="ALL", - temp_output_sub_dir=f"stack_all_{target_name}", + temp_output_sub_dir=f"stack_all_{TARGET_NAME}", ), ] @@ -284,7 +281,7 @@ sextractor_config_path=sextractor_anet_config["config_path"], use_weight=False, ), - ImageSaver(output_dir_name=f"anet", use_existing_weight=False), + ImageSaver(output_dir_name="anet", use_existing_weight=False), Sextractor( **sextractor_autoastrometry_config, write_regions_bool=True, @@ -306,7 +303,7 @@ subtract_bkg=False, cache=False, center_type="ALL", - temp_output_sub_dir=f"stack_all_{target_name}", + temp_output_sub_dir=f"stack_all_{TARGET_NAME}", ), ] @@ -324,7 +321,7 @@ write_masked_pixels_to_file=True, output_dir="mask1", ), - ImageSaver(output_dir_name=f"noisemask_{board_id}"), + ImageSaver(output_dir_name=f"noisemask_{BOARD_ID}"), ] # process_proc = [ImageDebatcher(), # ImageSelector(("OBSTYPE", ["SCIENCE"])), @@ -347,7 +344,7 @@ cache=True, center_type="MANUAL", ), - ImageSaver(output_dir_name=f"stack_{target_name}"), + ImageSaver(output_dir_name=f"stack_{TARGET_NAME}"), ] photcal = [ @@ -356,35 +353,35 @@ ImageBatcher(["BOARD_ID"]), Sextractor( **sextractor_photometry_config, - output_sub_dir=f"phot_{board_id}_{target_name}", + output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", checkimage_type="BACKGROUND_RMS", ), PhotCalibrator( ref_catalog_generator=winter_photometric_catalog_generator, - temp_output_sub_dir=f"phot_{board_id}_{target_name}", + temp_output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", write_regions=True, cache=True, ), # ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}") - ImageSaver(output_dir_name=f"phot_{target_name}"), + ImageSaver(output_dir_name=f"phot_{TARGET_NAME}"), ] photcal_indiv = [ - ImageSelector(("BOARD_ID", board_id)), + ImageSelector(("BOARD_ID", BOARD_ID)), ImageDebatcher(), ImageBatcher(["UTCTIME"]), Sextractor( **sextractor_photometry_config, - output_sub_dir=f"phot_{board_id}_{target_name}", + output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", checkimage_type="BACKGROUND_RMS", ), PhotCalibrator( ref_catalog_generator=winter_photometric_catalog_generator, - temp_output_sub_dir=f"phot_{board_id}_{target_name}", + temp_output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", write_regions=True, cache=True, ), - ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}"), + ImageSaver(output_dir_name=f"phot_{BOARD_ID}_{TARGET_NAME}"), ] stack_multiboard = [ @@ -394,7 +391,7 @@ include_scamp=False, subtract_bkg=False, cache=True, - temp_output_sub_dir=f"multiboard_stack_{target_name}", + temp_output_sub_dir=f"multiboard_stack_{TARGET_NAME}", center_type="MANUAL", ) ] diff --git a/mirar/pipelines/winter/models/_fields.py b/mirar/pipelines/winter/models/_fields.py index 969167b1d..ca02eb1b8 100644 --- a/mirar/pipelines/winter/models/_fields.py +++ b/mirar/pipelines/winter/models/_fields.py @@ -2,10 +2,8 @@ Models for the 'fields' table """ import time -import urllib.request from typing import ClassVar -import pandas as pd from pydantic import Field from sqlalchemy import REAL, Column, Insert, Integer, Select from sqlalchemy.orm import Mapped, relationship diff --git a/mirar/pipelines/winter/models/_proc.py b/mirar/pipelines/winter/models/_proc.py index fe892cd90..29f2d961c 100644 --- a/mirar/pipelines/winter/models/_proc.py +++ b/mirar/pipelines/winter/models/_proc.py @@ -5,15 +5,7 @@ from typing import ClassVar from pydantic import Field, validator -from sqlalchemy import ( # event, - REAL, - VARCHAR, - Column, - Double, - ForeignKey, - Integer, - Sequence, -) +from sqlalchemy import REAL, VARCHAR, Column, ForeignKey, Integer, Sequence # event, from sqlalchemy.orm import Mapped, relationship from mirar.pipelines.winter.models._raw import Raw diff --git a/mirar/pipelines/winter/models/_ref_components.py b/mirar/pipelines/winter/models/_ref_components.py index e76e4c31d..8fb6a0f49 100644 --- a/mirar/pipelines/winter/models/_ref_components.py +++ b/mirar/pipelines/winter/models/_ref_components.py @@ -5,7 +5,7 @@ from pydantic import Field from sqlalchemy import VARCHAR, Column, Double, Float, Integer -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, relationship from mirar.pipelines.winter.models.base_model import WinterBase, dec_field, ra_field from mirar.processors.sqldatabase.base_model import BaseDB diff --git a/mirar/pipelines/winter/models/base_model.py b/mirar/pipelines/winter/models/base_model.py index 62f8f5d79..1b6f6b3f5 100644 --- a/mirar/pipelines/winter/models/base_model.py +++ b/mirar/pipelines/winter/models/base_model.py @@ -8,7 +8,7 @@ from mirar.processors.sqldatabase.base_model import BaseTable -db_name = "winter" +DB_NAME = "winter" class WinterBase(DeclarativeBase, BaseTable): @@ -16,7 +16,7 @@ class WinterBase(DeclarativeBase, BaseTable): Parent class for summer database """ - db_name = db_name + DB_NAME = DB_NAME ra_field: float = Field(title="RA (degrees)", ge=0.0, le=360.0) diff --git a/mirar/processors/__init__.py b/mirar/processors/__init__.py index 60653314d..38669a20b 100644 --- a/mirar/processors/__init__.py +++ b/mirar/processors/__init__.py @@ -1,3 +1,6 @@ +""" +This module contains the processors that are used to process the raw data +""" # import logging from mirar.processors.base_processor import BaseImageProcessor from mirar.processors.bias import BiasCalibrator diff --git a/mirar/references/ukirt.py b/mirar/references/ukirt.py index 5bb4255ce..d9ddd25e7 100644 --- a/mirar/references/ukirt.py +++ b/mirar/references/ukirt.py @@ -547,7 +547,8 @@ def get_reference(self, image: Image) -> tuple[PrimaryHDU, PrimaryHDU]: logger.debug(f"Reading image from {url}") with fits.open(url, ignore_missing_simple=True) as ukirt_hdulist: ukirt_image = Image( - header=ukirt_hdulist[0].header, data=ukirt_hdulist[0].data + header=ukirt_hdulist[0].header, # pylint: disable=no-member + data=ukirt_hdulist[0].data, # pylint: disable=no-member ) if self.check_local_database & ~qexists: From e30801032fd91d5b0d197ba9beae04318f648a68 Mon Sep 17 00:00:00 2001 From: Robert Stein Date: Thu, 29 Jun 2023 22:08:29 -0700 Subject: [PATCH 86/86] Fix typo --- mirar/pipelines/winter/models/base_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mirar/pipelines/winter/models/base_model.py b/mirar/pipelines/winter/models/base_model.py index 1b6f6b3f5..84fec76ca 100644 --- a/mirar/pipelines/winter/models/base_model.py +++ b/mirar/pipelines/winter/models/base_model.py @@ -16,7 +16,7 @@ class WinterBase(DeclarativeBase, BaseTable): Parent class for summer database """ - DB_NAME = DB_NAME + db_name = DB_NAME ra_field: float = Field(title="RA (degrees)", ge=0.0, le=360.0)