From 9a75d878b4623dc67c9df28dd1d56f5bc7cf3946 Mon Sep 17 00:00:00 2001 From: Viraj Karambelkar Date: Fri, 30 Jun 2023 00:29:07 -0500 Subject: [PATCH] Wirc full pipeline (#412) Co-authored-by: Robert Stein --- mirar/catalog/gaia.py | 45 +- mirar/io.py | 8 + mirar/paths.py | 5 +- mirar/pipelines/__init__.py | 22 +- mirar/pipelines/sedmv2/blocks.py | 6 +- mirar/pipelines/summer/__init__.py | 3 + mirar/pipelines/summer/blocks.py | 14 +- .../pipelines/summer/config/files/astrom.sex | 1 - .../summer/config/files/photomCat.sex | 4 +- mirar/pipelines/summer/generator.py | 26 ++ mirar/pipelines/winter/__init__.py | 3 + mirar/pipelines/winter/blocks.py | 408 +++++++++++++++++- mirar/pipelines/winter/config/__init__.py | 37 ++ .../pipelines/winter/config/files/Scorr.param | 134 ++++++ .../winter/config/files/astrom.param | 30 ++ .../pipelines/winter/config/files/astrom.sex | 111 +++++ .../winter/config/files/astrom_anet.sex | 111 +++++ .../winter/config/files/config.swarp | 102 +++++ .../winter/config/files/default.conv | 5 + .../pipelines/winter/config/files/default.nnw | 28 ++ .../winter/config/files/photom.param | 28 ++ .../winter/config/files/photom.psfex | 58 +++ .../winter/config/files/photomCat.sex | 97 +++++ .../pipelines/winter/config/files/scamp.conf | 142 ++++++ .../pipelines/winter/config/files/sex.config | 74 ++++ mirar/pipelines/winter/config/files/sex.conv | 5 + .../pipelines/winter/config/files/temp.param | 9 + mirar/pipelines/winter/fix_headers.py | 46 ++ mirar/pipelines/winter/generator.py | 57 ++- mirar/pipelines/winter/load_winter_image.py | 243 +++++++++++ mirar/pipelines/winter/models/_fields.py | 2 - mirar/pipelines/winter/models/_proc.py | 10 +- .../winter/models/_ref_components.py | 2 +- mirar/pipelines/winter/models/base_model.py | 4 +- mirar/pipelines/winter/winter_pipeline.py | 46 +- mirar/pipelines/wirc/blocks.py | 98 ++++- mirar/pipelines/wirc/generator.py | 35 +- mirar/pipelines/wirc/load_wirc_image.py | 85 ++-- mirar/pipelines/wirc/wirc_files/__init__.py | 7 + .../wirc/wirc_files/files/focusloop.sex | 0 .../wirc/wirc_files/files/matchcat.sex | 107 +++++ mirar/processors/__init__.py | 8 +- mirar/processors/anet/__init__.py | 6 - mirar/processors/anet/anet_processor.py | 99 ----- .../processors/astromatic/config/__init__.py | 3 + mirar/processors/astromatic/psfex/__init__.py | 3 + .../astromatic/sextractor/settings.py | 15 +- .../astromatic/sextractor/sextractor.py | 32 +- .../astromatic/sextractor/sourceextractor.py | 43 +- mirar/processors/astromatic/swarp/swarp.py | 131 +++++- mirar/processors/astrometry/__init__.py | 0 mirar/processors/astrometry/anet/__init__.py | 6 + .../processors/{ => astrometry}/anet/anet.py | 22 +- .../astrometry/anet/anet_processor.py | 167 +++++++ .../autoastrometry/__init__.py | 5 +- .../autoastrometry/autoastrometry.py | 23 +- .../autoastrometry_processor.py | 4 +- .../autoastrometry/crossmatch.py | 4 +- .../{ => astrometry}/autoastrometry/detect.py | 53 ++- .../{ => astrometry}/autoastrometry/errors.py | 0 .../{ => astrometry}/autoastrometry/io.py | 9 +- .../autoastrometry/reference.py | 8 +- .../autoastrometry/sources.py | 0 .../{ => astrometry}/autoastrometry/utils.py | 0 mirar/processors/astrometry/utils.py | 70 +++ mirar/processors/base_processor.py | 7 +- mirar/processors/bias.py | 5 +- mirar/processors/dark.py | 13 +- mirar/processors/flat.py | 48 ++- mirar/processors/mask.py | 242 ++++++++++- mirar/processors/photcal.py | 223 +++++----- mirar/processors/sky.py | 8 +- mirar/processors/utils/image_loader.py | 42 +- mirar/processors/utils/image_saver.py | 14 +- mirar/processors/utils/multi_ext_parser.py | 57 ++- mirar/references/ukirt.py | 3 +- tests/test_summer_pipeline.py | 40 +- tests/test_wfau_references.py | 42 +- tests/test_wirc_pipeline.py | 104 ++--- 79 files changed, 3263 insertions(+), 554 deletions(-) create mode 100644 mirar/pipelines/winter/config/__init__.py create mode 100644 mirar/pipelines/winter/config/files/Scorr.param create mode 100644 mirar/pipelines/winter/config/files/astrom.param create mode 100644 mirar/pipelines/winter/config/files/astrom.sex create mode 100644 mirar/pipelines/winter/config/files/astrom_anet.sex create mode 100644 mirar/pipelines/winter/config/files/config.swarp create mode 100644 mirar/pipelines/winter/config/files/default.conv create mode 100644 mirar/pipelines/winter/config/files/default.nnw create mode 100644 mirar/pipelines/winter/config/files/photom.param create mode 100644 mirar/pipelines/winter/config/files/photom.psfex create mode 100644 mirar/pipelines/winter/config/files/photomCat.sex create mode 100644 mirar/pipelines/winter/config/files/scamp.conf create mode 100644 mirar/pipelines/winter/config/files/sex.config create mode 100644 mirar/pipelines/winter/config/files/sex.conv create mode 100644 mirar/pipelines/winter/config/files/temp.param create mode 100644 mirar/pipelines/winter/fix_headers.py create mode 100644 mirar/pipelines/winter/load_winter_image.py create mode 100644 mirar/pipelines/wirc/wirc_files/files/focusloop.sex create mode 100644 mirar/pipelines/wirc/wirc_files/files/matchcat.sex delete mode 100644 mirar/processors/anet/__init__.py delete mode 100644 mirar/processors/anet/anet_processor.py create mode 100644 mirar/processors/astrometry/__init__.py create mode 100644 mirar/processors/astrometry/anet/__init__.py rename mirar/processors/{ => astrometry}/anet/anet.py (80%) create mode 100644 mirar/processors/astrometry/anet/anet_processor.py rename mirar/processors/{ => astrometry}/autoastrometry/__init__.py (58%) rename mirar/processors/{ => astrometry}/autoastrometry/autoastrometry.py (97%) rename mirar/processors/{ => astrometry}/autoastrometry/autoastrometry_processor.py (95%) rename mirar/processors/{ => astrometry}/autoastrometry/crossmatch.py (99%) rename mirar/processors/{ => astrometry}/autoastrometry/detect.py (82%) rename mirar/processors/{ => astrometry}/autoastrometry/errors.py (100%) rename mirar/processors/{ => astrometry}/autoastrometry/io.py (98%) rename mirar/processors/{ => astrometry}/autoastrometry/reference.py (97%) rename mirar/processors/{ => astrometry}/autoastrometry/sources.py (100%) rename mirar/processors/{ => astrometry}/autoastrometry/utils.py (100%) create mode 100644 mirar/processors/astrometry/utils.py diff --git a/mirar/catalog/gaia.py b/mirar/catalog/gaia.py index f7db5fd17..af175f8fb 100644 --- a/mirar/catalog/gaia.py +++ b/mirar/catalog/gaia.py @@ -6,6 +6,7 @@ import astropy.table import astropy.units as u +import numpy as np from astropy.coordinates import SkyCoord from astroquery.gaia import Gaia @@ -26,18 +27,40 @@ def __init__( self, *args, filter_name: str = "j", - ph_qual_cut: bool = False, snr_threshold: float = 5, trim: bool = False, image_catalog_path: Optional[str] = None, + acceptable_j_ph_quals: str | list[str] = None, + acceptable_h_ph_quals: str | list[str] = None, + acceptable_k_ph_quals: str | list[str] = None, **kwargs, ): super().__init__(*args, filter_name=filter_name, **kwargs) - self.ph_qual_cut = ph_qual_cut + self.trim = trim self.image_catalog_path = image_catalog_path self.snr_threshold = snr_threshold + if isinstance(acceptable_j_ph_quals, str): + acceptable_j_ph_quals = [acceptable_j_ph_quals] + if isinstance(acceptable_h_ph_quals, str): + acceptable_h_ph_quals = [acceptable_h_ph_quals] + if isinstance(acceptable_k_ph_quals, str): + acceptable_k_ph_quals = [acceptable_k_ph_quals] + + self.acceptable_ph_quals = { + "j": acceptable_j_ph_quals, + "h": acceptable_h_ph_quals, + "k": acceptable_k_ph_quals, + } + + if self.acceptable_ph_quals[self.filter_name.lower()] is None: + self.acceptable_ph_quals[self.filter_name.lower()] = ["A"] + + for filt in self.acceptable_ph_quals: + if self.acceptable_ph_quals[filt] is None: + self.acceptable_ph_quals[filt] = ["A", "B", "C"] + logger.debug(f"Sextractor catalog path is {self.image_catalog_path}") def get_catalog( @@ -62,14 +85,9 @@ def get_catalog( f"AND tmass.{self.filter_name}_m > {self.min_mag:.2f} " f"AND tmass.{self.filter_name}_m < {self.max_mag:.2f} " f"AND tbest.number_of_mates=0 " - f"AND tbest.number_of_neighbours=1" + f"AND tbest.number_of_neighbours=1;" ) - if self.ph_qual_cut: - cmd += "AND tmass.ph_qual='AAA';" - else: - cmd += ";" - job = Gaia.launch_job_async(cmd, dump_to_file=False) src_list = job.get_results() src_list["ph_qual"] = src_list["ph_qual"].astype(str) @@ -80,6 +98,17 @@ def get_catalog( src_list["magnitude_err"] = src_list[f"{self.filter_name.lower()}_msigcom"] logger.info(f"Found {len(src_list)} sources in Gaia") + j_phquals = [x[0] for x in src_list["ph_qual"]] + h_phquals = [x[0] for x in src_list["ph_qual"]] + k_phquals = [x[0] for x in src_list["ph_qual"]] + + j_phmask = np.array([x in self.acceptable_ph_quals["j"] for x in j_phquals]) + h_phmask = np.array([x in self.acceptable_ph_quals["h"] for x in h_phquals]) + k_phmask = np.array([x in self.acceptable_ph_quals["k"] for x in k_phquals]) + + phmask = j_phmask & h_phmask & k_phmask + + src_list = src_list[phmask] src_list = src_list[src_list["magnitude_err"] < 1.086 / self.snr_threshold] if self.trim: if self.image_catalog_path is None: diff --git a/mirar/io.py b/mirar/io.py index 22fb8e71f..c03f57b75 100644 --- a/mirar/io.py +++ b/mirar/io.py @@ -11,6 +11,8 @@ from astropy.io import fits from astropy.utils.exceptions import AstropyUserWarning +from mirar.paths import BASE_NAME_KEY, RAW_IMG_KEY + def create_fits(data: np.ndarray, header: fits.Header | None) -> fits.PrimaryHDU: """ @@ -70,6 +72,12 @@ def open_fits(path: str | Path) -> tuple[np.ndarray, fits.Header]: data = hdu.data header = hdu.header + if BASE_NAME_KEY not in header: + header[BASE_NAME_KEY] = Path(path).name + + if RAW_IMG_KEY not in header.keys(): + header[RAW_IMG_KEY] = path + return data, header diff --git a/mirar/paths.py b/mirar/paths.py index 2c8dedaf8..005908f47 100644 --- a/mirar/paths.py +++ b/mirar/paths.py @@ -238,6 +238,7 @@ def get_astrometry_keys() -> list: UNC_IMG_KEY = "UNCPATH" PROC_HISTORY_KEY = "CALSTEPS" PROC_FAIL_KEY = "PROCFAIL" +ASTROMETRY_FILE_KEY = "ASTRFILE" LATEST_SAVE_KEY = "SAVEPATH" LATEST_WEIGHT_SAVE_KEY = "WGHTPATH" SEXTRACTOR_HEADER_KEY = "SRCCAT" @@ -252,6 +253,7 @@ def get_astrometry_keys() -> list: GAIN_KEY = "GAIN" EXPTIME_KEY = "AEXPTIME" ZP_KEY = "ZP" +SATURATE_KEY = "SATURATE" PSF_FLUX_KEY = "psf_flux" PSF_FLUXUNC_KEY = "psf_fluxunc" MAG_PSF_KEY = "magpsf" @@ -261,13 +263,14 @@ def get_astrometry_keys() -> list: CAND_NAME_KEY = "objectId" CAND_RA_KEY = "ra" CAND_DEC_KEY = "dec" +FITS_MASK_KEY = "MASKFITS" sextractor_checkimg_keys = { "BACKGROUND": "BKGPT", "BACKGROUND_RMS": "BKGRMS", "MINIBACKGROUND": "MINIBKG", "MINIBACK_RMS": "MINIBGRM", } - +STACKED_COMPONENT_IMAGES_KEY = "COMPENTS" APFLUX_PREFIX_KEY = "fluxap" APFLUXUNC_PREFIX_KEY = "fluxucap" APMAG_PREFIX_KEY = "magap" diff --git a/mirar/pipelines/__init__.py b/mirar/pipelines/__init__.py index 422887fec..4730ba594 100644 --- a/mirar/pipelines/__init__.py +++ b/mirar/pipelines/__init__.py @@ -1,3 +1,6 @@ +""" +Central location for all pipelines. This is where you should add new pipelines. +""" import logging from mirar.errors import ProcessorError @@ -14,19 +17,30 @@ class PipelineConfigError(ProcessorError, KeyError): - pass + """ + Error raised when a pipeline is not found + """ -def get_pipeline(instrument, selected_configurations=None, *args, **kwargs): +def get_pipeline(instrument, selected_configurations=None, *args, **kwargs) -> Pipeline: + """ + Function to get pipeline + + :param instrument: Name of instrument + :param selected_configurations: Configurations to use + :param args: args + :param kwargs: kwargs + :return: pipeline + """ try: pipeline = Pipeline.pipelines[instrument.lower()] logger.info(f"Found {instrument} pipeline") - except KeyError: + except KeyError as exc: err = ( f"Unrecognised pipeline {instrument}. " f"Available pipelines are: {Pipeline.pipelines.keys()}" ) logger.error(err) - raise PipelineConfigError(err) + raise PipelineConfigError(err) from exc return pipeline(selected_configurations=selected_configurations, *args, **kwargs) diff --git a/mirar/pipelines/sedmv2/blocks.py b/mirar/pipelines/sedmv2/blocks.py index 6cd106120..c5b3712fc 100644 --- a/mirar/pipelines/sedmv2/blocks.py +++ b/mirar/pipelines/sedmv2/blocks.py @@ -23,10 +23,10 @@ ) from mirar.pipelines.sedmv2.load_sedmv2_image import load_raw_sedmv2_image from mirar.processors import BiasCalibrator, FlatCalibrator -from mirar.processors.anet import AstrometryNet from mirar.processors.astromatic import PSFex, Sextractor, Swarp +from mirar.processors.astrometry.anet import AstrometryNet from mirar.processors.csvlog import CSVLog -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import ( CandidateAperturePhotometry, @@ -79,7 +79,7 @@ ] # pylint: disable=duplicate-code reduce = [ - MaskPixels(mask_path=sedmv2_mask_path), + MaskPixelsFromPath(mask_path=sedmv2_mask_path), BiasCalibrator(), ImageSelector(("OBSTYPE", ["FLAT", "SCIENCE"])), ImageBatcher(split_key="filter"), diff --git a/mirar/pipelines/summer/__init__.py b/mirar/pipelines/summer/__init__.py index 66eb04b02..daa168a95 100644 --- a/mirar/pipelines/summer/__init__.py +++ b/mirar/pipelines/summer/__init__.py @@ -1 +1,4 @@ +""" +Pipline for SUMMER data +""" from mirar.pipelines.summer.summer_pipeline import SummerPipeline diff --git a/mirar/pipelines/summer/blocks.py b/mirar/pipelines/summer/blocks.py index cc892b6fe..228be078d 100644 --- a/mirar/pipelines/summer/blocks.py +++ b/mirar/pipelines/summer/blocks.py @@ -23,6 +23,7 @@ from mirar.pipelines.summer.generator import ( summer_astrometric_catalog_generator, summer_photometric_catalog_generator, + summer_photometric_img_catalog_purifier, summer_reference_image_generator, summer_reference_image_resampler, summer_reference_psfex, @@ -35,7 +36,7 @@ from mirar.pipelines.summer.models import Exposure, Proc, Raw from mirar.processors import BiasCalibrator, FlatCalibrator from mirar.processors.astromatic import PSFex, Scamp, Sextractor, Swarp -from mirar.processors.autoastrometry import AutoAstrometry +from mirar.processors.astrometry.autoastrometry import AutoAstrometry from mirar.processors.candidates.candidate_detector import DetectCandidates from mirar.processors.candidates.utils import DataframeWriter, RegionsWriter from mirar.processors.cosmic_rays import LACosmicCleaner @@ -44,7 +45,7 @@ DatabaseImageExporter as PSQLDatabaseImageExporter, ) from mirar.processors.database.database_modifier import ModifyImageDatabaseSeq -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import CandidateAperturePhotometry from mirar.processors.photometry.psf_photometry import CandidatePSFPhotometry @@ -123,7 +124,7 @@ duplicate_protocol="replace", q3c_bool=False, ), - MaskPixels(mask_path=summer_mask_path), + MaskPixelsFromPath(mask_path=summer_mask_path), DatabaseImageExporter(db_table=Raw, duplicate_protocol="replace", q3c_bool=False), ImageSelector(("OBSTYPE", ["BIAS", "FLAT", "SCIENCE"])), ] @@ -133,7 +134,7 @@ ] test_cr = [ - MaskPixels(mask_path=summer_mask_path), + MaskPixelsFromPath(mask_path=summer_mask_path), BiasCalibrator(), ImageSelector(("OBSTYPE", ["FLAT", "SCIENCE"])), ImageBatcher(split_key="filter"), @@ -179,7 +180,10 @@ checkimage_type="BACKGROUND_RMS", **sextractor_photometry_config ), - PhotCalibrator(ref_catalog_generator=summer_photometric_catalog_generator), + PhotCalibrator( + ref_catalog_generator=summer_photometric_catalog_generator, + image_photometric_catalog_purifier=summer_photometric_img_catalog_purifier, + ), ImageSaver( output_dir_name="processed", # TODO: work out why this was ever here... diff --git a/mirar/pipelines/summer/config/files/astrom.sex b/mirar/pipelines/summer/config/files/astrom.sex index b5aca201c..231837b55 100644 --- a/mirar/pipelines/summer/config/files/astrom.sex +++ b/mirar/pipelines/summer/config/files/astrom.sex @@ -58,7 +58,6 @@ PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: SATUR_KEY SATURATE # keyword for saturation level (in ADUs) -SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation MAG_ZEROPOINT 0 # magnitude zero-point MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) diff --git a/mirar/pipelines/summer/config/files/photomCat.sex b/mirar/pipelines/summer/config/files/photomCat.sex index fb391e219..d6e901398 100644 --- a/mirar/pipelines/summer/config/files/photomCat.sex +++ b/mirar/pipelines/summer/config/files/photomCat.sex @@ -54,8 +54,8 @@ PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , # -SATUR_KEY SATURATE # keyword for saturation level (in ADUs) -SATUR_LEVEL 25000 # level (in ADUs) at which arises saturation +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 1000000000.0 # level (in ADUs) at which arises saturation MAG_ZEROPOINT 0 # magnitude zero-point MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) diff --git a/mirar/pipelines/summer/generator.py b/mirar/pipelines/summer/generator.py index 82eeacf32..8efe76fef 100644 --- a/mirar/pipelines/summer/generator.py +++ b/mirar/pipelines/summer/generator.py @@ -4,6 +4,8 @@ """ import logging +from astropy.table import Table + from mirar.catalog import BaseCatalog, Gaia2Mass from mirar.catalog.vizier import PS1, SkyMapper from mirar.catalog.vizier.sdss import SDSS, NotInSDSSError, in_sdss @@ -36,10 +38,34 @@ def summer_astrometric_catalog_generator(image: Image) -> Gaia2Mass: trim=True, image_catalog_path=temp_cat_path, filter_name="j", + acceptable_j_ph_quals=["A", "B", "C"], ) return cat +def summer_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Table: + """ + Default function to purify the photometric image catalog + """ + edge_width_pixels = 100 + fwhm_threshold_arcsec = 4.0 + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + def summer_photometric_catalog_generator(image: Image) -> BaseCatalog: """ Generate a photometric calibration catalog for SUMMER images diff --git a/mirar/pipelines/winter/__init__.py b/mirar/pipelines/winter/__init__.py index 5dfbb5a05..43782b5c2 100644 --- a/mirar/pipelines/winter/__init__.py +++ b/mirar/pipelines/winter/__init__.py @@ -1 +1,4 @@ +""" +Pipeline for WINTER data +""" from mirar.pipelines.winter.winter_pipeline import WINTERPipeline diff --git a/mirar/pipelines/winter/blocks.py b/mirar/pipelines/winter/blocks.py index 0550925b3..a70f3629a 100644 --- a/mirar/pipelines/winter/blocks.py +++ b/mirar/pipelines/winter/blocks.py @@ -1,6 +1,45 @@ -from mirar.pipelines.winter.generator import winter_reference_generator +""" +Module for WINTER data reduction +""" +from mirar.paths import FITS_MASK_KEY +from mirar.pipelines.winter.config import ( + sextractor_anet_config, + sextractor_autoastrometry_config, + sextractor_photometry_config, + swarp_config_path, +) +from mirar.pipelines.winter.generator import ( + scamp_config_path, + winter_astrometric_catalog_generator, + winter_photometric_catalog_generator, + winter_reference_generator, +) +from mirar.pipelines.winter.load_winter_image import ( + load_proc_winter_image, + load_raw_winter_image, + load_stacked_winter_image, +) +from mirar.processors.astromatic import Scamp +from mirar.processors.astromatic.sextractor.sextractor import ( + Sextractor, + sextractor_checkimg_map, +) +from mirar.processors.astromatic.swarp.swarp import Swarp +from mirar.processors.astrometry.anet.anet_processor import AstrometryNet +from mirar.processors.csvlog import CSVLog +from mirar.processors.dark import DarkCalibrator +from mirar.processors.mask import MaskPixelsFromPath, WriteMaskedCoordsToFile +from mirar.processors.photcal import PhotCalibrator from mirar.processors.reference import GetReferenceImage -from mirar.processors.utils import ImageDebatcher, ImageSaver +from mirar.processors.sky import NightSkyMedianCalibrator, SkyFlatCalibrator +from mirar.processors.utils import ( + ImageBatcher, + ImageDebatcher, + ImageLoader, + ImageSaver, + ImageSelector, +) +from mirar.processors.utils.multi_ext_parser import MultiExtParser refbuild = [ ImageDebatcher(), @@ -9,3 +48,368 @@ ), ImageSaver(output_dir_name="stacked_ref"), ] + +BOARD_ID = 4 +TARGET_NAME = "EMGW_gal1" +split = [ + MultiExtParser( + input_sub_dir="raw/", + extension_num_header_key="BOARD_ID", + only_extract_num=BOARD_ID, + output_sub_dir=f"raw_split_{BOARD_ID}", + ) +] + +split_all_boards = [ + MultiExtParser( + input_sub_dir="raw/", + extension_num_header_key="BOARD_ID", + output_sub_dir="raw_split", + ) +] +load = [ + ImageLoader( + input_sub_dir=f"raw_split_{BOARD_ID}", load_image=load_raw_winter_image + ), + ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), +] + +load_all_boards = [ + ImageLoader(input_sub_dir="raw_split", load_image=load_raw_winter_image), + ImageSelector(("OBSTYPE", ["FOCUS", "DARK", "FLAT", "SCIENCE"])), +] + +load_proc = [ + ImageLoader(input_sub_dir=f"skysub_{BOARD_ID}", load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE")), +] + +load_dark = [ + ImageLoader(input_sub_dir=f"darkcal_{BOARD_ID}", load_image=load_raw_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}")), +] + +load_anet = [ + ImageLoader(input_sub_dir=f"anet_{BOARD_ID}", load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE")), +] + +load_stack = [ + ImageLoader(input_sub_dir=f"anet_{BOARD_ID}", load_image=load_proc_winter_image), + ImageSelector(("TARGNAME", f"{TARGET_NAME}"), ("OBSTYPE", "SCIENCE")), + ImageBatcher("EXPTIME"), +] + +load_multiboard_stack = [ + ImageLoader( + input_sub_dir=f"stack_all_{TARGET_NAME}", load_image=load_stacked_winter_image + ), + ImageSelector( + ("TARGNAME", f"{TARGET_NAME}"), + ("OBSTYPE", "SCIENCE"), + ), +] +# ImageBatcher("COADDS")] +log = ( + split + + load + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] +) + +log_all_boards = ( + split_all_boards + + load_all_boards + + [ + CSVLog( + export_keys=[ + "FILTER", + "UTCTIME", + "EXPTIME", + "OBSTYPE", + "UNIQTYPE", + "BOARD_ID", + "OBSCLASS", + "TARGET", + "FILTER", + "BASENAME", + "TARGNAME", + "RADEG", + "DECDEG", + "MEDCOUNT", + "STDDEV", + "T_ROIC", + ] + ) + ] +) + +dark_cal = [ + ImageSelector(("BOARD_ID", f"{BOARD_ID}")), + ImageBatcher(["BOARD_ID", "EXPTIME"]), + WriteMaskedCoordsToFile(output_dir="mask_raw"), + DarkCalibrator(cache_sub_dir=f"calibration_{BOARD_ID}"), + ImageSaver(output_dir_name=f"darkcal_{BOARD_ID}"), + ImageDebatcher(), +] + +dark_cal_all_boards = [ + ImageBatcher(["BOARD_ID", "EXPTIME"]), + WriteMaskedCoordsToFile(output_dir="mask_raw"), + DarkCalibrator(cache_sub_dir="calibration"), + ImageSaver(output_dir_name="darkcal"), + ImageDebatcher(), +] + +flat_cal = [ + # ImageSelector(("OBSTYPE", ["FOCUS", "SCIENCE", "FLAT"])), + # ImageSelector(("TARGNAME", ["INTERESTING"])), + # ImageBatcher(["BOARD_ID", "FILTER"]), + # FlatCalibrator(flat_mask_key=FITS_MASK_KEY, + # cache_sub_dir=f"calibration_{board_id}" + # ), + ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{TARGET_NAME}")), + ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir=f"skycals_{BOARD_ID}"), + ImageSelector(("OBSTYPE", ["SCIENCE"])), + ImageSaver(output_dir_name=f"skyflatcal_{BOARD_ID}"), + # ImageSelector(("OBSTYPE", ["SCIENCE"])), + # Sextractor(**sextractor_astrometry_config, + # write_regions_bool=True, + # output_sub_dir="sextractor", + # cache=True), + # ImageSelector(("TARGNAME", [""])), + # ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), + NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), + ImageSaver(output_dir_name=f"skysub_{BOARD_ID}"), + # Sextractor(**sextractor_astrometry_config, + # write_regions_bool=True, + # output_sub_dir="sextractor", + # cache=True), + # AutoAstrometry(catalog="tmc", pixel_scale=1.0, pa=0, inv=True, + # write_crosscheck_files=True), + # ImageSaver(output_dir_name="skysub"), + # Sextractor(**sextractor_astrometry_config, + # write_regions_bool=True, + # output_sub_dir="sextractor"), + # MultiExtParser(input_sub_dir="raw/mef/"), + # SplitImage(), + # MaskPixelsFromPath(mask_path=winter_mask_path), + # DarkCalibrator(), + # SkyFlatCalibrator(), + # NightSkyMedianCalibrator(), +] + +flat_cal_all_boards = [ + ImageSelector(("OBSTYPE", ["SCIENCE"]), ("TARGNAME", f"{TARGET_NAME}")), + ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME", "EXPTIME"]), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir="skycals"), + ImageSelector(("OBSTYPE", ["SCIENCE"])), + ImageSaver(output_dir_name="skyflatcal"), + NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), + ImageSaver(output_dir_name="skysub"), +] + +process = dark_cal + flat_cal +process_proc = [ + ImageDebatcher(), + AstrometryNet( + output_sub_dir=f"anet_{BOARD_ID}", + scale_bounds=[25, 40], + scale_units="amw", + use_sextractor=True, + parity="neg", + search_radius_deg=1.0, + # sextractor_config_path=sextractor_autoastrometry_config[ + # 'config_path'], + use_weight=False, + ), + ImageSaver(output_dir_name=f"anet_{BOARD_ID}", use_existing_weight=False), + Sextractor( + **sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="scamp", + ), + Scamp( + temp_output_sub_dir="scamp", + ref_catalog_generator=winter_astrometric_catalog_generator, + scamp_config_path=scamp_config_path, + cache=False, + ), + ImageDebatcher(), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=True, + subtract_bkg=False, + cache=True, + center_type="ALL", + temp_output_sub_dir=f"stack_all_{TARGET_NAME}", + ), +] + +process_proc_all_boards = [ + ImageDebatcher(), + ImageBatcher(["UTCTIME", "BOARD_ID"]), + AstrometryNet( + output_sub_dir="anet", + scale_bounds=[25, 40], + scale_units="amw", + use_sextractor=True, + parity="neg", + search_radius_deg=1.0, + sextractor_config_path=sextractor_anet_config["config_path"], + use_weight=False, + ), + ImageSaver(output_dir_name="anet", use_existing_weight=False), + Sextractor( + **sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="scamp", + ), + Scamp( + temp_output_sub_dir="scamp", + ref_catalog_generator=winter_astrometric_catalog_generator, + scamp_config_path=scamp_config_path, + cache=False, + ), + ImageDebatcher(), + ImageBatcher(["BOARD_ID", "FILTER", "TARGNAME"]), + # ImageSaver(output_dir_name="pre-swarp"), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=True, + subtract_bkg=False, + cache=False, + center_type="ALL", + temp_output_sub_dir=f"stack_all_{TARGET_NAME}", + ), +] + +process_noise = [ + Sextractor( + **sextractor_autoastrometry_config, + write_regions_bool=True, + output_sub_dir="mask_sextractor", + checkimage_type="SEGMENTATION", + cache=True, + verbose_type="FULL", + ), + MaskPixelsFromPath( + mask_path_key=sextractor_checkimg_map["SEGMENTATION"], + write_masked_pixels_to_file=True, + output_dir="mask1", + ), + ImageSaver(output_dir_name=f"noisemask_{BOARD_ID}"), +] +# process_proc = [ImageDebatcher(), +# ImageSelector(("OBSTYPE", ["SCIENCE"])), +# Sextractor(**sextractor_autoastrometry_config, +# write_regions_bool=True, +# output_sub_dir="sextractor", +# cache=False), +# AutoAstrometry(catalog="tmc", pixel_scale=1.07, +# write_crosscheck_files=True, +# inv=False, +# ), ] + +stack_proc = [ + ImageBatcher(["TARGNAME", "FILTER"]), + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=True, + cache=True, + center_type="MANUAL", + ), + ImageSaver(output_dir_name=f"stack_{TARGET_NAME}"), +] + +photcal = [ + # ImageSelector(("BOARD_ID", board_id)), + ImageDebatcher(), + ImageBatcher(["BOARD_ID"]), + Sextractor( + **sextractor_photometry_config, + output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", + checkimage_type="BACKGROUND_RMS", + ), + PhotCalibrator( + ref_catalog_generator=winter_photometric_catalog_generator, + temp_output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", + write_regions=True, + cache=True, + ), + # ImageSaver(output_dir_name=f"phot_{board_id}_{target_name}") + ImageSaver(output_dir_name=f"phot_{TARGET_NAME}"), +] + +photcal_indiv = [ + ImageSelector(("BOARD_ID", BOARD_ID)), + ImageDebatcher(), + ImageBatcher(["UTCTIME"]), + Sextractor( + **sextractor_photometry_config, + output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", + checkimage_type="BACKGROUND_RMS", + ), + PhotCalibrator( + ref_catalog_generator=winter_photometric_catalog_generator, + temp_output_sub_dir=f"phot_{BOARD_ID}_{TARGET_NAME}", + write_regions=True, + cache=True, + ), + ImageSaver(output_dir_name=f"phot_{BOARD_ID}_{TARGET_NAME}"), +] + +stack_multiboard = [ + Swarp( + swarp_config_path=swarp_config_path, + calculate_dims_in_swarp=True, + include_scamp=False, + subtract_bkg=False, + cache=True, + temp_output_sub_dir=f"multiboard_stack_{TARGET_NAME}", + center_type="MANUAL", + ) +] +# commissioning = \ +# log + process_proc + +commissioning = log + process + +commissioning_dark = log + dark_cal +commissioning_proc = load_proc + process_proc +commissioning_flat = load_dark + flat_cal +commissioning_reduce = log + dark_cal + flat_cal +commissioning_stack = load_stack + stack_proc +commissioning_multiboard_stack = load_multiboard_stack + stack_multiboard +commissioning_noise = load_anet + process_noise +commissioning_photcal = load_multiboard_stack + photcal +commissioning_photcal_indiv = load_anet + photcal_indiv +full_commissioning = log + process + process_proc # + stack_proc +full_commissioning_all_boards = ( + log_all_boards + dark_cal_all_boards + flat_cal_all_boards + process_proc_all_boards +) diff --git a/mirar/pipelines/winter/config/__init__.py b/mirar/pipelines/winter/config/__init__.py new file mode 100644 index 000000000..4ae05a6dc --- /dev/null +++ b/mirar/pipelines/winter/config/__init__.py @@ -0,0 +1,37 @@ +from pathlib import Path + +PIPELINE_NAME = "winter" + +winter_file_dir = Path(__file__).parent.joinpath("files") + +sextractor_astrometry_config = { + "config_path": winter_file_dir.joinpath("photomCat.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("astrom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + + +sextractor_photometry_config = { + "config_path": winter_file_dir.joinpath("photomCat.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("photom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + + +sextractor_anet_config = { + "config_path": winter_file_dir.joinpath("astrom_anet.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("astrom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + +sextractor_autoastrometry_config = { + "config_path": winter_file_dir.joinpath("astrom.sex"), + "filter_path": winter_file_dir.joinpath("default.conv"), + "parameter_path": winter_file_dir.joinpath("astrom.param"), + "starnnw_path": winter_file_dir.joinpath("default.nnw"), +} + +swarp_config_path = winter_file_dir.joinpath("swarp.config") diff --git a/mirar/pipelines/winter/config/files/Scorr.param b/mirar/pipelines/winter/config/files/Scorr.param new file mode 100644 index 000000000..d116900f0 --- /dev/null +++ b/mirar/pipelines/winter/config/files/Scorr.param @@ -0,0 +1,134 @@ +NUMBER + +#FLUX_ISO +#FLUXERR_ISO +#MAG_ISO +#MAGERR_ISO + +#FLUX_ISOCOR +#FLUXERR_ISOCOR +#MAG_ISOCOR +#MAGERR_ISOCOR + +FLUX_APER(1) +FLUXERR_APER(1) +#MAG_APER(1) +#MAGERR_APER(1) + +FLUX_AUTO +FLUXERR_AUTO +#MAG_AUTO +#MAGERR_AUTO + +#FLUX_BEST +#FLUXERR_BEST +#MAG_BEST +#MAGERR_BEST + +#KRON_RADIUS +FLUX_RADIUS +#BACKGROUND + +#THRESHOLD +#MU_THRESHOLD +FLUX_MAX +#MU_MAX +#ISOAREA_IMAGE +#ISOAREA_WORLD + +#XMIN_IMAGE +#YMIN_IMAGE +#XMAX_IMAGE +#YMAX_IMAGE + +X_IMAGE +Y_IMAGE +XPEAK_IMAGE +YPEAK_IMAGE +#XWIN_IMAGE +#YWIN_IMAGE +ERRX2_IMAGE +ERRY2_IMAGE +#ERRAWIN_IMAGE +#ERRBWIN_IMAGE +#ERRTHETAWIN_IMAGE +#X_WORLD +#Y_WORLD +#ALPHA_SKY +#DELTA_SKY +ALPHA_J2000 +DELTA_J2000 +#ALPHA_B1950 +#DELTA_B1950 + +#X2_IMAGE +#Y2_IMAGE +#XY_IMAGE +#X2_WORLD +#Y2_WORLD +#XY_WORLD + +#CXX_IMAGE +#CYY_IMAGE +#CXY_IMAGE +#CXX_WORLD +#CYY_WORLD +#CXY_WORLD + +A_IMAGE +B_IMAGE +#A_WORLD +#B_WORLD + +#THETA_IMAGE +#THETA_WORLD +#THETA_SKY +#THETA_J2000 +#THETA_B1950 + +ELONGATION +#ELLIPTICITY + +#ERRX2_IMAGE +#ERRY2_IMAGE +#ERRXY_IMAGE +#ERRX2_WORLD +#ERRY2_WORLD +#ERRXY_WORLD + +#ERRCXX_IMAGE +#ERRCYY_IMAGE +#ERRCXY_IMAGE +#ERRCXX_WORLD +#ERRCYY_WORLD +#ERRCXY_WORLD + +#ERRA_IMAGE +#ERRB_IMAGE +#ERRA_WORLD +#ERRB_WORLD + +#ERRTHETA_IMAGE +#ERRTHETA_WORLD +#ERRTHETA_SKY +#ERRTHETA_J2000 +#ERRTHETA_B1950 + +FWHM_IMAGE +FWHM_WORLD + +#ISO0 +#ISO1 +#ISO2 +#ISO3 +#ISO4 +#ISO5 +#ISO6 +#ISO7 + +FLAGS +#IMAFLAGS_ISO +#NIMAFLAGS_ISO(1) + +#CLASS_STAR +#VIGNET(5,5) diff --git a/mirar/pipelines/winter/config/files/astrom.param b/mirar/pipelines/winter/config/files/astrom.param new file mode 100644 index 000000000..5f72f7204 --- /dev/null +++ b/mirar/pipelines/winter/config/files/astrom.param @@ -0,0 +1,30 @@ +#VECTOR_ASSOC(10) +ALPHAWIN_J2000 +DELTAWIN_J2000 +X_IMAGE +Y_IMAGE +ELONGATION +ELLIPTICITY +XWIN_IMAGE +YWIN_IMAGE +ERRAWIN_IMAGE +ERRBWIN_IMAGE +FLUX_RADIUS +FWHM_WORLD +FWHM_IMAGE +FLUX_AUTO +FLUXERR_AUTO +FLUX_MAX +MAG_AUTO +MAGERR_AUTO +FLAGS +BACKGROUND +CLASS_STAR +FLUX_APER(5) +FLUXERR_APER(5) +MAG_APER(5) +MAGERR_APER(5) +VIGNET(41,41) +SNR_WIN +X +Y diff --git a/mirar/pipelines/winter/config/files/astrom.sex b/mirar/pipelines/winter/config/files/astrom.sex new file mode 100644 index 000000000..605f80cc8 --- /dev/null +++ b/mirar/pipelines/winter/config/files/astrom.sex @@ -0,0 +1,111 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/astrom.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 10 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 16 # Number of deblending sub-thresholds +DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + +#------------------------------ Photometry ----------------------------------- + +#PHOT_APERTURES 6.0,10.0,14.0,18.0,22.0 # MAG_APER aperture diameter(s) in pixels +PHOT_APERTURES 4.0, 6.0, 8.0, 10.0 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +#PHOT_AUTOPARAMS 2.5,3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0,3.5 # MAG_PETRO parameters: , + # + # +PHOT_AUTOPARAMS 1.0,2.0 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: , + + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +#PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 1.6 # detector gain in e-/ADU + +PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 2 # stellar FWHM in arcsec +STARNNW_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 256 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------- ASSOCiation --------------------------------- + +#ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +#ASSOC_DATA 5,6,7,8,9,10,11,12,13,14 # columns of the data to replicate (0=all) +#ASSOC_PARAMS 3,4 # columns of xpos,ypos[,mag] +#ASSOC_RADIUS 3.0 # cross-matching radius (pixels) +#ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, +# # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +#ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/pipelines/winter/config/files/astrom_anet.sex b/mirar/pipelines/winter/config/files/astrom_anet.sex new file mode 100644 index 000000000..071a1ac8c --- /dev/null +++ b/mirar/pipelines/winter/config/files/astrom_anet.sex @@ -0,0 +1,111 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE ASCII_HEAD # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/astrom.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 3 # or , in mag.arcsec-2 +ANALYSIS_THRESH 3 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.005 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE NONE # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + +#------------------------------ Photometry ----------------------------------- + +#PHOT_APERTURES 6.0,10.0,14.0,18.0,22.0 # MAG_APER aperture diameter(s) in pixels +PHOT_APERTURES 4.0, 6.0, 8.0, 10.0 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +#PHOT_AUTOPARAMS 2.5,3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0,3.5 # MAG_PETRO parameters: , + # + # +PHOT_AUTOPARAMS 1.0,2.0 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: , + + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +#PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 1.6 # detector gain in e-/ADU + +PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 2 # stellar FWHM in arcsec +STARNNW_NAME /Users/viraj/winter_telescope/mirar/mirar/pipelines/winter/config/files/default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 256 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------- ASSOCiation --------------------------------- + +#ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +#ASSOC_DATA 5,6,7,8,9,10,11,12,13,14 # columns of the data to replicate (0=all) +#ASSOC_PARAMS 3,4 # columns of xpos,ypos[,mag] +#ASSOC_RADIUS 3.0 # cross-matching radius (pixels) +#ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, +# # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +#ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/pipelines/winter/config/files/config.swarp b/mirar/pipelines/winter/config/files/config.swarp new file mode 100644 index 000000000..91c9265b1 --- /dev/null +++ b/mirar/pipelines/winter/config/files/config.swarp @@ -0,0 +1,102 @@ +# Default configuration file for SWarp 2.19.1 +# EB 2011-05-22 +# +#----------------------------------- Output ----------------------------------- +IMAGEOUT_NAME coadd.fits # Output filename +WEIGHTOUT_NAME coadd.weight.fits # Output weight-map filename + +HEADER_ONLY N # Only a header as an output file (Y/N)? +HEADER_SUFFIX .head # Filename extension for additional headers + +#------------------------------- Input Weights -------------------------------- + +WEIGHT_TYPE NONE # BACKGROUND,MAP_RMS,MAP_VARIANCE + +#RESCALE_WEIGHTS N # or MAP_WEIGHT +WEIGHT_SUFFIX .weight.fits # Suffix to use for weight-maps +WEIGHT_IMAGE # Weightmap filename if suffix not used + # (all or for each weight-map) + +#------------------------------- Co-addition ---------------------------------- + +COMBINE N # Combine resampled images (Y/N)? +COMBINE_TYPE MEDIAN # MEDIAN,AVERAGE,MIN,MAX,WEIGHTED,CHI2 + # or SUM +#CLIP_AMPFRAC 0.3 # +#CLIP_SIGMA 4.0 +#CLIP_WRITELOG N +#CLIP_LOGNAME clipped.log +#BLANK_BADPIXELS N + +#-------------------------------- Astrometry ---------------------------------- + +CELESTIAL_TYPE NATIVE # NATIVE, PIXEL, EQUATORIAL, + # GALACTIC,ECLIPTIC, or SUPERGALACTIC +PROJECTION_TYPE TAN # Any WCS projection code or NONE +PROJECTION_ERR 0.001 # Maximum projection error (in output + # pixels), or 0 for no approximation +CENTER_TYPE MANUAL # MANUAL, ALL or MOST +CENTER 00:00:00.0, +00:00:00.0 # Coordinates of the image center +PIXELSCALE_TYPE MANUAL # MANUAL,FIT,MIN,MAX or MEDIAN +PIXEL_SCALE 0.0 # Pixel scale +IMAGE_SIZE 0 # Image size (0 = AUTOMATIC) + +#-------------------------------- Resampling ---------------------------------- + +RESAMPLE Y # Resample input images (Y/N)? +RESAMPLE_DIR . # Directory path for resampled images +RESAMPLE_SUFFIX .resamp.fits # filename extension for resampled images + +RESAMPLING_TYPE LANCZOS3 # NEAREST,BILINEAR,LANCZOS2,LANCZOS3 + # or LANCZOS4 (1 per axis) +OVERSAMPLING 0 # Oversampling in each dimension + # (0 = automatic) +INTERPOLATE N # Interpolate bad input pixels (Y/N)? + # (all or for each image) + +FSCALASTRO_TYPE FIXED # NONE,FIXED, or VARIABLE +FSCALE_KEYWORD FLXSCALE # FITS keyword for the multiplicative + # factor applied to each input image +FSCALE_DEFAULT 1.0 # Default FSCALE value if not in header + +GAIN_KEYWORD GAIN # FITS keyword for effect. gain (e-/ADU) +GAIN_DEFAULT 0.0 # Default gain if no FITS keyword found +SATLEV_KEYWORD SATURATE +SATLEV_DEFAULT 40000 + +#--------------------------- Background subtraction --------------------------- + +SUBTRACT_BACK Y # Subtraction sky background (Y/N)? + # (all or for each image) + +BACK_TYPE AUTO # AUTO or MANUAL + # (all or for each image) +BACK_DEFAULT 0.0 # Default background value in MANUAL + # (all or for each image) +BACK_SIZE 128 # Background mesh size (pixels) + # (all or for each image) +BACK_FILTERSIZE 6 # Background map filter range (meshes) + # (all or for each image) + +#------------------------------ Memory management ----------------------------- + +VMEM_DIR . # Directory path for swap files +VMEM_MAX 2047 # Maximum amount of virtual memory (MB) +MEM_MAX 256 # Maximum amount of usable RAM (MB) +COMBINE_BUFSIZE 256 # RAM dedicated to co-addition(MB) + +#------------------------------ Miscellaneous --------------------------------- + +DELETE_TMPFILES Y # Delete temporary resampled FITS files + # (Y/N)? +COPY_KEYWORDS OBJECT # List of FITS keywords to propagate + # from the input to the output headers +WRITE_FILEINFO N # Write information about each input + # file in the output image header? +WRITE_XML N # Write XML file (Y/N)? +XML_NAME swarp.xml # Filename for XML output +VERBOSE_TYPE NORMAL # QUIET,NORMAL or FULL + +NTHREADS 0 # Number of simultaneous threads for + # the SMP version of SWarp + # 0 = automatic diff --git a/mirar/pipelines/winter/config/files/default.conv b/mirar/pipelines/winter/config/files/default.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/mirar/pipelines/winter/config/files/default.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/mirar/pipelines/winter/config/files/default.nnw b/mirar/pipelines/winter/config/files/default.nnw new file mode 100644 index 000000000..4b521bd43 --- /dev/null +++ b/mirar/pipelines/winter/config/files/default.nnw @@ -0,0 +1,28 @@ +NNW +# Neural Network Weights for the SExtractor star/galaxy classifier (V1.3) +# inputs: 9 for profile parameters + 1 for seeing. +# outputs: ``Stellarity index'' (0.0 to 1.0) +# Seeing FWHM range: from 0.025 to 5.5'' (images must have 1.5 < FWHM < 5 pixels) +# Optimized for Moffat profiles with 2<= beta <= 4. + + 3 10 10 1 + +-1.56604e+00 -2.48265e+00 -1.44564e+00 -1.24675e+00 -9.44913e-01 -5.22453e-01 4.61342e-02 8.31957e-01 2.15505e+00 2.64769e-01 + 3.03477e+00 2.69561e+00 3.16188e+00 3.34497e+00 3.51885e+00 3.65570e+00 3.74856e+00 3.84541e+00 4.22811e+00 3.27734e+00 + +-3.22480e-01 -2.12804e+00 6.50750e-01 -1.11242e+00 -1.40683e+00 -1.55944e+00 -1.84558e+00 -1.18946e-01 5.52395e-01 -4.36564e-01 -5.30052e+00 + 4.62594e-01 -3.29127e+00 1.10950e+00 -6.01857e-01 1.29492e-01 1.42290e+00 2.90741e+00 2.44058e+00 -9.19118e-01 8.42851e-01 -4.69824e+00 +-2.57424e+00 8.96469e-01 8.34775e-01 2.18845e+00 2.46526e+00 8.60878e-02 -6.88080e-01 -1.33623e-02 9.30403e-02 1.64942e+00 -1.01231e+00 + 4.81041e+00 1.53747e+00 -1.12216e+00 -3.16008e+00 -1.67404e+00 -1.75767e+00 -1.29310e+00 5.59549e-01 8.08468e-01 -1.01592e-02 -7.54052e+00 + 1.01933e+01 -2.09484e+01 -1.07426e+00 9.87912e-01 6.05210e-01 -6.04535e-02 -5.87826e-01 -7.94117e-01 -4.89190e-01 -8.12710e-02 -2.07067e+01 +-5.31793e+00 7.94240e+00 -4.64165e+00 -4.37436e+00 -1.55417e+00 7.54368e-01 1.09608e+00 1.45967e+00 1.62946e+00 -1.01301e+00 1.13514e-01 + 2.20336e-01 1.70056e+00 -5.20105e-01 -4.28330e-01 1.57258e-03 -3.36502e-01 -8.18568e-02 -7.16163e+00 8.23195e+00 -1.71561e-02 -1.13749e+01 + 3.75075e+00 7.25399e+00 -1.75325e+00 -2.68814e+00 -3.71128e+00 -4.62933e+00 -2.13747e+00 -1.89186e-01 1.29122e+00 -7.49380e-01 6.71712e-01 +-8.41923e-01 4.64997e+00 5.65808e-01 -3.08277e-01 -1.01687e+00 1.73127e-01 -8.92130e-01 1.89044e+00 -2.75543e-01 -7.72828e-01 5.36745e-01 +-3.65598e+00 7.56997e+00 -3.76373e+00 -1.74542e+00 -1.37540e-01 -5.55400e-01 -1.59195e-01 1.27910e-01 1.91906e+00 1.42119e+00 -4.35502e+00 + +-1.70059e+00 -3.65695e+00 1.22367e+00 -5.74367e-01 -3.29571e+00 2.46316e+00 5.22353e+00 2.42038e+00 1.22919e+00 -9.22250e-01 -2.32028e+00 + + + 0.00000e+00 + 1.00000e+00 diff --git a/mirar/pipelines/winter/config/files/photom.param b/mirar/pipelines/winter/config/files/photom.param new file mode 100644 index 000000000..2e275fa46 --- /dev/null +++ b/mirar/pipelines/winter/config/files/photom.param @@ -0,0 +1,28 @@ +#VECTOR_ASSOC(10) +ALPHAWIN_J2000 +DELTAWIN_J2000 +X_IMAGE +Y_IMAGE +ELONGATION +ELLIPTICITY +XWIN_IMAGE +YWIN_IMAGE +ERRAWIN_IMAGE +ERRBWIN_IMAGE +FLUX_RADIUS +FWHM_WORLD +FWHM_IMAGE +FLUX_AUTO +FLUXERR_AUTO +FLUX_MAX +MAG_AUTO +MAGERR_AUTO +FLAGS +BACKGROUND +CLASS_STAR +FLUX_APER(7) +FLUXERR_APER(7) +MAG_APER(7) +MAGERR_APER(7) +VIGNET(41,41) +SNR_WIN diff --git a/mirar/pipelines/winter/config/files/photom.psfex b/mirar/pipelines/winter/config/files/photom.psfex new file mode 100644 index 000000000..7f1f88c11 --- /dev/null +++ b/mirar/pipelines/winter/config/files/photom.psfex @@ -0,0 +1,58 @@ +# Default configuration file for PSFEx 3.17.1 +# EB 2014-09-19 +# + +#-------------------------------- PSF model ---------------------------------- + +BASIS_TYPE PIXEL_AUTO # NONE, PIXEL, GAUSS-LAGUERRE or FILE +BASIS_NUMBER 20 # Basis number or parameter +PSF_SAMPLING 1.0 # Sampling step in pixel units (0.0 = auto) +PSF_ACCURACY 0.01 # Accuracy to expect from PSF "pixel" values +PSF_SIZE 41, 41 # Image size of the PSF model + +#------------------------- Point source measurements ------------------------- + +CENTER_KEYS X_IMAGE,Y_IMAGE # Catalogue parameters for source pre-centering +PHOTFLUX_KEY FLUX_APER(3) # Catalogue parameter for photometric norm. +PHOTFLUXERR_KEY FLUXERR_APER(3) # Catalogue parameter for photometric error + +#----------------------------- PSF variability ------------------------------- + +PSFVAR_KEYS X_IMAGE,Y_IMAGE # Catalogue or FITS (preceded by :) params +PSFVAR_GROUPS 1,1 # Group tag for each context key +PSFVAR_DEGREES 3 # Polynom degree for each group + +#----------------------------- Sample selection ------------------------------ + +SAMPLE_AUTOSELECT Y # Automatically select the FWHM (Y/N) ? +SAMPLEVAR_TYPE SEEING # File-to-file PSF variability: NONE or SEEING +SAMPLE_FWHMRANGE 1.5, 14 # Allowed FWHM range +SAMPLE_VARIABILITY 0.5 # Allowed FWHM variability (1.0 = 100%) +SAMPLE_MINSN 40 # Minimum S/N for a source to be used #was 20 +SAMPLE_MAXELLIP 0.5 # Maximum (A-B)/(A+B) for a source to be used + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_DEV PNG # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_TYPE NONE #FWHM,ELLIPTICITY,COUNTS, COUNT_FRACTION, CHI2, RESIDUALS + # or NONE +CHECKPLOT_NAME NONE #fwhm, ellipticity, counts, countfrac, chi2, resi + +#------------------------------ Check-Images --------------------------------- + +CHECKIMAGE_TYPE NONE #CHI,PROTOTYPES,SAMPLES,RESIDUALS,SNAPSHOTS + # or MOFFAT,-MOFFAT,-SYMMETRICAL +CHECKIMAGE_NAME chi.fits,proto.fits,samp.fits,resi.fits,snap.fits + # Check-image filenames + +#----------------------------- Miscellaneous --------------------------------- + +PSF_DIR # Where to write PSFs (empty=same as input) +VERBOSE_TYPE NORMAL # can be QUIET,NORMAL,LOG or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME psfex.xml # Filename for XML output +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of PSFEx + # 0 = automatic + diff --git a/mirar/pipelines/winter/config/files/photomCat.sex b/mirar/pipelines/winter/config/files/photomCat.sex new file mode 100644 index 000000000..fb391e219 --- /dev/null +++ b/mirar/pipelines/winter/config/files/photomCat.sex @@ -0,0 +1,97 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /home/kde/reduxScripts/gattini/drp_commissioning/astrometry/photomCat.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 10 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 5 # or , in mag.arcsec-2 +ANALYSIS_THRESH 5 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /home/kde/reduxScripts/gattini/drp_commissioning/astrometry/default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN N # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 2, 3, 4, 5, 6, 7, 8 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +#PHOT_AUTOPARAMS 1.0,2.0 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 1.0,2.0 # MAG_PETRO parameters: , + # + # + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 25000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 1 # detector gain in e-/ADU +PIXEL_SCALE 0.466 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 2 # stellar FWHM in arcsec +STARNNW_NAME /home/kde/reduxScripts/gattini/drp_commissioning/astrometry/default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 256 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE BACKGROUND,BACKGROUND_RMS # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME sources.fits # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +NTHREADS 1 #Number of threads to use +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/pipelines/winter/config/files/scamp.conf b/mirar/pipelines/winter/config/files/scamp.conf new file mode 100644 index 000000000..df968f20a --- /dev/null +++ b/mirar/pipelines/winter/config/files/scamp.conf @@ -0,0 +1,142 @@ +# Default configuration file for SCAMP 2.0.4 +# EB 2018-01-23 +# + +#----------------------------- Field grouping --------------------------------- + +#FGROUP_RADIUS 0.3 # Max dist (deg) between field groups + +#---------------------------- Reference catalogs ------------------------------ + +REF_SERVER cocat1.u-strasbg.fr # Internet addresses of catalog servers +REF_PORT 80 # Ports to connect to catalog servers +CDSCLIENT_EXEC aclient_cgi # CDSclient executable +ASTREF_CATALOG FILE # NONE, FILE, USNO-A1,USNO-A2,USNO-B1, + # GSC-1.3,GSC-2.2,GSC-2.3, + # TYCHO-2, UCAC-1,UCAC-2,UCAC-3,UCAC-4, + # NOMAD-1, PPMX, CMC-14, 2MASS, DENIS-3, + # SDSS-R3,SDSS-R5,SDSS-R6,SDSS-R7, + # SDSS-R8, SDSS-R9 +ASTREF_BAND DEFAULT # Photom. band for astr.ref.magnitudes + # or DEFAULT, BLUEST, or REDDEST +ASTREFCAT_NAME astrefcat.cat # Local astrometric reference catalogs +ASTREFCENT_KEYS ra, dec # Local ref.cat. centroid parameters +ASTREFERR_KEYS ra_errdeg, dec_errdeg #e_RAJ2000, e_DEJ2000 + # Local ref.cat. err. ellipse params +ASTREFMAG_KEY phot_rp_mean_mag # Local ref.cat. magnitude parameter +ASTREFMAGERR_KEY phot_rp_mean_mag_error # Local ref.cat. mag. error parameter +ASTREFOBSDATE_KEY ref_epoch # Local ref.cat. obs. date parameter +ASTREFMAG_LIMITS -99.0,99.0 # Select magnitude range in ASTREF_BAND +SAVE_REFCATALOG N # Save ref catalogs in FITS-LDAC format? +REFOUT_CATPATH . # Save path for reference catalogs + +#--------------------------- Merged output catalogs --------------------------- + +MERGEDOUTCAT_TYPE NONE # NONE, ASCII_HEAD, ASCII, FITS_LDAC +MERGEDOUTCAT_NAME merged.cat # Merged output catalog filename + +#--------------------------- Full output catalogs --------------------------- + +FULLOUTCAT_TYPE NONE # NONE, ASCII_HEAD, ASCII, FITS_LDAC +FULLOUTCAT_NAME full.cat # Full output catalog filename + +#----------------------------- Pattern matching ------------------------------- + +MATCH Y # Do pattern-matching (Y/N) ? +MATCH_NMAX 0 # Max.number of detections for MATCHing + # (0=auto) +PIXSCALE_MAXERR 1.2 # Max scale-factor uncertainty +POSANGLE_MAXERR 1.0 # Max position-angle uncertainty (deg) +POSITION_MAXERR 0.15 # Max positional uncertainty (arcmin) +MATCH_RESOL 0 # Matching resolution (arcsec); 0=auto +MATCH_FLIPPED N # Allow matching with flipped axes? +MOSAIC_TYPE UNCHANGED # UNCHANGED, SAME_CRVAL, SHARE_PROJAXIS, + # FIX_FOCALPLANE or LOOSE +FIXFOCALPLANE_NMIN 1 # Min number of dets for FIX_FOCALPLANE + +#---------------------------- Cross-identification ---------------------------- + +CROSSID_RADIUS 1.0 # Cross-id initial radius (arcsec) + +#---------------------------- Astrometric solution ---------------------------- + +SOLVE_ASTROM Y # Compute astrometric solution (Y/N) ? +PROJECTION_TYPE SAME # SAME, TPV or TAN +ASTRINSTRU_KEY # FITS keyword(s) defining the astrom +STABILITY_TYPE INSTRUMENT # EXPOSURE, PRE-DISTORTED or INSTRUMENT +CENTROID_KEYS XWIN_IMAGE,YWIN_IMAGE # Cat. parameters for centroiding +CENTROIDERR_KEYS ERRAWIN_IMAGE,ERRBWIN_IMAGE,ERRTHETAWIN_IMAGE + # Cat. params for centroid err ellipse +DISTORT_KEYS XWIN_IMAGE,YWIN_IMAGE # Cat. parameters or FITS keywords +DISTORT_GROUPS 1,1 # Polynom group for each context key +DISTORT_DEGREES 3 # Polynom degree for each group +FOCDISTORT_DEGREE 3 # Polynom degree for focal plane coords +#ASTREF_WEIGHT 1.0 # Relative weight of ref.astrom.cat. +#ASTRACCURACY_TYPE SIGMA-PIXEL # SIGMA-PIXEL, SIGMA-ARCSEC, + # or TURBULENCE-ARCSEC +#ASTRACCURACY_KEY ASTRACCU # FITS keyword for ASTR_ACCURACY param. +#ASTR_ACCURACY 0.01 # Astrom. uncertainty floor parameter +#ASTRCLIP_NSIGMA 3.0 # Astrom. clipping threshold in sigmas +#COMPUTE_PARALLAXES N # Compute trigonom. parallaxes (Y/N)? +#COMPUTE_PROPERMOTIONS Y # Compute proper motions (Y/N)? +#CORRECT_COLOURSHIFTS N # Correct for colour shifts (Y/N)? +#INCLUDE_ASTREFCATALOG Y # Include ref.cat in prop.motions (Y/N)? +#ASTR_FLAGSMASK 0x00fc # Astrometry rejection mask on SEx FLAGS +#ASTR_IMAFLAGSMASK 0x0 # Astrometry rejection mask on IMAFLAGS + +#---------------------------- Photometric solution ---------------------------- + +SOLVE_PHOTOM Y # Compute photometric solution (Y/N) ? +MAGZERO_OUT 0.0 # Magnitude zero-point(s) in output +MAGZERO_INTERR 0.01 # Internal mag.zero-point accuracy +MAGZERO_REFERR 0.03 # Photom.field mag.zero-point accuracy +PHOTINSTRU_KEY # FITS keyword(s) defining the photom. +MAGZERO_KEY PHOT_C # FITS keyword for the mag zero-point +EXPOTIME_KEY EXPTIME # FITS keyword for the exposure time (s) +AIRMASS_KEY AIRMASS # FITS keyword for the airmass +EXTINCT_KEY PHOT_K # FITS keyword for the extinction coeff +PHOTOMFLAG_KEY PHOTFLAG # FITS keyword for the photometry flag +PHOTFLUX_KEY FLUX_AUTO # Catalog param. for the flux measurement +PHOTFLUXERR_KEY FLUXERR_AUTO # Catalog parameter for the flux error +PHOTCLIP_NSIGMA 5.0 # Photom.clipping threshold in sigmas +PHOT_ACCURACY 1e-3 # Photometric uncertainty floor (frac.) +PHOT_FLAGSMASK 0x00fc # Photometry rejection mask on SEx FLAGS +PHOT_IMAFLAGSMASK 0x0 # Photometry rejection mask on IMAFLAGS + +#------------------------------- Check-plots ---------------------------------- + +CHECKPLOT_CKEY SCAMPCOL # FITS keyword for PLPLOT field colour +CHECKPLOT_DEV PSC # NULL, XWIN, TK, PS, PSC, XFIG, PNG, + # JPEG, AQT, PDF or SVG +CHECKPLOT_RES 0 # Check-plot resolution (0 = default) +CHECKPLOT_ANTIALIAS Y # Anti-aliasing using convert (Y/N) ? +CHECKPLOT_TYPE FGROUPS,DISTORTION,ASTR_INTERROR2D,ASTR_INTERROR1D,ASTR_REFERROR2D,ASTR_REFERROR1D,ASTR_CHI2,PHOT_ERROR +CHECKPLOT_NAME fgroups,distort,astr_interror2d,astr_interror1d,astr_referror2d,astr_referror1d,astr_chi2,psphot_error # Check-plot filename(s) + +#------------------------------- Check-images --------------------------------- + +CHECKIMAGE_TYPE NONE # NONE, AS_PAIR, AS_REFPAIR, or AS_XCORR +CHECKIMAGE_NAME check.fits # Check-image filename(s) + +#------------------------------ Miscellaneous --------------------------------- + +SN_THRESHOLDS 10.0,100.0 # S/N thresholds (in sigmas) for all and + # high-SN sample +FWHM_THRESHOLDS 1.0,10.0 # FWHM thresholds (in pixels) for sources +ELLIPTICITY_MAX 0.5 # Max. source ellipticity +FLAGS_MASK 0x00f0 # Global rejection mask on SEx FLAGS +WEIGHTFLAGS_MASK 0x00ff # Global rejec. mask on SEx FLAGS_WEIGHT +IMAFLAGS_MASK 0x0 # Global rejec. mask on SEx IMAFLAGS_ISO +AHEADER_GLOBAL scamp.ahead # Filename of the global INPUT header +AHEADER_SUFFIX .ahead # Filename extension for additional + # INPUT headers +HEADER_SUFFIX .head # Filename extension for OUTPUT headers +HEADER_TYPE NORMAL # NORMAL or FOCAL_PLANE +VERBOSE_TYPE NORMAL # QUIET, NORMAL, LOG or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME scamp.xml # Filename for XML output +#XSL_URL file:///scr2/kde/scamp/share/scamp/scamp.xsl + # Filename for XSL style-sheet +NTHREADS 1 # Number of simultaneous threads for + # the SMP version of SCAMP + # 0 = automatic diff --git a/mirar/pipelines/winter/config/files/sex.config b/mirar/pipelines/winter/config/files/sex.config new file mode 100644 index 000000000..42ce7fc99 --- /dev/null +++ b/mirar/pipelines/winter/config/files/sex.config @@ -0,0 +1,74 @@ + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME temp.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME /Users/viraj/winter_telescope/mirar/mirar/processors/astromatic/config/temp.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 5 # minimum number of pixels above threshold +DETECT_THRESH 3 # or , in mag.arcsec-2 +ANALYSIS_THRESH 3 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME /Users/viraj/winter_telescope/mirar/mirar/processors/astromatic/config/sex.conv # name of the file containing the filter + +DEBLEND_NTHRESH 16 # Number of deblending sub-thresholds +DEBLEND_MINCONT 0.02 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + + + +MAG_ZEROPOINT 0.0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 0.0 # detector gain in e-/ADU +PIXEL_SCALE 1.0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 1.2 # stellar FWHM in arcsec +STARNNW_NAME default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 3 # Background filter: or , + +BACKPHOTO_TYPE GLOBAL # can be GLOBAL or LOCAL + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE NONE # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME check.fits # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 3000 # number of objects in stack +MEMORY_PIXSTACK 300000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation diff --git a/mirar/pipelines/winter/config/files/sex.conv b/mirar/pipelines/winter/config/files/sex.conv new file mode 100644 index 000000000..2590b9cba --- /dev/null +++ b/mirar/pipelines/winter/config/files/sex.conv @@ -0,0 +1,5 @@ +CONV NORM +# 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. +1 2 1 +2 4 2 +1 2 1 diff --git a/mirar/pipelines/winter/config/files/temp.param b/mirar/pipelines/winter/config/files/temp.param new file mode 100644 index 000000000..7a5bccb76 --- /dev/null +++ b/mirar/pipelines/winter/config/files/temp.param @@ -0,0 +1,9 @@ +X_IMAGE +Y_IMAGE +ALPHA_J2000 +DELTA_J2000 +MAG_AUTO +MAGERR_AUTO +ELLIPTICITY +FWHM_IMAGE +FLAGS diff --git a/mirar/pipelines/winter/fix_headers.py b/mirar/pipelines/winter/fix_headers.py new file mode 100644 index 000000000..979364c0d --- /dev/null +++ b/mirar/pipelines/winter/fix_headers.py @@ -0,0 +1,46 @@ +""" +Script to fix headers of images based on a log file +""" +import argparse + +import numpy as np +import pandas as pd +from astropy.io import fits + +from mirar.paths import get_output_path + + +def fix_headers(logfile, night, keyword_list, sub_dir="raw"): + """ + Function to fix headers of images based on a log file + """ + obslog = pd.read_csv(logfile) + for ind, image_basename in enumerate(obslog["BASENAME"]): + for i in range(5): + if f"_{i}.fits" in image_basename: + image_basename = image_basename.replace(f"_{i}.fits", ".fits") + image_filename = get_output_path( + base_name=image_basename, dir_root=sub_dir, sub_dir="winter/" + night + ) + img_hdulist = fits.open(image_filename, "update") + img_header = img_hdulist[0].header + for key in keyword_list: + if obslog.loc[ind][key] is np.nan: + img_header[key] = "" + else: + img_header[key] = obslog.loc[ind][key] + + img_hdulist.close() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Fix headers of images") + parser.add_argument("logfile", type=str, help="CSV file with image metadata") + parser.add_argument("night", type=str, help="Night of observation") + parser.add_argument( + "--sub_dir", type=str, default="raw", help="Subdirectory of images" + ) + parser.add_argument("-k", "--keywords", type=str, nargs="+") + args = parser.parse_args() + + fix_headers(args.logfile, args.night, args.keywords, args.sub_dir) diff --git a/mirar/pipelines/winter/generator.py b/mirar/pipelines/winter/generator.py index c52048ebc..6ff3365f1 100644 --- a/mirar/pipelines/winter/generator.py +++ b/mirar/pipelines/winter/generator.py @@ -6,6 +6,7 @@ from typing import Type import numpy as np +from astropy.table import Table from mirar.catalog import Gaia2Mass from mirar.data import Image @@ -23,6 +24,8 @@ winter_dir = os.path.dirname(__file__) astromatic_config_dir = os.path.join(winter_dir, "config/") swarp_config_path = os.path.join(astromatic_config_dir, "config.swarp") +winter_mask_path = os.path.join(winter_dir, "winter_mask.fits") +scamp_config_path = os.path.join(winter_dir, "scamp.conf") def winter_reference_generator(image: Image, db_table: Type[BaseDB] = RefStacks): @@ -97,7 +100,7 @@ def winter_photometric_catalog_generator(image: Image) -> Gaia2Mass: filter_name = image["FILTER"] search_radius_arcmin = ( np.max([image["NAXIS1"], image["NAXIS2"]]) * np.abs(image["CD1_1"]) * 60 - ) + ) / 2.0 return Gaia2Mass( min_mag=10, max_mag=20, @@ -107,6 +110,29 @@ def winter_photometric_catalog_generator(image: Image) -> Gaia2Mass: ) +def winter_ref_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Table: + """ + Default function to purify the photometric image catalog + """ + edge_width_pixels = 100 + fwhm_threshold_arcsec = 4.0 + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + def winter_reference_phot_calibrator(image: Image, **kwargs) -> PhotCalibrator: """ Generates a resampler for reference images @@ -114,18 +140,15 @@ def winter_reference_phot_calibrator(image: Image, **kwargs) -> PhotCalibrator: :param kwargs: kwargs :return: Swarp processor """ - x_lower_limit = 0 - y_lower_limit = 0 - x_upper_limit = image.header["NAXIS1"] - y_upper_limit = image.header["NAXIS2"] + # x_lower_limit = 0 + # y_lower_limit = 0 + # x_upper_limit = image.header["NAXIS1"] + # y_upper_limit = image.header["NAXIS2"] return PhotCalibrator( ref_catalog_generator=winter_photometric_catalog_generator, - x_lower_limit=x_lower_limit, - x_upper_limit=x_upper_limit, - y_lower_limit=y_lower_limit, - y_upper_limit=y_upper_limit, write_regions=True, + image_photometric_catalog_purifier=winter_ref_photometric_img_catalog_purifier, **kwargs, ) @@ -148,18 +171,10 @@ def ref_sextractor(image: Image): ) -def ref_phot_calibrator(image: Image): +def winter_astrometric_catalog_generator(_) -> Gaia2Mass: """ - Generates a photcalibrator instance for reference images to get photometry - Args: - image: - - Returns: + Function to crossmatch WIRC to GAIA/2mass for astrometry + :return: catalogue """ - logger.debug(image) - return PhotCalibrator( - ref_catalog_generator=winter_photometric_catalog_generator, - write_regions=True, - fwhm_threshold_arcsec=3, - ) + return Gaia2Mass(min_mag=10, max_mag=20, search_radius_arcmin=15) diff --git a/mirar/pipelines/winter/load_winter_image.py b/mirar/pipelines/winter/load_winter_image.py new file mode 100644 index 000000000..b4dd37f92 --- /dev/null +++ b/mirar/pipelines/winter/load_winter_image.py @@ -0,0 +1,243 @@ +""" +Module for loading raw WINTER images and ensuring they have the correct format +""" +import logging +import os +from pathlib import Path + +import astropy +import numpy as np +from astropy.io import fits +from astropy.stats import sigma_clipped_stats +from astropy.time import Time + +from mirar.paths import ( + BASE_NAME_KEY, + COADD_KEY, + PROC_FAIL_KEY, + PROC_HISTORY_KEY, + RAW_IMG_KEY, +) + +logger = logging.getLogger(__name__) + + +def mask_datasec(data: np.ndarray, header: astropy.io.fits.Header) -> np.ndarray: + """ + Function to mask the data section of an image + """ + datasec = header["DATASEC"].replace("[", "").replace("]", "").split(",") + datasec_xmin = int(datasec[0].split(":")[0]) + datasec_xmax = int(datasec[0].split(":")[1]) + datasec_ymin = int(datasec[1].split(":")[0]) + datasec_ymax = int(datasec[1].split(":")[1]) + + data[:, :datasec_xmin] = np.nan + data[:, datasec_xmax:] = np.nan + data[:datasec_ymin, :] = np.nan + data[datasec_ymax:, :] = np.nan + return data + + +def load_raw_winter_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: + """ + Function to load a raw WIRC image + + :param path: path of file + :return: data and header of image + """ + logger.info(f"Loading {path}") + with fits.open(path) as img: + # pylint: disable=E1101 + data = img[0].data + header = img[0].header + + header["UNIQTYPE"] = f"{header['OBSTYPE']}_{header['BOARD_ID']}" + + # if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: + # header["OBSTYPE"] = "calibration" + + # if '065456' in path or '064257' in path: + # if '073730' in path or '075817' in path or '080024' in path: + # if '_082' in path: + basename = os.path.basename(path) + timestamp = basename.split(".fits")[0].split("_")[1] + date = timestamp.split("-")[0] + time = timestamp.split("-")[1] + + if "RADEG" not in header.keys(): + header["RADEG"] = header["RA"] + header["DECDEG"] = header["DEC"] + + header["UTCTIME"] = ( + f"{date[:4]}-{date[4:6]}-{date[6:]}T" f"{time[:2]}:{time[2:4]}:{time[4:]}" + ) + logger.info(header["UTCTIME"]) + header["MJD-OBS"] = Time(header["UTCTIME"]).mjd + + # if '085739' in path or '-09' in path: + # if Time("2023-06-13T09:12:01") >= Time(header["UTCTIME"]) \ + # >= Time("2023-06-13T08:57:39"): + # if header['TARGNAME'] == 'm16': + # header["OBSTYPE"] = "SCIENCE" + # header['TARGNAME'] = 'INTERESTING' + # elif header["OBSTYPE"] != "DARK": + # header["OBSTYPE"] = "OTHER" + header["OBSCLASS"] = ["science", "calibration"][ + header["OBSTYPE"] in ["DARK", "FLAT"] + ] + # if header["OBSTYPE"] == "TEST" and ("_mef" not in path): + # header["OBSTYPE"] = "FLAT" + header["EXPTIME"] = np.rint(header["EXPTIME"]) + header[BASE_NAME_KEY] = os.path.basename(path) + if RAW_IMG_KEY not in header.keys(): + header[RAW_IMG_KEY] = path + header["TARGET"] = header["OBSTYPE"].lower() + + if (header["FILTERID"] == "dark") & (header["OBSTYPE"] != "BIAS"): + header["OBSTYPE"] = "DARK" + header["TARGET"] = "dark" + + if ".weight" in path: + header["OBSTYPE"] = "WEIGHT" + header["RA"] = header["RADEG"] + header["DEC"] = header["DECDEG"] + # elif '053618' in path: + # header["FILTER"] = "Hs" + # elif '053936' in path: + # header['FILTER'] = 'Y' + # elif Time(header["UTCTIME"]) >= Time("2023-06-10T06:50:29"): + # header["FILTER"] = "Hs" + # else: + # header["FILTER"] = "J" + + if COADD_KEY not in header.keys(): + logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") + header[COADD_KEY] = 1 + + header[PROC_HISTORY_KEY] = "" + header[PROC_FAIL_KEY] = "" + + filter_dict = {"J": 1, "H": 2, "Ks": 3} + + if "FILTERID" not in header.keys(): + header["FILTERID"] = filter_dict[header["FILTER"]] + if "FIELDID" not in header.keys(): + header["FIELDID"] = 99999 + if "PROGPI" not in header.keys(): + header["PROGPI"] = "Kasliwal" + if "PROGID" not in header.keys(): + header["PROGID"] = 0 + + if "CTYPE1" not in header: + header["CTYPE1"] = "RA---TAN" + if "CTYPE2" not in header: + header["CTYPE2"] = "DEC--TAN" + # if 'RADEG' in header.keys(): + # header['CRVAL1'] = header['RADEG'] + # if 'DECDEG' in header.keys(): + # header['CRVAL2'] = header['DECDEG'] + data = data.astype(float) + # data[data > 40000.0] = np.nan + # data[:, :250] = np.nan + # data[:, 1800:] = np.nan + # data[:20, :] = np.nan + # data[1060:, :] = np.nan + + header["FILTER"] = header["FILTERID"] + if "DATASEC" in header.keys(): + data = mask_datasec(data, header) + del header["DATASEC"] + # TODO: check if this is necessary for short exposures + # (Non-linearity/1700 counts) + data[data > 40000] = np.nan + if header["BOARD_ID"] == 0: + # data[:500, 700:1500] = np.nan + data[1075:, :] = np.nan + data[:, 1950:] = np.nan + data[:20, :] = np.nan + + if header["BOARD_ID"] == 1: + pass + + if header["BOARD_ID"] == 2: + data[1085:, :] = np.nan + data[:, 1970:] = np.nan + data[:55, :] = np.nan + data[:, :20] = np.nan + + if header["BOARD_ID"] == 3: + data[1085:, :] = np.nan + data[:, 1970:] = np.nan + data[:55, :] = np.nan + data[:, :20] = np.nan + + if header["BOARD_ID"] == 4: + # data[610:, :280] = np.nan + data[:, 1948:] = np.nan + data[:, :61] = np.nan + data[:20, :] = np.nan + data[1060:, :] = np.nan + data[:, 999:1002] = np.nan + + if header["BOARD_ID"] == 5: + # data[740:, 1270: 1850] = np.nan + data[1072:, :] = np.nan + data[:, 1940:] = np.nan + data[:15, :] = np.nan + data[680:, 1180:] = np.nan + # data[data > 25000] = np.nan + + _, med, std = sigma_clipped_stats(data, sigma=3.0, maxiters=5) + header["MEDCOUNT"] = med + header["STDDEV"] = std + # if header['RADEG']>300: + # header['TARGNAME'] = 'other' + + if "weight" in path: + header["OBSTYPE"] = "weight" + + return data, header + + +def load_proc_winter_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: + """ + Load proc image + """ + logger.info(f"Loading {path}") + with fits.open(path) as img: + data = img[0].data + header = img[0].header + if "weight" in path: + header["OBSTYPE"] = "weight" + + header["FILTER"] = header["FILTERID"] + + return data, header + + +def load_stacked_winter_image( + path: str | Path, +) -> tuple[np.array, astropy.io.fits.Header]: + """ + Load proc image + """ + logger.info(f"Loading {path}") + with fits.open(path) as img: + data = img[0].data + header = img[0].header + if "weight" in path: + header["OBSTYPE"] = "weight" + + header["OBSCLASS"] = "weight" + header["COADDS"] = 1 + header["TARGET"] = "science" + header["CALSTEPS"] = "" + header["PROCFAIL"] = 1 + header["RAWPATH"] = "" + header["BASENAME"] = os.path.basename(path) + header["TARGNAME"] = "weight" + if "UTCTIME" not in header.keys(): + header["UTCTIME"] = "2023-06-14T00:00:00" + + return data, header diff --git a/mirar/pipelines/winter/models/_fields.py b/mirar/pipelines/winter/models/_fields.py index 969167b1d..ca02eb1b8 100644 --- a/mirar/pipelines/winter/models/_fields.py +++ b/mirar/pipelines/winter/models/_fields.py @@ -2,10 +2,8 @@ Models for the 'fields' table """ import time -import urllib.request from typing import ClassVar -import pandas as pd from pydantic import Field from sqlalchemy import REAL, Column, Insert, Integer, Select from sqlalchemy.orm import Mapped, relationship diff --git a/mirar/pipelines/winter/models/_proc.py b/mirar/pipelines/winter/models/_proc.py index fe892cd90..29f2d961c 100644 --- a/mirar/pipelines/winter/models/_proc.py +++ b/mirar/pipelines/winter/models/_proc.py @@ -5,15 +5,7 @@ from typing import ClassVar from pydantic import Field, validator -from sqlalchemy import ( # event, - REAL, - VARCHAR, - Column, - Double, - ForeignKey, - Integer, - Sequence, -) +from sqlalchemy import REAL, VARCHAR, Column, ForeignKey, Integer, Sequence # event, from sqlalchemy.orm import Mapped, relationship from mirar.pipelines.winter.models._raw import Raw diff --git a/mirar/pipelines/winter/models/_ref_components.py b/mirar/pipelines/winter/models/_ref_components.py index e76e4c31d..8fb6a0f49 100644 --- a/mirar/pipelines/winter/models/_ref_components.py +++ b/mirar/pipelines/winter/models/_ref_components.py @@ -5,7 +5,7 @@ from pydantic import Field from sqlalchemy import VARCHAR, Column, Double, Float, Integer -from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.orm import Mapped, relationship from mirar.pipelines.winter.models.base_model import WinterBase, dec_field, ra_field from mirar.processors.sqldatabase.base_model import BaseDB diff --git a/mirar/pipelines/winter/models/base_model.py b/mirar/pipelines/winter/models/base_model.py index 62f8f5d79..84fec76ca 100644 --- a/mirar/pipelines/winter/models/base_model.py +++ b/mirar/pipelines/winter/models/base_model.py @@ -8,7 +8,7 @@ from mirar.processors.sqldatabase.base_model import BaseTable -db_name = "winter" +DB_NAME = "winter" class WinterBase(DeclarativeBase, BaseTable): @@ -16,7 +16,7 @@ class WinterBase(DeclarativeBase, BaseTable): Parent class for summer database """ - db_name = db_name + db_name = DB_NAME ra_field: float = Field(title="RA (degrees)", ge=0.0, le=360.0) diff --git a/mirar/pipelines/winter/winter_pipeline.py b/mirar/pipelines/winter/winter_pipeline.py index 308f9f17d..aba4a2175 100644 --- a/mirar/pipelines/winter/winter_pipeline.py +++ b/mirar/pipelines/winter/winter_pipeline.py @@ -3,20 +3,52 @@ """ import logging +from mirar.downloader.caltech import download_via_ssh from mirar.pipelines.base_pipeline import Pipeline -from mirar.pipelines.winter.blocks import refbuild +from mirar.pipelines.winter.blocks import ( + commissioning, + commissioning_dark, + commissioning_flat, + commissioning_multiboard_stack, + commissioning_noise, + commissioning_photcal, + commissioning_photcal_indiv, + commissioning_proc, + commissioning_reduce, + commissioning_stack, + full_commissioning, + full_commissioning_all_boards, + log, + refbuild, +) +from mirar.pipelines.winter.config import PIPELINE_NAME logger = logging.getLogger(__name__) class WINTERPipeline(Pipeline): """ - Pipeline for building reference images in the IR from WFAU + Pipeline for processing WINTER data """ name = "winter" - all_pipeline_configurations = {"refbuild": refbuild} + all_pipeline_configurations = { + "refbuild": refbuild, + "commissioning": commissioning, + "log": log, + "commissioning_dark": commissioning_dark, + "commissioning_stack": commissioning_stack, + "commissioning_flat": commissioning_flat, + "commissioning_reduce": commissioning_reduce, + "commissioning_proc": commissioning_proc, + "commissioning_multiboard_stack": commissioning_multiboard_stack, + "full_commissioning": full_commissioning, + "commissioning_noise": commissioning_noise, + "commissioning_photcal": commissioning_photcal, + "commissioning_photcal_indiv": commissioning_photcal_indiv, + "full_commissioning_all_boards": full_commissioning_all_boards, + } gain = 1.0 non_linear_level = 65535 @@ -27,4 +59,10 @@ def _load_raw_image(path: str): @staticmethod def download_raw_images_for_night(night: str): - pass + download_via_ssh( + server="winter.caltech.edu", + base_dir="/data/loki/raw_data/winter", + night=night, + pipeline=PIPELINE_NAME, + server_sub_dir="raw", + ) diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index 602429b35..f8367557f 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -2,9 +2,11 @@ Module containing standard processing blocks for WIRC """ from mirar.catalog.kowalski import PS1, TMASS +from mirar.paths import FITS_MASK_KEY, LATEST_SAVE_KEY, RAW_IMG_KEY, SATURATE_KEY from mirar.pipelines.wirc.generator import ( wirc_astrometric_catalog_generator, wirc_photometric_catalog_generator, + wirc_photometric_img_catalog_purifier, wirc_reference_image_generator, wirc_reference_image_resampler, wirc_reference_psfex, @@ -17,6 +19,7 @@ scamp_fp_path, sextractor_astrometry_config, sextractor_candidate_config, + sextractor_photometry_config, sextractor_reference_config, swarp_sp_path, wirc_candidate_schema_path, @@ -25,7 +28,11 @@ from mirar.processors.alerts import AvroPacketMaker, SendToFritz from mirar.processors.astromatic import Scamp, Sextractor, Swarp from mirar.processors.astromatic.psfex import PSFex -from mirar.processors.autoastrometry import AutoAstrometry +from mirar.processors.astromatic.scamp.scamp import scamp_header_key +from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map +from mirar.processors.astromatic.swarp.swarp import GetSwarpComponentImages +from mirar.processors.astrometry.autoastrometry import AutoAstrometry +from mirar.processors.astrometry.utils import AstrometryFromFile from mirar.processors.candidates.candidate_detector import DetectCandidates from mirar.processors.candidates.candidate_extractor import ( ForcedPhotometryCandidateTable, @@ -37,7 +44,12 @@ from mirar.processors.database.database_exporter import DatabaseDataframeExporter from mirar.processors.database.database_importer import DatabaseHistoryImporter from mirar.processors.flat import SkyFlatCalibrator -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import ( + MaskAboveThreshold, + MaskPixelsFromPath, + MaskPixelsFromWCS, + WriteMaskedCoordsToFile, +) from mirar.processors.photcal import PhotCalibrator from mirar.processors.photometry.aperture_photometry import ( CandidateAperturePhotometry, @@ -49,18 +61,23 @@ ) from mirar.processors.reference import ProcessReference from mirar.processors.sky import NightSkyMedianCalibrator -from mirar.processors.utils import ImageLoader, ImageSaver -from mirar.processors.utils.image_selector import ( +from mirar.processors.utils import ( + HeaderAnnotator, ImageBatcher, ImageDebatcher, + ImageLoader, + ImageSaver, ImageSelector, ) +from mirar.processors.utils.image_loader import LoadImageFromHeader from mirar.processors.xmatch import XMatch from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)] +# load_raw = [ImageLoader(input_sub_dir="firstpassstack", +# load_image=load_raw_wirc_image)] -reduce = [ +log = [ CSVLog( export_keys=[ "OBJECT", @@ -71,13 +88,20 @@ "OBSTYPE", "OBSCLASS", ] - ), - MaskPixels(mask_path=wirc_mask_path), - ImageSelector(("exptime", "45.0")), - DarkCalibrator(), + ) +] + +masking = [MaskPixelsFromPath(mask_path=wirc_mask_path)] + +dark_calibration = [ImageSelector(("exptime", "45.0")), DarkCalibrator()] + +reduction = [ + ImageSaver(output_dir_name="darkcal"), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), ImageDebatcher(), ImageSelector(("obsclass", "science")), - ImageBatcher(split_key="filter"), + # ImageSelector(("object", "ZTF18aavqmki")), + ImageBatcher(split_key=["filter", "object"]), SkyFlatCalibrator(), NightSkyMedianCalibrator(), AutoAstrometry(catalog="tmc"), @@ -85,13 +109,61 @@ Scamp( ref_catalog_generator=wirc_astrometric_catalog_generator, scamp_config_path=scamp_fp_path, + cache=True, + ), + ImageSaver(output_dir_name="firstpass"), + Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True), + ImageSaver(output_dir_name="firstpassstack"), + # ImageSelector(("BASENAME", "image0125.fits_stack.fits")), + Sextractor( + output_sub_dir="firstpasssextractor", + **sextractor_astrometry_config, + checkimage_type="SEGMENTATION", + cache=True, + ), + MaskPixelsFromPath( + mask_path_key=sextractor_checkimg_map["SEGMENTATION"], + write_masked_pixels_to_file=True, + output_dir="mask1", + ), + ImageSaver(output_dir_name="mask1", write_mask=True), + MaskAboveThreshold( + threshold_key=SATURATE_KEY, write_masked_pixels_to_file=True, output_dir="mask2" + ), + ImageSaver(output_dir_name="mask2", write_mask=True), + WriteMaskedCoordsToFile(output_dir="mask_stack"), + GetSwarpComponentImages( + load_image=load_raw_wirc_image, + copy_header_keys=FITS_MASK_KEY, + ), + LoadImageFromHeader( + header_key=RAW_IMG_KEY, + copy_header_keys=[scamp_header_key, FITS_MASK_KEY], + load_image=load_raw_wirc_image, + ), + AstrometryFromFile(astrometry_file_key=scamp_header_key), + ImageSaver(output_dir_name="firstpassastrom", write_mask=True), + MaskPixelsFromWCS( + write_masked_pixels_to_file=True, + output_dir="mask_secondpass", + only_write_mask=True, + ), + ImageSaver(output_dir_name="firstpassmasked", write_mask=True), + SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY), + NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY), + Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), + Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True), + Sextractor(output_sub_dir="final_sextractor", **sextractor_photometry_config), + PhotCalibrator( + ref_catalog_generator=wirc_photometric_catalog_generator, + image_photometric_catalog_purifier=wirc_photometric_img_catalog_purifier, + write_regions=True, ), - Swarp(swarp_config_path=swarp_sp_path), - Sextractor(output_sub_dir="final_sextractor", **sextractor_astrometry_config), - PhotCalibrator(ref_catalog_generator=wirc_photometric_catalog_generator), ImageSaver(output_dir_name="final"), ] +reduce = log + masking + dark_calibration + reduction + reference = [ ProcessReference( ref_image_generator=wirc_reference_image_generator, diff --git a/mirar/pipelines/wirc/generator.py b/mirar/pipelines/wirc/generator.py index 5ab3fc33f..01a8991f8 100644 --- a/mirar/pipelines/wirc/generator.py +++ b/mirar/pipelines/wirc/generator.py @@ -18,13 +18,39 @@ logger = logging.getLogger(__name__) +def wirc_photometric_img_catalog_purifier(catalog, image): + """ + Function to purify the photometric catalog + + :return: purified catalog + """ + edge_width_pixels = 100 + fwhm_threshold_arcsec = 4.0 + + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + def wirc_astrometric_catalog_generator(_) -> Gaia2Mass: """ Function to crossmatch WIRC to GAIA/2mass for astrometry :return: catalogue """ - return Gaia2Mass(min_mag=10, max_mag=20, search_radius_arcmin=30) + return Gaia2Mass(min_mag=10, max_mag=20, search_radius_arcmin=10) def wirc_photometric_catalog_generator(image: Image) -> Gaia2Mass: @@ -36,7 +62,12 @@ def wirc_photometric_catalog_generator(image: Image) -> Gaia2Mass: """ filter_name = image["FILTER"] return Gaia2Mass( - min_mag=10, max_mag=20, search_radius_arcmin=30, filter_name=filter_name + min_mag=10, + max_mag=20, + search_radius_arcmin=10, + filter_name=filter_name, + acceptable_h_ph_quals=["A"], + acceptable_k_ph_quals=["A"], ) diff --git a/mirar/pipelines/wirc/load_wirc_image.py b/mirar/pipelines/wirc/load_wirc_image.py index 1f83745a6..5e507a894 100644 --- a/mirar/pipelines/wirc/load_wirc_image.py +++ b/mirar/pipelines/wirc/load_wirc_image.py @@ -2,24 +2,19 @@ Module for loading raw WIRC images and ensuring they have the correct format """ import logging -import os from pathlib import Path import astropy import numpy as np -from astropy.io import fits from astropy.time import Time -from mirar.paths import ( - BASE_NAME_KEY, - COADD_KEY, - PROC_FAIL_KEY, - PROC_HISTORY_KEY, - RAW_IMG_KEY, -) +from mirar.io import open_fits +from mirar.paths import COADD_KEY, PROC_FAIL_KEY, PROC_HISTORY_KEY, SATURATE_KEY logger = logging.getLogger(__name__) +WIRC_NONLINEAR_LEVEL = 30000 + def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Header]: """ @@ -28,47 +23,47 @@ def load_raw_wirc_image(path: str | Path) -> tuple[np.array, astropy.io.fits.Hea :param path: path of file :return: data and header of image """ - with fits.open(path) as img: - # pylint: disable=E1101 - data = img[0].data - header = img[0].header - # pylint: enable=E1101 - header["FILTER"] = header["AFT"].split("__")[0] - - if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: - header["OBSTYPE"] = "calibration" + data, header = open_fits(path) + header["FILTER"] = header["AFT"].split("__")[0] + if "COADDS" in header.keys(): + header["DETCOADD"] = header["COADDS"] + if SATURATE_KEY not in header: + header[SATURATE_KEY] = WIRC_NONLINEAR_LEVEL * header["DETCOADD"] + if header["OBJECT"] in ["acquisition", "pointing", "focus", "none"]: + header["OBSTYPE"] = "calibration" - header["OBSCLASS"] = ["calibration", "science"][header["OBSTYPE"] == "object"] + header["OBSCLASS"] = ["calibration", "science"][header["OBSTYPE"] == "object"] - header[BASE_NAME_KEY] = os.path.basename(path) - header[RAW_IMG_KEY] = path - header["TARGET"] = header["OBJECT"].lower() + header["TARGET"] = header["OBJECT"].lower() + if "MJD-OBS" in header.keys(): + header["UTCTIME"] = Time(header["MJD-OBS"], format="mjd").isot + else: header["UTCTIME"] = header["UTSHUT"] header["MJD-OBS"] = Time(header["UTSHUT"]).mjd - if COADD_KEY not in header.keys(): - logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") - header[COADD_KEY] = 1 + if COADD_KEY not in header.keys(): + logger.debug(f"No {COADD_KEY} entry. Setting coadds to 1.") + header[COADD_KEY] = 1 - header[PROC_HISTORY_KEY] = "" - header[PROC_FAIL_KEY] = "" + header[PROC_HISTORY_KEY] = "" + header[PROC_FAIL_KEY] = "" - filter_dict = {"J": 1, "H": 2, "Ks": 3} + filter_dict = {"J": 1, "H": 2, "Ks": 3} - if "FILTERID" not in header.keys(): - header["FILTERID"] = filter_dict[header["FILTER"]] - if "FIELDID" not in header.keys(): - header["FIELDID"] = 99999 - if "PROGPI" not in header.keys(): - header["PROGPI"] = "Kasliwal" - if "PROGID" not in header.keys(): - header["PROGID"] = 0 - if "ZP" not in header.keys(): - if "TMC_ZP" in header.keys(): - header["ZP"] = header["TMC_ZP"] - header["ZP_std"] = header["TMC_ZPSD"] - if "ZP_AUTO" in header.keys(): - header["ZP"] = header["ZP_AUTO"] - header["ZP_std"] = header["ZP_AUTO_std"] - data = data.astype(float) - data[data == 0.0] = np.nan + if "FILTERID" not in header.keys(): + header["FILTERID"] = filter_dict[header["FILTER"]] + if "FIELDID" not in header.keys(): + header["FIELDID"] = 99999 + if "PROGPI" not in header.keys(): + header["PROGPI"] = "Kasliwal" + if "PROGID" not in header.keys(): + header["PROGID"] = 0 + if "ZP" not in header.keys(): + if "TMC_ZP" in header.keys(): + header["ZP"] = header["TMC_ZP"] + header["ZP_std"] = header["TMC_ZPSD"] + if "ZP_AUTO" in header.keys(): + header["ZP"] = header["ZP_AUTO"] + header["ZP_std"] = header["ZP_AUTO_std"] + data = data.astype(float) + data[data == 0.0] = np.nan return data, header diff --git a/mirar/pipelines/wirc/wirc_files/__init__.py b/mirar/pipelines/wirc/wirc_files/__init__.py index b78bff7ac..8db669ff5 100644 --- a/mirar/pipelines/wirc/wirc_files/__init__.py +++ b/mirar/pipelines/wirc/wirc_files/__init__.py @@ -17,6 +17,13 @@ "starnnw_path": wirc_file_dir.joinpath("default.nnw"), } +sextractor_photometry_config = { + "config_path": wirc_file_dir.joinpath("matchcat.sex"), + "filter_path": wirc_file_dir.joinpath("default.conv"), + "parameter_path": wirc_file_dir.joinpath("astrom.param"), + "starnnw_path": wirc_file_dir.joinpath("default.nnw"), +} + scamp_fp_path = wirc_file_dir.joinpath("scamp_fp.conf") swarp_sp_path = wirc_file_dir.joinpath("second_pass.swarp") diff --git a/mirar/pipelines/wirc/wirc_files/files/focusloop.sex b/mirar/pipelines/wirc/wirc_files/files/focusloop.sex new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/pipelines/wirc/wirc_files/files/matchcat.sex b/mirar/pipelines/wirc/wirc_files/files/matchcat.sex new file mode 100644 index 000000000..a4877668b --- /dev/null +++ b/mirar/pipelines/wirc/wirc_files/files/matchcat.sex @@ -0,0 +1,107 @@ +# Default configuration file for SExtractor 2.5.0 +# EB 2006-07-14 +# + +#-------------------------------- Catalog ------------------------------------ + +CATALOG_NAME test.cat # name of the output catalog +CATALOG_TYPE FITS_LDAC # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, + # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC +PARAMETERS_NAME matchCat.param # name of the file containing catalog contents + +#------------------------------- Extraction ---------------------------------- + +DETECT_TYPE CCD # CCD (linear) or PHOTO (with gamma correction) +DETECT_MINAREA 10 # minimum number of pixels above threshold +THRESH_TYPE RELATIVE # threshold type: RELATIVE (in sigmas) or ABSOLUTE (in ADUs) + +DETECT_THRESH 2 # or , in mag.arcsec-2 +ANALYSIS_THRESH 2 # or , in mag.arcsec-2 + +FILTER Y # apply filter for detection (Y or N)? +FILTER_NAME default.conv # name of the file containing the filter + +DEBLEND_NTHRESH 32 # Number of deblending sub-thresholds +DEBLEND_MINCONT 1e-4 # Minimum contrast parameter for deblending + +CLEAN Y # Clean spurious detections? (Y or N)? +CLEAN_PARAM 1.0 # Cleaning efficiency + +MASK_TYPE CORRECT # type of detection MASKing: can be one of + # NONE, BLANK or CORRECT + +#-------------------------------- WEIGHTing ---------------------------------- + +WEIGHT_TYPE MAP_WEIGHT # type of WEIGHTing: NONE, BACKGROUND, + # MAP_RMS, MAP_VAR or MAP_WEIGHT +WEIGHT_IMAGE weight.fits # weight-map filename +WEIGHT_GAIN Y # modulate gain (E/ADU) with weights? (Y/N) +WEIGHT_THRESH # weight threshold[s] for bad pixels + + +#------------------------------ Photometry ----------------------------------- + +PHOT_APERTURES 6.0,10.0,14.0,18.0 # MAG_APER aperture diameter(s) in pixels +PHOT_FLUXFRAC 0.5 # flux fraction[s] used for FLUX_RADIUS +PHOT_AUTOPARAMS 2.5,3.5 # MAG_AUTO parameters: , +PHOT_PETROPARAMS 2.0,3.5 # MAG_PETRO parameters: , + # + # + +#PHOT_APERTURES 5 # MAG_APER aperture diameter(s) in pixels +#PHOT_AUTOPARAMS 2.5, 3.5 # MAG_AUTO parameters: , +#PHOT_PETROPARAMS 2.0, 3.5 # MAG_PETRO parameters: , + # + +SATUR_KEY SATURATE # keyword for saturation level (in ADUs) +SATUR_LEVEL 60000 # level (in ADUs) at which arises saturation + +MAG_ZEROPOINT 0 # magnitude zero-point +MAG_GAMMA 4.0 # gamma of emulsion (for photographic scans) +GAIN 5.467 # detector gain in e-/ADU + +PIXEL_SCALE 0 # size of pixel in arcsec (0=use FITS WCS info) + +#------------------------- Star/Galaxy Separation ---------------------------- + +SEEING_FWHM 0.7 # stellar FWHM in arcsec +STARNNW_NAME default.nnw # Neural-Network_Weight table filename + +#------------------------------ Background ----------------------------------- + +BACK_SIZE 64 # Background mesh: or , +BACK_FILTERSIZE 6 # Background filter: or , + +BACK_TYPE AUTO # AUTO or MANUAL +BACKPHOTO_TYPE LOCAL # can be GLOBAL or LOCAL + +#------------------------------- ASSOCiation --------------------------------- + +#ASSOC_NAME sky.list # name of the ASCII file to ASSOCiate +#ASSOC_DATA 5,6,7,8,9,10 # columns of the data to replicate (0=all) +#ASSOC_PARAMS 3,4 # columns of xpos,ypos[,mag] +#ASSOC_RADIUS 3.0 # cross-matching radius (pixels) +#ASSOC_TYPE NEAREST # ASSOCiation method: FIRST, NEAREST, MEAN, + # MAG_MEAN, SUM, MAG_SUM, MIN or MAX +#ASSOCSELEC_TYPE MATCHED # ASSOC selection type: ALL, MATCHED or -MATCHED + + +#------------------------------ Check Image ---------------------------------- + +CHECKIMAGE_TYPE BACKGROUND, BACKGROUND_RMS # can be NONE, BACKGROUND, BACKGROUND_RMS, + # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND, + # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, + # or APERTURES +CHECKIMAGE_NAME check.fits # Filename for the check-image + +#--------------------- Memory (change with caution!) ------------------------- + +MEMORY_OBJSTACK 10000 # number of objects in stack +MEMORY_PIXSTACK 5000000 # number of pixels in stack +MEMORY_BUFSIZE 1024 # number of lines in buffer + +#----------------------------- Miscellaneous --------------------------------- + +VERBOSE_TYPE NORMAL # can be QUIET, NORMAL or FULL +WRITE_XML N # Write XML file (Y/N)? +XML_NAME sex.xml # Filename for XML output diff --git a/mirar/processors/__init__.py b/mirar/processors/__init__.py index 9cfd025f1..38669a20b 100644 --- a/mirar/processors/__init__.py +++ b/mirar/processors/__init__.py @@ -1,9 +1,12 @@ +""" +This module contains the processors that are used to process the raw data +""" # import logging from mirar.processors.base_processor import BaseImageProcessor from mirar.processors.bias import BiasCalibrator from mirar.processors.dark import DarkCalibrator from mirar.processors.flat import FlatCalibrator, SkyFlatCalibrator -from mirar.processors.mask import MaskPixels +from mirar.processors.mask import MaskPixelsFromPath from mirar.processors.utils.image_saver import ImageSaver # @@ -16,7 +19,8 @@ # processor = BaseProcessor.subclasses[processor_name] # except KeyError: # err = f"Processor type '{processor_name}' not recognised. " \ -# f"The following processors are available: {BaseProcessor.subclasses.keys()}" +# f"The following processors are available: +# {BaseProcessor.subclasses.keys()}" # logger.error(err) # raise KeyError(err) # diff --git a/mirar/processors/anet/__init__.py b/mirar/processors/anet/__init__.py deleted file mode 100644 index 9f13a24c2..000000000 --- a/mirar/processors/anet/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -""" -Central module for running anet.net solve -""" - -from mirar.processors.anet.anet import run_astrometry_net -from mirar.processors.anet.anet_processor import AstrometryNet diff --git a/mirar/processors/anet/anet_processor.py b/mirar/processors/anet/anet_processor.py deleted file mode 100644 index 1f6b463ad..000000000 --- a/mirar/processors/anet/anet_processor.py +++ /dev/null @@ -1,99 +0,0 @@ -""" -Module containing a processor to run astrometry.net -""" -import logging -import os -from pathlib import Path -from typing import Optional - -from astropy.io import fits - -from mirar.data import ImageBatch -from mirar.paths import BASE_NAME_KEY, get_output_dir, get_temp_path -from mirar.processors.anet.anet import run_astrometry_net_single -from mirar.processors.base_processor import BaseImageProcessor - -logger = logging.getLogger(__name__) - - -ASTROMETRY_TIMEOUT = 900 # astrometry cmd execute timeout, in seconds - - -class AstrometryNet(BaseImageProcessor): - """Processor to run astrometry.net""" - - base_key = "a-net" - - def __init__( - self, - output_sub_dir: str, # = "a-net" - scale_bounds: Optional[tuple | list] = None, # limits on scale (lower, upper) - scale_units: Optional[str] = None, # scale units ('degw', 'amw') - downsample: Optional[float | int] = None, - timeout: Optional[ - float - ] = ASTROMETRY_TIMEOUT, # astrometry cmd execute timeout, in seconds - ): - super().__init__() - - self.output_sub_dir = output_sub_dir - self.scale_bounds = scale_bounds - self.scale_units = scale_units - self.downsample = downsample - self.timeout = timeout - - def __str__(self) -> str: - return "Processor to perform astrometric calibration via astrometry.net." - - def get_anet_output_dir(self) -> Path: - """ - Get the directory to output - - :return: output directory - """ - return get_output_dir(self.output_sub_dir, self.night_sub_dir) - - def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: - anet_out_dir = self.get_anet_output_dir() - cache = False - try: - os.makedirs(anet_out_dir) - except OSError: - pass - - for i, image in enumerate(batch): - temp_path = get_temp_path(anet_out_dir, image[BASE_NAME_KEY]) - if not os.path.exists(temp_path): - self.save_fits(image, temp_path) - - temp_files = [temp_path] - - run_astrometry_net_single( - img=temp_path, - output_dir=anet_out_dir, - scale_bounds=self.scale_bounds, - scale_units=self.scale_units, - downsample=self.downsample, - timeout=self.timeout, - ) - - newname = anet_out_dir.joinpath(Path(str(temp_path).split("temp_")[1])) - solved = fits.open(newname) - hdr = solved[0].header # pylint: disable=no-member - - del hdr["HISTORY"] - - fits.writeto( # pylint: disable=no-member - newname, - fits.open(temp_files[0])[0].data, # pylint: disable=no-member - hdr, - overwrite=True, - ) # pylint: disable=no-member - batch[i] = self.open_fits(newname) # pylint: disable=no-member - - if not cache: - for temp_file in temp_files: - os.remove(temp_file) - logger.info(f"Deleted temporary file {temp_file}") - - return batch diff --git a/mirar/processors/astromatic/config/__init__.py b/mirar/processors/astromatic/config/__init__.py index e775cdf8d..ab2590fbb 100644 --- a/mirar/processors/astromatic/config/__init__.py +++ b/mirar/processors/astromatic/config/__init__.py @@ -1,3 +1,6 @@ +""" +Module for configs of astrometry +""" import os astromatic_config_dir = os.path.dirname(__file__) diff --git a/mirar/processors/astromatic/psfex/__init__.py b/mirar/processors/astromatic/psfex/__init__.py index 0592e9eaf..eca8f375c 100644 --- a/mirar/processors/astromatic/psfex/__init__.py +++ b/mirar/processors/astromatic/psfex/__init__.py @@ -1 +1,4 @@ +""" +Module for running PSFex +""" from mirar.processors.astromatic.psfex.psfex import PSFex diff --git a/mirar/processors/astromatic/sextractor/settings.py b/mirar/processors/astromatic/sextractor/settings.py index c53c8aa62..050c9e978 100644 --- a/mirar/processors/astromatic/sextractor/settings.py +++ b/mirar/processors/astromatic/sextractor/settings.py @@ -12,6 +12,9 @@ def write_param_file(param_path: str = default_param_path): + """ + Write a default parameter file for sextractor + """ params = """X_IMAGE Y_IMAGE ALPHA_J2000 @@ -26,6 +29,9 @@ def write_param_file(param_path: str = default_param_path): def write_conv_file(conv_path: str = default_conv_path): + """ + Write a default convolution file for sextractor + """ convol = """CONV NORM # 3x3 ``all-ground'' convolution mask with FWHM = 2 pixels. 1 2 1 @@ -40,13 +46,16 @@ def write_config_file( param_path: str = default_param_path, conv_path: str = default_conv_path, config_path: str = default_config_path, - saturation: float = 55000.0, + saturation_key: str = "SATURATE", ): + """ + Write a default configuration file for sextractor + """ configs = f""" #-------------------------------- Catalog ------------------------------------ CATALOG_NAME temp.cat # name of the output catalog -CATALOG_TYPE ASCII_HEAD # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, +CATALOG_TYPE ASCII_HEAD # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT, # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC PARAMETERS_NAME {param_path} # name of the file containing catalog contents @@ -114,7 +123,7 @@ def write_config_file( VERBOSE_TYPE QUIET # can be QUIET, NORMAL or FULL WRITE_XML N # Write XML file (Y/N)? XML_NAME sex.xml # Filename for XML output -SATUR_LEVEL {saturation} # level (in ADUs) at which arises saturation +SATUR_KEY {saturation_key} # keyword for saturation level (in ADUs) """ with open(config_path, "w") as pf: pf.write(configs) diff --git a/mirar/processors/astromatic/sextractor/sextractor.py b/mirar/processors/astromatic/sextractor/sextractor.py index 136ad0ed9..53ccfb786 100644 --- a/mirar/processors/astromatic/sextractor/sextractor.py +++ b/mirar/processors/astromatic/sextractor/sextractor.py @@ -18,7 +18,6 @@ get_temp_path, ) from mirar.processors.astromatic.sextractor.sourceextractor import ( - default_saturation, parse_checkimage, run_sextractor_single, ) @@ -35,6 +34,7 @@ "BACKGROUND_RMS": "BKGRMS", "MINIBACKGROUND": "MINIBKG", "MINIBACK_RMS": "MINIBGRM", + "SEGMENTATION": "SEGMAP", } @@ -53,7 +53,7 @@ def __init__( parameter_path: str, filter_path: str, starnnw_path: str, - saturation: float = default_saturation, + saturation: float = None, verbose_type: str = "QUIET", checkimage_name: Optional[str | list] = None, checkimage_type: Optional[str | list] = None, @@ -63,6 +63,24 @@ def __init__( mag_zp: Optional[float] = None, write_regions_bool: bool = False, ): + """ + :param output_sub_dir: subdirectory to output sextractor files + :param config_path: path to sextractor config file + :param parameter_path: path to sextractor parameter file + :param filter_path: path to sextractor filter file + :param starnnw_path: path to sextractor starnnw file + :param saturation: saturation level for sextractor. Leave to None if not known, + no saturation will be applied + :param verbose_type: verbose type for sextractor + :param checkimage_type: type of checkimage to output + :param checkimage_name: name of checkimage to output. Leave to None to use + pipeline defaults in sextractor_checkimage_map for output name (recommended). + :param gain: gain for sextractor. Leave to None if not known. + :param dual: whether to run sextractor in dual mode + :param cache: whether to cache sextractor output + :param mag_zp: magnitude zero point for sextractor. Leave to None if not known. + :param write_regions_bool: whether to write regions file for ds9 + """ # pylint: disable=too-many-arguments super().__init__() self.output_sub_dir = output_sub_dir @@ -87,7 +105,7 @@ def __str__(self) -> str: f"and save detected sources to the '{self.output_sub_dir}' directory." ) - def get_sextractor_output_dir(self) -> str: + def get_sextractor_output_dir(self) -> Path: """ Get the directory to output @@ -138,12 +156,14 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: sextractor_out_dir, image[BASE_NAME_KEY].replace(".fits", ".cat") ) - _, self.checkimage_name = parse_checkimage( - checkimage_name=self.checkimage_name, + _, checkimage_name = parse_checkimage( + checkimage_name=None, checkimage_type=self.checkimage_type, image=os.path.join(sextractor_out_dir, image[BASE_NAME_KEY]), ) + logger.debug(f"Sextractor checkimage name is {checkimage_name}") + output_cat, checkimage_name = run_sextractor_single( img=temp_path, config=self.config, @@ -154,7 +174,7 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: saturation=self.saturation, weight_image=mask_path, verbose_type=self.verbose_type, - checkimage_name=self.checkimage_name, + checkimage_name=checkimage_name, checkimage_type=self.checkimage_type, gain=self.gain, catalog_name=output_cat, diff --git a/mirar/processors/astromatic/sextractor/sourceextractor.py b/mirar/processors/astromatic/sextractor/sourceextractor.py index 98b9c1db6..eadb04482 100644 --- a/mirar/processors/astromatic/sextractor/sourceextractor.py +++ b/mirar/processors/astromatic/sextractor/sourceextractor.py @@ -7,13 +7,15 @@ from typing import Optional from mirar.processors.astromatic.config import astromatic_config_dir +from mirar.processors.candidates.utils.regions_writer import write_regions_file from mirar.utils import ExecutionError, execute +from mirar.utils.ldac_tools import get_table_from_ldac logger = logging.getLogger(__name__) # sextractor_cmd = os.getenv("SEXTRACTOR_CMD") -default_saturation = 1.0e10 +default_saturation = 10000000000.0 default_config_path = os.path.join(astromatic_config_dir, "astrom.sex") default_param_path = os.path.join(astromatic_config_dir, "astrom.param") default_filter_name = os.path.join(astromatic_config_dir, "default.conv") @@ -79,8 +81,8 @@ def parse_checkimage( ) logger.error(err) raise ValueError(err) - else: - cmd += f"-CHECKIMAGE_NAME {','.join(checkimage_name)}" + + cmd += f"-CHECKIMAGE_NAME {','.join(checkimage_name)}" else: if image is not None: @@ -135,20 +137,21 @@ def run_sextractor_single( parameters_name: str = default_param_path, filter_name: str = default_filter_name, starnnw_name: str = default_starnnw_path, - saturation: float = default_saturation, + saturation: float = None, weight_image: Optional[str] = None, verbose_type: str = "QUIET", checkimage_name: Optional[str | list] = None, checkimage_type: Optional[str | list] = None, gain: Optional[float] = None, mag_zp: Optional[float] = None, + write_regions: bool = False, ): """ Function to run sextractor in single mode Args: - img: - output_dir: - catalog_name: + img: The image to run sextractor on + output_dir: The directory to output the catalog to + catalog_name: The name of the catalog to output. config: parameters_name: filter_name: @@ -158,9 +161,9 @@ def run_sextractor_single( verbose_type: checkimage_name: checkimage_type: - gain: - mag_zp: - + gain: The gain to use for the catalog + mag_zp: The magnitude zero point to use for the catalog + write_regions: Whether to write ds9 regions for the objects in the catalog Returns: """ @@ -207,6 +210,21 @@ def run_sextractor_single( except ExecutionError as e: raise SextractorError(e) + if write_regions: + output_catalog = get_table_from_ldac(catalog_name) + + x_coords = output_catalog["X_IMAGE"] + y_coords = output_catalog["Y_IMAGE"] + + regions_path = catalog_name.as_posix() + ".reg" + + write_regions_file( + regions_path=regions_path, + x_coords=x_coords, + y_coords=y_coords, + system="image", + region_radius=5, + ) return catalog_name, checkimage_name @@ -219,7 +237,7 @@ def run_sextractor_dual( parameters_name: str = default_param_path, filter_name: str = default_filter_name, starnnw_name: str = default_starnnw_path, - saturation: float = default_saturation, + saturation: float = None, weight_image: Optional[str] = None, verbose_type: str = "QUIET", checkimage_name: Optional[str | list] = None, @@ -275,6 +293,9 @@ def run_sextractor_dual( if starnnw_name is not None: cmd += f"-STARNNW_NAME {starnnw_name} " + if mag_zp is not None: + cmd += f" -MAG_ZEROPOINT {mag_zp}" + checkimage_cmd, checkimage_name = parse_checkimage( checkimage_type=checkimage_type, checkimage_name=checkimage_name, diff --git a/mirar/processors/astromatic/swarp/swarp.py b/mirar/processors/astromatic/swarp/swarp.py index 7e7d217d0..2c766b6f4 100644 --- a/mirar/processors/astromatic/swarp/swarp.py +++ b/mirar/processors/astromatic/swarp/swarp.py @@ -3,18 +3,23 @@ """ import logging import os +from collections.abc import Callable from pathlib import Path from typing import Optional import numpy as np +from astropy.io.fits import Header from astropy.wcs import WCS -from mirar.data import ImageBatch +from mirar.data import Image, ImageBatch from mirar.errors import ProcessorError +from mirar.io import open_fits from mirar.paths import ( BASE_NAME_KEY, + LATEST_SAVE_KEY, LATEST_WEIGHT_SAVE_KEY, RAW_IMG_KEY, + STACKED_COMPONENT_IMAGES_KEY, SWARP_FLUX_SCALING_KEY, all_astrometric_keywords, copy_temp_file, @@ -23,6 +28,7 @@ ) from mirar.processors.astromatic.scamp.scamp import scamp_header_key from mirar.processors.base_processor import BaseImageProcessor +from mirar.processors.utils.image_saver import ImageSaver from mirar.utils import execute logger = logging.getLogger(__name__) @@ -49,6 +55,7 @@ def run_swarp( subtract_bkg: bool = False, flux_scaling_keyword: str = None, cache: bool = False, + center_type: str = None, ): """ Wrapper to resample and stack images with swarp @@ -128,6 +135,9 @@ def run_swarp( if np.logical_and(center_ra is not None, center_dec is not None): swarp_command += f" -CENTER_TYPE MANUAL -CENTER {center_ra},{center_dec}" + else: + swarp_command += f" -CENTER_TYPE {center_type}" + if x_imgpixsize is not None: swarp_command += f" -IMAGE_SIZE {x_imgpixsize}" if y_imgpixsize is not None: @@ -162,6 +172,7 @@ def __init__( x_imgpixsize: Optional[float] = None, y_imgpixsize: Optional[float] = None, propogate_headerlist: Optional[list] = None, + center_type: Optional[str] = None, center_ra: Optional[float] = None, center_dec: Optional[float] = None, gain: Optional[float] = None, @@ -180,18 +191,23 @@ def __init__( temp_output_sub_dir: str output sub-directory pixscale: float - Pixel scale in degrees + Pixel scale in degrees. If None, set as median of the pixel scales + of input images. x_imgpixsize: float - X-dimension in pixels + X-dimension in pixels. If None, set as max x-size of input images. If + you want a stacked image covering all input images, set + calculate_dims_in_swarp to True instead. y_imgpixsize: float - Y-dimension in pixels + Y-dimension in pixels. If None, set as max y-size of input images. propogate_headerlist: list List of header keywords to propagate. Recommended to leave None, the processor will take care of it. center_ra: float - Desired central RA of output image + Desired central RA of output image. If None, set as the median of the + input images. center_dec: - Desired central Dec of output image + Desired central Dec of output image. If None, set as the median of the + input images. gain: float Gain include_scamp: bool @@ -241,6 +257,9 @@ def __init__( self.subtract_bkg = subtract_bkg self.flux_scaling_factor = flux_scaling_factor self.calculate_dims_in_swarp = calculate_dims_in_swarp + self.center_type = center_type + if self.center_type is not None: + assert self.center_type in ["MOST", "ALL", "MANUAL"] def __str__(self) -> str: return "Processor to apply swarp to images, stacking them together." @@ -301,6 +320,11 @@ def _apply_to_images( ) logger.debug(f"Saving to {output_image_path}") + try: + component_images_list = [x[LATEST_SAVE_KEY] for x in batch] + except KeyError: + component_images_list = None + all_pixscales = [] all_imgpixsizes = [] all_ras = [] @@ -398,6 +422,13 @@ def _apply_to_images( x_imgpixsize_to_use = None y_imgpixsize_to_use = None + if self.center_type is None: + self.center_type = "MANUAL" + + if self.center_type != "MANUAL": + center_dec_to_use = None + center_ra_to_use = None + output_image_weight_path = output_image_path.with_suffix(".weight.fits") run_swarp( @@ -410,6 +441,7 @@ def _apply_to_images( x_imgpixsize=x_imgpixsize_to_use, y_imgpixsize=y_imgpixsize_to_use, propogate_headerlist=self.propogate_headerlist, + center_type=self.center_type, center_ra=center_ra_to_use, center_dec=center_dec_to_use, combine=self.combine, @@ -456,10 +488,17 @@ def _apply_to_images( key.strip() not in all_astrometric_keywords, ): if key not in new_image.keys(): - new_image[key] = batch[0][key] + try: + new_image[key] = batch[0][key] + except ValueError: + continue new_image["COADDS"] = np.sum([x["COADDS"] for x in batch]) new_image[RAW_IMG_KEY] = ",".join([x[RAW_IMG_KEY] for x in batch]) + + if component_images_list is not None: + new_image[STACKED_COMPONENT_IMAGES_KEY] = ",".join(component_images_list) + new_image[BASE_NAME_KEY] = output_image_path.name new_image[LATEST_WEIGHT_SAVE_KEY] = output_image_weight_path.as_posix() self.save_fits(new_image, output_image_path) @@ -471,3 +510,81 @@ def _apply_to_images( logger.debug(f"Deleted temporary file {temp_file}") return ImageBatch([new_image]) + + +class GetSwarpComponentImages(BaseImageProcessor): + """ + Get the component images used to make a swarp stack + """ + + base_key = "swarp_component_images" + + def __init__( + self, + load_image: Callable[[str], [np.ndarray, Header]] = open_fits, + header_key=STACKED_COMPONENT_IMAGES_KEY, + copy_header_keys: str | list[str] = None, + ): + super().__init__() + self.load_image = load_image + self.header_key = header_key + + if isinstance(copy_header_keys, str): + copy_header_keys = [copy_header_keys] + self.copy_header_keys = copy_header_keys + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + if len(batch) > 1: + raise NotImplementedError( + "GetSwarpComponentImages only works on a batch containing a " + "single images. Consider adding an ImageDebatcher before " + "this processor." + ) + component_batch = ImageBatch() + image = batch[0] + component_images_list = image[self.header_key].split(",") + + for component_image_path in component_images_list: + if not Path(component_image_path).exists(): + raise FileNotFoundError( + f"Component image {component_image_path} not found. " + f"Are you sure it was saved using ImageSaver to this path just " + f"before the Swarp processor that stacked it?" + ) + component_data, component_header = self.load_image(component_image_path) + component_image = Image(component_data, component_header) + if self.copy_header_keys is not None: + for key in self.copy_header_keys: + if key in image.keys(): + component_image[key] = image[key] + component_batch.append(component_image) + logger.info(f"Loaded {len(component_batch)} component images") + return component_batch + + def check_prerequisites( + self, + ): + mask = np.array([isinstance(x, Swarp) for x in self.preceding_steps]) + if np.sum(mask) == 0: + err = ( + f"{self.__module__} requires {Swarp} as a prerequisite. " + f"However, the following steps were found: {self.preceding_steps}." + ) + logger.error(err) + raise ValueError(err) + + index = np.argmax(mask) + + preceding_step = self.preceding_steps[index - 1] + + if not isinstance(preceding_step, ImageSaver): + err = ( + f"{self.__module__} requires an {ImageSaver} to be used to save the " + f"component images immediately before {Swarp} is run. " + f"However, the following steps were found: {self.preceding_steps}." + ) + logger.error(err) + raise ValueError(err) diff --git a/mirar/processors/astrometry/__init__.py b/mirar/processors/astrometry/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/mirar/processors/astrometry/anet/__init__.py b/mirar/processors/astrometry/anet/__init__.py new file mode 100644 index 000000000..fdef8e9df --- /dev/null +++ b/mirar/processors/astrometry/anet/__init__.py @@ -0,0 +1,6 @@ +""" +Central module for running anet.net solve +""" + +from mirar.processors.astrometry.anet.anet import run_astrometry_net +from mirar.processors.astrometry.anet.anet_processor import AstrometryNet diff --git a/mirar/processors/anet/anet.py b/mirar/processors/astrometry/anet/anet.py similarity index 80% rename from mirar/processors/anet/anet.py rename to mirar/processors/astrometry/anet/anet.py index 384a82771..98abb304a 100644 --- a/mirar/processors/anet/anet.py +++ b/mirar/processors/astrometry/anet/anet.py @@ -42,6 +42,14 @@ def run_astrometry_net_single( scale_units: Optional[str] = None, # scale units ('degw', 'amw') downsample: Optional[float | int] = None, # downsample by factor of __ timeout: Optional[float] = None, # astrometry cmd execute timeout, in seconds + use_sextractor: bool = False, + sextractor_path: str = "sex", + search_radius_deg: float = 5, + parity: str = None, + sextractor_config_path: str = None, + x_image_key: str = "X_IMAGE", + y_image_key: str = "Y_IMAGE", + sort_key_name: str = "FLUX_AUTO", ): """ function to run astrometry.net locally on one image, with options to adjust settings @@ -74,8 +82,20 @@ def run_astrometry_net_single( # cmd with a ra, dec first guess (speeds up solution) header = fits.open(img)[0].header # pylint: disable=no-member ra_req, dec_req = header["RA"], header["DEC"] # requested ra, dec + if use_sextractor: + cmd += f"--use-source-extractor --source-extractor-path '{sextractor_path}' " + + if sextractor_config_path is not None: + cmd += f"--source-extractor-config {sextractor_config_path} " + + cmd += f"-X {x_image_key} -Y {y_image_key} -s {sort_key_name} " + + if parity is not None: + assert parity in ["pos", "neg"] + cmd += f"--parity {parity} " + cmd_loc = ( - cmd + f"--ra {ra_req}, --dec {dec_req} --radius 5 " + cmd + f"--ra {ra_req}, --dec {dec_req} --radius {search_radius_deg} " ) # radius takes on units of ra, dec try: diff --git a/mirar/processors/astrometry/anet/anet_processor.py b/mirar/processors/astrometry/anet/anet_processor.py new file mode 100644 index 000000000..2eb09e953 --- /dev/null +++ b/mirar/processors/astrometry/anet/anet_processor.py @@ -0,0 +1,167 @@ +""" +Module containing a processor to run astrometry.net +""" +import logging +import os +from pathlib import Path +from typing import Optional + +from astropy.io import fits + +from mirar.data import ImageBatch +from mirar.errors import ProcessorError +from mirar.paths import ( + BASE_NAME_KEY, + LATEST_WEIGHT_SAVE_KEY, + get_output_dir, + get_temp_path, +) +from mirar.processors.astrometry.anet.anet import run_astrometry_net_single +from mirar.processors.base_processor import BaseImageProcessor + +logger = logging.getLogger(__name__) + +ASTROMETRY_TIMEOUT = 900 # astrometry cmd execute timeout, in seconds + + +class AstrometryNetError(ProcessorError): + """ + Class for errors in astrometry.net + """ + + +class AstrometryNet(BaseImageProcessor): + """Processor to run astrometry.net""" + + base_key = "a-net" + + def __init__( + self, + output_sub_dir: str, # = "a-net" + scale_bounds: Optional[tuple | list] = None, + # limits on scale (lower, upper) + scale_units: Optional[str] = None, # scale units ('degw', 'amw') + downsample: Optional[float | int] = None, + timeout: Optional[ + float + ] = ASTROMETRY_TIMEOUT, # astrometry cmd execute timeout, in seconds + use_sextractor: bool = False, + sextractor_path: str = "sex", + search_radius_deg: float = 5.0, + parity: str = None, + sextractor_config_path: str = None, + x_image_key: str = "X_IMAGE", + y_image_key: str = "Y_IMAGE", + sort_key_name: str = "FLUX_AUTO", + use_weight: bool = True, + ): + """ + :param output_sub_dir: subdirectory to output astrometry.net results + :param scale_bounds: limits on scale (lower, upper) + :param scale_units: scale units ('degw', 'amw') + :param downsample: downsample by factor of __ + :param timeout: astrometry cmd execute timeout, in seconds + :param use_sextractor: use sextractor to find sources + :param sextractor_path: path to sextractor executable (e.g. sex) + :param search_radius_deg: search radius in degrees + :param parity: parity of the image, if known (e.g. "odd" or "even") + :param sextractor_config_path: path to sextractor config file, NOTE that you + cannot specify other config files (param, conv, nnw, etc.)to astrometry-net. + Make sure to set the config file to use the correct filter, etc. + :param x_image_key: key for x-image coordinate in sextractor catalog + :param y_image_key: key for y-image coordinate in sextractor catalog + :param sort_key_name: key for sorting sextractor catalog + """ + super().__init__() + + self.output_sub_dir = output_sub_dir + self.scale_bounds = scale_bounds + self.scale_units = scale_units + self.downsample = downsample + self.timeout = timeout + self.use_sextractor = use_sextractor + self.sextractor_path = sextractor_path + self.search_radius_deg = search_radius_deg + self.parity = parity + + self.x_image_key = x_image_key + self.y_image_key = y_image_key + self.sort_key_name = sort_key_name + self.use_weight = use_weight + self.sextractor_config_path = sextractor_config_path + + def __str__(self) -> str: + return "Processor to perform astrometric calibration via astrometry.net." + + def get_anet_output_dir(self) -> Path: + """ + Get the directory to output + + :return: output directory + """ + return get_output_dir(self.output_sub_dir, self.night_sub_dir) + + def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: + anet_out_dir = self.get_anet_output_dir() + cache = False + try: + os.makedirs(anet_out_dir) + except OSError: + pass + + for i, image in enumerate(batch): + temp_path = get_temp_path(anet_out_dir, image[BASE_NAME_KEY]) + if not os.path.exists(temp_path): + self.save_fits(image, temp_path) + + temp_files = [temp_path] + sextractor_path = f"{self.sextractor_path}" + if self.use_sextractor & self.use_weight: + weight_image = image[LATEST_WEIGHT_SAVE_KEY] + sextractor_path = ( + f"{self.sextractor_path} -WEIGHT_TYPE MAP_WEIGHT" + + f" -WEIGHT_IMAGE {weight_image}" + ) + + run_astrometry_net_single( + img=temp_path, + output_dir=anet_out_dir, + scale_bounds=self.scale_bounds, + scale_units=self.scale_units, + downsample=self.downsample, + timeout=self.timeout, + use_sextractor=self.use_sextractor, + sextractor_path=sextractor_path, + sextractor_config_path=self.sextractor_config_path, + search_radius_deg=self.search_radius_deg, + parity=self.parity, + x_image_key=self.x_image_key, + y_image_key=self.y_image_key, + sort_key_name=self.sort_key_name, + ) + + newname = anet_out_dir.joinpath(Path(str(temp_path).split("temp_")[1])) + if not newname.exists(): + raise AstrometryNetError( + f"AstrometryNet did not run successfully - no output " + f"file {newname} found." + ) + solved = fits.open(newname) + hdr = solved[0].header # pylint: disable=no-member + + del hdr["HISTORY"] + + fits.writeto( # pylint: disable=no-member + newname, + fits.open(temp_files[0])[0].data, # pylint: disable=no-member + hdr, + overwrite=True, + ) # pylint: disable=no-member + batch[i] = self.open_fits(newname) # pylint: disable=no-member + + if not cache: + for temp_file in temp_files: + os.remove(temp_file) + logger.info(f"Deleted temporary file {temp_file}") + + return batch diff --git a/mirar/processors/autoastrometry/__init__.py b/mirar/processors/astrometry/autoastrometry/__init__.py similarity index 58% rename from mirar/processors/autoastrometry/__init__.py rename to mirar/processors/astrometry/autoastrometry/__init__.py index bdb973b44..e60d22d9f 100644 --- a/mirar/processors/autoastrometry/__init__.py +++ b/mirar/processors/astrometry/autoastrometry/__init__.py @@ -4,4 +4,7 @@ author: Daniel Perley (dperley@astro.caltech.edu) last significant modifications 2012-04-23 """ -from mirar.processors.autoastrometry.autoastrometry_processor import AutoAstrometry + +from mirar.processors.astrometry.autoastrometry.autoastrometry_processor import ( + AutoAstrometry, +) diff --git a/mirar/processors/autoastrometry/autoastrometry.py b/mirar/processors/astrometry/autoastrometry/autoastrometry.py similarity index 97% rename from mirar/processors/autoastrometry/autoastrometry.py rename to mirar/processors/astrometry/autoastrometry/autoastrometry.py index a9ed6c8e4..d288dff04 100644 --- a/mirar/processors/autoastrometry/autoastrometry.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry.py @@ -33,30 +33,32 @@ write_param_file, ) from mirar.processors.astromatic.sextractor.sourceextractor import default_saturation -from mirar.processors.autoastrometry.crossmatch import ( +from mirar.processors.astrometry.autoastrometry.crossmatch import ( crosscheck_source_lists, distance_match, ) -from mirar.processors.autoastrometry.detect import ( +from mirar.processors.astrometry.autoastrometry.detect import ( DEFAULT_MAX_FWHM, DEFAULT_MIN_FWHM, get_img_src_list, ) -from mirar.processors.autoastrometry.errors import ( +from mirar.processors.astrometry.autoastrometry.errors import ( AstrometryCrossmatchError, + AstrometryReferenceError, AstrometrySourceError, + AstrometryURLError, ) -from mirar.processors.autoastrometry.io import ( +from mirar.processors.astrometry.autoastrometry.io import ( export_src_lists, parse_header, write_region_file, write_text_file, ) -from mirar.processors.autoastrometry.reference import ( +from mirar.processors.astrometry.autoastrometry.reference import ( get_ref_sources_from_catalog, get_ref_sources_from_catalog_astroquery, ) -from mirar.processors.autoastrometry.utils import median, stdev +from mirar.processors.astrometry.autoastrometry.utils import median, stdev logger = logging.getLogger(__name__) @@ -244,7 +246,12 @@ def autoastrometry( center_dec=center_dec, box_size_arcsec=box_size_arcsec, ) - except TimeoutError: + except ( + TimeoutError, + AstrometryURLError, + AstrometrySourceError, + AstrometryReferenceError, + ): ref_src_list, n_ref, ref_density = get_ref_sources_from_catalog_astroquery( catalog=catalog, center_ra=center_ra, @@ -614,7 +621,7 @@ def run_autoastrometry_single( write_param_file() - write_config_file(saturation=saturation) + write_config_file() logger.debug(f"Outfile is {outfile}") fit_info = autoastrometry( filename=img_path, diff --git a/mirar/processors/autoastrometry/autoastrometry_processor.py b/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py similarity index 95% rename from mirar/processors/autoastrometry/autoastrometry_processor.py rename to mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py index 87da6b9e1..21b9ee202 100644 --- a/mirar/processors/autoastrometry/autoastrometry_processor.py +++ b/mirar/processors/astrometry/autoastrometry/autoastrometry_processor.py @@ -6,7 +6,9 @@ from mirar.data import ImageBatch from mirar.paths import BASE_NAME_KEY, get_output_dir -from mirar.processors.autoastrometry.autoastrometry import run_autoastrometry_single +from mirar.processors.astrometry.autoastrometry.autoastrometry import ( + run_autoastrometry_single, +) from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/crossmatch.py b/mirar/processors/astrometry/autoastrometry/crossmatch.py similarity index 99% rename from mirar/processors/autoastrometry/crossmatch.py rename to mirar/processors/astrometry/autoastrometry/crossmatch.py index ff9e94fc8..37462c0de 100644 --- a/mirar/processors/autoastrometry/crossmatch.py +++ b/mirar/processors/astrometry/autoastrometry/crossmatch.py @@ -8,7 +8,7 @@ import numpy as np -from mirar.processors.autoastrometry.sources import ( +from mirar.processors.astrometry.autoastrometry.sources import ( BaseSource, SextractorSource, distance, @@ -16,7 +16,7 @@ position_angle, quickdistance, ) -from mirar.processors.autoastrometry.utils import median, mode, stdev, unique +from mirar.processors.astrometry.autoastrometry.utils import median, mode, stdev, unique logger = logging.getLogger(__name__) diff --git a/mirar/processors/autoastrometry/detect.py b/mirar/processors/astrometry/autoastrometry/detect.py similarity index 82% rename from mirar/processors/autoastrometry/detect.py rename to mirar/processors/astrometry/autoastrometry/detect.py index 4d3f3cbcf..a8def018e 100644 --- a/mirar/processors/autoastrometry/detect.py +++ b/mirar/processors/astrometry/autoastrometry/detect.py @@ -6,6 +6,9 @@ from pathlib import Path from typing import Optional +from astropy.io import fits + +from mirar.paths import SEXTRACTOR_HEADER_KEY from mirar.processors.astromatic.sextractor.settings import ( default_config_path, default_conv_path, @@ -16,9 +19,13 @@ default_saturation, run_sextractor_single, ) -from mirar.processors.autoastrometry.errors import AstrometrySourceError -from mirar.processors.autoastrometry.sources import SextractorSource, compare_mag -from mirar.processors.autoastrometry.utils import median, mode +from mirar.processors.astrometry.autoastrometry.errors import AstrometrySourceError +from mirar.processors.astrometry.autoastrometry.sources import ( + SextractorSource, + compare_mag, +) +from mirar.processors.astrometry.autoastrometry.utils import median, mode +from mirar.utils.ldac_tools import get_table_from_ldac logger = logging.getLogger(__name__) @@ -68,16 +75,32 @@ def get_img_src_list( except FileNotFoundError: pass - run_sextractor_single( - img=img_path, - output_dir=os.path.dirname(output_catalog), - config=config_path, - saturation=saturation, - catalog_name=output_catalog, - parameters_name=default_param_path, - filter_name=default_conv_path, - starnnw_name=default_starnnw_path, - ) + header = fits.getheader(img_path) + sextractor_catalog_path = None + if SEXTRACTOR_HEADER_KEY in header.keys(): + sextractor_catalog_path = fits.getval(img_path, SEXTRACTOR_HEADER_KEY) + + if sextractor_catalog_path is not None: + logger.info("Using existing sextractor catalog") + output_catalog = sextractor_catalog_path + output_catalog_table = get_table_from_ldac(output_catalog) + output_catalog_table.write( + output_catalog.replace(".cat", ".ascii.cat"), + format="ascii.ecsv", + overwrite=True, + ) + output_catalog = output_catalog.replace(".cat", ".ascii.cat") + else: + run_sextractor_single( + img=img_path, + output_dir=os.path.dirname(output_catalog), + config=config_path, + saturation=saturation, + catalog_name=output_catalog, + parameters_name=default_param_path, + filter_name=default_conv_path, + starnnw_name=default_starnnw_path, + ) # Read in the sextractor catalog with open(output_catalog, "rb") as cat: @@ -106,6 +129,10 @@ def get_img_src_list( if line[0] == "#": continue + # TODO : make more generic to skip column names + if line[0] == "X": + continue + src = SextractorSource(line) # process the line into an object n_src_init += 1 diff --git a/mirar/processors/autoastrometry/errors.py b/mirar/processors/astrometry/autoastrometry/errors.py similarity index 100% rename from mirar/processors/autoastrometry/errors.py rename to mirar/processors/astrometry/autoastrometry/errors.py diff --git a/mirar/processors/autoastrometry/io.py b/mirar/processors/astrometry/autoastrometry/io.py similarity index 98% rename from mirar/processors/autoastrometry/io.py rename to mirar/processors/astrometry/autoastrometry/io.py index 3501735ea..ae10d2df1 100644 --- a/mirar/processors/autoastrometry/io.py +++ b/mirar/processors/astrometry/autoastrometry/io.py @@ -10,8 +10,11 @@ import numpy as np from astropy.io import fits -from mirar.processors.autoastrometry.sources import BaseSource, SextractorSource -from mirar.processors.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg +from mirar.processors.astrometry.autoastrometry.sources import ( + BaseSource, + SextractorSource, +) +from mirar.processors.astrometry.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg logger = logging.getLogger(__name__) @@ -349,7 +352,7 @@ def write_region_file( out.write("image\n") for i, src in enumerate(src_list): out.write( - f"point({src.ra_deg:.3f},{src.dec_deg:.3f}) " + f"point({src.x:.3f},{src.y:.3f}) " f"# point=boxcircle text={{{i + 1}}}\n" ) diff --git a/mirar/processors/autoastrometry/reference.py b/mirar/processors/astrometry/autoastrometry/reference.py similarity index 97% rename from mirar/processors/autoastrometry/reference.py rename to mirar/processors/astrometry/autoastrometry/reference.py index e784117db..43ebf2875 100644 --- a/mirar/processors/autoastrometry/reference.py +++ b/mirar/processors/astrometry/autoastrometry/reference.py @@ -12,12 +12,12 @@ from astropy.coordinates import SkyCoord from astroquery.vizier import Vizier -from mirar.processors.autoastrometry.errors import ( +from mirar.processors.astrometry.autoastrometry.errors import ( AstrometryReferenceError, AstrometryURLError, ) -from mirar.processors.autoastrometry.sources import BaseSource, compare_mag -from mirar.processors.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg +from mirar.processors.astrometry.autoastrometry.sources import BaseSource, compare_mag +from mirar.processors.astrometry.autoastrometry.utils import dec_str_2_deg, ra_str_2_deg logger = logging.getLogger(__name__) @@ -334,7 +334,7 @@ def get_ref_sources_from_catalog( f"scat?catalog={trycat}&ra={center_ra}" f"&dec={center_dec}&system=J2000&rad=-90" ) - + logger.debug(f"Trying {testqueryurl}") with urllib.request.urlopen(testqueryurl, timeout=30) as check: checklines = check.readlines() logger.debug(f"Found {len(checklines)}") diff --git a/mirar/processors/autoastrometry/sources.py b/mirar/processors/astrometry/autoastrometry/sources.py similarity index 100% rename from mirar/processors/autoastrometry/sources.py rename to mirar/processors/astrometry/autoastrometry/sources.py diff --git a/mirar/processors/autoastrometry/utils.py b/mirar/processors/astrometry/autoastrometry/utils.py similarity index 100% rename from mirar/processors/autoastrometry/utils.py rename to mirar/processors/astrometry/autoastrometry/utils.py diff --git a/mirar/processors/astrometry/utils.py b/mirar/processors/astrometry/utils.py new file mode 100644 index 000000000..308489a39 --- /dev/null +++ b/mirar/processors/astrometry/utils.py @@ -0,0 +1,70 @@ +""" +Module containing processors to add astrometry headers to images +""" +import logging +import os + +from astropy.io import fits + +from mirar.data import ImageBatch +from mirar.paths import ASTROMETRY_FILE_KEY, BASE_NAME_KEY, get_astrometry_keys +from mirar.processors.base_processor import BaseImageProcessor + +logger = logging.getLogger(__name__) + + +class AstrometryFromFile(BaseImageProcessor): + """ + Processor to add astrometry headers to images from file. + """ + + base_key = "astrometry_from_file" + + def __init__(self, astrometry_file_key: str = ASTROMETRY_FILE_KEY): + super().__init__() + self.astrometry_file_key = astrometry_file_key + + def __str__(self) -> str: + return "Processor to add astrometry headers to images from file." + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + astrometry_keys = get_astrometry_keys() + new_batch = ImageBatch() + for image in batch: + astrometry_file = image[self.astrometry_file_key] + + if not os.path.exists(astrometry_file): + raise FileNotFoundError( + f"Could not find astrometry file " + f"{astrometry_file}. Are you for sure running " + f"scamp with cache=True?" + ) + for key in astrometry_keys: + if key in image.header.keys(): + logger.debug(f"Removing {key} from {image[BASE_NAME_KEY]}") + del image.header[key] + + logger.info( + f"Adding astrometry headers from {astrometry_file} " + f"to {image[BASE_NAME_KEY]}" + ) + + with open(astrometry_file, "r") as f: + header_data = f.read() + # Scamp v 2.10.0 writes this annoying character in a comment + header_data = header_data.replace("é", "e") + + astrometry_header = fits.Header.fromstring(header_data, sep="\n") + for k in astrometry_header: + if (k == "HISTORY") | (k == "COMMENT"): + continue + + logger.debug(f"Adding {k} to {astrometry_header[k]}") + image.header.append( + (k, astrometry_header[k], astrometry_header.comments[k]) + ) + new_batch.append(image) + return new_batch diff --git a/mirar/processors/base_processor.py b/mirar/processors/base_processor.py index e2702aa44..45921453e 100644 --- a/mirar/processors/base_processor.py +++ b/mirar/processors/base_processor.py @@ -305,19 +305,22 @@ def save_fits( logger.info(f"Saving to {path}") save_to_path(data, header, path) - def save_weight_image(self, image: Image, img_path: Path) -> Path: + def save_weight_image( + self, image: Image, img_path: Path, use_existing: bool = True + ) -> Path: """ Saves a weight image :param image: Weight image :param img_path: Path of parent image + :param use_existing: If True, will use existing weight image if it exists :return: Path of weight image """ weight_path = get_weight_path(img_path) header = image.get_header() weight_found = False - if LATEST_WEIGHT_SAVE_KEY in header.keys(): + if use_existing & (LATEST_WEIGHT_SAVE_KEY in header.keys()): existing_weightpath = Path(image[LATEST_WEIGHT_SAVE_KEY]) logger.info(f"WGHTPATH {existing_weightpath}") if existing_weightpath.exists(): diff --git a/mirar/processors/bias.py b/mirar/processors/bias.py index f94fca6c5..ecf7d5c27 100644 --- a/mirar/processors/bias.py +++ b/mirar/processors/bias.py @@ -8,7 +8,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError -from mirar.paths import BIAS_FRAME_KEY, LATEST_SAVE_KEY +from mirar.paths import BIAS_FRAME_KEY, LATEST_SAVE_KEY, SATURATE_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -57,7 +57,8 @@ def _apply_to_images( data = data - master_bias.get_data() image.set_data(data) image[BIAS_FRAME_KEY] = master_bias[LATEST_SAVE_KEY] - + if SATURATE_KEY in image.header: + image[SATURATE_KEY] -= np.nanmedian(master_bias.get_data()) return batch def make_image( diff --git a/mirar/processors/dark.py b/mirar/processors/dark.py index f266701a1..777286b13 100644 --- a/mirar/processors/dark.py +++ b/mirar/processors/dark.py @@ -8,6 +8,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError +from mirar.paths import SATURATE_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -60,6 +61,10 @@ def _apply_to_images( data = data - (master_dark.get_data() * image["EXPTIME"]) image.set_data(data) + if SATURATE_KEY in image.header: + image[SATURATE_KEY] -= ( + np.nanmedian(master_dark.get_data()) * image["EXPTIME"] + ) return batch def make_image( @@ -78,12 +83,18 @@ def make_image( darks = np.zeros((nx, ny, n_frames)) + individual_dark_exptimes = [] for i, img in enumerate(images): dark_exptime = img["EXPTIME"] darks[:, :, i] = img.get_data() / dark_exptime + individual_dark_exptimes.append(str(dark_exptime)) logger.info(f"Median combining {n_frames} darks") - master_dark = Image(np.nanmedian(darks, axis=2), header=images[0].get_header()) + master_dark_header = images[0].get_header() + master_dark_header["EXPTIME"] = 1.0 + master_dark_header["NCOMBINE"] = n_frames + master_dark_header["INDIVEXP"] = ",".join(individual_dark_exptimes) + master_dark = Image(np.nanmedian(darks, axis=2), header=master_dark_header) return master_dark diff --git a/mirar/processors/flat.py b/mirar/processors/flat.py index 7b18eab2d..13c8fdec1 100644 --- a/mirar/processors/flat.py +++ b/mirar/processors/flat.py @@ -2,14 +2,16 @@ Module containing processors for flat calibration """ import logging +import os.path import sys from collections.abc import Callable import numpy as np +from astropy.io import fits from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError -from mirar.paths import FLAT_FRAME_KEY, LATEST_SAVE_KEY +from mirar.paths import BASE_NAME_KEY, FLAT_FRAME_KEY, LATEST_SAVE_KEY from mirar.processors.base_processor import ProcessorPremadeCache, ProcessorWithCache from mirar.processors.utils.image_selector import select_from_images @@ -44,6 +46,7 @@ def __init__( y_max: int = sys.maxsize, flat_nan_threshold: float = 0.0, select_flat_images: Callable[[ImageBatch], ImageBatch] = default_select_flat, + flat_mask_key: str = None, **kwargs, ): super().__init__(*args, **kwargs) @@ -53,6 +56,7 @@ def __init__( self.y_max = y_max self.flat_nan_threshold = flat_nan_threshold self.select_cache_images = select_flat_images + self.flat_mask_key = flat_mask_key def __str__(self) -> str: return "Creates a flat image, divides other images by this image." @@ -83,6 +87,8 @@ def make_image( ) -> Image: images = self.select_cache_images(images) + logger.debug(f"Found {len(images)} suitable flats in batch") + n_frames = len(images) if n_frames == 0: err = f"Found {n_frames} suitable flats in batch" @@ -94,12 +100,39 @@ def make_image( flats = np.zeros((nx, ny, n_frames)) for i, img in enumerate(images): + data = img.get_data() + + if self.flat_mask_key is not None: + if self.flat_mask_key not in img.header.keys(): + err = ( + f"Image {img} does not have a mask with key " + f"{self.flat_mask_key}" + ) + logger.error(err) + raise KeyError(err) + + mask_file = img[self.flat_mask_key] + logger.info(f"Masking flat {img[BASE_NAME_KEY]} with mask {mask_file}") + if not os.path.exists(mask_file): + err = f"Mask file {mask_file} does not exist" + logger.error(err) + raise FileNotFoundError(err) + with fits.open(mask_file) as mask_img: + mask = mask_img[0].data + mask = mask > 0 + logger.info( + f"Masking {np.sum(mask)} pixels in flat " + f"{img[BASE_NAME_KEY]}" + ) + data[mask] = np.nan + median = np.nanmedian( - img.get_data()[self.x_min : self.x_max, self.y_min : self.y_max] + data[self.x_min : self.x_max, self.y_min : self.y_max] ) - flats[:, :, i] = img.get_data() / median + flats[:, :, i] = data / median logger.info(f"Median combining {n_frames} flats") + master_flat = np.nanmedian(flats, axis=2) return Image(master_flat, header=images[0].get_header()) @@ -110,8 +143,13 @@ class SkyFlatCalibrator(FlatCalibrator): Processor to do flat calibration using sky flats """ - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, select_flat_images=self.select_sky_flat) + def __init__(self, flat_mask_key=None, *args, **kwargs): + super().__init__( + *args, + **kwargs, + select_flat_images=self.select_sky_flat, + flat_mask_key=flat_mask_key, + ) @staticmethod def select_sky_flat( diff --git a/mirar/processors/mask.py b/mirar/processors/mask.py index decbe86a4..838688544 100644 --- a/mirar/processors/mask.py +++ b/mirar/processors/mask.py @@ -5,8 +5,12 @@ from pathlib import Path import numpy as np +from astropy.coordinates import SkyCoord +from astropy.io import fits +from astropy.wcs import WCS -from mirar.data import ImageBatch +from mirar.data import Image, ImageBatch +from mirar.paths import BASE_NAME_KEY, FITS_MASK_KEY, get_output_dir from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) @@ -15,40 +19,240 @@ MASK_VALUE = np.nan -class MaskPixels(BaseImageProcessor): +class BaseMask(BaseImageProcessor): + """ + Base class for masking processors + """ + + def __init__( + self, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + only_write_mask: bool = False, + ): + super().__init__() + self.write_masked_pixels_to_file = write_masked_pixels_to_file + self.output_dir = output_dir + self.only_write_mask = only_write_mask + + def get_mask(self, image) -> np.ndarray: + """ + Function to get the mask for a given image + """ + raise NotImplementedError + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + for image in batch: + data = image.get_data() + logger.debug(f"Masking {image[BASE_NAME_KEY]}") + mask = self.get_mask(image) + + if not self.only_write_mask: + data[mask] = MASK_VALUE + image.set_data(data) + + logger.info(f"Masked {np.sum(mask)} pixels in {image[BASE_NAME_KEY]}") + + if self.write_masked_pixels_to_file: + mask_directory = get_output_dir(self.output_dir, self.night_sub_dir) + if not mask_directory.exists(): + mask_directory.mkdir(parents=True) + mask_file_path = mask_directory / f"{image[BASE_NAME_KEY]}_mask.fits" + + mask_image = Image(data=mask.astype(int), header=image.get_header()) + self.save_fits(mask_image, mask_file_path) + + image[FITS_MASK_KEY] = mask_file_path.as_posix() + return batch + + +class MaskPixelsFromPath(BaseMask): """ Processor to apply bias calibration """ - base_key = "mask" + base_key = "maskfrompath" - def __init__(self, mask_path: str | Path): - super().__init__() + def __init__( + self, + mask_path: str | Path = None, + mask_path_key: str = None, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + only_write_mask: bool = False, + ): + super().__init__( + write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir, + only_write_mask=only_write_mask, + ) self.mask = None - self.mask_path = Path(mask_path) + self.mask_path = mask_path + self.mask_path_key = mask_path_key + if mask_path is None and mask_path_key is None: + raise ValueError("Must specify either mask_path or mask_path_key") + if mask_path is not None and mask_path_key is not None: + raise ValueError("Must specify either mask_path or mask_path_key, not both") def __str__(self) -> str: return f"Processor to mask bad pixels using a pre-defined map: {self.mask_path}" - def get_mask(self): + def get_mask(self, image) -> np.ndarray: """ loads mask if needed, and returns it :return: mask """ - if self.mask is None: + # if self.mask is None: # why is this needed? + if self.mask_path is not None: self.mask = self.open_fits(self.mask_path) - return self.mask + elif self.mask_path_key is not None: + logger.debug(f"Loading mask from {image[self.mask_path_key]}") + self.mask = self.open_fits(image[self.mask_path_key]) + mask = self.mask.get_data() + mask = mask != 0 + return mask - def _apply_to_images( + +class MaskAboveThreshold(BaseMask): + """ + Processor to mask pixels above a threshold + """ + + base_key = "maskthresh" + + def __init__( self, - batch: ImageBatch, - ) -> ImageBatch: - for image in batch: - data = image.get_data() - mask = self.get_mask().get_data() - mask = mask != 0 - data[mask] = MASK_VALUE - image.set_data(data) + threshold: float = None, + threshold_key: str = None, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + only_write_mask: bool = False, + ): + """ + :param threshold: threshold to mask above + :param threshold_key: key to use to get threshold from image header + """ + super().__init__( + write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir, + only_write_mask=only_write_mask, + ) + self.threshold = threshold + self.threshold_key = threshold_key + self.write_masked_pixels_to_file = write_masked_pixels_to_file + if threshold is None and threshold_key is None: + raise ValueError("Must specify either threshold or threshold_key") + if threshold is not None and threshold_key is not None: + raise ValueError("Must specify either threshold or threshold_key, not both") - return batch + def __str__(self) -> str: + return f"Processor to mask pixels above a threshold: {self.threshold}" + + def get_mask(self, image) -> np.ndarray: + """ + Returns a mask for pixels above a threshold + + :return: mask + """ + if self.threshold is None: + self.threshold = image.get_header()[self.threshold_key] + mask = image.get_data() > self.threshold + return mask + + +class MaskPixelsFromWCS(BaseMask): + """ + Processor to mask pixels from a file where WCS coordinates of masked pixels are + given + """ + + base_key = "maskwcs" + + def __init__( + self, + mask_pixels_ra: float | list[float] = None, + mask_pixels_dec: float | list[float] = None, + mask_file_key: str = FITS_MASK_KEY, + write_masked_pixels_to_file: bool = False, + output_dir: str | Path = "mask", + only_write_mask: bool = False, + ): + super().__init__( + write_masked_pixels_to_file=write_masked_pixels_to_file, + output_dir=output_dir, + only_write_mask=only_write_mask, + ) + self.mask_pixels_ra = mask_pixels_ra + self.mask_pixels_dec = mask_pixels_dec + self.mask_file_key = mask_file_key + + if self.mask_pixels_ra is not None: + self.mask_file_key = None + + def __str__(self) -> str: + return "Processor to mask pixels using a list of RA/Dec." + + def get_mask(self, image) -> np.ndarray: + """ + loads mask if needed, and returns it + + :return: mask + """ + wcs = WCS(image.get_header()) + if self.mask_file_key is not None: + mask_file_path = image.get_header()[self.mask_file_key] + with fits.open(mask_file_path) as mask_image: + mask = mask_image[0].data + mask_wcs = WCS(mask_image[0].header) + + masked_pixel_x, masked_pixel_y = np.where(mask) + mask_pixel_coords = mask_wcs.pixel_to_world(masked_pixel_y, masked_pixel_x) + mask_pixels_ra = mask_pixel_coords.ra.deg + mask_pixels_dec = mask_pixel_coords.dec.deg + else: + mask_pixels_ra = self.mask_pixels_ra + mask_pixels_dec = self.mask_pixels_dec + logger.debug(f"Masking {mask_pixels_ra} ras and {mask_pixels_dec} decs") + + mask_pixel_coords = SkyCoord(mask_pixels_ra, mask_pixels_dec, unit="deg") + mask_pixels_x, mask_pixels_y = wcs.world_to_pixel(mask_pixel_coords) + mask_pixels_x = mask_pixels_x.astype(int) + mask_pixels_y = mask_pixels_y.astype(int) + mask = np.zeros(image.get_data().shape, dtype=bool) + mask_in_image = np.logical_and( + mask_pixels_x >= 0, mask_pixels_x < mask.shape[1] + ) & np.logical_and(mask_pixels_y >= 0, mask_pixels_y < mask.shape[0]) + mask_pixels_x = mask_pixels_x[mask_in_image] + mask_pixels_y = mask_pixels_y[mask_in_image] + mask[mask_pixels_y, mask_pixels_x] = True + return mask + + +class WriteMaskedCoordsToFile(BaseMask): + """ + Processor to write masked coordinates to a file + """ + + base_key = "writemaskedcoords" + + def __init__(self, output_dir: str | Path = "mask", only_write_mask: bool = False): + super().__init__( + write_masked_pixels_to_file=True, + output_dir=output_dir, + only_write_mask=only_write_mask, + ) + + def get_mask(self, image) -> np.ndarray: + mask = np.zeros(image.get_data().shape, dtype=bool) + + # For some reason, MASK_VALUE == np.nan returns False. Issue/Feature of numpy? + # This is a workaround + if np.isnan(MASK_VALUE): + mask[np.isnan(image.get_data())] = True + else: + mask[image.get_data() == MASK_VALUE] = True + return mask diff --git a/mirar/processors/photcal.py b/mirar/processors/photcal.py index ac40fde75..bd4643ad9 100644 --- a/mirar/processors/photcal.py +++ b/mirar/processors/photcal.py @@ -11,11 +11,12 @@ from astropy.coordinates import SkyCoord from astropy.io import fits from astropy.stats import sigma_clip, sigma_clipped_stats +from astropy.table import Table from mirar.catalog.base_catalog import BaseCatalog from mirar.data import Image, ImageBatch from mirar.errors import ProcessorError -from mirar.paths import copy_temp_file, get_output_dir, get_output_path +from mirar.paths import BASE_NAME_KEY, copy_temp_file, get_output_dir, get_output_path from mirar.processors.astromatic.sextractor.sextractor import ( SEXTRACTOR_HEADER_KEY, Sextractor, @@ -60,6 +61,29 @@ class PhotometryCalculationError(PhotometryError): """Error related to the photometric calibration""" +def default_photometric_img_catalog_purifier(catalog: Table, image: Image) -> Table: + """ + Default function to purify the photometric image catalog + """ + edge_width_pixels = 100 + fwhm_threshold_arcsec = 4.0 + x_lower_limit = edge_width_pixels + x_upper_limit = image.get_data().shape[1] - edge_width_pixels + y_lower_limit = edge_width_pixels + y_upper_limit = image.get_data().shape[0] - edge_width_pixels + + clean_mask = ( + (catalog["FLAGS"] == 0) + & (catalog["FWHM_WORLD"] < fwhm_threshold_arcsec / 3600.0) + & (catalog["X_IMAGE"] > x_lower_limit) + & (catalog["X_IMAGE"] < x_upper_limit) + & (catalog["Y_IMAGE"] > y_lower_limit) + & (catalog["Y_IMAGE"] < y_upper_limit) + ) + + return catalog[clean_mask] + + class PhotCalibrator(BaseImageProcessor): """ Photometric calibrator processor @@ -72,11 +96,9 @@ def __init__( ref_catalog_generator: Callable[[Image], BaseCatalog], temp_output_sub_dir: str = "phot", redo: bool = True, - x_lower_limit: float = 100, - x_upper_limit: float = 2800, # Are these floats or ints? - y_lower_limit: float = 100, - y_upper_limit: float = 2800, - fwhm_threshold_arcsec: float = 4.0, + image_photometric_catalog_purifier: Callable[ + [Table, Image], Table + ] = default_photometric_img_catalog_purifier, num_matches_threshold: int = 5, write_regions: bool = False, cache: bool = False, @@ -85,15 +107,9 @@ def __init__( self.redo = redo # What is this for? self.ref_catalog_generator = ref_catalog_generator self.temp_output_sub_dir = temp_output_sub_dir - self.x_lower_limit = x_lower_limit - self.x_upper_limit = x_upper_limit - self.y_lower_limit = y_lower_limit - self.y_upper_limit = y_upper_limit + self.image_photometric_catalog_purifier = image_photometric_catalog_purifier self.cache = cache - # Why is this here not in catalog? - self.fwhm_threshold_arcsec = fwhm_threshold_arcsec - self.num_matches_threshold = num_matches_threshold self.write_regions = write_regions @@ -108,60 +124,25 @@ def get_phot_output_dir(self): return get_output_dir(self.temp_output_sub_dir, self.night_sub_dir) def calculate_zeropoint( - self, ref_cat_path: Path, img_cat_path: Path, img_filt + self, + ref_cat: Table, + clean_img_cat: Table, ) -> list[dict]: """ Function to calculate zero point from two catalogs Args: - ref_cat_path: Path to reference ldac catalog - img_cat_path: Path to image ldac catalog + ref_cat: Reference catalog table + clean_img_cat: Catalog of sources from image to xmatch with ref_cat Returns: """ - ref_cat_with_flagged = get_table_from_ldac(ref_cat_path) - img_cat = get_table_from_ldac(img_cat_path) - - if len(ref_cat_with_flagged) == 0: - err = "No sources found in reference catalog" - logger.error(err) - raise PhotometryReferenceError(err) - - if str(ref_cat_path).split(".")[-2] == "ps1": - # this reference catalog is from ps1 - # remove sources with SATURATED flag - ref_cat = self.remove_sat_ps1(ref_cat_with_flagged, img_filt) - - else: - # reference not ps1, no flags to check - ref_cat = ref_cat_with_flagged - ref_coords = SkyCoord(ra=ref_cat["ra"], dec=ref_cat["dec"], unit=(u.deg, u.deg)) - clean_mask = ( - (img_cat["FLAGS"] == 0) - & (img_cat["FWHM_WORLD"] < self.fwhm_threshold_arcsec / 3600.0) - & (img_cat["X_IMAGE"] > self.x_lower_limit) - & (img_cat["X_IMAGE"] < self.x_upper_limit) - & (img_cat["Y_IMAGE"] > self.y_lower_limit) - & (img_cat["Y_IMAGE"] < self.y_upper_limit) - ) - img_coords = SkyCoord( - ra=img_cat["ALPHAWIN_J2000"], - dec=img_cat["DELTAWIN_J2000"], - unit=(u.deg, u.deg), - ) - clean_img_cat = img_cat[clean_mask] - logger.debug(f"Found {len(clean_img_cat)} clean sources in image.") clean_img_coords = SkyCoord( ra=clean_img_cat["ALPHAWIN_J2000"], dec=clean_img_cat["DELTAWIN_J2000"], unit=(u.deg, u.deg), ) - if len(clean_img_coords) == 0: - err = "No clean sources found in image" - logger.error(err) - raise PhotometrySourceError(err) - idx, d2d, _ = ref_coords.match_to_catalog_sky(clean_img_coords) match_mask = d2d < 1.0 * u.arcsec matched_ref_cat = ref_cat[match_mask] @@ -170,45 +151,6 @@ def calculate_zeropoint( f"Cross-matched {len(matched_img_cat)} sources from catalog to the image." ) - if self.write_regions: - ref_regions_path = get_output_path( - base_name="ref.reg", - dir_root=self.temp_output_sub_dir, - sub_dir=self.night_sub_dir, - ) - cleaned_img_regions_path = get_output_path( - base_name="cleaned_img.reg", - dir_root=self.temp_output_sub_dir, - sub_dir=self.night_sub_dir, - ) - img_regions_path = get_output_path( - base_name="img.reg", - dir_root=self.temp_output_sub_dir, - sub_dir=self.night_sub_dir, - ) - - write_regions_file( - regions_path=ref_regions_path, - x_coords=ref_coords.ra.deg, - y_coords=ref_coords.dec.deg, - system="wcs", - region_radius=2.0 / 3600, - ) - write_regions_file( - regions_path=cleaned_img_regions_path, - x_coords=clean_img_coords.ra.deg, - y_coords=clean_img_coords.dec.deg, - system="wcs", - region_radius=2.0 / 3600, - ) - write_regions_file( - regions_path=img_regions_path, - x_coords=img_coords.ra.deg, - y_coords=img_coords.dec.deg, - system="wcs", - region_radius=2.0 / 3600, - ) - if len(matched_img_cat) < self.num_matches_threshold: err = ( "Not enough cross-matched sources " @@ -296,9 +238,79 @@ def _apply_to_images( image["FWHM_MED"] = fwhm_med image["FWHM_STD"] = fwhm_std - zp_dicts = self.calculate_zeropoint( - ref_cat_path, temp_cat_path, image.header["FILTER"] - ) + ref_cat = get_table_from_ldac(ref_cat_path) + img_cat = get_table_from_ldac(temp_cat_path) + + if len(ref_cat) == 0: + err = "No sources found in reference catalog" + logger.error(err) + raise PhotometryReferenceError(err) + + clean_img_cat = self.image_photometric_catalog_purifier(img_cat, image) + logger.debug(f"Found {len(clean_img_cat)} clean sources in image.") + + if len(clean_img_cat) == 0: + err = "No clean sources found in image" + logger.error(err) + raise PhotometrySourceError(err) + + if self.write_regions: + ref_coords = SkyCoord( + ra=ref_cat["ra"], dec=ref_cat["dec"], unit=(u.deg, u.deg) + ) + + img_coords = SkyCoord( + ra=img_cat["ALPHAWIN_J2000"], + dec=img_cat["DELTAWIN_J2000"], + unit=(u.deg, u.deg), + ) + + clean_img_coords = SkyCoord( + ra=clean_img_cat["ALPHAWIN_J2000"], + dec=clean_img_cat["DELTAWIN_J2000"], + unit=(u.deg, u.deg), + ) + + ref_regions_path = get_output_path( + base_name=image.header[BASE_NAME_KEY] + "ref.reg", + dir_root=self.temp_output_sub_dir, + sub_dir=self.night_sub_dir, + ) + cleaned_img_regions_path = get_output_path( + base_name=image.header[BASE_NAME_KEY] + "cleaned_img.reg", + dir_root=self.temp_output_sub_dir, + sub_dir=self.night_sub_dir, + ) + img_regions_path = get_output_path( + base_name=image.header[BASE_NAME_KEY] + "img.reg", + dir_root=self.temp_output_sub_dir, + sub_dir=self.night_sub_dir, + ) + + write_regions_file( + regions_path=ref_regions_path, + x_coords=ref_coords.ra.deg, + y_coords=ref_coords.dec.deg, + system="wcs", + region_radius=2.0 / 3600, + ) + write_regions_file( + regions_path=cleaned_img_regions_path, + x_coords=clean_img_coords.ra.deg, + y_coords=clean_img_coords.dec.deg, + system="wcs", + region_radius=2.0 / 3600, + ) + write_regions_file( + regions_path=img_regions_path, + x_coords=img_coords.ra.deg, + y_coords=img_coords.dec.deg, + system="wcs", + region_radius=2.0 / 3600, + ) + + zp_dicts = self.calculate_zeropoint(ref_cat, clean_img_cat) + aperture_diameters = [] zp_values = [] for zpvals in zp_dicts: @@ -314,7 +326,8 @@ def _apply_to_images( aperture_diameters.append(med_fwhm_pix) zp_values.append(image["ZP_AUTO"]) - if sextractor_checkimg_map["BACKGROUND_RMS"] in image: + if sextractor_checkimg_map["BACKGROUND_RMS"] in image.header.keys(): + logger.info("Calculating limiting magnitudes from background RMS file") limmags = self.get_maglim( image[sextractor_checkimg_map["BACKGROUND_RMS"]], zp_values, @@ -389,6 +402,9 @@ def get_maglim( return maglim def get_sextractor_module(self) -> Sextractor: + """ + Get the Sextractor module from the preceding steps + """ mask = [isinstance(x, Sextractor) for x in self.preceding_steps] return np.array(self.preceding_steps)[mask][-1] @@ -452,18 +468,3 @@ def get_sextractor_apertures(self) -> list[float]: line = aperture_lines[0].replace("PHOT_APERTURES", " ").split("#")[0] return [float(x) for x in line.split(",") if x not in [""]] - - def remove_sat_ps1(self, catalog, filt: str): - """ - remove ps1 sources flagged as "SATURATED" - """ - logger.info(f"original ps1 table length: {len(catalog)}") - logger.info("removing ps1 sources with SATURATED flag...") - sat_flag = 4096 # SATURATED value - column = catalog[str(filt) + "Flags"] - check = (column & sat_flag) / sat_flag - # check != 0 means this flag is there - # check == 0 means this flag is not there - clean_cat = catalog[np.where(check == 0)[0]] - logger.info(f"found {len(clean_cat)} columns without this flag \n") - return clean_cat diff --git a/mirar/processors/sky.py b/mirar/processors/sky.py index ff452ca8b..fce32cfb9 100644 --- a/mirar/processors/sky.py +++ b/mirar/processors/sky.py @@ -6,6 +6,7 @@ import numpy as np from mirar.data import ImageBatch +from mirar.paths import SATURATE_KEY from mirar.processors.base_processor import ProcessorPremadeCache from mirar.processors.flat import SkyFlatCalibrator @@ -28,7 +29,9 @@ def _apply_to_images( mask = master_sky.get_data() <= self.flat_nan_threshold if np.sum(mask) > 0: - master_sky[mask] = np.nan + data = master_sky.get_data() + data[mask] = np.nan + master_sky.set_data(data) for image in batch: data = image.get_data() @@ -40,6 +43,9 @@ def _apply_to_images( header.append( ("SKMEDSUB", subtract_median, "Median sky level subtracted"), end=True ) + if SATURATE_KEY in image.header: + # image[SATURATE_KEY] -= subtract_median + image[SATURATE_KEY] = 25000 image.set_data(data) image.set_header(header) diff --git a/mirar/processors/utils/image_loader.py b/mirar/processors/utils/image_loader.py index 530716522..d0ce156d1 100644 --- a/mirar/processors/utils/image_loader.py +++ b/mirar/processors/utils/image_loader.py @@ -13,7 +13,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError from mirar.io import open_fits -from mirar.paths import RAW_IMG_SUB_DIR, base_raw_dir, core_fields +from mirar.paths import RAW_IMG_KEY, RAW_IMG_SUB_DIR, base_raw_dir, core_fields from mirar.processors.base_processor import BaseImageProcessor logger = logging.getLogger(__name__) @@ -110,3 +110,43 @@ def unzip(zipped_list: list[str]) -> list[str]: os.rename(file, unzipped_list[i]) return unzipped_list + + +class LoadImageFromHeader(BaseImageProcessor): + """ + Class to load images from header information + """ + + base_key = "load_from_header" + + def __init__( + self, + header_key: str = RAW_IMG_KEY, + copy_header_keys: str | list[str] = None, + load_image: Callable[[str], [np.ndarray, astropy.io.fits.Header]] = open_fits, + ): + super().__init__() + self.header_key = header_key + self.copy_header_keys = copy_header_keys + self.load_image = load_image + if isinstance(self.copy_header_keys, str): + self.copy_header_keys = [self.copy_header_keys] + + def __str__(self): + return f"Processor to load images from header key {self.header_key}" + + def _apply_to_images( + self, + batch: ImageBatch, + ) -> ImageBatch: + new_batch = ImageBatch() + for image in batch: + new_image_file = image.header[self.header_key] + new_image_data, new_header = self.load_image(new_image_file) + new_image = Image(new_image_data, new_header) + if self.copy_header_keys is not None: + for key in self.copy_header_keys: + new_image.header[key] = image.header[key] + new_batch.append(new_image) + + return new_batch diff --git a/mirar/processors/utils/image_saver.py b/mirar/processors/utils/image_saver.py index 1666cddcb..5c8abc041 100644 --- a/mirar/processors/utils/image_saver.py +++ b/mirar/processors/utils/image_saver.py @@ -1,6 +1,7 @@ """ Module for saving images """ +import logging import shutil from pathlib import Path @@ -14,6 +15,8 @@ ) from mirar.processors.base_processor import BaseImageProcessor +logger = logging.getLogger(__name__) + class ImageSaver(BaseImageProcessor): """ @@ -27,11 +30,13 @@ def __init__( output_dir_name: str, write_mask: bool = True, output_dir: str | Path = base_output_dir, + use_existing_weight: bool = True, ): super().__init__() self.output_dir_name = output_dir_name self.write_mask = write_mask self.output_dir = Path(output_dir) + self.use_existing_weight = use_existing_weight def __str__(self): return f"Processor to save images to the '{self.output_dir_name}' subdirectory" @@ -53,7 +58,10 @@ def _apply_to_images( image[LATEST_SAVE_KEY] = str(path) if self.write_mask: weight_image_found, mask_path = False, "" - if LATEST_WEIGHT_SAVE_KEY in image.header.keys(): + if self.use_existing_weight & ( + LATEST_WEIGHT_SAVE_KEY in image.header.keys() + ): + logger.debug(f"Searching for existing weight image") existing_weightpath = Path(image[LATEST_WEIGHT_SAVE_KEY]) if existing_weightpath.exists(): weight_image_found = True @@ -67,7 +75,9 @@ def _apply_to_images( shutil.copy(existing_weightpath, mask_path) if not weight_image_found: - mask_path = self.save_weight_image(image, img_path=path) + mask_path = self.save_weight_image( + image, img_path=path, use_existing=self.use_existing_weight + ) image[LATEST_WEIGHT_SAVE_KEY] = str(mask_path) self.save_fits(image, path) diff --git a/mirar/processors/utils/multi_ext_parser.py b/mirar/processors/utils/multi_ext_parser.py index b3dc7dd51..4775ef98f 100644 --- a/mirar/processors/utils/multi_ext_parser.py +++ b/mirar/processors/utils/multi_ext_parser.py @@ -13,7 +13,7 @@ from mirar.data import Image, ImageBatch from mirar.errors import ImageNotFoundError from mirar.io import open_fits -from mirar.paths import RAW_IMG_SUB_DIR, base_raw_dir +from mirar.paths import RAW_IMG_SUB_DIR, base_raw_dir, get_output_dir, get_output_path from mirar.processors.base_processor import BaseImageProcessor from mirar.processors.utils.image_loader import unzip @@ -30,15 +30,33 @@ class MultiExtParser(BaseImageProcessor): def __init__( self, input_sub_dir: str = RAW_IMG_SUB_DIR, + output_sub_dir: str = "raw_split", input_img_dir: str = base_raw_dir, load_image: Callable[[str], [np.ndarray, astropy.io.fits.Header]] = open_fits, skip_first: bool = False, + extension_num_header_key: str = None, + only_extract_num: int = None, ): + """ + :param input_sub_dir: subdirectory to look for images + :param output_sub_dir: subdirectory to save split single extenion images + :param input_img_dir: parent directory of input_sub_dir + :param load_image: function to load image + :param extension_num_header_key: If provided, will use the corresponding value + in the header to identify an extension_number for every image and save the file + as <>_{extension_number}.fits. If None, will serially number the extensions. + :param only_extract_num: If provided, will only extract the extension with this + number. If None, will extract all extensions. extension_number is calculated + as described in extension_num_header_key. + """ super().__init__() self.input_sub_dir = input_sub_dir self.input_img_dir = input_img_dir self.load_image = load_image self.skip_first = skip_first + self.output_sub_dir = output_sub_dir + self.extension_num_header_key = extension_num_header_key + self.only_extract_num = only_extract_num def __str__(self): return ( @@ -56,10 +74,16 @@ def parse(self, path: str) -> list: ex: /[instrument]/[night]/raw/mef/ """ + output_dir = get_output_dir( + dir_root=self.output_sub_dir, sub_dir=self.night_sub_dir + ) + if not output_dir.exists(): + output_dir.mkdir(parents=True) + new_paths = [] with astropy.io.fits.open(path) as hdu: num_ext = len(hdu) - logger.info(f"This file has {num_ext} extensions.") + logger.info(f"This file - {path} - has {num_ext} extensions.") hdr0 = hdu[0].header # pylint: disable=no-member # zip hdr0's values and comments @@ -74,17 +98,36 @@ def parse(self, path: str) -> list: data = hdu[ext].data hdrext = hdu[ext].header + extension_num_str = str(ext) + if self.extension_num_header_key is not None: + extension_num_str = hdrext[self.extension_num_header_key] + + if self.only_extract_num is not None: + if int(extension_num_str) != self.only_extract_num: + continue + # append hdr0 to hdrext for count, key in enumerate(list(hdr0.keys())): hdrext.append((key, zipped[count][0], zipped[count][1])) # save to new file with 1 extension - notmefpath = path.split("/mef/")[0] + path.split("/mef")[1] - newpath = notmefpath.split(".fits")[0] + "_" + str(ext) + ".fits" + # notmefpath = path.split("/mef/")[0] + path.split("/mef")[1] + + splitfile_basename = ( + f"{os.path.basename(path).split('.fits')[0]}_" + f"{extension_num_str}.fits" + ) + + splitfile_path = get_output_path( + base_name=splitfile_basename, + dir_root=self.output_sub_dir, + sub_dir=self.night_sub_dir, + ) + # newpath = notmefpath.split(".fits")[0] + "_" + str(ext) + ".fits" astropy.io.fits.writeto( - newpath, data, hdrext, overwrite=True + splitfile_path, data, hdrext, overwrite=True ) # pylint: disable=no-member - new_paths.append(newpath) + new_paths.append(splitfile_path) return new_paths @@ -96,7 +139,7 @@ def _apply_to_images(self, batch: ImageBatch) -> ImageBatch: def load_from_dir( - input_dir: str | Path, parse_f: Callable[[str | Path], Image] + input_dir: str | Path, parse_f: Callable[[list[str | Path]], Image] ) -> ImageBatch: """ Function to parse all MEF images in a directory diff --git a/mirar/references/ukirt.py b/mirar/references/ukirt.py index 5bb4255ce..d9ddd25e7 100644 --- a/mirar/references/ukirt.py +++ b/mirar/references/ukirt.py @@ -547,7 +547,8 @@ def get_reference(self, image: Image) -> tuple[PrimaryHDU, PrimaryHDU]: logger.debug(f"Reading image from {url}") with fits.open(url, ignore_missing_simple=True) as ukirt_hdulist: ukirt_image = Image( - header=ukirt_hdulist[0].header, data=ukirt_hdulist[0].data + header=ukirt_hdulist[0].header, # pylint: disable=no-member + data=ukirt_hdulist[0].data, # pylint: disable=no-member ) if self.check_local_database & ~qexists: diff --git a/tests/test_summer_pipeline.py b/tests/test_summer_pipeline.py index 6fa341b99..683845528 100644 --- a/tests/test_summer_pipeline.py +++ b/tests/test_summer_pipeline.py @@ -10,27 +10,30 @@ logger = logging.getLogger(__name__) expected_zp = { - "ZP_2.0": 24.379148900858567, - "ZP_2.0_std": 0.07320184249312667, + "ZP_2.0": 24.376073571395878, + "ZP_2.0_std": 0.07935580417736623, "ZP_2.0_nstars": 30, - "ZP_3.0": 25.072568754704793, - "ZP_3.0_std": 0.06529106709538676, + "ZP_3.0": 25.075430540593466, + "ZP_3.0_std": 0.06447897389004999, "ZP_3.0_nstars": 30, - "ZP_4.0": 25.449809682718914, - "ZP_4.0_std": 0.06168474184669056, + "ZP_4.0": 25.449352268727623, + "ZP_4.0_std": 0.061933883191802645, "ZP_4.0_nstars": 30, - "ZP_5.0": 25.66054034665426, - "ZP_5.0_std": 0.06357885150281098, + "ZP_5.0": 25.662642403793335, + "ZP_5.0_std": 0.06341347711205561, "ZP_5.0_nstars": 30, - "ZP_6.0": 25.779508960596722, - "ZP_6.0_std": 0.06472718057714481, + "ZP_6.0": 25.781888934326172, + "ZP_6.0_std": 0.0643293713949796, "ZP_6.0_nstars": 30, - "ZP_7.0": 25.851914108149213, - "ZP_7.0_std": 0.06594640100455139, + "ZP_7.0": 25.854481351725262, + "ZP_7.0_std": 0.06547451826311122, "ZP_7.0_nstars": 30, - "ZP_8.0": 25.89904772872925, - "ZP_8.0_std": 0.06650479236590319, + "ZP_8.0": 25.902204645029705, + "ZP_8.0_std": 0.06612088851303272, "ZP_8.0_nstars": 30, + "ZP_AUTO": 25.96681058044434, + "ZP_AUTO_std": 0.07490968980093107, + "ZP_AUTO_nstars": 30, } pipeline = get_pipeline( @@ -83,3 +86,12 @@ def test_pipeline(self): new_res, new_errorstack = pipeline.reduce_images( dataset=Dataset(ImageBatch()), catch_all_errors=False ) + + new_header = new_res[0][0].get_header() + + new_exp = "expected_zp = { \n" + for header_key in new_header.keys(): + if "ZP_" in header_key: + new_exp += f' "{header_key}": {new_header[header_key]}, \n' + new_exp += "}" + print(new_exp) diff --git a/tests/test_wfau_references.py b/tests/test_wfau_references.py index 10b8a768a..cfea35445 100644 --- a/tests/test_wfau_references.py +++ b/tests/test_wfau_references.py @@ -11,29 +11,29 @@ TEST_WINTER_FIELD_ID = 5842 expected_header = { - "ZP_2.0": 24.52299118041992, - "ZP_2.0_std": 0.0678536668419838, - "ZP_2.0_nstars": 617, - "ZP_4.0": 25.24053764343262, - "ZP_4.0_std": 0.04157085344195366, - "ZP_4.0_nstars": 616, - "ZP_5.0": 25.32686233520508, - "ZP_5.0_std": 0.03873402997851372, - "ZP_5.0_nstars": 615, - "ZP_8.0": 25.4289379119873, - "ZP_8.0_std": 0.03831901773810387, - "ZP_8.0_nstars": 616, - "ZP_10.0": 25.45285606384277, - "ZP_10.0_std": 0.03835428133606911, - "ZP_10.0_nstars": 615, + "ZP_2.0": 24.51775550842285, + "ZP_2.0_std": 0.06673479825258255, + "ZP_2.0_nstars": 522, + "ZP_4.0": 25.237825393676758, + "ZP_4.0_std": 0.041513219475746155, + "ZP_4.0_nstars": 522, + "ZP_5.0": 25.324222564697266, + "ZP_5.0_std": 0.03828829526901245, + "ZP_5.0_nstars": 520, + "ZP_8.0": 25.427032470703125, + "ZP_8.0_std": 0.038392938673496246, + "ZP_8.0_nstars": 521, + "ZP_10.0": 25.450834274291992, + "ZP_10.0_std": 0.0386686846613884, + "ZP_10.0_nstars": 521, "COADDS": 4, - "RA0_0": 282.0557287433173, - "DEC0_0": 45.35327585919234, + "RA0_0": 282.05572874331733, + "DEC0_0": 45.353275859192344, "RA1_0": 282.0582425176955, "DEC1_0": 45.82890665679109, - "RA0_1": 281.3771483296388, - "DEC0_1": 45.35301704115277, - "RA1_1": 281.3738911418988, + "RA0_1": 281.37714832963877, + "DEC0_1": 45.353017041152775, + "RA1_1": 281.37389114189875, "DEC1_1": 45.82864350802002, "FIELDID": 5842, "SUBDETID": 0, @@ -94,7 +94,7 @@ def test_pipeline(self): ) header = new_res[0][0].get_header() - print("{") + print("expected_header = {") for key, value in expected_header.items(): print(f""""{key}": {header[key]},""") print("}") diff --git a/tests/test_wirc_pipeline.py b/tests/test_wirc_pipeline.py index 7a1fcd811..1164c00c9 100644 --- a/tests/test_wirc_pipeline.py +++ b/tests/test_wirc_pipeline.py @@ -6,33 +6,12 @@ from mirar.data import Dataset, ImageBatch from mirar.downloader.get_test_data import get_test_data_dir -from mirar.pipelines.wirc.generator import ( - wirc_astrometric_catalog_generator, - wirc_photometric_catalog_generator, -) +from mirar.pipelines.wirc.blocks import log, masking, reduction from mirar.pipelines.wirc.load_wirc_image import load_raw_wirc_image -from mirar.pipelines.wirc.wirc_files import ( - scamp_fp_path, - sextractor_astrometry_config, - swarp_sp_path, - wirc_mask_path, -) from mirar.pipelines.wirc.wirc_pipeline import WircPipeline -from mirar.processors.astromatic import Scamp, Sextractor, Swarp -from mirar.processors.autoastrometry import AutoAstrometry -from mirar.processors.csvlog import CSVLog from mirar.processors.dark import MasterDarkCalibrator -from mirar.processors.flat import MasterFlatCalibrator -from mirar.processors.mask import MaskPixels -from mirar.processors.photcal import PhotCalibrator -from mirar.processors.sky import MasterSkyCalibrator -from mirar.processors.utils import ImageSaver from mirar.processors.utils.image_loader import ImageLoader -from mirar.processors.utils.image_selector import ( - ImageBatcher, - ImageDebatcher, - ImageSelector, -) +from mirar.processors.utils.image_selector import ImageSelector from mirar.testing import BaseTestCase logger = logging.getLogger(__name__) @@ -40,23 +19,20 @@ test_data_dir = get_test_data_dir() expected_zp = { - "ZP_2.0": 25.908477783203125, - "ZP_2.0_std": 0.11761965602636337, - "ZP_2.0_nstars": 12, - "ZP_4.0": 27.235090255737305, - "ZP_4.0_std": 0.1022496446967125, - "ZP_4.0_nstars": 12, - "ZP_5.0": 27.5942440032959, - "ZP_5.0_std": 0.09162138402462006, - "ZP_5.0_nstars": 12, - "ZP_8.0": 28.16297149658203, - "ZP_8.0_std": 0.07523621618747711, - "ZP_8.0_nstars": 12, - "ZP_10.0": 28.324859619140625, - "ZP_10.0_std": 0.07975760102272034, - "ZP_10.0_nstars": 12, - "ZP_AUTO": 28.434249877929688, - "ZP_AUTO_std": 0.14610908925533295, + "ZP_6.0": 27.760366439819336, + "ZP_6.0_std": 0.2747618854045868, + "ZP_6.0_nstars": 13, + "ZP_10.0": 28.341157913208008, + "ZP_10.0_std": 0.09141725301742554, + "ZP_10.0_nstars": 11, + "ZP_14.0": 28.417837142944336, + "ZP_14.0_std": 0.13541148602962494, + "ZP_14.0_nstars": 13, + "ZP_18.0": 28.475311279296875, + "ZP_18.0_std": 0.10811392217874527, + "ZP_18.0_nstars": 13, + "ZP_AUTO": 28.48914909362793, + "ZP_AUTO_std": 0.09673704952001572, "ZP_AUTO_nstars": 12, } @@ -73,41 +49,19 @@ def get_cal_path(name: str) -> str: return os.path.join(test_data_dir, f"wirc/cals/test_{name}.fits") -test_configuration = [ - ImageLoader( - input_img_dir=test_data_dir, input_sub_dir="raw", load_image=load_raw_wirc_image - ), - CSVLog( - export_keys=[ - "OBJECT", - "FILTER", - "UTSHUT", - "EXPTIME", - "COADDS", - "OBSTYPE", - "OBSCLASS", - ], - ), - MaskPixels(mask_path=wirc_mask_path), - ImageSelector(("exptime", "45.0")), - MasterDarkCalibrator(get_cal_path("dark")), - ImageDebatcher(), - ImageSelector(("obsclass", "science")), - ImageBatcher(split_key="filter"), - MasterFlatCalibrator(get_cal_path("flat")), - MasterSkyCalibrator(get_cal_path("sky")), - ImageSelector(("object", "ZTF21aagppzg"), ("filter", "J")), - AutoAstrometry(catalog="tmc"), - Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), - Scamp( - ref_catalog_generator=wirc_astrometric_catalog_generator, - scamp_config_path=scamp_fp_path, - ), - Swarp(swarp_config_path=swarp_sp_path), - Sextractor(output_sub_dir="final_sextractor", **sextractor_astrometry_config), - PhotCalibrator(ref_catalog_generator=wirc_photometric_catalog_generator), - ImageSaver(output_dir_name="final"), -] +test_configuration = ( + [ + ImageLoader( + input_img_dir=test_data_dir, + input_sub_dir="raw", + load_image=load_raw_wirc_image, + ), + ] + + log + + masking + + [ImageSelector(("exptime", "45.0")), MasterDarkCalibrator(get_cal_path("dark"))] + + reduction +) pipeline = WircPipeline(night="20210330", selected_configurations="test") pipeline.add_configuration(configuration_name="test", configuration=test_configuration)