diff --git a/mirar/pipelines/wirc/blocks.py b/mirar/pipelines/wirc/blocks.py index 249ea1cbe..a8ece7f08 100644 --- a/mirar/pipelines/wirc/blocks.py +++ b/mirar/pipelines/wirc/blocks.py @@ -10,9 +10,11 @@ OBSCLASS_KEY, RAW_IMG_KEY, SATURATE_KEY, + TARGET_KEY, ) from mirar.pipelines.wirc.generator import ( annotate_target_coordinates, + label_stack_id, wirc_astrometric_catalog_generator, wirc_photometric_catalog_generator, wirc_reference_generator, @@ -72,6 +74,7 @@ ImageDebatcher, ImageLoader, ImageRebatcher, + ImageRejector, ImageSaver, ImageSelector, ) @@ -86,14 +89,26 @@ ] log = [ + ImageRejector((BASE_NAME_KEY, "_diff.fits")), + ImageRejector(("object", "test")), + ImageDebatcher(), + CustomImageBatchModifier(label_stack_id), + ImageRebatcher("stackid"), + CustomImageBatchModifier(annotate_target_coordinates), ImageRebatcher("UTSHUT"), CSVLog( export_keys=[ + TARGET_KEY, + "TARGRA", + "TARGDEC", + "TARGNUM", + "STACKID", "OBJECT", "FILTER", "UTSHUT", "EXPTIME", "COADDS", + "TELFOCUS", OBSCLASS_KEY, BASE_NAME_KEY, "CRVAL1", @@ -109,19 +124,17 @@ MaskPixelsFromPath(mask_path=wirc_mask_path), ] -dark_calibration = [ImageBatcher("EXPTIME"), DarkCalibrator()] +dark_calibration = [ImageRebatcher("EXPTIME"), DarkCalibrator()] reduction = [ ImageSaver(output_dir_name="darkcal"), - HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), - ImageDebatcher(), ImageSelector((OBSCLASS_KEY, "science")), - ImageBatcher(split_key=["filter", "object"]), - CustomImageBatchModifier(annotate_target_coordinates), + HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY), + ImageRebatcher("stackid"), SkyFlatCalibrator(cache_sub_dir="firstpasscal"), NightSkyMedianCalibrator(cache_sub_dir="firstpasscal"), - ImageBatcher(BASE_NAME_KEY), + ImageRebatcher(BASE_NAME_KEY), AutoAstrometry(catalog="tmc"), Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config), Scamp( @@ -131,7 +144,7 @@ temp_output_sub_dir="firstpassscamp", timeout=120.0, ), - ImageRebatcher(split_key=["filter", "object"]), + ImageRebatcher(split_key=["stackid"]), ImageSaver(output_dir_name="firstpass"), Swarp( swarp_config_path=swarp_sp_path, diff --git a/mirar/pipelines/wirc/generator/__init__.py b/mirar/pipelines/wirc/generator/__init__.py index 9587ac08d..c7864c900 100644 --- a/mirar/pipelines/wirc/generator/__init__.py +++ b/mirar/pipelines/wirc/generator/__init__.py @@ -16,4 +16,5 @@ wirc_reference_psfex, wirc_reference_sextractor, ) +from mirar.pipelines.wirc.generator.stacks import label_stack_id from mirar.pipelines.wirc.generator.target import annotate_target_coordinates diff --git a/mirar/pipelines/wirc/generator/references.py b/mirar/pipelines/wirc/generator/references.py index 7eb2a2672..2872bf4e6 100644 --- a/mirar/pipelines/wirc/generator/references.py +++ b/mirar/pipelines/wirc/generator/references.py @@ -35,8 +35,8 @@ def wirc_reference_generator(image: Image): filtername = image["FILTER"] - if filtername not in ["J", "H", "Ks"]: - raise ValueError(f"Filter {filtername} not recognized for WINTER") + if filtername not in ["J", "H", "K"]: + raise ValueError(f"Filter {filtername} not recognized for WIRC") cache_ref_stack = False diff --git a/mirar/pipelines/wirc/generator/stacks.py b/mirar/pipelines/wirc/generator/stacks.py new file mode 100644 index 000000000..4249d3cab --- /dev/null +++ b/mirar/pipelines/wirc/generator/stacks.py @@ -0,0 +1,67 @@ +""" +Module to group images based on the target coordinates into planned stack groups +""" + +from astropy import coordinates as coords +from astropy import units as u +from astropy.coordinates import Angle + +from mirar.data import ImageBatch +from mirar.paths import OBSCLASS_KEY +from mirar.processors.utils.image_selector import split_images_into_batches + +MAX_RADIUS_DEG = 0.2 + + +def label_stack_id(batch: ImageBatch) -> ImageBatch: + """ + Label the stack id of the images in the batch + :param batch: Original batch of images + :return: Labeled batch of images + """ + + ras = [] + decs = [] + + for image in batch: + + if image[OBSCLASS_KEY] != "science": + image["targnum"] = -1 + continue + + target_ra = Angle(image["RA"], unit="hourangle").degree + target_dec = Angle(image["DEC"], unit="degree").degree + + position = coords.SkyCoord(target_ra, target_dec, unit="deg") + + match = None + + if len(ras) > 0: + idx, d2d, _ = position.match_to_catalog_sky( + coords.SkyCoord(ra=ras, dec=decs, unit="deg") + ) + + mask = d2d < MAX_RADIUS_DEG * u.deg + if mask: + match = idx + + if match is None: + ras.append(target_ra) + decs.append(target_dec) + image["targnum"] = int(len(ras) - 1) + else: + image["targnum"] = int(match) + + new_batches = split_images_into_batches( + batch, ["targnum", "filter", "telfocus", "exptime"] + ) + + combined_batch = ImageBatch() + + for i, split_batch in enumerate(new_batches): + label = f"stack{i}" + for image in split_batch: + image["stackid"] = label + combined_batch.append(image) + + return combined_batch diff --git a/mirar/pipelines/wirc/generator/target.py b/mirar/pipelines/wirc/generator/target.py index 743ee5629..d5b9beef7 100644 --- a/mirar/pipelines/wirc/generator/target.py +++ b/mirar/pipelines/wirc/generator/target.py @@ -5,7 +5,7 @@ from astropy.coordinates import Angle from mirar.data import ImageBatch -from mirar.paths import TIME_KEY +from mirar.paths import TARGET_KEY, TIME_KEY def annotate_target_coordinates(image_batch: ImageBatch) -> ImageBatch: @@ -24,8 +24,13 @@ def annotate_target_coordinates(image_batch: ImageBatch) -> ImageBatch: target_ra = Angle(first_image["RA"], unit="hourangle").degree target_dec = Angle(first_image["DEC"], unit="degree").degree + # In case one of the dithers is mis-named, we'll use the most common name + names = [x[TARGET_KEY] for x in image_batch] + most_common_name = max(set(names), key=names.count) + for image in image_batch: image["TARGRA"] = target_ra image["TARGDEC"] = target_dec + image[TARGET_KEY] = most_common_name return image_batch diff --git a/mirar/references/wfcam/wfcam_query.py b/mirar/references/wfcam/wfcam_query.py index 73445bb5b..f3476370e 100644 --- a/mirar/references/wfcam/wfcam_query.py +++ b/mirar/references/wfcam/wfcam_query.py @@ -479,13 +479,13 @@ def download_wfcam_archive_images( into a database. :param components_table: Table to use for the components database. :param duplicate_protocol: Protocol to follow if the image already exists locally. - :param q3c_bool: Is q3c setup? :param undeprecated_compids_file: Path to the file with the list of undeprecated component ids. :return imagepaths: List of paths to the downloaded images. """ # ukirt_query = UkidssClass() wfau_query.database = survey_name + wfau_query.TIMEOUT = 300.0 # First get a list with details of the images that overlap with the # coordinates.