Add development from wircdev

mirar/pipelines/wirc/blocks.py

@@ -10,9 +10,11 @@
     OBSCLASS_KEY,
     RAW_IMG_KEY,
     SATURATE_KEY,
+    TARGET_KEY,
 )
 from mirar.pipelines.wirc.generator import (
     annotate_target_coordinates,
+    label_stack_id,
     wirc_astrometric_catalog_generator,
     wirc_photometric_catalog_generator,
     wirc_reference_generator,
@@ -72,6 +74,7 @@
     ImageDebatcher,
     ImageLoader,
     ImageRebatcher,
+    ImageRejector,
     ImageSaver,
     ImageSelector,
 )
@@ -86,14 +89,26 @@
 ]
 
 log = [
+    ImageRejector((BASE_NAME_KEY, "_diff.fits")),
+    ImageRejector(("object", "test")),
+    ImageDebatcher(),
+    CustomImageBatchModifier(label_stack_id),
+    ImageRebatcher("stackid"),
+    CustomImageBatchModifier(annotate_target_coordinates),
     ImageRebatcher("UTSHUT"),
     CSVLog(
         export_keys=[
+            TARGET_KEY,
+            "TARGRA",
+            "TARGDEC",
+            "TARGNUM",
+            "STACKID",
             "OBJECT",
             "FILTER",
             "UTSHUT",
             "EXPTIME",
             "COADDS",
+            "TELFOCUS",
             OBSCLASS_KEY,
             BASE_NAME_KEY,
             "CRVAL1",
@@ -109,19 +124,17 @@
     MaskPixelsFromPath(mask_path=wirc_mask_path),
 ]
 
-dark_calibration = [ImageBatcher("EXPTIME"), DarkCalibrator()]
+dark_calibration = [ImageRebatcher("EXPTIME"), DarkCalibrator()]
 
 
 reduction = [
     ImageSaver(output_dir_name="darkcal"),
-    HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY),
-    ImageDebatcher(),
     ImageSelector((OBSCLASS_KEY, "science")),
-    ImageBatcher(split_key=["filter", "object"]),
-    CustomImageBatchModifier(annotate_target_coordinates),
+    HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY),
+    ImageRebatcher("stackid"),
     SkyFlatCalibrator(cache_sub_dir="firstpasscal"),
     NightSkyMedianCalibrator(cache_sub_dir="firstpasscal"),
-    ImageBatcher(BASE_NAME_KEY),
+    ImageRebatcher(BASE_NAME_KEY),
     AutoAstrometry(catalog="tmc"),
     Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config),
     Scamp(
@@ -131,7 +144,7 @@
         temp_output_sub_dir="firstpassscamp",
         timeout=120.0,
     ),
-    ImageRebatcher(split_key=["filter", "object"]),
+    ImageRebatcher(split_key=["stackid"]),
     ImageSaver(output_dir_name="firstpass"),
     Swarp(
         swarp_config_path=swarp_sp_path,

mirar/pipelines/wirc/generator/__init__.py

@@ -16,4 +16,5 @@
     wirc_reference_psfex,
     wirc_reference_sextractor,
 )
+from mirar.pipelines.wirc.generator.stacks import label_stack_id
 from mirar.pipelines.wirc.generator.target import annotate_target_coordinates

mirar/pipelines/wirc/generator/references.py

@@ -35,8 +35,8 @@ def wirc_reference_generator(image: Image):
 
     filtername = image["FILTER"]
 
-    if filtername not in ["J", "H", "Ks"]:
-        raise ValueError(f"Filter {filtername} not recognized for WINTER")
+    if filtername not in ["J", "H", "K"]:
+        raise ValueError(f"Filter {filtername} not recognized for WIRC")
 
     cache_ref_stack = False
 

mirar/pipelines/wirc/generator/stacks.py

@@ -0,0 +1,67 @@
+"""
+Module to group images based on the target coordinates into planned stack groups
+"""
+
+from astropy import coordinates as coords
+from astropy import units as u
+from astropy.coordinates import Angle
+
+from mirar.data import ImageBatch
+from mirar.paths import OBSCLASS_KEY
+from mirar.processors.utils.image_selector import split_images_into_batches
+
+MAX_RADIUS_DEG = 0.2
+
+
+def label_stack_id(batch: ImageBatch) -> ImageBatch:
+    """
+    Label the stack id of the images in the batch
+    :param batch: Original batch of images
+    :return: Labeled batch of images
+    """
+
+    ras = []
+    decs = []
+
+    for image in batch:
+
+        if image[OBSCLASS_KEY] != "science":
+            image["targnum"] = -1
+            continue
+
+        target_ra = Angle(image["RA"], unit="hourangle").degree
+        target_dec = Angle(image["DEC"], unit="degree").degree
+
+        position = coords.SkyCoord(target_ra, target_dec, unit="deg")
+
+        match = None
+
+        if len(ras) > 0:
+            idx, d2d, _ = position.match_to_catalog_sky(
+                coords.SkyCoord(ra=ras, dec=decs, unit="deg")
+            )
+
+            mask = d2d < MAX_RADIUS_DEG * u.deg
+            if mask:
+                match = idx
+
+        if match is None:
+            ras.append(target_ra)
+            decs.append(target_dec)
+            image["targnum"] = int(len(ras) - 1)
+        else:
+            image["targnum"] = int(match)
+
+    new_batches = split_images_into_batches(
+        batch, ["targnum", "filter", "telfocus", "exptime"]
+    )
+
+    combined_batch = ImageBatch()
+
+    for i, split_batch in enumerate(new_batches):
+        label = f"stack{i}"
+        for image in split_batch:
+            image["stackid"] = label
+            combined_batch.append(image)
+
+    return combined_batch

mirar/pipelines/wirc/generator/target.py

@@ -5,7 +5,7 @@
 from astropy.coordinates import Angle
 
 from mirar.data import ImageBatch
-from mirar.paths import TIME_KEY
+from mirar.paths import TARGET_KEY, TIME_KEY
 
 
 def annotate_target_coordinates(image_batch: ImageBatch) -> ImageBatch:
@@ -24,8 +24,13 @@ def annotate_target_coordinates(image_batch: ImageBatch) -> ImageBatch:
     target_ra = Angle(first_image["RA"], unit="hourangle").degree
     target_dec = Angle(first_image["DEC"], unit="degree").degree
 
+    # In case one of the dithers is mis-named, we'll use the most common name
+    names = [x[TARGET_KEY] for x in image_batch]
+    most_common_name = max(set(names), key=names.count)
+
     for image in image_batch:
         image["TARGRA"] = target_ra
         image["TARGDEC"] = target_dec
+        image[TARGET_KEY] = most_common_name
 
     return image_batch

mirar/references/wfcam/wfcam_query.py

@@ -479,13 +479,13 @@ def download_wfcam_archive_images(
     into a database.
     :param components_table: Table to use for the components database.
     :param duplicate_protocol: Protocol to follow if the image already exists locally.
-    :param q3c_bool: Is q3c setup?
     :param undeprecated_compids_file: Path to the file with the list of undeprecated
     component ids.
     :return imagepaths: List of paths to the downloaded images.
     """
     # ukirt_query = UkidssClass()
     wfau_query.database = survey_name
+    wfau_query.TIMEOUT = 300.0
 
     # First get a list with details of the images that overlap with the
     # coordinates.