Wirc, working (#949)

.github/workflows/continuous_integration.yml

@@ -21,7 +21,7 @@ jobs:
   # This workflow contains a single job called "build"
   ci:
     # The type of runner that the job will run on
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
 
     # Specify the python versions to test
     strategy:
@@ -60,10 +60,6 @@ jobs:
           auto-update-conda: true
           python-version: ${{ matrix.python-version }}
 
-      - name: Install poetry
-        shell: bash -el {0}
-        run: conda install poetry
-
       # Runs a set of commands using the runners shell
       - name: set up ssh
         uses: webfactory/ssh-agent@v0.5.4

.github/workflows/mac_m1_install.yml

@@ -33,7 +33,7 @@ jobs:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
       - uses: actions/checkout@v3
 
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         env:
           CONDA_CHANNELS: conda-forge
           CONDA_SUBDIR: osx-64

mirar/pipelines/wirc/blocks.py

@@ -3,21 +3,19 @@
 """
 
 # pylint: disable=duplicate-code
-from mirar.catalog.kowalski import PS1, TMASS
 from mirar.paths import (
+    BASE_NAME_KEY,
     FITS_MASK_KEY,
     LATEST_SAVE_KEY,
     OBSCLASS_KEY,
     RAW_IMG_KEY,
-    REF_IMG_KEY,
     SATURATE_KEY,
-    SCI_IMG_KEY,
-    SOURCE_NAME_KEY,
 )
 from mirar.pipelines.wirc.generator import (
+    annotate_target_coordinates,
     wirc_astrometric_catalog_generator,
     wirc_photometric_catalog_generator,
-    wirc_reference_image_generator,
+    wirc_reference_generator,
     wirc_reference_image_resampler,
     wirc_reference_psfex,
     wirc_reference_sextractor,
@@ -26,34 +24,25 @@
 )
 from mirar.pipelines.wirc.load_wirc_image import load_raw_wirc_image
 from mirar.pipelines.wirc.wirc_files import (
-    prv_candidate_cols,
     psfex_path,
     scamp_fp_path,
     sextractor_astrometry_config,
     sextractor_candidate_config,
     sextractor_photometry_config,
     sextractor_reference_config,
     swarp_sp_path,
-    wirc_avro_schema_path,
     wirc_mask_path,
 )
-from mirar.pipelines.wirc.wirc_files.models import (
-    CANDIDATE_PREFIX,
-    NAME_START,
-    Candidate,
-)
 from mirar.processors.astromatic import Scamp, Sextractor, Swarp
 from mirar.processors.astromatic.psfex import PSFex
 from mirar.processors.astromatic.scamp.scamp import SCAMP_HEADER_KEY
 from mirar.processors.astromatic.sextractor.sextractor import sextractor_checkimg_map
 from mirar.processors.astromatic.swarp import ReloadSwarpComponentImages
 from mirar.processors.astrometry.autoastrometry import AutoAstrometry
 from mirar.processors.astrometry.utils import AstrometryFromFile
-from mirar.processors.avro import IPACAvroExporter
+from mirar.processors.catalog_limiting_mag import CatalogLimitingMagnitudeCalculator
 from mirar.processors.csvlog import CSVLog
 from mirar.processors.dark import DarkCalibrator
-from mirar.processors.database.database_inserter import DatabaseSourceInserter
-from mirar.processors.database.database_selector import DatabaseHistorySelector
 from mirar.processors.flat import SkyFlatCalibrator
 from mirar.processors.mask import (
     MaskAboveThreshold,
@@ -62,37 +51,42 @@
     MaskPixelsFromWCS,
     WriteMaskedCoordsToFile,
 )
-from mirar.processors.photcal.photcalibrator import PhotCalibrator
+from mirar.processors.photcal import PhotCalibrator
 from mirar.processors.photometry import AperturePhotometry, PSFPhotometry
 from mirar.processors.reference import ProcessReference
 from mirar.processors.sky import NightSkyMedianCalibrator
-from mirar.processors.skyportal import SkyportalCandidateUploader
-from mirar.processors.skyportal.client import SkyportalClient
 from mirar.processors.sources import (
-    CandidateNamer,
     ForcedPhotometryDetector,
+    ImageUpdater,
+    JSONExporter,
+    ParquetWriter,
     SourceWriter,
     ZOGYSourceDetector,
 )
 from mirar.processors.sources.source_table_modifier import CustomSourceTableModifier
 from mirar.processors.sources.utils import RegionsWriter
 from mirar.processors.utils import (
+    CustomImageBatchModifier,
     HeaderAnnotator,
     ImageBatcher,
     ImageDebatcher,
     ImageLoader,
+    ImageRebatcher,
     ImageSaver,
     ImageSelector,
 )
 from mirar.processors.utils.image_loader import LoadImageFromHeader
-from mirar.processors.xmatch import XMatch
 from mirar.processors.zogy.zogy import ZOGY, ZOGYPrepare
 
 load_raw = [ImageLoader(input_sub_dir="raw", load_image=load_raw_wirc_image)]
-# load_raw = [ImageLoader(input_sub_dir="firstpassstack",
-# load_image=load_raw_wirc_image)]
+
+load_stack = [
+    ImageLoader(input_sub_dir="final", load_image=load_raw_wirc_image),
+    ImageBatcher(split_key=[BASE_NAME_KEY]),
+]
 
 log = [
+    ImageRebatcher("UTSHUT"),
     CSVLog(
         export_keys=[
             "OBJECT",
@@ -101,34 +95,46 @@
             "EXPTIME",
             "COADDS",
             OBSCLASS_KEY,
+            BASE_NAME_KEY,
         ]
-    )
+    ),
+    ImageDebatcher(),
 ]
 
-masking = [MaskPixelsFromPath(mask_path=wirc_mask_path)]
+masking = [
+    ImageSelector((OBSCLASS_KEY, ["science", "dark"])),
+    MaskPixelsFromPath(mask_path=wirc_mask_path),
+]
+
+dark_calibration = [ImageBatcher("EXPTIME"), DarkCalibrator()]
 
-dark_calibration = [ImageSelector(("exptime", "45.0")), DarkCalibrator()]
 
 reduction = [
     ImageSaver(output_dir_name="darkcal"),
     HeaderAnnotator(input_keys=LATEST_SAVE_KEY, output_key=RAW_IMG_KEY),
     ImageDebatcher(),
     ImageSelector((OBSCLASS_KEY, "science")),
-    # ImageSelector(("object", "ZTF18aavqmki")),
     ImageBatcher(split_key=["filter", "object"]),
-    SkyFlatCalibrator(),
-    NightSkyMedianCalibrator(),
+    CustomImageBatchModifier(annotate_target_coordinates),
+    SkyFlatCalibrator(cache_sub_dir="firstpasscal"),
+    NightSkyMedianCalibrator(cache_sub_dir="firstpasscal"),
+    ImageBatcher(BASE_NAME_KEY),
     AutoAstrometry(catalog="tmc"),
     Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config),
     Scamp(
         ref_catalog_generator=wirc_astrometric_catalog_generator,
         scamp_config_path=scamp_fp_path,
         cache=True,
+        temp_output_sub_dir="firstpassscamp",
     ),
+    ImageRebatcher(split_key=["filter", "object"]),
     ImageSaver(output_dir_name="firstpass"),
-    Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True),
+    Swarp(
+        swarp_config_path=swarp_sp_path,
+        calculate_dims_in_swarp=True,
+        temp_output_sub_dir="firstpassswarp",
+    ),
     ImageSaver(output_dir_name="firstpassstack"),
-    # ImageSelector(("BASENAME", "image0125.fits_stack.fits")),
     Sextractor(
         output_sub_dir="firstpasssextractor",
         **sextractor_astrometry_config,
@@ -148,11 +154,11 @@
     WriteMaskedCoordsToFile(output_dir="mask_stack"),
     ReloadSwarpComponentImages(
         load_image=load_raw_wirc_image,
-        copy_header_keys=FITS_MASK_KEY,
+        copy_header_keys=[FITS_MASK_KEY, "TARGRA", "TARGDEC"],
     ),
     LoadImageFromHeader(
         header_key=RAW_IMG_KEY,
-        copy_header_keys=[SCAMP_HEADER_KEY, FITS_MASK_KEY],
+        copy_header_keys=[SCAMP_HEADER_KEY, FITS_MASK_KEY, "TARGRA", "TARGDEC"],
         load_image=load_raw_wirc_image,
     ),
     AstrometryFromFile(astrometry_file_key=SCAMP_HEADER_KEY),
@@ -163,27 +169,43 @@
         only_write_mask=True,
     ),
     ImageSaver(output_dir_name="firstpassmasked", write_mask=True),
-    SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY),
-    NightSkyMedianCalibrator(flat_mask_key=FITS_MASK_KEY),
+    SkyFlatCalibrator(flat_mask_key=FITS_MASK_KEY, cache_sub_dir="secondpasscal"),
+    NightSkyMedianCalibrator(
+        flat_mask_key=FITS_MASK_KEY, cache_sub_dir="secondpasscal"
+    ),
     Sextractor(output_sub_dir="postprocess", **sextractor_astrometry_config),
-    Swarp(swarp_config_path=swarp_sp_path, calculate_dims_in_swarp=True),
-    Sextractor(output_sub_dir="final_sextractor", **sextractor_photometry_config),
+    Swarp(
+        swarp_config_path=swarp_sp_path,
+        calculate_dims_in_swarp=True,
+        temp_output_sub_dir="secondpassswarp",
+    ),
+    ImageSaver(output_dir_name="stack"),
+    Sextractor(
+        **sextractor_photometry_config,
+        output_sub_dir="final_sextractor",
+        checkimage_type="BACKGROUND_RMS",
+    ),
     PhotCalibrator(
         ref_catalog_generator=wirc_photometric_catalog_generator,
         write_regions=True,
+        temp_output_sub_dir="photcal",
+    ),
+    CatalogLimitingMagnitudeCalculator(
+        sextractor_mag_key_name="MAG_AUTO", write_regions=True
     ),
     ImageSaver(output_dir_name="final"),
 ]
 
 reduce = log + masking + dark_calibration + reduction
 
 reference = [
+    ImageRebatcher(split_key=[BASE_NAME_KEY]),
     ProcessReference(
-        ref_image_generator=wirc_reference_image_generator,
+        ref_image_generator=wirc_reference_generator,
         swarp_resampler=wirc_reference_image_resampler,
         sextractor=wirc_reference_sextractor,
         ref_psfex=wirc_reference_psfex,
-    )
+    ),
 ]
 
 subtract = [
@@ -196,9 +218,10 @@
 
 export_candidates_from_header = [
     ForcedPhotometryDetector(ra_header_key="TARGRA", dec_header_key="TARGDEC"),
+    RegionsWriter(output_dir_name="diffs"),
 ]
 
-candidate_photometry = [
+calculate_photometry = [
     AperturePhotometry(
         aper_diameters=[16, 70],
         phot_cutout_half_size=100,
@@ -209,82 +232,41 @@
     PSFPhotometry(),
 ]
 
-detect_candidates = [
-    ZOGYSourceDetector(
-        output_sub_dir="subtract",
-        **sextractor_candidate_config,
-    ),
+export_photometry = [
+    SourceWriter(output_dir_name="photometry"),
+    JSONExporter(output_dir_name="photometry"),
+    ParquetWriter(output_dir_name="photometry"),
+    ImageUpdater(modify_dir_name="diffs"),
 ]
-#
+
+forced_photometry = (
+    export_candidates_from_header + calculate_photometry + export_photometry
+)
+
 image_photometry = [
     Sextractor(**sextractor_reference_config, output_sub_dir="subtract", cache=False),
     PSFex(config_path=psfex_path, output_sub_dir="photometry", norm_fits=True),
     ImageSaver(output_dir_name="photometry"),
-    # AperturePhotometry(
-    #     aper_diameters=[16],
-    #     bkg_in_diameters=[25],
-    #     bkg_out_diameters=[40],
-    #     col_suffix_list=[""],
-    #     phot_cutout_size=100,
-    #     target_ra_key="TARGRA",
-    #     target_dec_key="TARGDEC",
-    # ),
-    # PSFPhotometry(),
 ]
 
-process_candidates = [
-    RegionsWriter(output_dir_name="candidates"),
-    PSFPhotometry(),
-    AperturePhotometry(
-        aper_diameters=[16, 70],
-        phot_cutout_half_size=100,
-        bkg_in_diameters=[25, 90],
-        bkg_out_diameters=[40, 100],
-        col_suffix_list=["", "big"],
-    ),
-    # SourceWriter(output_dir_name="candidates"),
-    CustomSourceTableModifier(modifier_function=wirc_source_table_filter_annotator),
-    XMatch(catalog=TMASS(num_sources=3, search_radius_arcmin=0.5)),
-    XMatch(catalog=PS1(num_sources=3, search_radius_arcmin=0.5)),
-    SourceWriter(output_dir_name="kowalski"),
-    HeaderAnnotator(input_keys=[LATEST_SAVE_KEY], output_key="diffimgname"),
-    HeaderAnnotator(input_keys=[SCI_IMG_KEY], output_key="sciimgname"),
-    HeaderAnnotator(input_keys=[REF_IMG_KEY], output_key="refimgname"),
-    CandidateNamer(
-        db_table=Candidate,
-        base_name=CANDIDATE_PREFIX,
-        name_start=NAME_START,
-        db_output_columns=[SOURCE_NAME_KEY],
-        crossmatch_radius_arcsec=2.0,
-    ),
-    DatabaseHistorySelector(
-        crossmatch_radius_arcsec=2.0,
-        history_duration_days=500.0,
-        db_table=Candidate,
-        db_output_columns=[SOURCE_NAME_KEY] + prv_candidate_cols,
-    ),
-    DatabaseSourceInserter(db_table=Candidate, duplicate_protocol="replace"),
-    SourceWriter(output_dir_name="candidates"),
-    # EdgeCandidatesMask(edge_boundary_size=100)
-    # FilterCandidates(),
-]
+imsub = reference + subtract + forced_photometry
 
-package_candidates = [
-    HeaderAnnotator(input_keys=["MJD-OBS"], output_key="mjd"),
-    IPACAvroExporter(
-        base_name="WIRC",
-        avro_schema_path=wirc_avro_schema_path,
-    ),
-    SkyportalCandidateUploader(
-        origin="WIRCTEST",
-        group_ids=[1431],
-        fritz_filter_id=74,
-        instrument_id=5,
-        stream_id=1005,
-        update_thumbnails=True,
-        skyportal_client=SkyportalClient(base_url="https://fritz.science/api/"),
-    ),
-]
+candidates = (
+    [
+        ImageLoader(input_sub_dir="diffs"),
+        ZOGYSourceDetector(
+            output_sub_dir="subtract",
+            **sextractor_candidate_config,
+        ),
+        RegionsWriter(output_dir_name="candidates"),
+    ]
+    + calculate_photometry
+    + [
+        CustomSourceTableModifier(modifier_function=wirc_source_table_filter_annotator),
+        JSONExporter(output_dir_name="candidates"),
+        ParquetWriter(output_dir_name="candidates"),
+        SourceWriter(output_dir_name="candidates"),
+    ]
+)
 
-candidates = detect_candidates + process_candidates + package_candidates
-imsub = reference + subtract + candidates
+test = reduction + imsub