From 9d3c92d2d7288cbd5820c4a51739bc4b815e4a9c Mon Sep 17 00:00:00 2001 From: Toby Jennings Date: Wed, 4 Dec 2024 11:42:43 -0800 Subject: [PATCH] Update hsc_micro example steps to reflect upstream changes --- examples/build_hsc_micro.yaml | 5 +++++ examples/example_hsc_micro.yaml | 13 +++---------- examples/example_hsc_micro_steps.yaml | 11 +++++++++++ examples/start_hsc_micro.yaml | 2 +- tests/db/test_micro.py | 6 +++--- 5 files changed, 23 insertions(+), 14 deletions(-) diff --git a/examples/build_hsc_micro.yaml b/examples/build_hsc_micro.yaml index fe0f36ac3..c22710ed1 100644 --- a/examples/build_hsc_micro.yaml +++ b/examples/build_hsc_micro.yaml @@ -11,6 +11,8 @@ Campaign: butler_repo: '/repo/main' prod_area: 'output/archive' data_query: "instrument = 'HSC' AND exposure in (30504, 30502) AND detector in (45, 46, 47, 48)" + # This custom campaign demonstrates the creation of an hsc_micro using pipeline + # definitions as of this lsst_version. lsst_version: w_2024_24 bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" @@ -23,6 +25,9 @@ Campaign: split_method: split_by_query split_min_groups: 2 # Add another predefined step + # NOTE: the characterizeImage and calibrate steps were deprecated and + # replaced with a single calibrateImage step; this demonstrates + # using the old steps with a legacy lsst version. - Step: name: characterizeImage spec_block: hsc_micro_characterizeImage diff --git a/examples/example_hsc_micro.yaml b/examples/example_hsc_micro.yaml index 74fb83d40..1419e6cbd 100644 --- a/examples/example_hsc_micro.yaml +++ b/examples/example_hsc_micro.yaml @@ -16,24 +16,17 @@ split_method: split_by_query split_min_groups: 2 - Step: - name: characterizeImage - spec_block: hsc_micro_characterizeImage + name: calibrateImage + spec_block: hsc_micro_calibrateImage prerequisites: ['isr'] child_config: split_method: split_by_query split_min_groups: 2 - - Step: - name: calibrate - spec_block: hsc_micro_calibrate - prerequisites: ['characterizeImage'] - child_config: - split_method: split_by_query - split_min_groups: 2 data: butler_repo: '/repo/main' prod_area: 'output/archive' data_query: "instrument = 'HSC' AND exposure in (30504, 30502) AND detector in (45, 46, 47, 48)" - lsst_version: w_2023_46 + lsst_version: w_2025_01 bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" - Specification: diff --git a/examples/example_hsc_micro_steps.yaml b/examples/example_hsc_micro_steps.yaml index 71e4c971a..56b80ee74 100644 --- a/examples/example_hsc_micro_steps.yaml +++ b/examples/example_hsc_micro_steps.yaml @@ -8,6 +8,16 @@ split_dataset: raw split_field: exposure - SpecBlock: + name: hsc_micro_calibrateImage + includes: ["step"] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#calibrateImage" + child_config: + base_query: "instrument = 'HSC'" + split_dataset: raw + split_field: exposure +- SpecBlock: + # deprecated name: hsc_micro_characterizeImage includes: ["step"] data: @@ -17,6 +27,7 @@ split_dataset: raw split_field: exposure - SpecBlock: + # deprecated name: hsc_micro_calibrate includes: ["step"] data: diff --git a/examples/start_hsc_micro.yaml b/examples/start_hsc_micro.yaml index 16daeda52..5c92d6c35 100644 --- a/examples/start_hsc_micro.yaml +++ b/examples/start_hsc_micro.yaml @@ -8,7 +8,7 @@ Campaign: campaign_source: HSC/raw/RC2 campaign_ancillary_inputs: "HSC/calib,HSC/masks,HSC/fgcmcal/lut/RC2,refcats,skymaps" data: - lsst_version: w_2024_38 + lsst_version: w_2025_01 prod_area: MUST_OVERRIDE butler_repo: '/repo/main' data_query: "instrument = 'HSC' AND exposure in (30504, 30502) AND detector in (45, 46, 47, 48)" diff --git a/tests/db/test_micro.py b/tests/db/test_micro.py index 6b0bc2f97..f927aae91 100644 --- a/tests/db/test_micro.py +++ b/tests/db/test_micro.py @@ -58,7 +58,7 @@ async def test_micro_db( session, "examples/example_hsc_micro.yaml", "hsc_micro_panda", - "w_2023_41", + "w_2025_01", "hsc_micro_panda#campaign", ) @@ -78,7 +78,7 @@ async def test_micro_db( changed, status = await interface.process( session, - "hsc_micro_panda/w_2023_41", + "hsc_micro_panda/w_2025_01", fake_status=StatusEnum.accepted, ) @@ -89,7 +89,7 @@ async def test_micro_db( session, remaining_only=False, ) - assert len(jobs) == 6 + assert len(jobs) == 4 changed, status = await campaign.run_check( session,