From 0ce130dc5032f2593e0c3bbd214553d61f9ed296 Mon Sep 17 00:00:00 2001 From: Trys McCann Date: Fri, 1 Nov 2024 11:06:59 -0700 Subject: [PATCH 1/2] Add steps to run coadd level source injection. --- examples/example_HSC_DRP-RC2.yaml | 20 +++++++++++ examples/example_HSC_DRP-RC2_steps.yaml | 16 +++++++++ examples/example_HSC_DRP-RC2_subset.yaml | 12 +++++++ .../example_HSC_DRP-RC2_subset_steps.yaml | 34 ++++++++++++++----- 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/examples/example_HSC_DRP-RC2.yaml b/examples/example_HSC_DRP-RC2.yaml index cb9f85f24..0dd394d00 100644 --- a/examples/example_HSC_DRP-RC2.yaml +++ b/examples/example_HSC_DRP-RC2.yaml @@ -53,6 +53,26 @@ - 9615 - 9697 - 9813 + - Step: + name: injected_step3 + spec_block: HSC_DRP-RC2_injected_step3 + prerequisites: ['step3'] + child_config: + split_method: split_by_vals + split_vals: + - 9615 + - 9697 + - 9813 + - Step: + name: injected_coadd_analysis + spec_block: HSC_DRP-RC2_injected_coadd_analysis + prerequisites: ['injected_step3'] + child_config: + split_method: split_by_vals + split_vals: + - 9615 + - 9697 + - 9813 - Step: name: step4 spec_block: HSC_DRP-RC2_step4 diff --git a/examples/example_HSC_DRP-RC2_steps.yaml b/examples/example_HSC_DRP-RC2_steps.yaml index f29f2aa09..3d4aaef7f 100644 --- a/examples/example_HSC_DRP-RC2_steps.yaml +++ b/examples/example_HSC_DRP-RC2_steps.yaml @@ -39,6 +39,22 @@ child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_field: tract +- SpecBlock: + name: HSC_DRP-RC2_injected_step3 + includes: ['step'] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2+injected_deepCoadd.yaml#injected_step3" + child_config: + base_query: "instrument='HSC' and skymap='hsc_rings_v1'" + split_field: tract +- SpecBlock: + name: HSC_DRP-RC2_injected_coadd_analysis + includes: ['step'] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2+injected_deepCoadd.yaml#injected_coadd_analysis" + child_config: + base_query: "instrument='HSC' and skymap='hsc_rings_v1'" + split_field: tract - SpecBlock: name: HSC_DRP-RC2_step4 includes: ['step'] diff --git a/examples/example_HSC_DRP-RC2_subset.yaml b/examples/example_HSC_DRP-RC2_subset.yaml index 368379249..68b42174a 100644 --- a/examples/example_HSC_DRP-RC2_subset.yaml +++ b/examples/example_HSC_DRP-RC2_subset.yaml @@ -44,6 +44,18 @@ prerequisites: ['step2d'] child_config: split_method: no_split + - Step: + name: injected_step3 + spec_block: HSC_DRP-RC2_subset_injected_step3 + prerequisites: ['step3'] + child_config: + split_method: no_split + - Step: + name: injected_coadd_analysis + spec_block: HSC_DRP-RC2_subset_injected_coadd_analysis + prerequisites: ['injected_step3'] + child_config: + split_method: no_split - Step: name: step4 spec_block: HSC_DRP-RC2_subset_step4 diff --git a/examples/example_HSC_DRP-RC2_subset_steps.yaml b/examples/example_HSC_DRP-RC2_subset_steps.yaml index 3478b93f8..6ac650d9c 100644 --- a/examples/example_HSC_DRP-RC2_subset_steps.yaml +++ b/examples/example_HSC_DRP-RC2_subset_steps.yaml @@ -2,7 +2,7 @@ name: HSC_DRP-RC2_subset_step1 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep1" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep1" child_config: base_query: "" split_method: no_split @@ -10,7 +10,7 @@ name: HSC_DRP-RC2_subset_step2a includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2a" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2a" child_config: base_query: "" split_method: no_split @@ -18,7 +18,7 @@ name: HSC_DRP-RC2_subset_step2b includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2b" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2b" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1' AND tract = 9813" split_method: no_split @@ -26,7 +26,7 @@ name: HSC_DRP-RC2_subset_step2c includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2c" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2c" child_config: base_query: "" split_method: no_split @@ -34,7 +34,7 @@ name: HSC_DRP-RC2_subset_step2d includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2d" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2d" child_config: base_query: "" split_method: no_split @@ -42,15 +42,31 @@ name: HSC_DRP-RC2_subset_step3 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep3" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep3" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1' AND tract=9813 AND patch=40" split_method: no_split +- SpecBlock: + name: HSC_DRP-RC2_subset_injected_step3 + includes: ['step'] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset+injected_deepCoadd.yaml#injected_step3" + child_config: + base_query: "instrument='HSC' and skymap='hsc_rings_v1'" + split_method: no_split +- SpecBlock: + name: HSC_DRP-RC2_subset_injected_coadd_analysis + includes: ['step'] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset+injected_deepCoadd.yaml#injected_coadd_analysis" + child_config: + base_query: "instrument='HSC' and skymap='hsc_rings_v1'" + split_method: no_split - SpecBlock: name: HSC_DRP-RC2_subset_step4 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep4" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep4" child_config: base_query: "skymap = 'hsc_rings_v1' AND tract=9813 AND patch=40" split_method: no_split @@ -58,7 +74,7 @@ name: HSC_DRP-RC2_subset_step5 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep5" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep5" child_config: base_query: "skymap = 'hsc_rings_v1' AND tract=9813 AND patch=40 AND band in ('g', 'r', 'i', 'z', 'y')" split_method: no_split @@ -66,7 +82,7 @@ name: HSC_DRP-RC2_subset_step8 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep8" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep8" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_method: no_split From cad5a526879b7babeb6296860dd58bf358263e0e Mon Sep 17 00:00:00 2001 From: Orion Eiger Date: Wed, 13 Nov 2024 11:34:48 -0800 Subject: [PATCH 2/2] Set up test for source injection steps --- examples/example_HSC_DRP-RC2.yaml | 82 +----------------------- examples/example_HSC_DRP-RC2_steps.yaml | 75 ---------------------- examples/example_start_RC2_template.yaml | 24 +++---- src/lsst/cmservice/models/queue.py | 5 ++ 4 files changed, 19 insertions(+), 167 deletions(-) diff --git a/examples/example_HSC_DRP-RC2.yaml b/examples/example_HSC_DRP-RC2.yaml index 0dd394d00..da74543e4 100644 --- a/examples/example_HSC_DRP-RC2.yaml +++ b/examples/example_HSC_DRP-RC2.yaml @@ -6,7 +6,7 @@ name: HSC_DRP-RC2_campaign includes: ["base_campaign"] collections: - campaign_source: HSC/raw/RC2 + campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2" campaign_ancillary_inputs: - HSC/calib - HSC/masks @@ -14,49 +14,9 @@ - refcats - skymaps steps: - - Step: - name: step1 - spec_block: HSC_DRP-RC2_step1 - child_config: - split_method: split_by_query - split_min_groups: 3 - - - Step: - name: step2a - spec_block: HSC_DRP-RC2_step2a - prerequisites: ['step1'] - child_config: - split_method: no_split - - Step: - name: step2b - spec_block: HSC_DRP-RC2_step2b - prerequisites: ['step2a'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - - Step: - name: step2cde - spec_block: HSC_DRP-RC2_step2cde - prerequisites: ['step2b'] - child_config: - split_method: no_split - - Step: - name: step3 - spec_block: HSC_DRP-RC2_step3 - prerequisites: ['step2cde'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - Step: name: injected_step3 spec_block: HSC_DRP-RC2_injected_step3 - prerequisites: ['step3'] child_config: split_method: split_by_vals split_vals: @@ -73,46 +33,8 @@ - 9615 - 9697 - 9813 - - Step: - name: step4 - spec_block: HSC_DRP-RC2_step4 - prerequisites: ['step3'] - child_config: - split_method: split_by_query - split_min_groups: 3 - data: - extra_qgraph_options: > - -c parameters:sasquatch_dataset_identifier=HSC/RC2 - -c parameters:sasquatch_timestamp_version=reference_package_timestamp - --dataset-query-constraint finalVisitSummary - - - Step: - name: step5 - spec_block: HSC_DRP-RC2_step5 - prerequisites: ['step4'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - - Step: - name: step6 - spec_block: HSC_DRP-RC2_step6 - prerequisites: ['step4'] - child_config: - split_method: split_by_query - split_min_groups: 3 - split_dataset: calexp - split_field: visit - - Step: - name: step7 - spec_block: HSC_DRP-RC2_step7 - prerequisites: ['step3'] - child_config: - split_method: no_split data: - butler_repo: '/repo/main+sasquatch_dev' + butler_repo: '/repo/main' prod_area: 'output/archive' # Change this to /sdf/group/shared/campaigns data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1' and band in ('g', 'r', 'i', 'z', 'y')" bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" diff --git a/examples/example_HSC_DRP-RC2_steps.yaml b/examples/example_HSC_DRP-RC2_steps.yaml index 3d4aaef7f..cd57a448f 100644 --- a/examples/example_HSC_DRP-RC2_steps.yaml +++ b/examples/example_HSC_DRP-RC2_steps.yaml @@ -1,44 +1,3 @@ -- SpecBlock: - name: HSC_DRP-RC2_step1 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step1" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: raw - split_field: exposure -- SpecBlock: - name: HSC_DRP-RC2_step2a - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2a" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split -- SpecBlock: - name: HSC_DRP-RC2_step2b - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2b" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step2cde - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2cde" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split -- SpecBlock: - name: HSC_DRP-RC2_step3 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step3" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract - SpecBlock: name: HSC_DRP-RC2_injected_step3 includes: ['step'] @@ -55,37 +14,3 @@ child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step4 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step4" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: calexp - split_field: visit -- SpecBlock: - name: HSC_DRP-RC2_step5 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step5" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step6 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step6" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: calexp - split_field: visit -- SpecBlock: - name: HSC_DRP-RC2_step7 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step7" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split diff --git a/examples/example_start_RC2_template.yaml b/examples/example_start_RC2_template.yaml index 4fed4b005..eadc16c35 100644 --- a/examples/example_start_RC2_template.yaml +++ b/examples/example_start_RC2_template.yaml @@ -1,22 +1,22 @@ Production: name: HSC_DRP-RC2 Campaign: - name: w_2024_26_DM-45061 - spec_name: HSC_DRP-RC2_panda + name: test_source_injection + spec_name: HSC_DRP-RC2_htcondor collections: - out: HSC/runs/RC2/w_2024_26/DM-45061 - campaign_source: HSC/raw/RC2 + out: u/eiger/DM-41828 + campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2" data: - lsst_version: w_2024_26 + lsst_version: w_2024_38 prod_area: /sdf/group/rubin/shared/campaigns/HSC-RC2/output/archive -# custom_lsst_setup: setup -j -r /sdf/group/rubin/u/yusra/lsst_devel/ctrl_bps -# -# butler_repo: '/repo/main+sasquatch_dev' -# data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'" -# bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" -# bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" -# + custom_lsst_setup: "setup -j -r /sdf/home/m/mccann/repos/drp_pipe;setup -j -r /sdf/home/m/mccann/repos/source_injection;" + + butler_repo: '/repo/main' + data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'" + bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" + bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" + # Examples of how to override step specific stuff # child_config: # step1: diff --git a/src/lsst/cmservice/models/queue.py b/src/lsst/cmservice/models/queue.py index 65e9cb027..bd9ebbda2 100644 --- a/src/lsst/cmservice/models/queue.py +++ b/src/lsst/cmservice/models/queue.py @@ -68,3 +68,8 @@ class QueueUpdate(QueueBase): time_updated: datetime | None = None # When processing of this element completed time_finished: datetime | None = None + + # Id of node to add + node_id: int + # Which type of node to add + node_level: LevelEnum