DM-41828: Add steps to run coadd level source injection.

examples/example_HSC_DRP-RC2.yaml

@@ -6,7 +6,7 @@
       name: HSC_DRP-RC2_campaign
       includes: ["base_campaign"]
       collections:
-          campaign_source: HSC/raw/RC2
+          campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2"
           campaign_ancillary_inputs:
               - HSC/calib
               - HSC/masks
@@ -15,84 +15,26 @@
               - skymaps
       steps:
           - Step:
-                name: step1
-                spec_block: HSC_DRP-RC2_step1
-                child_config:
-                    split_method: split_by_query
-                    split_min_groups: 3
-
-          - Step:
-                name: step2a
-                spec_block: HSC_DRP-RC2_step2a
-                prerequisites: ['step1']
-                child_config:
-                    split_method: no_split
-          - Step:
-                name: step2b
-                spec_block: HSC_DRP-RC2_step2b
-                prerequisites: ['step2a']
+                name: injected_step3
+                spec_block: HSC_DRP-RC2_injected_step3
                 child_config:
                     split_method: split_by_vals
                     split_vals:
                         - 9615
                         - 9697
                         - 9813
           - Step:
-                name: step2cde
-                spec_block: HSC_DRP-RC2_step2cde
-                prerequisites: ['step2b']
-                child_config:
-                    split_method: no_split
-          - Step:
-                name: step3
-                spec_block: HSC_DRP-RC2_step3
-                prerequisites: ['step2cde']
+                name: injected_coadd_analysis
+                spec_block: HSC_DRP-RC2_injected_coadd_analysis
+                prerequisites: ['injected_step3']
                 child_config:
                     split_method: split_by_vals
                     split_vals:
                         - 9615
                         - 9697
                         - 9813
-          - Step:
-                name: step4
-                spec_block: HSC_DRP-RC2_step4
-                prerequisites: ['step3']
-                child_config:
-                    split_method: split_by_query
-                    split_min_groups: 3
-                data:
-                    extra_qgraph_options: >
-                        -c parameters:sasquatch_dataset_identifier=HSC/RC2
-                        -c parameters:sasquatch_timestamp_version=reference_package_timestamp
-                        --dataset-query-constraint finalVisitSummary
-
-          - Step:
-                name: step5
-                spec_block: HSC_DRP-RC2_step5
-                prerequisites: ['step4']
-                child_config:
-                    split_method: split_by_vals
-                    split_vals:
-                        - 9615
-                        - 9697
-                        - 9813
-          - Step:
-                name: step6
-                spec_block: HSC_DRP-RC2_step6
-                prerequisites: ['step4']
-                child_config:
-                    split_method: split_by_query
-                    split_min_groups: 3
-                    split_dataset: calexp
-                    split_field: visit
-          - Step:
-                name: step7
-                spec_block: HSC_DRP-RC2_step7
-                prerequisites: ['step3']
-                child_config:
-                    split_method: no_split
       data:
-          butler_repo: '/repo/main+sasquatch_dev'
+          butler_repo: '/repo/main'
           prod_area: 'output/archive'  # Change this to /sdf/group/shared/campaigns
           data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1' and band in ('g', 'r', 'i', 'z', 'y')"
           bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml"

examples/example_HSC_DRP-RC2_steps.yaml

@@ -1,75 +1,16 @@
 - SpecBlock:
-      name: HSC_DRP-RC2_step1
+      name: HSC_DRP-RC2_injected_step3
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step1"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_dataset: raw
-          split_field: exposure
-- SpecBlock:
-      name: HSC_DRP-RC2_step2a
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2a"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_method: no_split
-- SpecBlock:
-      name: HSC_DRP-RC2_step2b
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2b"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_field: tract
-- SpecBlock:
-      name: HSC_DRP-RC2_step2cde
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2cde"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_method: no_split
-- SpecBlock:
-      name: HSC_DRP-RC2_step3
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step3"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2+injected_deepCoadd.yaml#injected_step3"
       child_config:
           base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
           split_field: tract
 - SpecBlock:
-      name: HSC_DRP-RC2_step4
+      name: HSC_DRP-RC2_injected_coadd_analysis
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step4"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_dataset: calexp
-          split_field: visit
-- SpecBlock:
-      name: HSC_DRP-RC2_step5
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step5"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2+injected_deepCoadd.yaml#injected_coadd_analysis"
       child_config:
           base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
           split_field: tract
-- SpecBlock:
-      name: HSC_DRP-RC2_step6
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step6"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_dataset: calexp
-          split_field: visit
-- SpecBlock:
-      name: HSC_DRP-RC2_step7
-      includes: ['step']
-      data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step7"
-      child_config:
-          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
-          split_method: no_split

examples/example_HSC_DRP-RC2_subset.yaml

@@ -44,6 +44,18 @@
                 prerequisites: ['step2d']
                 child_config:
                     split_method: no_split
+          - Step:
+                name: injected_step3
+                spec_block: HSC_DRP-RC2_subset_injected_step3
+                prerequisites: ['step3']
+                child_config:
+                    split_method: no_split
+          - Step:
+                name: injected_coadd_analysis
+                spec_block: HSC_DRP-RC2_subset_injected_coadd_analysis
+                prerequisites: ['injected_step3']
+                child_config:
+                    split_method: no_split
           - Step:
                 name: step4
                 spec_block: HSC_DRP-RC2_subset_step4

examples/example_HSC_DRP-RC2_subset_steps.yaml

@@ -2,71 +2,87 @@
       name: HSC_DRP-RC2_subset_step1
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep1"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep1"
       child_config:
           base_query: ""
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step2a
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2a"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2a"
       child_config:
           base_query: ""
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step2b
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2b"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2b"
       child_config:
           base_query: "instrument='HSC' and skymap='hsc_rings_v1' AND tract = 9813"
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step2c
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2c"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2c"
       child_config:
           base_query: ""
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step2d
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2d"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2d"
       child_config:
           base_query: ""
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step3
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep3"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep3"
       child_config:
           base_query: "instrument='HSC' and skymap='hsc_rings_v1' AND tract=9813 AND patch=40"
           split_method: no_split
+- SpecBlock:
+      name: HSC_DRP-RC2_subset_injected_step3
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset+injected_deepCoadd.yaml#injected_step3"
+      child_config:
+          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
+          split_method: no_split
+- SpecBlock:
+      name: HSC_DRP-RC2_subset_injected_coadd_analysis
+      includes: ['step']
+      data:
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset+injected_deepCoadd.yaml#injected_coadd_analysis"
+      child_config:
+          base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
+          split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step4
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep4"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep4"
       child_config:
           base_query: "skymap = 'hsc_rings_v1' AND tract=9813 AND patch=40"
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step5
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep5"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep5"
       child_config:
           base_query: "skymap = 'hsc_rings_v1' AND tract=9813 AND patch=40 AND band in ('g', 'r', 'i', 'z', 'y')"
           split_method: no_split
 - SpecBlock:
       name: HSC_DRP-RC2_subset_step8
       includes: ['step']
       data:
-          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep8"
+          pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep8"
       child_config:
           base_query: "instrument='HSC' and skymap='hsc_rings_v1'"
           split_method: no_split

examples/example_start_RC2_template.yaml

@@ -1,22 +1,22 @@
 Production:
     name: HSC_DRP-RC2
 Campaign:
-    name: w_2024_26_DM-45061
-    spec_name: HSC_DRP-RC2_panda
+    name: test_source_injection
+    spec_name: HSC_DRP-RC2_htcondor
     collections:
-        out: HSC/runs/RC2/w_2024_26/DM-45061
-        campaign_source: HSC/raw/RC2
+        out: u/eiger/DM-41828
+        campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2"
     data:
-        lsst_version: w_2024_26
+        lsst_version: w_2024_38
         prod_area: /sdf/group/rubin/shared/campaigns/HSC-RC2/output/archive
 
-#       custom_lsst_setup: setup -j -r /sdf/group/rubin/u/yusra/lsst_devel/ctrl_bps
-#
-#       butler_repo: '/repo/main+sasquatch_dev'
-#       data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'"
-#       bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml"
-#       bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml"
-#
+        custom_lsst_setup: "setup -j -r /sdf/home/m/mccann/repos/drp_pipe;setup -j -r /sdf/home/m/mccann/repos/source_injection;"
+
+        butler_repo: '/repo/main'
+        data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'"
+        bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml"
+        bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml"
+
 # Examples of how to override step specific stuff
 #   child_config:
 #       step1:

src/lsst/cmservice/models/queue.py

@@ -68,3 +68,8 @@ class QueueUpdate(QueueBase):
     time_updated: datetime | None = None
     # When processing of this element completed
     time_finished: datetime | None = None
+
+    # Id of node to add
+    node_id: int
+    # Which type of node to add
+    node_level: LevelEnum