diff --git a/examples/example_HSC_DRP-RC2.yaml b/examples/example_HSC_DRP-RC2.yaml index cb9f85f24..da74543e4 100644 --- a/examples/example_HSC_DRP-RC2.yaml +++ b/examples/example_HSC_DRP-RC2.yaml @@ -6,7 +6,7 @@ name: HSC_DRP-RC2_campaign includes: ["base_campaign"] collections: - campaign_source: HSC/raw/RC2 + campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2" campaign_ancillary_inputs: - HSC/calib - HSC/masks @@ -15,22 +15,8 @@ - skymaps steps: - Step: - name: step1 - spec_block: HSC_DRP-RC2_step1 - child_config: - split_method: split_by_query - split_min_groups: 3 - - - Step: - name: step2a - spec_block: HSC_DRP-RC2_step2a - prerequisites: ['step1'] - child_config: - split_method: no_split - - Step: - name: step2b - spec_block: HSC_DRP-RC2_step2b - prerequisites: ['step2a'] + name: injected_step3 + spec_block: HSC_DRP-RC2_injected_step3 child_config: split_method: split_by_vals split_vals: @@ -38,61 +24,17 @@ - 9697 - 9813 - Step: - name: step2cde - spec_block: HSC_DRP-RC2_step2cde - prerequisites: ['step2b'] - child_config: - split_method: no_split - - Step: - name: step3 - spec_block: HSC_DRP-RC2_step3 - prerequisites: ['step2cde'] + name: injected_coadd_analysis + spec_block: HSC_DRP-RC2_injected_coadd_analysis + prerequisites: ['injected_step3'] child_config: split_method: split_by_vals split_vals: - 9615 - 9697 - 9813 - - Step: - name: step4 - spec_block: HSC_DRP-RC2_step4 - prerequisites: ['step3'] - child_config: - split_method: split_by_query - split_min_groups: 3 - data: - extra_qgraph_options: > - -c parameters:sasquatch_dataset_identifier=HSC/RC2 - -c parameters:sasquatch_timestamp_version=reference_package_timestamp - --dataset-query-constraint finalVisitSummary - - - Step: - name: step5 - spec_block: HSC_DRP-RC2_step5 - prerequisites: ['step4'] - child_config: - split_method: split_by_vals - split_vals: - - 9615 - - 9697 - - 9813 - - Step: - name: step6 - spec_block: HSC_DRP-RC2_step6 - prerequisites: ['step4'] - child_config: - split_method: split_by_query - split_min_groups: 3 - split_dataset: calexp - split_field: visit - - Step: - name: step7 - spec_block: HSC_DRP-RC2_step7 - prerequisites: ['step3'] - child_config: - split_method: no_split data: - butler_repo: '/repo/main+sasquatch_dev' + butler_repo: '/repo/main' prod_area: 'output/archive' # Change this to /sdf/group/shared/campaigns data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1' and band in ('g', 'r', 'i', 'z', 'y')" bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" diff --git a/examples/example_HSC_DRP-RC2_steps.yaml b/examples/example_HSC_DRP-RC2_steps.yaml index f29f2aa09..cd57a448f 100644 --- a/examples/example_HSC_DRP-RC2_steps.yaml +++ b/examples/example_HSC_DRP-RC2_steps.yaml @@ -1,75 +1,16 @@ - SpecBlock: - name: HSC_DRP-RC2_step1 + name: HSC_DRP-RC2_injected_step3 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step1" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: raw - split_field: exposure -- SpecBlock: - name: HSC_DRP-RC2_step2a - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2a" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split -- SpecBlock: - name: HSC_DRP-RC2_step2b - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2b" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step2cde - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step2cde" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split -- SpecBlock: - name: HSC_DRP-RC2_step3 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step3" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2+injected_deepCoadd.yaml#injected_step3" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_field: tract - SpecBlock: - name: HSC_DRP-RC2_step4 + name: HSC_DRP-RC2_injected_coadd_analysis includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step4" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: calexp - split_field: visit -- SpecBlock: - name: HSC_DRP-RC2_step5 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step5" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2+injected_deepCoadd.yaml#injected_coadd_analysis" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_field: tract -- SpecBlock: - name: HSC_DRP-RC2_step6 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step6" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_dataset: calexp - split_field: visit -- SpecBlock: - name: HSC_DRP-RC2_step7 - includes: ['step'] - data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#step7" - child_config: - base_query: "instrument='HSC' and skymap='hsc_rings_v1'" - split_method: no_split diff --git a/examples/example_HSC_DRP-RC2_subset.yaml b/examples/example_HSC_DRP-RC2_subset.yaml index 368379249..68b42174a 100644 --- a/examples/example_HSC_DRP-RC2_subset.yaml +++ b/examples/example_HSC_DRP-RC2_subset.yaml @@ -44,6 +44,18 @@ prerequisites: ['step2d'] child_config: split_method: no_split + - Step: + name: injected_step3 + spec_block: HSC_DRP-RC2_subset_injected_step3 + prerequisites: ['step3'] + child_config: + split_method: no_split + - Step: + name: injected_coadd_analysis + spec_block: HSC_DRP-RC2_subset_injected_coadd_analysis + prerequisites: ['injected_step3'] + child_config: + split_method: no_split - Step: name: step4 spec_block: HSC_DRP-RC2_subset_step4 diff --git a/examples/example_HSC_DRP-RC2_subset_steps.yaml b/examples/example_HSC_DRP-RC2_subset_steps.yaml index 3478b93f8..6ac650d9c 100644 --- a/examples/example_HSC_DRP-RC2_subset_steps.yaml +++ b/examples/example_HSC_DRP-RC2_subset_steps.yaml @@ -2,7 +2,7 @@ name: HSC_DRP-RC2_subset_step1 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep1" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep1" child_config: base_query: "" split_method: no_split @@ -10,7 +10,7 @@ name: HSC_DRP-RC2_subset_step2a includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2a" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2a" child_config: base_query: "" split_method: no_split @@ -18,7 +18,7 @@ name: HSC_DRP-RC2_subset_step2b includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2b" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2b" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1' AND tract = 9813" split_method: no_split @@ -26,7 +26,7 @@ name: HSC_DRP-RC2_subset_step2c includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2c" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2c" child_config: base_query: "" split_method: no_split @@ -34,7 +34,7 @@ name: HSC_DRP-RC2_subset_step2d includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep2d" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep2d" child_config: base_query: "" split_method: no_split @@ -42,15 +42,31 @@ name: HSC_DRP-RC2_subset_step3 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep3" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep3" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1' AND tract=9813 AND patch=40" split_method: no_split +- SpecBlock: + name: HSC_DRP-RC2_subset_injected_step3 + includes: ['step'] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset+injected_deepCoadd.yaml#injected_step3" + child_config: + base_query: "instrument='HSC' and skymap='hsc_rings_v1'" + split_method: no_split +- SpecBlock: + name: HSC_DRP-RC2_subset_injected_coadd_analysis + includes: ['step'] + data: + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset+injected_deepCoadd.yaml#injected_coadd_analysis" + child_config: + base_query: "instrument='HSC' and skymap='hsc_rings_v1'" + split_method: no_split - SpecBlock: name: HSC_DRP-RC2_subset_step4 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep4" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep4" child_config: base_query: "skymap = 'hsc_rings_v1' AND tract=9813 AND patch=40" split_method: no_split @@ -58,7 +74,7 @@ name: HSC_DRP-RC2_subset_step5 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep5" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep5" child_config: base_query: "skymap = 'hsc_rings_v1' AND tract=9813 AND patch=40 AND band in ('g', 'r', 'i', 'z', 'y')" split_method: no_split @@ -66,7 +82,7 @@ name: HSC_DRP-RC2_subset_step8 includes: ['step'] data: - pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2.yaml#nightlyStep8" + pipeline_yaml: "${DRP_PIPE_DIR}/pipelines/HSC/DRP-RC2_subset.yaml#nightlyStep8" child_config: base_query: "instrument='HSC' and skymap='hsc_rings_v1'" split_method: no_split diff --git a/examples/example_start_RC2_template.yaml b/examples/example_start_RC2_template.yaml index 4fed4b005..eadc16c35 100644 --- a/examples/example_start_RC2_template.yaml +++ b/examples/example_start_RC2_template.yaml @@ -1,22 +1,22 @@ Production: name: HSC_DRP-RC2 Campaign: - name: w_2024_26_DM-45061 - spec_name: HSC_DRP-RC2_panda + name: test_source_injection + spec_name: HSC_DRP-RC2_htcondor collections: - out: HSC/runs/RC2/w_2024_26/DM-45061 - campaign_source: HSC/raw/RC2 + out: u/eiger/DM-41828 + campaign_source: "HSC/runs/RC2/w_2024_38/DM-46429,HSC/raw/RC2" data: - lsst_version: w_2024_26 + lsst_version: w_2024_38 prod_area: /sdf/group/rubin/shared/campaigns/HSC-RC2/output/archive -# custom_lsst_setup: setup -j -r /sdf/group/rubin/u/yusra/lsst_devel/ctrl_bps -# -# butler_repo: '/repo/main+sasquatch_dev' -# data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'" -# bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" -# bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" -# + custom_lsst_setup: "setup -j -r /sdf/home/m/mccann/repos/drp_pipe;setup -j -r /sdf/home/m/mccann/repos/source_injection;" + + butler_repo: '/repo/main' + data_query: "instrument = 'HSC' and skymap = 'hsc_rings_v1'" + bps_wms_clustering_file: "${DRP_PIPE_DIR}/bps/clustering/HSC/DRP-RC2-clustering.yaml" + bps_wms_resources_file: "${DRP_PIPE_DIR}/bps/resources/HSC/DRP-RC2.yaml" + # Examples of how to override step specific stuff # child_config: # step1: diff --git a/src/lsst/cmservice/models/queue.py b/src/lsst/cmservice/models/queue.py index 65e9cb027..bd9ebbda2 100644 --- a/src/lsst/cmservice/models/queue.py +++ b/src/lsst/cmservice/models/queue.py @@ -68,3 +68,8 @@ class QueueUpdate(QueueBase): time_updated: datetime | None = None # When processing of this element completed time_finished: datetime | None = None + + # Id of node to add + node_id: int + # Which type of node to add + node_level: LevelEnum