Merge remote-tracking branch 'origin/master' into Low-Q2_training

benchmarks/backgrounds/Snakefile

@@ -38,6 +38,9 @@ rule backgrounds_ecal_backwards:
         proton_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/backgrounds/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/pythia8.306-1.0/100GeV/pythia8.306-1.0_ProtonBeamGas_100GeV_run001.edm4hep.root",
     output:
         directory("results/backgrounds/backwards_ecal")
+    log:
+        scheduler=".logs/results/backgrounds/backwards_ecal/scheduler.log",
+        worker=".logs/results/backgrounds/backwards_ecal/worker.log",
     threads: workflow.cores
     shell:
         """
@@ -49,10 +52,10 @@ cleanup() {{
 trap cleanup EXIT
 
 PORT=$RANDOM
-dask scheduler --port $PORT &
+dask scheduler --port $PORT 2>{log.scheduler} &
 export DASK_SCHEDULER=localhost:$PORT
 SCHEDULER_PID=$!
-dask worker tcp://$DASK_SCHEDULER --nworkers {threads} --nthreads 1 &
+dask worker tcp://$DASK_SCHEDULER --nworkers {threads} --nthreads 1 2>{log.worker} &
 WORKER_PID=$!
 env \
 MATPLOTLIBRC={input.matplotlibrc} \

benchmarks/backwards_ecal/Snakefile

@@ -18,7 +18,7 @@ rule backwards_ecal_sim:
         PARTICLE="(e-|pi-)",
         ENERGY="[0-9]+[kMG]eV",
         PHASE_SPACE="(3to50|45to135|130to177)deg",
-        INDEX="\d{4}",
+        INDEX=r"\d{4}",
     params:
         N_EVENTS=get_n_events,
         SEED=lambda wildcards: "1" + wildcards.INDEX,
@@ -51,7 +51,7 @@ rule backwards_ecal_recon:
     log:
         "sim_output/backwards_ecal/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root.log",
     wildcard_constraints:
-        INDEX="\d{4}",
+        INDEX=r"\d{4}",
     params:
         DETECTOR_CONFIG=lambda wildcards: wildcards.DETECTOR_CONFIG,
         EICRECON_HASH=get_spack_package_hash("eicrecon"),
@@ -64,22 +64,77 @@ exec env DETECTOR_CONFIG={params.DETECTOR_CONFIG} \
 """
 
 
-rule backwards_ecal_recon_many:
+rule backwards_ecal_local_sim_list:
     input:
         expand(
             "sim_output/backwards_ecal/{{DETECTOR_CONFIG}}/{{PARTICLE}}/{{ENERGY}}/{{PHASE_SPACE}}/{{PARTICLE}}_{{ENERGY}}_{{PHASE_SPACE}}.{INDEX:04d}.eicrecon.tree.edm4eic.root",
             INDEX=range(20),
         ),
     output:
-        touch("sim_output/backwards_ecal/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/flag"),
+        "listing/backwards_ecal/local/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}.lst",
+    run:
+        with open(output[0], "wt") as fp:
+            fp.write("\n".join(input))
+
+
+if config.get("stream_from_xrootd", True) not in [False, "", "0", "false"]:
+    rule backwards_ecal_campaign_sim_list:
+        output:
+            "listing/backwards_ecal/{CAMPAIGN}/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}.lst",
+        params:
+            search_path=lambda wildcards: f"EPIC/RECO/{wildcards.CAMPAIGN}/epic_craterlake/SINGLE/{wildcards.PARTICLE}/{wildcards.ENERGY}/{wildcards.PHASE_SPACE}/",
+        shell: """
+    xrdfs root://dtn-eic.jlab.org/ ls /work/eic2/{params.search_path} \
+      | awk '{{ print "root://dtn-eic.jlab.org/"$1; }}' \
+      | sort \
+      > {output}
+    if [ ! -s {output} ]; then
+      echo "Got an empty file listing for path \"\""
+      exit 1
+    fi
+    """
+else:
+    checkpoint backwards_ecal_campaign_sim_list_checkpoint:
+        output:
+            "listing/backwards_ecal/{CAMPAIGN}/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}.lst.orig",
+        params:
+            search_path=lambda wildcards: f"EPIC/RECO/{wildcards.CAMPAIGN}/epic_craterlake/SINGLE/{wildcards.PARTICLE}/{wildcards.ENERGY}/{wildcards.PHASE_SPACE}/",
+        shell: """
+    xrdfs root://dtn-eic.jlab.org/ ls /work/eic2/{params.search_path} \
+      | sed -e 's#^/work/eic2/##' \
+      | sort \
+      > {output}
+    if [ ! -s {output} ]; then
+      echo "Got an empty file listing for path \"\""
+      exit 1
+    fi
+    """
+
+    def get_backwards_ecal_campaign_sim_list(wildcards):
+        with checkpoints.backwards_ecal_campaign_sim_list_checkpoint.get(**wildcards).output[0].open() as fp:
+            return [line.rstrip() for line in fp.readlines()]
+
+    rule backwards_ecal_campaign_sim_list:
+        input:
+            # depend on paths from the file list
+            get_backwards_ecal_campaign_sim_list,
+            orig_list="listing/backwards_ecal/{CAMPAIGN}/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}.lst.orig",
+        output:
+            "listing/backwards_ecal/{CAMPAIGN}/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}.lst",
+        shell: """
+    cp {input.orig_list} {output}
+    """
+
+
+ruleorder: backwards_ecal_local_sim_list > backwards_ecal_campaign_sim_list
 
 
 DETECTOR_CONFIG=os.environ["DETECTOR_CONFIG"]
 
 rule backwards_ecal:
     input:
         expand(
-            "sim_output/backwards_ecal/" + DETECTOR_CONFIG + "/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/flag",
+            "listing/backwards_ecal/{{CAMPAIGN}}/" + DETECTOR_CONFIG + "/{PARTICLE}/{ENERGY}/{PHASE_SPACE}.lst",
             PARTICLE=["pi-", "e-"],
             ENERGY=[
                 "100MeV",
@@ -96,14 +151,36 @@ rule backwards_ecal:
         matplotlibrc=".matplotlibrc",
         script="benchmarks/backwards_ecal/backwards_ecal.py",
     output:
-        directory("results/backwards_ecal")
+        directory("results/backwards_ecal/{CAMPAIGN}/")
+    log:
+        scheduler=".logs/results/backwards_ecal/{CAMPAIGN}/scheduler.log",
+        worker=".logs/results/backwards_ecal/{CAMPAIGN}/worker.log",
+    threads: workflow.cores
     shell:
         """
+if [[ "{wildcards.CAMPAIGN}" == "local" ]]; then
+export PLOT_TITLE="Benchmark simulation"
+else
+export PLOT_TITLE="\\textbf{{ePIC}} Simulation {wildcards.CAMPAIGN}"
+fi
+
+set -m # monitor mode to prevent lingering processes
+cleanup() {{
+  echo Cleaning up
+  kill $WORKER_PID $SCHEDULER_PID
+}}
+trap cleanup EXIT
+
+PORT=$RANDOM
+dask scheduler --port $PORT 2>{log.scheduler} &
+export DASK_SCHEDULER=localhost:$PORT
+SCHEDULER_PID=$!
+dask worker tcp://$DASK_SCHEDULER --nworkers {threads} --nthreads 1 2>{log.worker} &
+WORKER_PID=$!
 env \
 MATPLOTLIBRC={input.matplotlibrc} \
 DETECTOR_CONFIG=""" + DETECTOR_CONFIG + """ \
-PLOT_TITLE=""" + DETECTOR_CONFIG + """ \
-INPUT_PATH_FORMAT=sim_output/backwards_ecal/""" + DETECTOR_CONFIG + """/{{particle}}/{{energy}}/130to177deg/{{particle}}_{{energy}}_130to177deg.{{ix:04d}}.eicrecon.tree.edm4eic.root \
+INPUT_PATH_FORMAT=listing/backwards_ecal/{wildcards.CAMPAIGN}/""" + DETECTOR_CONFIG + """/{{particle}}/{{energy}}/130to177deg.lst \
 OUTPUT_DIR={output} \
 python {input.script}
 """