From e10fa7abe6637522224aa28ca864906a7eaa78d3 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Fri, 10 Nov 2023 18:20:26 -0500 Subject: [PATCH 01/25] benchmarks/barrel_ecal/config.yml: run via Snakemake --- flake.lock | 242 +++++++++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 62 ++++++++++++++ 2 files changed, 304 insertions(+) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/flake.lock b/flake.lock new file mode 100644 index 00000000..d1ed2f8a --- /dev/null +++ b/flake.lock @@ -0,0 +1,242 @@ +{ + "nodes": { + "acts-src": { + "flake": false, + "locked": { + "lastModified": 1698354503, + "narHash": "sha256-stRoSNq+LspmLWYAf2mZ+8HrugwvGiekGDZ1e6JqN3A=", + "owner": "acts-project", + "repo": "acts", + "rev": "76826f208f5929d8326798c87263f2563d0ae7e9", + "type": "github" + }, + "original": { + "owner": "acts-project", + "ref": "v30.3.2", + "repo": "acts", + "type": "github" + } + }, + "algorithms-src": { + "flake": false, + "locked": { + "lastModified": 1702617573, + "narHash": "sha256-tS2vunbFt2bfy6UZ++QmTAajeQTsGGYPg62oQdpyEEE=", + "owner": "eic", + "repo": "algorithms", + "rev": "ed4ede1971fdd883d13714972346fb6f61362eb5", + "type": "github" + }, + "original": { + "owner": "eic", + "repo": "algorithms", + "rev": "ed4ede1971fdd883d13714972346fb6f61362eb5", + "type": "github" + } + }, + "dd4hep-src": { + "flake": false, + "locked": { + "lastModified": 1700579447, + "narHash": "sha256-GUt5Cdm/Bl74mNJxVdfhgO+jFDWcycfqxPEmfWtl9PE=", + "owner": "AIDASoft", + "repo": "DD4hep", + "rev": "04bb629ddab5344c8df69070c35573f2f8095c69", + "type": "github" + }, + "original": { + "owner": "AIDASoft", + "ref": "v01-27-01", + "repo": "DD4hep", + "type": "github" + } + }, + "edm4eic-src": { + "flake": false, + "locked": { + "lastModified": 1698095659, + "narHash": "sha256-tdDVzkchePTlhD3uLUjf8Ww09jiEXlMTDbb4W4Ao510=", + "owner": "eic", + "repo": "EDM4eic", + "rev": "f41acf12643455f8decd8acb46f3ac50fce2a83e", + "type": "github" + }, + "original": { + "owner": "eic", + "ref": "v4.0.0", + "repo": "EDM4eic", + "type": "github" + } + }, + "edm4hep-src": { + "flake": false, + "locked": { + "lastModified": 1700584051, + "narHash": "sha256-E3yR8zeXQGB66hKIT0k2jFjPyuT0h4Qel3Zmebk7R5A=", + "owner": "key4hep", + "repo": "EDM4hep", + "rev": "d3066e5e8898a39d05a66e37dc4ff8105545654c", + "type": "github" + }, + "original": { + "owner": "key4hep", + "ref": "v00-10-02", + "repo": "EDM4hep", + "type": "github" + } + }, + "eicrecon-src": { + "flake": false, + "locked": { + "lastModified": 1702058038, + "narHash": "sha256-mq2jAtdITmMcCaijOLu8/piTRe8Er7gNBW2+0L3/PLA=", + "owner": "eic", + "repo": "EICrecon", + "rev": "3ad7e59cc30a871f6ef219215131334df9badf7a", + "type": "github" + }, + "original": { + "owner": "eic", + "ref": "v1.9.0", + "repo": "EICrecon", + "type": "github" + } + }, + "epic-nix": { + "inputs": { + "acts-src": "acts-src", + "algorithms-src": "algorithms-src", + "dd4hep-src": "dd4hep-src", + "edm4eic-src": "edm4eic-src", + "edm4hep-src": "edm4hep-src", + "eicrecon-src": "eicrecon-src", + "epic-src": "epic-src", + "flake-compat": "flake-compat", + "jana2-src": "jana2-src", + "nixpkgs": "nixpkgs", + "podio-src": "podio-src", + "site-overlay": "site-overlay" + }, + "locked": { + "lastModified": 1702897038, + "narHash": "sha256-/oNB6dA7EuybtqoURkDSmxUiZLcA/OtBTtMfkEXVS3A=", + "owner": "veprbl", + "repo": "epic-nix", + "rev": "eee7bb30780b00c1fff6705bf948c0a4250de86c", + "type": "github" + }, + "original": { + "owner": "veprbl", + "repo": "epic-nix", + "type": "github" + } + }, + "epic-src": { + "flake": false, + "locked": { + "lastModified": 1702141064, + "narHash": "sha256-WIuERj3LKLYhNgIJb3dwqivrQGB6WQEHHkbOotE2ivg=", + "owner": "eic", + "repo": "epic", + "rev": "103f90e98ffe0cf522a6955fb35ef775958858f8", + "type": "github" + }, + "original": { + "owner": "eic", + "ref": "23.12.0", + "repo": "epic", + "type": "github" + } + }, + "flake-compat": { + "flake": false, + "locked": { + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", + "owner": "edolstra", + "repo": "flake-compat", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", + "type": "github" + }, + "original": { + "owner": "edolstra", + "repo": "flake-compat", + "type": "github" + } + }, + "jana2-src": { + "flake": false, + "locked": { + "lastModified": 1700606162, + "narHash": "sha256-M92HRmfavVWJuuPZKoakEnNIEw8SaNQUTbKGTnRtlh8=", + "owner": "JeffersonLab", + "repo": "JANA2", + "rev": "3be28c29d7603f145d37dc76714c3e16074bcf58", + "type": "github" + }, + "original": { + "owner": "JeffersonLab", + "ref": "v2.1.2", + "repo": "JANA2", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1702444334, + "narHash": "sha256-vxZd9EjLzicTRCyffxqTu9Y5KJgBz8T/ZksxJfZ15js=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "9a81165eceee0aec8e12876372f20b689cda031d", + "type": "github" + }, + "original": { + "id": "nixpkgs", + "type": "indirect" + } + }, + "podio-src": { + "flake": false, + "locked": { + "lastModified": 1695623732, + "narHash": "sha256-vownh0TrIMfUpcaDraIs3Ys/pz/hbKF5KMwJZBOVW24=", + "owner": "AIDASoft", + "repo": "podio", + "rev": "babec745c12727cd5da590abbe1e3cf1ab8743ad", + "type": "github" + }, + "original": { + "owner": "AIDASoft", + "ref": "v00-17", + "repo": "podio", + "type": "github" + } + }, + "root": { + "inputs": { + "epic-nix": "epic-nix", + "nixpkgs": [ + "epic-nix", + "nixpkgs" + ] + } + }, + "site-overlay": { + "locked": { + "lastModified": 1692478388, + "narHash": "sha256-uVx/EVDRrXYjqHSZcpDbeNWG4mA0at11xi/dnN7xyEk=", + "owner": "veprbl", + "repo": "empty-overlay-flake", + "rev": "a29c5c5670303cd98cc976072af11e9ad7daecb3", + "type": "github" + }, + "original": { + "owner": "veprbl", + "repo": "empty-overlay-flake", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 00000000..21c758cb --- /dev/null +++ b/flake.nix @@ -0,0 +1,62 @@ +{ + inputs.epic-nix.url = "github:veprbl/epic-nix"; + + inputs.nixpkgs.follows = "epic-nix/nixpkgs"; + + outputs = { self, epic-nix, nixpkgs }: + let + + inherit (nixpkgs) lib; + supportedSystems = [ "x86_64-linux" "x86_64-darwin" ]; + + in + { + + devShells = lib.genAttrs supportedSystems + (system: + with import nixpkgs { + inherit system; + overlays = [ epic-nix.overlays.default ]; + }; + { + default = + mkShell { + buildInputs = + (builtins.attrValues epic-nix.packages.${system}) + ++ (with epic-nix.packages.${system}; [ + geant4.data.G4EMLOW + geant4.data.G4ENSDFSTATE + geant4.data.G4ENSDFSTATE + geant4.data.G4PARTICLEXS + geant4.data.G4PhotonEvaporation + ]) + ++ [ + nlohmann_json + snakemake + python3 + python3Packages.awkward + python3Packages.boto3 + python3Packages.bokeh + python3Packages.dask + python3Packages.dask-awkward + python3Packages.dask-histogram + python3Packages.distributed + python3Packages.matplotlib + python3Packages.notebook + python3Packages.numpy + python3Packages.pyhepmc + python3Packages.scipy + python3Packages.setuptools + python3Packages.uproot + ]; + shellHook = '' + unset JUPYTER_CONFIG_DIR JUPYTER_PATH + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${lib.makeLibraryPath [ fmt ]} + export S3_ACCESS_KEY="eicS3read" + export S3_SECRET_KEY="eicS3read" + ''; + }; + }); + + }; +} From 11a8db55e7772459f0cbcc09c5e3b1f7658c8b27 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Tue, 19 Dec 2023 12:44:15 -0500 Subject: [PATCH 02/25] add benchmarks/backgrounds --- .gitlab-ci.yml | 1 + Snakefile | 10 + benchmarks/backgrounds/Snakefile | 85 +++++ benchmarks/backgrounds/config.yml | 22 ++ benchmarks/backgrounds/ecal_backwards.org | 442 ++++++++++++++++++++++ benchmarks/backgrounds/org2py.awk | 22 ++ flake.lock | 242 ------------ flake.nix | 62 --- 8 files changed, 582 insertions(+), 304 deletions(-) create mode 100644 benchmarks/backgrounds/Snakefile create mode 100644 benchmarks/backgrounds/config.yml create mode 100644 benchmarks/backgrounds/ecal_backwards.org create mode 100644 benchmarks/backgrounds/org2py.awk delete mode 100644 flake.lock delete mode 100644 flake.nix diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 66f794e0..1a0ffb5b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,6 +128,7 @@ get_data: - runner_system_failure include: + - local: 'benchmarks/backgrounds/config.yml' - local: 'benchmarks/tracking_detectors/config.yml' - local: 'benchmarks/barrel_ecal/config.yml' - local: 'benchmarks/barrel_hcal/config.yml' diff --git a/Snakefile b/Snakefile index 3bd2c69c..8cdca119 100644 --- a/Snakefile +++ b/Snakefile @@ -1 +1,11 @@ +include: "benchmarks/backgrounds/Snakefile" include: "benchmarks/barrel_ecal/Snakefile" + +rule matplotlibrc: + output: + ".matplotlibrc", + run: + with open(output[0], "wt") as fp: + fp.write("backend: Agg\n") + # interactive mode prevents plt.show() from blocking + fp.write("interactive : True\n") diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile new file mode 100644 index 00000000..18abe77c --- /dev/null +++ b/benchmarks/backgrounds/Snakefile @@ -0,0 +1,85 @@ +import os + +from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider + + +S3 = S3RemoteProvider( + endpoint_url="https://dtn01.sdcc.bnl.gov:9000", + access_key_id=os.environ["S3_ACCESS_KEY"], + secret_access_key=os.environ["S3_SECRET_KEY"], +) + + +rule backgrounds_get: + input: + S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc"), + S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc"), + S3.remote("eictest/EPIC/FULL/23.11.0/epic_craterlake/DIS/NC/10x100/minQ2=10/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root"), + output: + "input/backgrounds/beam_gas_electron.hepmc", + "input/backgrounds/beam_gas_proton.hepmc", + dis_sim="input/backgrounds/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root", + run: + for src, dst in zip(input, output): + os.link(src, dst) + + +rule backgrounds_sim: + input: + "input/backgrounds/beam_gas_{BEAM}.hepmc", + output: + "sim/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root", + log: + "sim/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root.log", + params: + N_EVENTS=100 + shell: + """ +ddsim \ + --runType batch \ + --part.minimalKineticEnergy 100*GeV \ + --filter.tracker edep0 \ + -v WARNING \ + --numberOfEvents {params.N_EVENTS} \ + --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \ + --inputFiles {input} \ + --outputFile {output} +""" + + +rule backgrounds_org2py: + input: + notebook=workflow.source_path("ecal_backwards.org"), + converter=workflow.source_path("./org2py.awk"), + output: + "ecal_backwards.py" + shell: + """ +awk -f {input.converter} {input.notebook} > {output} +""" + +DETECTOR_CONFIG=os.environ["DETECTOR_CONFIG"] + +rule backgrounds_ecal_backwards: + input: + matplotlibrc=".matplotlibrc", + script="ecal_backwards.py", + electron_beam_gas_gen="input/backgrounds/beam_gas_electron.hepmc", + electron_beam_gas_sim="sim/" + DETECTOR_CONFIG + "/beam_gas_electron.edm4hep.root", + physics_signal_sim=rules.backgrounds_get.output.dis_sim, + proton_beam_gas_gen="input/backgrounds/beam_gas_proton.hepmc", + proton_beam_gas_sim="sim/" + DETECTOR_CONFIG + "/beam_gas_proton.edm4hep.root", + output: + directory("results/backgrounds/backwards_ecal") + shell: + """ +env \ +MATPLOTLIBRC={input.matplotlibrc} \ +ELECTRON_BEAM_GAS_GEN=$(realpath {input.electron_beam_gas_gen}) \ +ELECTRON_BEAM_GAS_SIM=$(realpath {input.electron_beam_gas_sim}) \ +PHYSICS_PROCESS_SIM=$(realpath {input.physics_signal_sim}) \ +PROTON_BEAM_GAS_GEN=$(realpath {input.proton_beam_gas_gen}) \ +PROTON_BEAM_GAS_SIM=$(realpath {input.proton_beam_gas_sim}) \ +OUTPUT_DIR={output} \ +python {input.script} +""" diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml new file mode 100644 index 00000000..513209a4 --- /dev/null +++ b/benchmarks/backgrounds/config.yml @@ -0,0 +1,22 @@ +sim:backgrounds: + extends: .det_benchmark + stage: simulate + script: + - snakemake --cores 1 backgrounds_get + - snakemake --cores 1 sim/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root + +bench:backgrounds_emcal_backwards: + extends: .det_benchmark + stage: benchmarks + needs: + - ["sim:backgrounds"] + script: + - snakemake --cores 1 backgrounds_ecal_backwards + +collect_results:backgrounds: + extends: .det_benchmark + stage: collect + needs: + - "bench:backgrounds_emcal_backwards" + script: + - ls -lrht diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org new file mode 100644 index 00000000..321d8081 --- /dev/null +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -0,0 +1,442 @@ +#+PROPERTY: header-args:jupyter-python :session /jpy:localhost#8888:eeemcal :async yes :results drawer :exports both + +#+TITLE: ePIC EEEMCal background rates study +#+AUTHOR: Dmitry Kalinkin +#+OPTIONS: d:t + +#+begin_src jupyter-python :results silent +import os +from pathlib import Path + +import awkward as ak +import boost_histogram as bh +import dask +import dask_awkward as dak +import dask_histogram as dh +import numpy as np +import uproot +import pyhepmc +#+end_src + +#+begin_src jupyter-python :results silent +from dask.distributed import Client + +client = Client(processes=False) +#+end_src + +* Plotting setup + +#+begin_src jupyter-python :results silent +import matplotlib as mpl +import matplotlib.pyplot as plt + +def setup_presentation_style(): + mpl.rcParams.update(mpl.rcParamsDefault) + plt.style.use('ggplot') + plt.rcParams.update({ + 'axes.labelsize': 8, + 'axes.titlesize': 9, + 'figure.titlesize': 9, + 'figure.figsize': (4, 3), + 'legend.fontsize': 7, + 'xtick.labelsize': 8, + 'ytick.labelsize': 8, + 'pgf.rcfonts': False, + }) + +setup_presentation_style() +#+end_src + +* Analysis + +** Input files + +#+begin_src jupyter-python :results silent +ELECTRON_BEAM_GAS_GEN=os.environ.get("ELECTRON_BEAM_GAS_GEN", "../beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc") +ELECTRON_BEAM_GAS_SIM=os.environ.get("ELECTRON_BEAM_GAS_SIM", "electron_beam_gas.edm4hep.root") +PHYSICS_PROCESS_SIM=os.environ.get("PHYSICS_PROCESS_SIM", "pythia8NCDIS_10x100_minQ2=1_beamEffects_xAngle=-0.025_hiDiv_1.edm4hep.root") +PROTON_BEAM_GAS_GEN=os.environ.get("PROTON_BEAM_GAS_GEN", "100GeV.hepmc") +PROTON_BEAM_GAS_SIM=os.environ.get("PROTON_BEAM_GAS_SIM", "proton+beam_gas_ep.edm4hep.root") + +output_dir=Path(os.environ.get("OUTPUT_DIR", "./")) +output_dir.mkdir(parents=True, exist_ok=True) +#+end_src + +#+begin_src jupyter-python :results silent +builder = ak.ArrayBuilder() +n = 0 +with pyhepmc.open(PROTON_BEAM_GAS_GEN) as f: + it = iter(f) + for event in f: + builder.begin_list() + for vertex in event.vertices: + assert event.length_unit.name == "MM" + with builder.record("Vector4D"): + builder.field("x") + builder.real(vertex.position.x) + builder.field("y") + builder.real(vertex.position.y) + builder.field("z") + builder.real(vertex.position.z) + builder.field("t") + builder.real(vertex.position.t) + builder.end_list() + n += 1 + if n > 10000: break + +vertices_proton_beam_gas = builder.snapshot() + +builder = ak.ArrayBuilder() +n = 0 +with pyhepmc.open(ELECTRON_BEAM_GAS_GEN) as f: + it = iter(f) + for event in f: + builder.begin_list() + for vertex in event.vertices: + assert event.length_unit.name == "MM" + with builder.record("Vector3D"): + builder.field("x") + builder.real(vertex.position.x) + builder.field("y") + builder.real(vertex.position.y) + builder.field("z") + builder.real(vertex.position.z) + builder.field("t") + builder.real(vertex.position.t) + builder.end_list() + n += 1 + if n > 10000: break + +vertices_electron_beam_gas = builder.snapshot() +#+end_src + +#+begin_src jupyter-python :results silent +def filter_name(name): + return "Hits." in name or "MCParticles." in name + +datasets = { + # https://wiki.bnl.gov/EPIC/index.php?title=Electron_Beam_Gas + "electron beam gas 10 GeV": { + "vertices": vertices_electron_beam_gas, + "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "cmap": "cool", + "rate": 3.2e6, # Hz + }, + "DIS 10x100, $Q^2 > 1$ GeV${}^2$": { + "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "rate": 184e3, # Hz + }, + # https://wiki.bnl.gov/EPIC/index.php?title=Hadron_Beam_Gas + "proton beam gas 100 GeV": { + "vertices": vertices_proton_beam_gas, + "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "cmap": "summer", + "rate": 31.9e3, # Hz + }, +} + +for ds in datasets.values(): + ds["events"].eager_compute_divisions() +#+end_src + +** Vertex distributions + +#+begin_src jupyter-python +for label, ds in datasets.items(): + if "vertices" not in ds: continue + vs = ds["vertices"] + weight = ds["rate"] / ak.num(ds["vertices"], axis=0) + plt.hist(ak.ravel(vs.t[:,0]), bins=100, histtype="step", label=label) +plt.minorticks_on() +plt.xlabel("vertex[0].t, mm") +plt.legend() +plt.savefig(output_dir / "vertex_time_distribution.pdf", bbox_inches="tight") +plt.show() + +for label, ds in datasets.items(): + if "vertices" not in ds: continue + vs = ds["vertices"] + weight = ds["rate"] / ak.num(ds["vertices"], axis=0) + plt.hist(ak.ravel(vs.z[:,0]), bins=100, histtype="step", label=label) +plt.minorticks_on() +plt.xlabel("vertex[0].z, mm") +plt.legend() +plt.savefig(output_dir / "vertex_z_distribution.pdf", bbox_inches="tight") +plt.show() + +for label, ds in datasets.items(): + if "vertices" not in ds: continue + vs = ds["vertices"] + cmap = ds["cmap"] + weight = ds["rate"] / ak.num(ds["vertices"], axis=0) + plt.hist2d(vs.z[:,0].to_numpy(), vs.x[:,0].to_numpy(), bins=(100, np.linspace(-130, 130, 160)), cmin=1e-30, label=label, cmap=cmap) + plt.plot([], color=mpl.colormaps[cmap](0.5), label=label) +plt.minorticks_on() +plt.xlabel("vertex[0].z, mm") +plt.ylabel("vertex[0].x, mm") +plt.legend() +plt.savefig(output_dir / "vertex_xz_distribution.pdf", bbox_inches="tight") +plt.show() + +for ix, (label, ds) in enumerate(datasets.items()): + if "vertices" not in ds: continue + vs = ds["vertices"] + cmap = ds["cmap"] + weight = ds["rate"] / ak.num(ds["vertices"], axis=0) + plt.hist2d(vs.z[:,0].to_numpy(), vs.y[:,0].to_numpy(), bins=(100, 100), cmin=1e-30, cmap=cmap) + plt.colorbar() + plt.minorticks_on() + plt.xlabel("vertex[0].z, mm") + plt.ylabel("vertex[0].y, mm") + plt.title(label) + plt.savefig(output_dir / f"vertex_yz_distribution_{ix}.pdf", bbox_inches="tight") + plt.show() +#+end_src + +** Simulation results + +#+begin_src jupyter-python +for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: + for dataset_ix, (label, ds) in enumerate(datasets.items()): + events = ds["events"] + + energy_sums = ak.sum(events[f"{collection_name}.energy"].head(10000), axis=1) + event_id = ak.argmax(energy_sums) + xs = events[f"{collection_name}.position.x"].head(event_id + 1)[event_id].to_numpy() + ys = events[f"{collection_name}.position.y"].head(event_id + 1)[event_id].to_numpy() + + bin_widths = [None, None] + for ix, vals in enumerate([xs, ys]): + centers = np.unique(vals) + diffs = centers[1:] - centers[:-1] + bin_widths[ix] = np.min(diffs[diffs > 0]) + print(f"bin_widths[{ix}]", bin_widths[ix]) + + bins = { + "EcalEndcapNHits": [np.arange(-750., 750., bin_width) for bin_width in bin_widths], + "EcalEndcapPHits": [np.arange(-1800., 1800., bin_width) for bin_width in bin_widths], + }[collection_name] + + plt.hist2d( + xs, + ys, + weights=events[f"{collection_name}.energy"].head(event_id + 1)[event_id].to_numpy(), + bins=bins, + cmin=1e-10, + ) + plt.colorbar().set_label("energy, GeV", loc="top") + plt.title(f"{label}, event_id={event_id}\n{collection_name}") + plt.xlabel("hit x, mm", loc="right") + plt.ylabel("hit y, mm", loc="top") + plt.savefig(output_dir / f"{collection_name}_event_display_{dataset_ix}.pdf", bbox_inches="tight") + plt.show() +#+end_src + +** Discovering number of cells + +Using HyperLogLog algorithm would be faster here, or actually load +DD4hep geometry and count sensitive volumes. + +#+begin_src jupyter-python +def unique(array): + if ak.backend(array) == "typetracer": + ak.typetracer.touch_data(array) + return array + return ak.from_numpy(np.unique(ak.to_numpy(ak.ravel(array)))) +unique_delayed = dask.delayed(unique) +len_delayed = dask.delayed(len) + +cellID_for_r = dict() + +for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: + r_axis = { + "EcalEndcapNHits": bh.axis.Regular(75, 0., 750.), + "EcalEndcapPHits": bh.axis.Regular(90, 0., 1800.), + }[collection_name] + ds = datasets["DIS 10x100, $Q^2 > 1$ GeV${}^2$"] + events = ds["events"] + + r = np.hypot( + ak.ravel(events[f"{collection_name}.position.x"]), + ak.ravel(events[f"{collection_name}.position.y"]), + ) + cellID = ak.ravel(events[f"{collection_name}.cellID"]) + + cellID_for_r[collection_name] = np.array(client.gather(client.compute([ + len_delayed(unique_delayed( + cellID[(r >= r_min) & (r < r_max)].map_partitions(unique) + )) + for r_min, r_max in zip(r_axis.edges[:-1], r_axis.edges[1:]) + ]))) + + print(cellID_for_r[collection_name]) + print(sum(cellID_for_r[collection_name])) + + plt.stairs( + cellID_for_r[collection_name], + r_axis.edges, + ) + + plt.title(f"{collection_name}") + plt.legend() + plt.xlabel("r, mm", loc="right") + dr = (r_axis.edges[1] - r_axis.edges[0]) + plt.ylabel(f"Number of towers per {dr} mm slice in $r$", loc="top") + plt.savefig(output_dir / f"{collection_name}_num_towers.pdf", bbox_inches="tight") + plt.show() +#+end_src + +** Plotting the rates + +#+begin_src jupyter-python +for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: + r_axis = { + "EcalEndcapNHits": bh.axis.Regular(75, 0., 750.), + "EcalEndcapPHits": bh.axis.Regular(90, 0., 1800.), + }[collection_name] + for edep_min in [0.005, 0.015, 0.050]: # GeV + for label, ds in datasets.items(): + events = ds["events"] + weight = ds["rate"] / len(events) + + r = np.hypot( + ak.ravel(events[f"{collection_name}.position.x"]), + ak.ravel(events[f"{collection_name}.position.y"]), + ) + edep = ak.ravel(events[f"{collection_name}.energy"]) + r = r[edep > edep_min] + + hist = dh.factory( + r, + axes=(r_axis,), + ).compute() + plt.stairs( + hist.values() * weight / cellID_for_r[collection_name], + hist.axes[0].edges, + label=f"{label}", + ) + + plt.title(f"for $E_{{dep.}} >$ {edep_min * 1000} MeV\n{collection_name}") + plt.legend() + plt.xlabel("r, mm", loc="right") + plt.ylabel("rate per tower, Hz", loc="top") + plt.yscale("log") + plt.savefig(output_dir / f"{collection_name}_hit_rate_vs_r_edep_min_{edep_min:.3f}.pdf", bbox_inches="tight") + plt.show() +#+end_src + +#+begin_src jupyter-python +for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: + for totedep_min in [-1, 0, 0.1, 0.5, 1.0, 5.0, 10.]: # GeV + for label, ds in datasets.items(): + events = ds["events"] + weight = ds["rate"] / len(events) + + z = ds["events"]["MCParticles.vertex.z"][:,1] + totedep = ak.sum(events[f"{collection_name}.energy"], axis=1) + z = z[totedep > totedep_min] + + hist = dh.factory( + z, + axes=(bh.axis.Regular(250, -7500., 17500.),), + ).compute() + plt.stairs( + hist.values() * weight, + hist.axes[0].edges, + label=f"{label}", + ) + + plt.title(rf"for events with $E_{{\mathrm{{dep. tot.}}}}$ $>$ {totedep_min} GeV" + f"\n{collection_name}") + plt.legend() + plt.xlabel("$z$ of the first interaction vertex, mm", loc="right") + plt.ylabel("rate, Hz", loc="top") + plt.yscale("log") + plt.savefig(output_dir / f"{collection_name}_hit_rate_vs_z_totedep_min_{totedep_min:.1f}.pdf", bbox_inches="tight") + plt.show() +#+end_src + +#+begin_src jupyter-python +num_towers_cache = { + "LumiSpecCAL": 200, + "LumiDirectPCAL": 1, + "ZDCHcal": 1470, + "LFHCAL": 578338, + "ZDC_WSi_": 187043, + "EcalBarrelScFi": 124205483, + "EcalEndcapP": 15037, + "ZDCEcal": 400, + "EcalEndcapPInsert": 536, + "HcalEndcapPInsert": 20251, + "B0ECal": 131, + "HcalEndcapN": 13800, + "HcalBarrel": 7680, + "EcalBarrelImaging": 5765469, + "EcalEndcapN": 2988, + "ZDC_PbSi_": 44344, +} + +fig_cmb = plt.figure() +ax_cmb = fig_cmb.gca() + +for edep_min in [0]: # GeV + for dataset_ix, (x_offset, (ds_label, ds)) in enumerate(zip(np.linspace(-0.3, 0.3, len(datasets)), datasets.items())): + events = ds["events"] + weight = ds["rate"] / len(events) + + labels = [] + values = [] + norms = [] + + for branch_name in events.fields: + if ".energy" not in branch_name: continue + if "ZDC_SiliconPix_Hits" in branch_name: continue + + edep = ak.ravel(events[branch_name]) + + #cellID = ak.ravel(events[branch_name.replace(".energy", ".cellID")]) + #num_towers = len(unique_delayed( + # cellID.map_partitions(unique) + #).compute()) + + num_towers = num_towers_cache[branch_name.replace("Hits.energy", "")] + + labels.append(branch_name.replace("Hits.energy", "")) + values.append(ak.count(edep[edep > edep_min])) + norms.append(num_towers if num_towers != 0 else np.nan) + + fig_cur = plt.figure() + ax_cur = fig_cur.gca() + + values, = dask.compute(values) + for ax, per_tower, offset, width in [ + (ax_cmb, True, x_offset, 2 * 0.3 / (len(datasets) - 1)), + (ax_cur, False, 0, 2 * 0.3), + ]: + ax.bar( + np.arange(len(labels)) + offset, + weight * np.array(values) / (np.array(norms) if per_tower else np.ones_like(norms)), + width=width, + tick_label=labels, + label=ds_label, + color=f"C{dataset_ix}", + ) + + plt.sca(ax_cur) + plt.legend() + plt.title(f"for $E_{{dep.}} >$ {edep_min * 1000} MeV") + plt.ylabel("rate, Hz", loc="top") + plt.yscale("log") + plt.xticks(rotation=90, ha='right') + fig_cur.savefig(f"rates_edep_min_{edep_min}_{dataset_ix}.pdf", bbox_inches="tight") + plt.show() + plt.close(fig_cur) + + plt.sca(ax_cmb) + plt.legend() + plt.title(f"for $E_{{dep.}} >$ {edep_min * 1000} MeV") + plt.ylabel("rate per tower, Hz", loc="top") + plt.yscale("log") + plt.xticks(rotation=90, ha='right') + fig_cmb.savefig(f"rates_edep_min_{edep_min}.pdf", bbox_inches="tight") + plt.show() +#+end_src diff --git a/benchmarks/backgrounds/org2py.awk b/benchmarks/backgrounds/org2py.awk new file mode 100644 index 00000000..0d23cf25 --- /dev/null +++ b/benchmarks/backgrounds/org2py.awk @@ -0,0 +1,22 @@ +BEGIN { + in_src = 0 + IGNORECASE=1 +} + +/^#\+begin_src\s+[^\s]*python/ { + in_src = 1 + match($0, /^ */) + spaces = RLENGTH + next +} + +/^#\+end_src/ { + in_src = 0 + next +} + +in_src { + re = "^ {" spaces "}" + gsub(re,"") + print +} diff --git a/flake.lock b/flake.lock deleted file mode 100644 index d1ed2f8a..00000000 --- a/flake.lock +++ /dev/null @@ -1,242 +0,0 @@ -{ - "nodes": { - "acts-src": { - "flake": false, - "locked": { - "lastModified": 1698354503, - "narHash": "sha256-stRoSNq+LspmLWYAf2mZ+8HrugwvGiekGDZ1e6JqN3A=", - "owner": "acts-project", - "repo": "acts", - "rev": "76826f208f5929d8326798c87263f2563d0ae7e9", - "type": "github" - }, - "original": { - "owner": "acts-project", - "ref": "v30.3.2", - "repo": "acts", - "type": "github" - } - }, - "algorithms-src": { - "flake": false, - "locked": { - "lastModified": 1702617573, - "narHash": "sha256-tS2vunbFt2bfy6UZ++QmTAajeQTsGGYPg62oQdpyEEE=", - "owner": "eic", - "repo": "algorithms", - "rev": "ed4ede1971fdd883d13714972346fb6f61362eb5", - "type": "github" - }, - "original": { - "owner": "eic", - "repo": "algorithms", - "rev": "ed4ede1971fdd883d13714972346fb6f61362eb5", - "type": "github" - } - }, - "dd4hep-src": { - "flake": false, - "locked": { - "lastModified": 1700579447, - "narHash": "sha256-GUt5Cdm/Bl74mNJxVdfhgO+jFDWcycfqxPEmfWtl9PE=", - "owner": "AIDASoft", - "repo": "DD4hep", - "rev": "04bb629ddab5344c8df69070c35573f2f8095c69", - "type": "github" - }, - "original": { - "owner": "AIDASoft", - "ref": "v01-27-01", - "repo": "DD4hep", - "type": "github" - } - }, - "edm4eic-src": { - "flake": false, - "locked": { - "lastModified": 1698095659, - "narHash": "sha256-tdDVzkchePTlhD3uLUjf8Ww09jiEXlMTDbb4W4Ao510=", - "owner": "eic", - "repo": "EDM4eic", - "rev": "f41acf12643455f8decd8acb46f3ac50fce2a83e", - "type": "github" - }, - "original": { - "owner": "eic", - "ref": "v4.0.0", - "repo": "EDM4eic", - "type": "github" - } - }, - "edm4hep-src": { - "flake": false, - "locked": { - "lastModified": 1700584051, - "narHash": "sha256-E3yR8zeXQGB66hKIT0k2jFjPyuT0h4Qel3Zmebk7R5A=", - "owner": "key4hep", - "repo": "EDM4hep", - "rev": "d3066e5e8898a39d05a66e37dc4ff8105545654c", - "type": "github" - }, - "original": { - "owner": "key4hep", - "ref": "v00-10-02", - "repo": "EDM4hep", - "type": "github" - } - }, - "eicrecon-src": { - "flake": false, - "locked": { - "lastModified": 1702058038, - "narHash": "sha256-mq2jAtdITmMcCaijOLu8/piTRe8Er7gNBW2+0L3/PLA=", - "owner": "eic", - "repo": "EICrecon", - "rev": "3ad7e59cc30a871f6ef219215131334df9badf7a", - "type": "github" - }, - "original": { - "owner": "eic", - "ref": "v1.9.0", - "repo": "EICrecon", - "type": "github" - } - }, - "epic-nix": { - "inputs": { - "acts-src": "acts-src", - "algorithms-src": "algorithms-src", - "dd4hep-src": "dd4hep-src", - "edm4eic-src": "edm4eic-src", - "edm4hep-src": "edm4hep-src", - "eicrecon-src": "eicrecon-src", - "epic-src": "epic-src", - "flake-compat": "flake-compat", - "jana2-src": "jana2-src", - "nixpkgs": "nixpkgs", - "podio-src": "podio-src", - "site-overlay": "site-overlay" - }, - "locked": { - "lastModified": 1702897038, - "narHash": "sha256-/oNB6dA7EuybtqoURkDSmxUiZLcA/OtBTtMfkEXVS3A=", - "owner": "veprbl", - "repo": "epic-nix", - "rev": "eee7bb30780b00c1fff6705bf948c0a4250de86c", - "type": "github" - }, - "original": { - "owner": "veprbl", - "repo": "epic-nix", - "type": "github" - } - }, - "epic-src": { - "flake": false, - "locked": { - "lastModified": 1702141064, - "narHash": "sha256-WIuERj3LKLYhNgIJb3dwqivrQGB6WQEHHkbOotE2ivg=", - "owner": "eic", - "repo": "epic", - "rev": "103f90e98ffe0cf522a6955fb35ef775958858f8", - "type": "github" - }, - "original": { - "owner": "eic", - "ref": "23.12.0", - "repo": "epic", - "type": "github" - } - }, - "flake-compat": { - "flake": false, - "locked": { - "lastModified": 1696426674, - "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", - "owner": "edolstra", - "repo": "flake-compat", - "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", - "type": "github" - }, - "original": { - "owner": "edolstra", - "repo": "flake-compat", - "type": "github" - } - }, - "jana2-src": { - "flake": false, - "locked": { - "lastModified": 1700606162, - "narHash": "sha256-M92HRmfavVWJuuPZKoakEnNIEw8SaNQUTbKGTnRtlh8=", - "owner": "JeffersonLab", - "repo": "JANA2", - "rev": "3be28c29d7603f145d37dc76714c3e16074bcf58", - "type": "github" - }, - "original": { - "owner": "JeffersonLab", - "ref": "v2.1.2", - "repo": "JANA2", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1702444334, - "narHash": "sha256-vxZd9EjLzicTRCyffxqTu9Y5KJgBz8T/ZksxJfZ15js=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "9a81165eceee0aec8e12876372f20b689cda031d", - "type": "github" - }, - "original": { - "id": "nixpkgs", - "type": "indirect" - } - }, - "podio-src": { - "flake": false, - "locked": { - "lastModified": 1695623732, - "narHash": "sha256-vownh0TrIMfUpcaDraIs3Ys/pz/hbKF5KMwJZBOVW24=", - "owner": "AIDASoft", - "repo": "podio", - "rev": "babec745c12727cd5da590abbe1e3cf1ab8743ad", - "type": "github" - }, - "original": { - "owner": "AIDASoft", - "ref": "v00-17", - "repo": "podio", - "type": "github" - } - }, - "root": { - "inputs": { - "epic-nix": "epic-nix", - "nixpkgs": [ - "epic-nix", - "nixpkgs" - ] - } - }, - "site-overlay": { - "locked": { - "lastModified": 1692478388, - "narHash": "sha256-uVx/EVDRrXYjqHSZcpDbeNWG4mA0at11xi/dnN7xyEk=", - "owner": "veprbl", - "repo": "empty-overlay-flake", - "rev": "a29c5c5670303cd98cc976072af11e9ad7daecb3", - "type": "github" - }, - "original": { - "owner": "veprbl", - "repo": "empty-overlay-flake", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 21c758cb..00000000 --- a/flake.nix +++ /dev/null @@ -1,62 +0,0 @@ -{ - inputs.epic-nix.url = "github:veprbl/epic-nix"; - - inputs.nixpkgs.follows = "epic-nix/nixpkgs"; - - outputs = { self, epic-nix, nixpkgs }: - let - - inherit (nixpkgs) lib; - supportedSystems = [ "x86_64-linux" "x86_64-darwin" ]; - - in - { - - devShells = lib.genAttrs supportedSystems - (system: - with import nixpkgs { - inherit system; - overlays = [ epic-nix.overlays.default ]; - }; - { - default = - mkShell { - buildInputs = - (builtins.attrValues epic-nix.packages.${system}) - ++ (with epic-nix.packages.${system}; [ - geant4.data.G4EMLOW - geant4.data.G4ENSDFSTATE - geant4.data.G4ENSDFSTATE - geant4.data.G4PARTICLEXS - geant4.data.G4PhotonEvaporation - ]) - ++ [ - nlohmann_json - snakemake - python3 - python3Packages.awkward - python3Packages.boto3 - python3Packages.bokeh - python3Packages.dask - python3Packages.dask-awkward - python3Packages.dask-histogram - python3Packages.distributed - python3Packages.matplotlib - python3Packages.notebook - python3Packages.numpy - python3Packages.pyhepmc - python3Packages.scipy - python3Packages.setuptools - python3Packages.uproot - ]; - shellHook = '' - unset JUPYTER_CONFIG_DIR JUPYTER_PATH - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${lib.makeLibraryPath [ fmt ]} - export S3_ACCESS_KEY="eicS3read" - export S3_SECRET_KEY="eicS3read" - ''; - }; - }); - - }; -} From a3b60b12c4e063213f80d9ed6fde86478ce9534b Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 31 Dec 2023 03:13:06 -0500 Subject: [PATCH 03/25] Snakefile: sim/ -> sim_output/ This is the one being symlinked to $LOCAL_DATA_PATH --- benchmarks/backgrounds/Snakefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index 18abe77c..cedaec61 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -28,9 +28,9 @@ rule backgrounds_sim: input: "input/backgrounds/beam_gas_{BEAM}.hepmc", output: - "sim/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root", + "sim_output/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root", log: - "sim/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root.log", + "sim_output/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root.log", params: N_EVENTS=100 shell: @@ -65,10 +65,10 @@ rule backgrounds_ecal_backwards: matplotlibrc=".matplotlibrc", script="ecal_backwards.py", electron_beam_gas_gen="input/backgrounds/beam_gas_electron.hepmc", - electron_beam_gas_sim="sim/" + DETECTOR_CONFIG + "/beam_gas_electron.edm4hep.root", + electron_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_electron.edm4hep.root", physics_signal_sim=rules.backgrounds_get.output.dis_sim, proton_beam_gas_gen="input/backgrounds/beam_gas_proton.hepmc", - proton_beam_gas_sim="sim/" + DETECTOR_CONFIG + "/beam_gas_proton.edm4hep.root", + proton_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_proton.edm4hep.root", output: directory("results/backgrounds/backwards_ecal") shell: From f09eb963382bcae93ed09ae43e4fe3e50a99c718 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 31 Dec 2023 06:59:20 -0500 Subject: [PATCH 04/25] add benchmarks/backgrounds/requirements.txt --- benchmarks/backgrounds/config.yml | 2 ++ benchmarks/backgrounds/requirements.txt | 7 +++++++ 2 files changed, 9 insertions(+) create mode 100644 benchmarks/backgrounds/requirements.txt diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index 513209a4..408b8471 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -11,6 +11,8 @@ bench:backgrounds_emcal_backwards: needs: - ["sim:backgrounds"] script: + - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps + - pip install -r benchmarks/backgrounds/requirements.txt - snakemake --cores 1 backgrounds_ecal_backwards collect_results:backgrounds: diff --git a/benchmarks/backgrounds/requirements.txt b/benchmarks/backgrounds/requirements.txt new file mode 100644 index 00000000..738152dd --- /dev/null +++ b/benchmarks/backgrounds/requirements.txt @@ -0,0 +1,7 @@ +awkward >= 2.4.0 +dask >= 2023 +dask_awkward +dask_histogram +distributed >= 2023 +pyhepmc +uproot >= 5.2.0 From 7055630c5aed6349b7fb65f75f7c7fb70c04e1a1 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 31 Dec 2023 21:40:02 -0500 Subject: [PATCH 05/25] backgrounds/config.yml: sim/ -> sim_output/ --- benchmarks/backgrounds/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index 408b8471..f24d0640 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -3,7 +3,7 @@ sim:backgrounds: stage: simulate script: - snakemake --cores 1 backgrounds_get - - snakemake --cores 1 sim/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root + - snakemake --cores 1 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root bench:backgrounds_emcal_backwards: extends: .det_benchmark From 4f61610ae029aa9af114b07ba9a339041a909b4c Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 3 Jan 2024 12:19:58 -0500 Subject: [PATCH 06/25] use dask-scheduler (provides multiprocesssing) --- benchmarks/backgrounds/Snakefile | 8 ++++++++ benchmarks/backgrounds/ecal_backwards.org | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index cedaec61..391cf149 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -71,8 +71,15 @@ rule backgrounds_ecal_backwards: proton_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_proton.edm4hep.root", output: directory("results/backgrounds/backwards_ecal") + threads: workflow.cores shell: """ +PORT=$RANDOM +dask scheduler --port $PORT & +export DASK_SCHEDULER=localhost:$PORT +SCHEDULER_PID=$! +dask worker tcp://$DASK_SCHEDULER --nworkers {threads} --nthreads 1 & +WORKER_PID=$! env \ MATPLOTLIBRC={input.matplotlibrc} \ ELECTRON_BEAM_GAS_GEN=$(realpath {input.electron_beam_gas_gen}) \ @@ -82,4 +89,5 @@ PROTON_BEAM_GAS_GEN=$(realpath {input.proton_beam_gas_gen}) \ PROTON_BEAM_GAS_SIM=$(realpath {input.proton_beam_gas_sim}) \ OUTPUT_DIR={output} \ python {input.script} +kill $WORKER_PID $SCHEDULER_PID """ diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index 321d8081..894fddef 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -21,7 +21,7 @@ import pyhepmc #+begin_src jupyter-python :results silent from dask.distributed import Client -client = Client(processes=False) +client = Client(os.environ.get("DASK_SCHEDULER", "localhost:8786")) #+end_src * Plotting setup From dddb0d030b65602ce885594f208b7125b7acf084 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 3 Jan 2024 13:31:15 -0500 Subject: [PATCH 07/25] ecal_backwards.org: handle case of insufficient events --- benchmarks/backgrounds/ecal_backwards.org | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index 894fddef..824584c2 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -209,7 +209,7 @@ for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: for ix, vals in enumerate([xs, ys]): centers = np.unique(vals) diffs = centers[1:] - centers[:-1] - bin_widths[ix] = np.min(diffs[diffs > 0]) + bin_widths[ix] = np.min(diffs[diffs > 0]) if np.sum(diffs > 0) > 0 else 1. print(f"bin_widths[{ix}]", bin_widths[ix]) bins = { From e4385c415e8cc8da2e8139820bf70f0f288754e5 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Wed, 3 Jan 2024 13:31:43 -0500 Subject: [PATCH 08/25] Snakefile: split get to avoid unnecessary downloads/simulations --- benchmarks/backgrounds/Snakefile | 25 +++++++++++++++++++------ benchmarks/backgrounds/config.yml | 1 - 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index 391cf149..de8d6180 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -10,18 +10,31 @@ S3 = S3RemoteProvider( ) -rule backgrounds_get: +rule backgrounds_get_beam_gas_electron: input: S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc"), - S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc"), - S3.remote("eictest/EPIC/FULL/23.11.0/epic_craterlake/DIS/NC/10x100/minQ2=10/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root"), output: "input/backgrounds/beam_gas_electron.hepmc", + run: + os.link(input[0], output[0]) + + +rule backgrounds_get_beam_gas_proton: + input: + S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc"), + output: "input/backgrounds/beam_gas_proton.hepmc", + run: + os.link(input[0], output[0]) + + +rule backgrounds_get_DIS: + input: + S3.remote("eictest/EPIC/FULL/23.11.0/epic_craterlake/DIS/NC/10x100/minQ2=10/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root"), + output: dis_sim="input/backgrounds/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root", run: - for src, dst in zip(input, output): - os.link(src, dst) + os.link(input[0], output[0]) rule backgrounds_sim: @@ -66,7 +79,7 @@ rule backgrounds_ecal_backwards: script="ecal_backwards.py", electron_beam_gas_gen="input/backgrounds/beam_gas_electron.hepmc", electron_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_electron.edm4hep.root", - physics_signal_sim=rules.backgrounds_get.output.dis_sim, + physics_signal_sim=rules.backgrounds_get_DIS.output[0], proton_beam_gas_gen="input/backgrounds/beam_gas_proton.hepmc", proton_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_proton.edm4hep.root", output: diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index f24d0640..c9cb5f85 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -2,7 +2,6 @@ sim:backgrounds: extends: .det_benchmark stage: simulate script: - - snakemake --cores 1 backgrounds_get - snakemake --cores 1 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root bench:backgrounds_emcal_backwards: From 9ae1eadbd1c0a5a0de6c85eb53b5a4b544329f0b Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Thu, 4 Jan 2024 14:24:52 -0500 Subject: [PATCH 09/25] backgrounds/config.yml: more threads --- benchmarks/backgrounds/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index c9cb5f85..36ccc8b2 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -2,7 +2,7 @@ sim:backgrounds: extends: .det_benchmark stage: simulate script: - - snakemake --cores 1 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root + - snakemake --cores 2 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root bench:backgrounds_emcal_backwards: extends: .det_benchmark @@ -12,7 +12,7 @@ bench:backgrounds_emcal_backwards: script: - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps - pip install -r benchmarks/backgrounds/requirements.txt - - snakemake --cores 1 backgrounds_ecal_backwards + - snakemake --cores 8 backgrounds_ecal_backwards collect_results:backgrounds: extends: .det_benchmark From 3987a9819926e0c8880d264ac19f376f7340de77 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Thu, 4 Jan 2024 17:15:29 -0500 Subject: [PATCH 10/25] ecal_backwards.org: use step_size (should work in uproot 5.2.0+) --- benchmarks/backgrounds/ecal_backwards.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index 824584c2..c3591fe6 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -118,18 +118,18 @@ datasets = { # https://wiki.bnl.gov/EPIC/index.php?title=Electron_Beam_Gas "electron beam gas 10 GeV": { "vertices": vertices_electron_beam_gas, - "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, step_size="128 MB"), "cmap": "cool", "rate": 3.2e6, # Hz }, "DIS 10x100, $Q^2 > 1$ GeV${}^2$": { - "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, step_size="128 MB"), "rate": 184e3, # Hz }, # https://wiki.bnl.gov/EPIC/index.php?title=Hadron_Beam_Gas "proton beam gas 100 GeV": { "vertices": vertices_proton_beam_gas, - "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, step_size="128 MB"), "cmap": "summer", "rate": 31.9e3, # Hz }, From ac8a1278b9009728ae1537ee7c9979283f04499d Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sat, 6 Jan 2024 13:14:57 -0500 Subject: [PATCH 11/25] Revert "ecal_backwards.org: use step_size (should work in uproot 5.2.0+)" This reverts commit 3987a9819926e0c8880d264ac19f376f7340de77. --- benchmarks/backgrounds/ecal_backwards.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index c3591fe6..824584c2 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -118,18 +118,18 @@ datasets = { # https://wiki.bnl.gov/EPIC/index.php?title=Electron_Beam_Gas "electron beam gas 10 GeV": { "vertices": vertices_electron_beam_gas, - "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, step_size="128 MB"), + "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), "cmap": "cool", "rate": 3.2e6, # Hz }, "DIS 10x100, $Q^2 > 1$ GeV${}^2$": { - "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, step_size="128 MB"), + "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), "rate": 184e3, # Hz }, # https://wiki.bnl.gov/EPIC/index.php?title=Hadron_Beam_Gas "proton beam gas 100 GeV": { "vertices": vertices_proton_beam_gas, - "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, step_size="128 MB"), + "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), "cmap": "summer", "rate": 31.9e3, # Hz }, From 5c5217088a4b30ea94814cf716b41d052870da77 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sat, 6 Jan 2024 16:23:28 -0500 Subject: [PATCH 12/25] store hepmc in $LOCAL_DATA_PATH --- benchmarks/backgrounds/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index 36ccc8b2..fb19fee3 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -2,6 +2,8 @@ sim:backgrounds: extends: .det_benchmark stage: simulate script: + - mkdir $LOCAL_DATA_PATH/input + - ln -s $LOCAL_DATA_PATH/input input - snakemake --cores 2 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root bench:backgrounds_emcal_backwards: @@ -10,6 +12,8 @@ bench:backgrounds_emcal_backwards: needs: - ["sim:backgrounds"] script: + - mkdir $LOCAL_DATA_PATH/input + - ln -s $LOCAL_DATA_PATH/input input - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps - pip install -r benchmarks/backgrounds/requirements.txt - snakemake --cores 8 backgrounds_ecal_backwards From 8f6febe83130f5dbb0bed1f691707aafa332f9fe Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sat, 6 Jan 2024 16:23:44 -0500 Subject: [PATCH 13/25] reduce steps_per_file --- benchmarks/backgrounds/ecal_backwards.org | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index 824584c2..2b95e6c3 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -118,18 +118,18 @@ datasets = { # https://wiki.bnl.gov/EPIC/index.php?title=Electron_Beam_Gas "electron beam gas 10 GeV": { "vertices": vertices_electron_beam_gas, - "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "events": uproot.dask({ELECTRON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=32), "cmap": "cool", "rate": 3.2e6, # Hz }, "DIS 10x100, $Q^2 > 1$ GeV${}^2$": { - "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "events": uproot.dask({PHYSICS_PROCESS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=32), "rate": 184e3, # Hz }, # https://wiki.bnl.gov/EPIC/index.php?title=Hadron_Beam_Gas "proton beam gas 100 GeV": { "vertices": vertices_proton_beam_gas, - "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=512), + "events": uproot.dask({PROTON_BEAM_GAS_SIM: "events"}, filter_name=filter_name, open_files=False, steps_per_file=32), "cmap": "summer", "rate": 31.9e3, # Hz }, From 606ed981e7025c3e6fb7315adab7cae9872c2d5a Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 7 Jan 2024 10:52:57 -0500 Subject: [PATCH 14/25] os.link -> os.rename --- benchmarks/backgrounds/Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index de8d6180..ac920ef8 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -16,7 +16,7 @@ rule backgrounds_get_beam_gas_electron: output: "input/backgrounds/beam_gas_electron.hepmc", run: - os.link(input[0], output[0]) + os.rename(input[0], output[0]) rule backgrounds_get_beam_gas_proton: @@ -25,7 +25,7 @@ rule backgrounds_get_beam_gas_proton: output: "input/backgrounds/beam_gas_proton.hepmc", run: - os.link(input[0], output[0]) + os.rename(input[0], output[0]) rule backgrounds_get_DIS: @@ -34,7 +34,7 @@ rule backgrounds_get_DIS: output: dis_sim="input/backgrounds/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root", run: - os.link(input[0], output[0]) + os.rename(input[0], output[0]) rule backgrounds_sim: From 2620492fe3eb8441082a1bac53070b7653bc262a Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sun, 7 Jan 2024 12:59:22 -0500 Subject: [PATCH 15/25] os.rename -> shutil.move --- benchmarks/backgrounds/Snakefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index ac920ef8..7d41387e 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -1,4 +1,5 @@ import os +import shutil from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider @@ -16,7 +17,7 @@ rule backgrounds_get_beam_gas_electron: output: "input/backgrounds/beam_gas_electron.hepmc", run: - os.rename(input[0], output[0]) + shutil.move(input[0], output[0]) rule backgrounds_get_beam_gas_proton: @@ -25,7 +26,7 @@ rule backgrounds_get_beam_gas_proton: output: "input/backgrounds/beam_gas_proton.hepmc", run: - os.rename(input[0], output[0]) + shutil.move(input[0], output[0]) rule backgrounds_get_DIS: @@ -34,7 +35,7 @@ rule backgrounds_get_DIS: output: dis_sim="input/backgrounds/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root", run: - os.rename(input[0], output[0]) + shutil.move(input[0], output[0]) rule backgrounds_sim: From 19d447205bd08d8f8356cabae3a3a7f25199b566 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Sat, 20 Jan 2024 15:29:13 -0500 Subject: [PATCH 16/25] don't try to create the directory the second time --- benchmarks/backgrounds/config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index fb19fee3..10e1ac0b 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -12,7 +12,6 @@ bench:backgrounds_emcal_backwards: needs: - ["sim:backgrounds"] script: - - mkdir $LOCAL_DATA_PATH/input - ln -s $LOCAL_DATA_PATH/input input - export PYTHONUSERBASE=$LOCAL_DATA_PATH/deps - pip install -r benchmarks/backgrounds/requirements.txt From 74ae54704f5815fdbd754d032fc48bba80b8f5fd Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 29 Jan 2024 14:35:03 -0500 Subject: [PATCH 17/25] fix: use eics3 instead of deprecated dtn01 --- benchmarks/backgrounds/Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index 7d41387e..14b36d1b 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -5,7 +5,7 @@ from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider S3 = S3RemoteProvider( - endpoint_url="https://dtn01.sdcc.bnl.gov:9000", + endpoint_url="https://eics3.sdcc.bnl.gov:9000", access_key_id=os.environ["S3_ACCESS_KEY"], secret_access_key=os.environ["S3_SECRET_KEY"], ) From 7d3d526a4741b8c68ea428f5ae4a8ec08a6bbb63 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Tue, 30 Jan 2024 18:06:25 -0500 Subject: [PATCH 18/25] debug: print timing --- benchmarks/backgrounds/org2py.awk | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/backgrounds/org2py.awk b/benchmarks/backgrounds/org2py.awk index 0d23cf25..b26864e8 100644 --- a/benchmarks/backgrounds/org2py.awk +++ b/benchmarks/backgrounds/org2py.awk @@ -7,6 +7,7 @@ BEGIN { in_src = 1 match($0, /^ */) spaces = RLENGTH + print "import time; import sys; sys.stderr.write(f\"{time.asctime()}\\n\")" next } From d86f5927f7b22971051c656da22c688b332bd07e Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 5 Feb 2024 16:36:32 -0500 Subject: [PATCH 19/25] fix legend position to not waste too much cpu --- benchmarks/backgrounds/ecal_backwards.org | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index 2b95e6c3..559f8c69 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -39,6 +39,7 @@ def setup_presentation_style(): 'figure.titlesize': 9, 'figure.figsize': (4, 3), 'legend.fontsize': 7, + 'legend.loc': 'upper right', 'xtick.labelsize': 8, 'ytick.labelsize': 8, 'pgf.rcfonts': False, From b86764274b399fbcbfa43670e269d3b05e8123b2 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 5 Feb 2024 18:13:06 -0500 Subject: [PATCH 20/25] render to png --- benchmarks/backgrounds/ecal_backwards.org | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index 559f8c69..c814a2fb 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -151,7 +151,7 @@ for label, ds in datasets.items(): plt.minorticks_on() plt.xlabel("vertex[0].t, mm") plt.legend() -plt.savefig(output_dir / "vertex_time_distribution.pdf", bbox_inches="tight") +plt.savefig(output_dir / "vertex_time_distribution.png", bbox_inches="tight") plt.show() for label, ds in datasets.items(): @@ -162,7 +162,7 @@ for label, ds in datasets.items(): plt.minorticks_on() plt.xlabel("vertex[0].z, mm") plt.legend() -plt.savefig(output_dir / "vertex_z_distribution.pdf", bbox_inches="tight") +plt.savefig(output_dir / "vertex_z_distribution.png", bbox_inches="tight") plt.show() for label, ds in datasets.items(): @@ -176,7 +176,7 @@ plt.minorticks_on() plt.xlabel("vertex[0].z, mm") plt.ylabel("vertex[0].x, mm") plt.legend() -plt.savefig(output_dir / "vertex_xz_distribution.pdf", bbox_inches="tight") +plt.savefig(output_dir / "vertex_xz_distribution.png", bbox_inches="tight") plt.show() for ix, (label, ds) in enumerate(datasets.items()): @@ -190,7 +190,7 @@ for ix, (label, ds) in enumerate(datasets.items()): plt.xlabel("vertex[0].z, mm") plt.ylabel("vertex[0].y, mm") plt.title(label) - plt.savefig(output_dir / f"vertex_yz_distribution_{ix}.pdf", bbox_inches="tight") + plt.savefig(output_dir / f"vertex_yz_distribution_{ix}.png", bbox_inches="tight") plt.show() #+end_src @@ -229,7 +229,7 @@ for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: plt.title(f"{label}, event_id={event_id}\n{collection_name}") plt.xlabel("hit x, mm", loc="right") plt.ylabel("hit y, mm", loc="top") - plt.savefig(output_dir / f"{collection_name}_event_display_{dataset_ix}.pdf", bbox_inches="tight") + plt.savefig(output_dir / f"{collection_name}_event_display_{dataset_ix}.png", bbox_inches="tight") plt.show() #+end_src @@ -283,7 +283,7 @@ for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: plt.xlabel("r, mm", loc="right") dr = (r_axis.edges[1] - r_axis.edges[0]) plt.ylabel(f"Number of towers per {dr} mm slice in $r$", loc="top") - plt.savefig(output_dir / f"{collection_name}_num_towers.pdf", bbox_inches="tight") + plt.savefig(output_dir / f"{collection_name}_num_towers.png", bbox_inches="tight") plt.show() #+end_src @@ -322,7 +322,7 @@ for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: plt.xlabel("r, mm", loc="right") plt.ylabel("rate per tower, Hz", loc="top") plt.yscale("log") - plt.savefig(output_dir / f"{collection_name}_hit_rate_vs_r_edep_min_{edep_min:.3f}.pdf", bbox_inches="tight") + plt.savefig(output_dir / f"{collection_name}_hit_rate_vs_r_edep_min_{edep_min:.3f}.png", bbox_inches="tight") plt.show() #+end_src @@ -352,7 +352,7 @@ for collection_name in ["EcalEndcapNHits", "EcalEndcapPHits"]: plt.xlabel("$z$ of the first interaction vertex, mm", loc="right") plt.ylabel("rate, Hz", loc="top") plt.yscale("log") - plt.savefig(output_dir / f"{collection_name}_hit_rate_vs_z_totedep_min_{totedep_min:.1f}.pdf", bbox_inches="tight") + plt.savefig(output_dir / f"{collection_name}_hit_rate_vs_z_totedep_min_{totedep_min:.1f}.png", bbox_inches="tight") plt.show() #+end_src @@ -428,7 +428,7 @@ for edep_min in [0]: # GeV plt.ylabel("rate, Hz", loc="top") plt.yscale("log") plt.xticks(rotation=90, ha='right') - fig_cur.savefig(f"rates_edep_min_{edep_min}_{dataset_ix}.pdf", bbox_inches="tight") + fig_cur.savefig(f"rates_edep_min_{edep_min}_{dataset_ix}.png", bbox_inches="tight") plt.show() plt.close(fig_cur) @@ -438,6 +438,6 @@ for edep_min in [0]: # GeV plt.ylabel("rate per tower, Hz", loc="top") plt.yscale("log") plt.xticks(rotation=90, ha='right') - fig_cmb.savefig(f"rates_edep_min_{edep_min}.pdf", bbox_inches="tight") + fig_cmb.savefig(f"rates_edep_min_{edep_min}.png", bbox_inches="tight") plt.show() #+end_src From 9288150449a6091d7e63f74052275ff200e6db51 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Tue, 6 Feb 2024 17:11:38 -0500 Subject: [PATCH 21/25] ecal_backwards.org: forward compat for new geometry --- benchmarks/backgrounds/ecal_backwards.org | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/benchmarks/backgrounds/ecal_backwards.org b/benchmarks/backgrounds/ecal_backwards.org index c814a2fb..a420358e 100644 --- a/benchmarks/backgrounds/ecal_backwards.org +++ b/benchmarks/backgrounds/ecal_backwards.org @@ -399,7 +399,11 @@ for edep_min in [0]: # GeV # cellID.map_partitions(unique) #).compute()) - num_towers = num_towers_cache[branch_name.replace("Hits.energy", "")] + key = branch_name.replace("Hits.energy", "") + if key not in num_towers_cache: + print(f"The \"{key}\" not in num_towers_cache. Skipping.") + continue + num_towers = num_towers_cache[key] labels.append(branch_name.replace("Hits.energy", "")) values.append(ak.count(edep[edep > edep_min])) From f8005503f5510cff8a1ddae77cb9d42485b51292 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 12 Feb 2024 10:53:43 -0500 Subject: [PATCH 22/25] benchmark/backgrounds: switch from minQ2=10 to 1 (requires running simulation) --- benchmarks/backgrounds/Snakefile | 15 +++++++++------ benchmarks/backgrounds/config.yml | 2 +- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index 14b36d1b..7f34ae66 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -31,20 +31,23 @@ rule backgrounds_get_beam_gas_proton: rule backgrounds_get_DIS: input: - S3.remote("eictest/EPIC/FULL/23.11.0/epic_craterlake/DIS/NC/10x100/minQ2=10/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root"), + S3.remote("eictest/EPIC/EVGEN/DIS/NC/{BEAM}/minQ2={MINQ2}/pythia8NCDIS_{BEAM}_minQ2={MINQ2}_{SUFFIX}.hepmc"), + wildcard_constraints: + BEAM="\d+x\d+", + MINQ2="\d+", output: - dis_sim="input/backgrounds/pythia8NCDIS_10x100_minQ2=10_beamEffects_xAngle=-0.025_hiDiv_1.0000.edm4hep.root", + "input/backgrounds/pythia8NCDIS_{BEAM}_minQ2={MINQ2}_{SUFFIX}.hepmc", run: shutil.move(input[0], output[0]) rule backgrounds_sim: input: - "input/backgrounds/beam_gas_{BEAM}.hepmc", + "input/backgrounds/{NAME}.hepmc", output: - "sim_output/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root", + "sim_output/{DETECTOR_CONFIG}/{NAME}.edm4hep.root", log: - "sim_output/{DETECTOR_CONFIG}/beam_gas_{BEAM}.edm4hep.root.log", + "sim_output/{DETECTOR_CONFIG}/{NAME}.edm4hep.root.log", params: N_EVENTS=100 shell: @@ -80,7 +83,7 @@ rule backgrounds_ecal_backwards: script="ecal_backwards.py", electron_beam_gas_gen="input/backgrounds/beam_gas_electron.hepmc", electron_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_electron.edm4hep.root", - physics_signal_sim=rules.backgrounds_get_DIS.output[0], + physics_signal_sim="sim_output/" + DETECTOR_CONFIG + "/pythia8NCDIS_10x100_minQ2=1_beamEffects_xAngle=-0.025_hiDiv_1.edm4hep.root", proton_beam_gas_gen="input/backgrounds/beam_gas_proton.hepmc", proton_beam_gas_sim="sim_output/" + DETECTOR_CONFIG + "/beam_gas_proton.edm4hep.root", output: diff --git a/benchmarks/backgrounds/config.yml b/benchmarks/backgrounds/config.yml index 10e1ac0b..84bbc7e0 100644 --- a/benchmarks/backgrounds/config.yml +++ b/benchmarks/backgrounds/config.yml @@ -4,7 +4,7 @@ sim:backgrounds: script: - mkdir $LOCAL_DATA_PATH/input - ln -s $LOCAL_DATA_PATH/input input - - snakemake --cores 2 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root + - snakemake --cores 2 sim_output/$DETECTOR_CONFIG/beam_gas_{electron,proton}.edm4hep.root sim_output/$DETECTOR_CONFIG/pythia8NCDIS_10x100_minQ2=1_beamEffects_xAngle=-0.025_hiDiv_vtxfix_1.edm4hep.root bench:backgrounds_emcal_backwards: extends: .det_benchmark From 49ed8bbe250331bf17323370a1ff83e11846887c Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 12 Feb 2024 12:22:04 -0500 Subject: [PATCH 23/25] Revert "debug: print timing" This reverts commit 7d3d526a4741b8c68ea428f5ae4a8ec08a6bbb63. --- benchmarks/backgrounds/org2py.awk | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmarks/backgrounds/org2py.awk b/benchmarks/backgrounds/org2py.awk index b26864e8..0d23cf25 100644 --- a/benchmarks/backgrounds/org2py.awk +++ b/benchmarks/backgrounds/org2py.awk @@ -7,7 +7,6 @@ BEGIN { in_src = 1 match($0, /^ */) spaces = RLENGTH - print "import time; import sys; sys.stderr.write(f\"{time.asctime()}\\n\")" next } From 93d8a479c30686cad1e0ff82d4127a8eac7cc9c3 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 12 Feb 2024 12:34:10 -0500 Subject: [PATCH 24/25] Snakefile: use XRootD by default --- Snakefile | 19 +++++++++++++++++++ benchmarks/backgrounds/Snakefile | 15 +++------------ config.yaml | 1 + 3 files changed, 23 insertions(+), 12 deletions(-) create mode 100644 config.yaml diff --git a/Snakefile b/Snakefile index 8cdca119..6e4bc637 100644 --- a/Snakefile +++ b/Snakefile @@ -1,3 +1,22 @@ +configfile: "config.yaml" + +if config["remote"] == "S3": + from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider + provider = S3RemoteProvider( + endpoint_url="https://eics3.sdcc.bnl.gov:9000", + access_key_id=os.environ["S3_ACCESS_KEY"], + secret_access_key=os.environ["S3_SECRET_KEY"], + ) + remote_path = lambda path: f"eictest/{path}" +elif config["remote"] == "XRootD": + from snakemake.remote.XRootD import RemoteProvider as XRootDRemoteProvider + provider = XRootDRemoteProvider( + stay_on_remote=False, + ) + remote_path = lambda path: f"root://dtn-eic.jlab.org//work/eic2/{path}" +else: + raise ValueError(f"Unexpected config[\"remote\"] = {config['remote']}") + include: "benchmarks/backgrounds/Snakefile" include: "benchmarks/barrel_ecal/Snakefile" diff --git a/benchmarks/backgrounds/Snakefile b/benchmarks/backgrounds/Snakefile index 7f34ae66..3e3b1c66 100644 --- a/benchmarks/backgrounds/Snakefile +++ b/benchmarks/backgrounds/Snakefile @@ -1,19 +1,10 @@ import os import shutil -from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider - - -S3 = S3RemoteProvider( - endpoint_url="https://eics3.sdcc.bnl.gov:9000", - access_key_id=os.environ["S3_ACCESS_KEY"], - secret_access_key=os.environ["S3_SECRET_KEY"], -) - rule backgrounds_get_beam_gas_electron: input: - S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc"), + provider.remote(remote_path("EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/beam_gas_ep_10GeV_foam_emin10keV_10Mevt_vtx.hepmc")), output: "input/backgrounds/beam_gas_electron.hepmc", run: @@ -22,7 +13,7 @@ rule backgrounds_get_beam_gas_electron: rule backgrounds_get_beam_gas_proton: input: - S3.remote("eictest/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc"), + provider.remote(remote_path("EPIC/EVGEN/BACKGROUNDS/BEAMGAS/proton/ProtonBeamGasEvents/100GeV/100GeV_1.hepmc")), output: "input/backgrounds/beam_gas_proton.hepmc", run: @@ -31,7 +22,7 @@ rule backgrounds_get_beam_gas_proton: rule backgrounds_get_DIS: input: - S3.remote("eictest/EPIC/EVGEN/DIS/NC/{BEAM}/minQ2={MINQ2}/pythia8NCDIS_{BEAM}_minQ2={MINQ2}_{SUFFIX}.hepmc"), + provider.remote(remote_path("EPIC/EVGEN/DIS/NC/{BEAM}/minQ2={MINQ2}/pythia8NCDIS_{BEAM}_minQ2={MINQ2}_{SUFFIX}.hepmc")), wildcard_constraints: BEAM="\d+x\d+", MINQ2="\d+", diff --git a/config.yaml b/config.yaml new file mode 100644 index 00000000..14f6faad --- /dev/null +++ b/config.yaml @@ -0,0 +1 @@ +remote: XRootD From 323bf916e6a17e3939fd71c684fffda82c5ef9c8 Mon Sep 17 00:00:00 2001 From: Dmitry Kalinkin Date: Mon, 12 Feb 2024 12:44:40 -0500 Subject: [PATCH 25/25] config.yaml: switch back to S3 missing files [dtn-eic.jlab.org:1094] / > ls /work/eic2/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron /work/eic2/EPIC/EVGEN/BACKGROUNDS/BEAMGAS/electron/23.05.2.edm4hep.root --- config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yaml b/config.yaml index 14f6faad..75d4ff68 100644 --- a/config.yaml +++ b/config.yaml @@ -1 +1 @@ -remote: XRootD +remote: S3