Merge pull request #152 from sophiemiddleton/merge_heac

Mu2eII_SM21: Merge head for event generators

CampaignConfig/mdc2020_beam.cfg

@@ -1,99 +1,16 @@
 [global]
-group = mu2e
-experiment = mu2e
-wrapper = file:///${FIFE_UTILS_DIR}/libexec/fife_wrap
-submitter = mu2e
-outdir_fcl = /pnfs/mu2e/persistent/datasets/phy-etc/cnf/mu2e/
-outdir_sim = /pnfs/mu2e/scratch/datasets/phy-sim/sim/mu2e/
-outdir_dts = /pnfs/mu2e/scratch/datasets/phy-sim/dts/mu2e/
-outdir_sim_tape = /pnfs/mu2e/tape/phy-sim/sim/mu2e/
-outdir_dts_tape = /pnfs/mu2e/tape/phy-sim/dts/mu2e/
-logdir_bck = /pnfs/mu2e/persistent/datasets/phy-etc/bck/mu2e/
-desc = MDC2020k
+includes = /mu2e/app/users/srsoleti/workDir/Production/CampaignConfig/mdc2020_prolog.cfg
+desc = MDC2020n_10pc
 run_number = 1201
-njobs = 500
-release = MDC2020k
-stage_name = override_me
-artRoot_dataset = override_me
-histRoot_dataset = override_me
-
-[env_pass]
-IFDH_DEBUG = 1
-SAM_EXPERIMENT = %(experiment)s
-OUTPUT_DATASET = bck.%(submitter)s.%(stage_name)s.%(desc)s.tbz
-ARTROOT_DATASET = %(artRoot_dataset)s
-HISTROOT_DATASET = %(histRoot_dataset)s
-
-[submit]
-debug = True
-G = %(group)s
-e = SAM_EXPERIMENT
-e_1 = IFDH_DEBUG
-e_2 = POMS4_CAMPAIGN_NAME
-e_3 = POMS4_CAMPAIGN_STAGE_NAME
-resource-provides = usage_model=DEDICATED,OPPORTUNISTIC
-generate-email-summary = True
-expected-lifetime = 23h
-memory = 2500MB
-email-to = srsoleti@fnal.gov
-# tar_file_name = dropbox:///mu2e/data/submitters/srsoleti/museTarball/tmp.sFo0Z6v306/Code.tar.bz2
-# use-cvmfs-dropbox = True
-l = '+SingularityImage=\"/cvmfs/singularity.opensciencegrid.org/fermilab/fnal-wn-sl7:latest\"'
-append_condor_requirements='(TARGET.HAS_SINGULARITY=?=true)'
-
-[job_setup]
-debug = True
-find_setups = False
-source_1 = /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
-source_2 = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/%(release)s/setup.sh
-setup_1 = dhtools
-setup_2 = ifdh_art v2_12_02 -q +e20:+prof
-setup_3 = ifdhc v2_5_14
-setup_4 = mu2etools
-setup_5 = mu2efiletools
-setup_6 = sqlite v3_34_01_00
-setup_7 = gcc v9_3_0
-ifdh_art = True
-postscript = [ -d 000 ] && sed -i "s/MU2EGRIDDSOWNER/%(submitter)s/g" [0-9]*/cnf.*.fcl
-postscript_2 = [ -d 000 ] && sed -i "s/MU2EGRIDDSCONF/%(desc)s/g" [0-9]*/cnf.*.fcl
-postscript_3 = [ -d 000 ] && mv [0-9]*/cnf.*.fcl* .
-postscript_4 = [ -f template.fcl ] && rm template.fcl
-
-[sam_consumer]
-limit = 1
-schema = xroot
-appvers = %(release)s
-appfamily = art
-appname = SimJob
-
-[job_output]
-addoutput = *.fcl
-add_to_dataset = cnf.%(submitter)s.%(stage_name)s.%(desc)s.fcl
-declare_metadata = True
-metadata_extractor = json
-add_location = True
-add_metadata = file_format=fcl
-hash = 2
-parallel = 5
-hash_alg = sha256
-
-[job_output_1]
-addoutput = *.tbz
-declare_metadata = False
-metadata_extractor = printJsonSave.sh
-add_location = True
-add_metadata = file_format=tbz
-add_to_dataset = bck.%(submitter)s.%(stage_name)s.%(desc)s.tbz
-hash = 2
-hash_alg = sha256
+njobs = 5000
+release = MDC2020
+release_v_o = n
 
 [job_output_2]
-addoutput = *.Beam.*.art
 declare_metadata = True
 metadata_extractor = printJsonSave.sh
 add_location = True
 add_metadata = file_format=art
-add_to_dataset = sim.%(submitter)s.Beam.%(desc)s.art
 hash = 2
 hash_alg = sha256
 
@@ -198,8 +115,14 @@ submit.n_files_per_job = 1
 sam_consumer.limit = 1
 
 job_output_1.dest = %(logdir_bck)s/%(stage_name)s/%(desc)s/tbz/
+
+job_output_2.add_output = *.Beam.*.art
+job_output_2.add_to_dataset = sim.%(submitter)s.Beam.%(desc)s.art
 job_output_2.dest = %(outdir_sim)s/Beam/%(desc)s/art
+
+job_output_3.addoutput = *.Neutrals.*.art
 job_output_3.dest = %(outdir_sim)s/Neutrals/%(desc)s/art
+job_output_3.add_to_dataset = sim.%(submitter)s.Neutrals.%(desc)s.art
 
 job_setup.postscript = samweb file-lineage parents `basename ${fname}` > parents.txt
 
@@ -228,11 +151,11 @@ global.stage_name = BeamSplitter
 job_output_1.dest = %(logdir_bck)s/%(stage_name)s/%(desc)s/tbz/
 job_output_2.addoutput = *.EleBeamCat.*.art
 job_output_2.add_to_dataset = sim.%(submitter)s.EleBeamCat.%(desc)s.art
-job_output_2.dest = %(outdir_sim)s/EleBeamCat/%(desc)s/art
+job_output_2.dest = %(outdir_sim_tape)s/EleBeamCat/%(desc)s/art
 
 job_output_3.addoutput = *.MuBeamCat.*.art
 job_output_3.add_to_dataset = sim.%(submitter)s.MuBeamCat.%(desc)s.art
-job_output_3.dest = %(outdir_sim)s/MuBeamCat/%(desc)s/art
+job_output_3.dest = %(outdir_sim_tape)s/MuBeamCat/%(desc)s/art
 
 global.artRoot_dataset = sim.%(submitter)s.MuBeamCat.%(desc)s.art,sim.%(submitter)s.EleBeamCat.%(desc)s.art
 
@@ -308,7 +231,7 @@ executable_2.name = generate_fcl
 executable_2.arg_4 = --aux=1:physics.filters.beamResampler.fileNames:inputs.txt
 executable_2.arg_5 = --events-per-job=400000
 executable_2.arg_6 = --run-number=%(run_number)s
-executable_2.arg_7 = --njobs=100
+executable_2.arg_7 = --njobs=1000
 executable_2.arg_8 = --embed
 executable_2.arg_9 = template.fcl
 
@@ -391,7 +314,7 @@ executable_2.name = generate_fcl
 executable_2.arg_4 = --aux=1:physics.filters.neutralsResampler.fileNames:inputs.txt
 executable_2.arg_5 = --events-per-job=400000
 executable_2.arg_6 = --run-number=%(run_number)s
-executable_2.arg_7 = --njobs=1000
+executable_2.arg_7 = --njobs=10000
 executable_2.arg_8 = --embed
 executable_2.arg_9 = template.fcl
 
@@ -427,10 +350,10 @@ job_setup.prescript_3 = echo 'outputs.out.fileName: "dts.DSOWNER.EleBeamFlashCat
 executable.name = true
 executable_2.name = catAndShuf.sh
 executable_2.arg_1 = inputs.txt
-executable_2.arg_2 = 10
+executable_2.arg_2 = 100
 executable_2.arg_3 = 500
 executable_2.arg_4 = EleBeamFlashCat
-executable_2.arg_5 = MDC2020k
+executable_2.arg_5 = %(desc)s
 executable_2.arg_6 = mu2e
 executable_2.arg_7 = template.fcl
 
@@ -643,12 +566,13 @@ job_setup.prescript_3 = echo 'outputs.out.fileName: "dts.DSOWNER.NeutralsFlashCa
 executable.name = true
 executable_2.name = catAndShuf.sh
 executable_2.arg_1 = inputs.txt
-executable_2.arg_2 = 10
+executable_2.arg_2 = 100
 executable_2.arg_3 = 1000
 executable_2.arg_4 = NeutralsFlashCat
-executable_2.arg_5 = MDC2020k
+executable_2.arg_5 = %(desc)s
 executable_2.arg_6 = mu2e
 executable_2.arg_7 = template.fcl
+
 job_setup.postscript = sed -i "s/MU2EGRIDDSOWNER/%(submitter)s/g" cnf.*.fcl
 job_setup.postscript_2 = sed -i "s/MU2EGRIDDSCONF/%(desc)s/g" cnf.*.fcl
 
@@ -760,7 +684,7 @@ job_setup.prescript_7 = echo 'physics.filters.TargetStopResampler.mu2e.MaxEvents
 executable.name = true
 executable_2.name = generate_fcl
 executable_2.arg_4 = --events-per-job=400000
-executable_2.arg_5 = --njobs=50
+executable_2.arg_5 = --njobs=500
 executable_2.arg_6 = --run-number=%(run_number)s
 executable_2.arg_7 = --auxinput=1:physics.filters.TargetStopResampler.fileNames:inputs.txt
 executable_2.arg_8 = --embed
@@ -821,12 +745,3 @@ job_setup.postscript = samweb file-lineage parents `basename ${fname}` > parents
 job_setup.getconfig = True
 job_setup.multifile = False
 job_setup.setup_local = True
-
-[executable]
-name = loggedMu2e.sh
-
-[executable_2]
-name = true
-arg_1 = --description=%(stage_name)s
-arg_2 = --dsconf=%(desc)s
-arg_3 = --dsowner=%(submitter)s

CampaignConfig/mdc2020_beam.ini

@@ -15,19 +15,19 @@ param_overrides="[]"
 test_param_overrides="[]"
 merge_overrides=False
 login_setup=mu2epro_login
-job_type=mu2e_mdc2020_production_jobtype
+job_type=mu2e_cvmfs_beam_jobtype
 stage_type=regular
 output_ancestor_depth=1
 
 [campaign_stage pot_fcl]
 param_overrides = [["--stage ", "pot_fcl"]]
 test_param_overrides = [["--stage ", "pot_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [campaign_stage pot]
 param_overrides = [["--stage ", "pot"]]
 test_param_overrides = [["--stage ", "pot"]]
-job_type = mu2e_mdc2020_production_jobtype
+job_type = mu2e_cvmfs_beam_jobtype
 
 [dependencies pot]
 campaign_stage_1 = pot_fcl
@@ -36,7 +36,7 @@ file_pattern_1 = %.fcl
 [campaign_stage beam_splitter_fcl]
 param_overrides = [["--stage ", "beam_splitter_fcl"]]
 test_param_overrides = [["--stage ", "beam_splitter_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies beam_splitter_fcl]
 campaign_stage_1 = pot
@@ -53,7 +53,7 @@ file_pattern_1 = %.fcl
 [campaign_stage neutrals_cat_fcl]
 param_overrides = [["--stage ", "neutrals_cat_fcl"]]
 test_param_overrides = [["--stage ", "neutrals_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies neutrals_cat_fcl]
 campaign_stage_1 = pot
@@ -70,7 +70,7 @@ file_pattern_1 = %.fcl
 [campaign_stage mubeam_resampler_fcl]
 param_overrides = [["--stage ", "mubeam_resampler_fcl"]]
 test_param_overrides = [["--stage ", "mubeam_resampler_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [campaign_stage mubeam_resampler]
 param_overrides = [["--stage ", "mubeam_resampler"]]
@@ -90,7 +90,7 @@ file_pattern_1 = %.fcl
 [campaign_stage elebeam_resampler_fcl]
 param_overrides = [["--stage ", "elebeam_resampler_fcl"]]
 test_param_overrides = [["--stage ", "elebeam_resampler_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [campaign_stage elebeam_resampler]
 param_overrides = [["--stage ", "elebeam_resampler"]]
@@ -107,7 +107,7 @@ file_pattern_1 = %.fcl
 [campaign_stage neutrals_resampler_fcl]
 param_overrides = [["--stage ", "neutrals_resampler_fcl"]]
 test_param_overrides = [["--stage ", "neutrals_resampler_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [campaign_stage neutrals_resampler]
 param_overrides = [["--stage ", "neutrals_resampler"]]
@@ -124,7 +124,7 @@ file_pattern_1 = %.fcl
 [campaign_stage elebeamflash_cat_fcl]
 param_overrides = [["--stage ", "elebeamflash_cat_fcl"]]
 test_param_overrides = [["--stage ", "elebeamflash_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies elebeamflash_cat_fcl]
 campaign_stage_1 = elebeam_resampler
@@ -141,7 +141,7 @@ file_pattern_1 = %.fcl
 [campaign_stage earlyelebeamflash_cat_fcl]
 param_overrides = [["--stage ", "earlyelebeamflash_cat_fcl"]]
 test_param_overrides = [["--stage ", "earlyelebeamflash_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies earlyelebeamflash_cat_fcl]
 campaign_stage_1 = elebeam_resampler
@@ -158,7 +158,7 @@ file_pattern_1 = %.fcl
 [campaign_stage mubeamflash_cat_fcl]
 param_overrides = [["--stage ", "mubeamflash_cat_fcl"]]
 test_param_overrides = [["--stage ", "mubeamflash_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies mubeamflash_cat_fcl]
 campaign_stage_1 = mubeam_resampler
@@ -175,7 +175,7 @@ file_pattern_1 = %.fcl
 [campaign_stage earlymubeamflash_cat_fcl]
 param_overrides = [["--stage ", "earlymubeamflash_cat_fcl"]]
 test_param_overrides = [["--stage ", "earlymubeamflash_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies earlymubeamflash_cat_fcl]
 campaign_stage_1 = mubeam_resampler
@@ -192,7 +192,7 @@ file_pattern_1 = %.fcl
 [campaign_stage ipastops_cat_fcl]
 param_overrides = [["--stage ", "ipastops_cat_fcl"]]
 test_param_overrides = [["--stage ", "ipastops_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies ipastops_cat_fcl]
 campaign_stage_1 = mubeam_resampler
@@ -209,7 +209,7 @@ file_pattern_1 = %.fcl
 [campaign_stage targetstops_cat_fcl]
 param_overrides = [["--stage ", "targetstops_cat_fcl"]]
 test_param_overrides = [["--stage ", "targetstops_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies targetstops_cat_fcl]
 campaign_stage_1 = mubeam_resampler
@@ -226,7 +226,7 @@ file_pattern_1 = %.fcl
 [campaign_stage neutralsflash_cat_fcl]
 param_overrides = [["--stage ", "neutralsflash_cat_fcl"]]
 test_param_overrides = [["--stage ", "neutralsflash_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies neutralsflash_cat_fcl]
 campaign_stage_1 = neutrals_resampler
@@ -243,7 +243,7 @@ file_pattern_1 = %.fcl
 [campaign_stage earlyneutralsflash_cat_fcl]
 param_overrides = [["--stage ", "earlyneutralsflash_cat_fcl"]]
 test_param_overrides = [["--stage ", "earlyneutralsflash_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies earlyneutralsflash_cat_fcl]
 campaign_stage_1 = neutrals_resampler
@@ -260,7 +260,7 @@ file_pattern_1 = %.fcl
 [campaign_stage muonstops_fcl]
 param_overrides = [["--stage ", "muonstops_fcl"]]
 test_param_overrides = [["--stage ", "muonstops_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies muonstops_fcl]
 campaign_stage_1 = targetstops_cat
@@ -277,7 +277,7 @@ file_pattern_1 = %.fcl
 [campaign_stage mustoppileup_fcl]
 param_overrides = [["--stage ", "mustoppileup_fcl"]]
 test_param_overrides = [["--stage ", "mustoppileup_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies mustoppileup_fcl]
 campaign_stage_1 = muonstops
@@ -294,7 +294,7 @@ file_pattern_1 = %.fcl
 [campaign_stage mustoppileup_cat_fcl]
 param_overrides = [["--stage ", "mustoppileup_cat_fcl"]]
 test_param_overrides = [["--stage ", "mustoppileup_cat_fcl"]]
-job_type = generate_fcl_beam_production_jobtype
+job_type = generate_fcl_cvmfs_beam_jobtype
 
 [dependencies mustoppileup_cat_fcl]
 campaign_stage_1 = mustoppileup
@@ -308,15 +308,15 @@ test_param_overrides = [["--stage ", "mustoppileup_cat"]]
 campaign_stage_1 = mustoppileup_cat_fcl
 file_pattern_1 = %.fcl
 
-[job_type mu2e_mdc2020_production_jobtype]
+[job_type mu2e_cvmfs_beam_jobtype]
 launch_script = fife_launch
-parameters = [["-c ", "/mu2e/app/users/srsoleti/workDir/Production/CampaignConfig/mdc2020_beam.cfg"]]
+parameters = [["-c ", "/cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/MDC2020m/link/Production/CampaignConfig/mdc2020_beam.cfg"]]
 output_file_patterns = %.art
 recoveries = [["proj_status",[["-Osubmit.dataset=","%(dataset)s"]]]]
 
-[job_type generate_fcl_beam_production_jobtype]
+[job_type generate_fcl_cvmfs_beam_jobtype]
 launch_script = fife_launch
-parameters = [["-c ", "/mu2e/app/users/srsoleti/workDir/Production/CampaignConfig/mdc2020_beam.cfg"]]
+parameters = [["-c ", "/cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/MDC2020m/link/Production/CampaignConfig/mdc2020_beam.cfg"]]
 output_file_patterns = %.fcl
 
 [login_setup mu2epro_login]