Merge pull request #112 from michaelmackenzie/Mu2eII_MergeMain

Mu2eII_SM21: Merge main into Mu2e-II branch

.muse

@@ -1,2 +1,2 @@
-PATH MDC2020
-PATH MDC2020/POMS
+PATH Scripts
+PATH Scripts/POMS

CampaignConfig/mdc2020_beam.cfg

@@ -3,12 +3,12 @@ group = mu2e
 experiment = mu2e
 wrapper = file:///${FIFE_UTILS_DIR}/libexec/fife_wrap
 submitter = mu2e
-outdir_fcl = /pnfs/mu2e/scratch/datasets/phy-etc/cnf/mu2e/
+outdir_fcl = /pnfs/mu2e/persistent/datasets/phy-etc/cnf/mu2e/
 outdir_sim = /pnfs/mu2e/scratch/datasets/phy-sim/sim/mu2e/
 outdir_dts = /pnfs/mu2e/scratch/datasets/phy-sim/dts/mu2e/
 outdir_sim_tape = /pnfs/mu2e/tape/phy-sim/sim/mu2e/
 outdir_dts_tape = /pnfs/mu2e/tape/phy-sim/dts/mu2e/
-logdir_bck = /pnfs/mu2e/scratch/datasets/phy-etc/bck/mu2e/
+logdir_bck = /pnfs/mu2e/persistent/datasets/phy-etc/bck/mu2e/
 desc = MDC2020k
 run_number = 1201
 njobs = 500
@@ -31,7 +31,7 @@ e = SAM_EXPERIMENT
 e_1 = IFDH_DEBUG
 e_2 = POMS4_CAMPAIGN_NAME
 e_3 = POMS4_CAMPAIGN_STAGE_NAME
-resource-provides = usage_model=DEDICATED,OPPORTUNISTIC,OFFSITE
+resource-provides = usage_model=DEDICATED,OPPORTUNISTIC
 generate-email-summary = True
 expected-lifetime = 23h
 memory = 2500MB
@@ -183,7 +183,7 @@ executable_2.name = generate_fcl
 executable_2.arg_4 = --run-number=%(run_number)s
 executable_2.arg_5 = --events-per-job=4000
 executable_2.arg_6 = --njobs=%(njobs)s
-executable_2.arg_7 = --embed
+executable_2.arg_7 = --include
 executable_2.arg_8 = Production/JobConfig/beam/POT.fcl
 
 job_setup.ifdh_art = False
@@ -217,7 +217,7 @@ executable.name = true
 executable_2.name = generate_fcl
 executable_2.arg_4 = --merge=500
 executable_2.arg_5 = --inputs=inputs.txt
-executable_2.arg_6 = --embed
+executable_2.arg_6 = --include
 executable_2.arg_7 = Production/JobConfig/beam/BeamSplitter.fcl
 
 job_setup.ifdh_art = False
@@ -255,7 +255,7 @@ job_setup.prescript_2 = dataset=sim.%(submitter)s.EleBeamCat.%(desc)s.art
 job_setup.prescript_3 = nfiles=`samCountFiles.sh $dataset`
 job_setup.prescript_4 = nevts=`samCountEvents.sh $dataset`
 job_setup.prescript_5 = let nskip=nevts/nfiles
-job_setup.prescript_6 = echo '#include "Production/JobConfig/beam/EleBeamResampler.fcl"' >> template.fcl
+job_setup.prescript_6 = echo '#include "Production/JobConfig/pileup/EleBeamResampler.fcl"' >> template.fcl
 job_setup.prescript_7 = echo 'physics.filters.beamResampler.mu2e.MaxEventsToSkip: '${nskip} >> template.fcl
 
 executable.name = true
@@ -300,7 +300,7 @@ job_setup.prescript_2 = dataset=sim.%(submitter)s.MuBeamCat.%(desc)s.art
 job_setup.prescript_3 = nfiles=`samCountFiles.sh $dataset`
 job_setup.prescript_4 = nevts=`samCountEvents.sh $dataset`
 job_setup.prescript_5 = let nskip=nevts/nfiles
-job_setup.prescript_6 = echo '#include "Production/JobConfig/beam/MuBeamResampler.fcl"' >> template.fcl
+job_setup.prescript_6 = echo '#include "Production/JobConfig/pileup/MuBeamResampler.fcl"' >> template.fcl
 job_setup.prescript_7 = echo 'physics.filters.beamResampler.mu2e.MaxEventsToSkip: '${nskip} >> template.fcl
 
 executable.name = true
@@ -383,7 +383,7 @@ job_setup.prescript_2 = dataset=sim.%(submitter)s.NeutralsCat.%(desc)s.art
 job_setup.prescript_3 = nfiles=`samCountFiles.sh $dataset`
 job_setup.prescript_4 = nevts=`samCountEvents.sh $dataset`
 job_setup.prescript_5 = let nskip=nevts/nfiles
-job_setup.prescript_6 = echo '#include "Production/JobConfig/beam/NeutralsResampler.fcl"' >> template.fcl
+job_setup.prescript_6 = echo '#include "Production/JobConfig/pileup/NeutralsResampler.fcl"' >> template.fcl
 job_setup.prescript_7 = echo 'physics.filters.neutralsResampler.mu2e.MaxEventsToSkip: '${nskip} >> template.fcl
 
 executable.name = true
@@ -718,7 +718,7 @@ executable_2.name = generate_fcl
 executable_2.arg_4 = --inputs=inputs.txt
 executable_2.arg_5 = --merge=500
 executable_2.arg_6 = --include
-executable_2.arg_7 = Production/JobConfig/beam/MuonStopSelector.fcl
+executable_2.arg_7 = Production/JobConfig/pileup/MuonStopSelector.fcl
 
 job_setup.ifdh_art = False
 

CampaignConfig/mdc2020_beam.ini

@@ -1,7 +1,7 @@
 [campaign]
 experiment = mu2e
 poms_role = production
-name = MDC2020f_beam
+name = MDC2020_beam_pileup
 campaign_stage_list = pot_fcl, pot, beam_cat_fcl, beam_cat, neutrals_cat_fcl, neutrals_cat, mubeam_resampler_fcl, mubeam_resampler, elebeam_resampler_fcl, elebeam_resampler, neutrals_resampler_fcl, neutrals_resampler, elebeamflash_cat_fcl, elebeamflash_cat, earlyelebeamflash_cat_fcl, earlyelebeamflash_cat, mubeamflash_cat_fcl, mubeamflash_cat, earlymubeamflash_cat_fcl, earlymubeamflash_cat, ipastops_cat_fcl, ipastops_cat, targetstops_cat_fcl, targetstops_cat, neutralsflash_cat_fcl, neutralsflash_cat, earlyneutralsflash_cat_fcl, earlyneutralsflash_cat, muonstops_fcl, muonstops, mustoppileup_fcl, mustoppileup, mustoppileup_cat_fcl, mustoppileup_cat
 
 [campaign_defaults]

CampaignConfig/mdc2020_corsika.cfg

@@ -3,12 +3,15 @@ group = mu2e
 experiment = mu2e
 wrapper = file:///${FIFE_UTILS_DIR}/libexec/fife_wrap
 submitter = mu2e
-outdir_fcl = /pnfs/mu2e/scratch/datasets/phy-etc/cnf/mu2e/
+outdir_fcl = /pnfs/mu2e/persistent/datasets/phy-etc/cnf/mu2e/
 outdir_sim = /pnfs/mu2e/scratch/datasets/phy-sim/sim/mu2e/
 outdir_dts = /pnfs/mu2e/scratch/datasets/phy-sim/dts/mu2e/
-logdir_bck = /pnfs/mu2e/scratch/datasets/phy-etc/bck/mu2e/
-desc = MDC2020j
-release = MDC2020j
+logdir_bck = /pnfs/mu2e/persistent/datasets/phy-etc/bck/mu2e/
+outdir_dig_tape = /pnfs/mu2e/tape/phy-sim/dig/mu2e/
+outdir_dts_tape = /pnfs/mu2e/tape/phy-sim/dts/mu2e/
+outdir_sim_tape = /pnfs/mu2e/tape/phy-sim/sim/mu2e/
+desc = MDC2020k
+release = MDC2020k
 output_dataset = override_me
 artRoot_dataset = override_me
 histRoot_dataset = override_me
@@ -28,7 +31,7 @@ e = SAM_EXPERIMENT
 e_1 = IFDH_DEBUG
 e_2 = POMS4_CAMPAIGN_NAME
 e_3 = POMS4_CAMPAIGN_STAGE_NAME
-resource-provides = usage_model=DEDICATED,OPPORTUNISTIC,OFFSITE
+resource-provides = usage_model=DEDICATED,OPPORTUNISTIC
 generate-email-summary = True
 expected-lifetime = 23h
 memory = 2000MB
@@ -42,7 +45,7 @@ append_condor_requirements='(TARGET.HAS_SINGULARITY=?=true)'
 debug = True
 find_setups = False
 source_1 = /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
-source_2 = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/setup.sh
+source_2 = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/%(release)s/setup.sh
 setup_1 = dhtools
 setup_2 = ifdh_art v2_12_02 -q +e20:+prof
 setup_3 = ifdhc v2_5_14
@@ -69,7 +72,6 @@ add_to_dataset = cnf.%(submitter)s.%(stage_name)s.%(desc)s.fcl
 declare_metadata = True
 metadata_extractor = json
 add_location = True
-filter_metadata = checksum
 add_metadata = file_format=fcl
 hash = 2
 parallel = 5
@@ -79,21 +81,18 @@ hash_alg = sha256
 addoutput = *.tbz
 add_to_dataset = bck.%(submitter)s.%(stage_name)s.%(desc)s.tbz
 declare_metadata = True
-metadata_extractor = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/MDC2020/POMS/printJsonSave.sh
+metadata_extractor = printJsonSave.sh
 add_location = True
 add_metadata = file_format=tbz
-filter_metadata = checksum
 hash = 2
 hash_alg = sha256
 
 [job_output_2]
 addoutput = *.art
-dest = %(outdir_sim)s
 declare_metadata = True
-metadata_extractor = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/MDC2020/POMS/printJsonSave.sh
+metadata_extractor = printJsonSave.sh
 add_location = True
 add_metadata = file_format=art
-filter_metadata = checksum
 hash = 2
 hash_alg = sha256
 
@@ -103,18 +102,15 @@ global.stage_name = CosmicDSStopsCORSIKA
 job_output.filter_metadata = parents,checksum
 job_output.dest = %(outdir_fcl)s/%(stage_name)s/%(desc)s/fcl/
 
-job_setup.prescript = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/MDC2020/POMS/fileNamesGenerator.sh 0 50 %(submitter)s > filenames.txt
-executable.name = generate_fcl
-executable.arg_1 = --description=CosmicDSStopsCORSIKA
-executable.arg_2 = --dsconf=%(desc)s
-executable.arg_3 = --dsowner=%(submitter)s
-executable.arg_4 = --inputs=filenames.txt
-executable.arg_5 = --merge-factor=1
-executable.arg_6 = --embed
-executable.arg_7 = JobConfig/cosmic/S1DSStopsCORSIKA.fcl
+job_setup.prescript = fileNamesGenerator.sh 0 50 %(submitter)s > filenames.txt
+executable.name = true
+executable_2.name = generate_fcl
+executable_2.arg_4 = --inputs=filenames.txt
+executable_2.arg_5 = --merge-factor=1
+executable_2.arg_6 = --include
+executable_2.arg_7 = JobConfig/cosmic/S1DSStopsCORSIKA.fcl
 job_setup.ifdh_art = False
 
-
 [stage_generation]
 global.stage_name = CosmicDSStopsCORSIKA
 job_output_2.add_to_dataset = sim.%(submitter)s.CosmicDSStopsCORSIKA.%(desc)s.art
@@ -130,16 +126,16 @@ job_setup.setup_9 = gcc v9_3_0
 submit.n_files_per_job = 1
 sam_consumer.limit = 1
 submit.dataset = cnf.%(submitter)s.CosmicDSStopsCORSIKA.%(desc)s.fcl
-executable.name = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/MDC2020/POMS/getFilename.sh
+executable.name = getFilename.sh
 job_setup.postscript_1 = samweb file-lineage parents `basename ${fname}` > parents.txt
 job_setup.postscript_2 = CORSIKA_EXE=`which corsika77400Linux_QGSJET_fluka`
 job_setup.postscript_3 = DATDIR=`dirname $CORSIKA_EXE`
 job_setup.postscript_4 = IFS='.' read -r -a FIELDS <<< "`cat filename.txt`"
-job_setup.postscript_5 = sed -e "s:_DATDIR_:$DATDIR/:" -e "s:_DIRECT_:`pwd`/:" -e "s:_SEED1_:$RANDOM:" -e "s:_SEED2_:$RANDOM:" -e "s:_NSHOW_:3000000:" -e "s:RUNNR   1:RUNNR   ${FIELDS[4]}:" /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/JobConfig/cosmic/CORSIKAInput.txt > corsika_conf.txt
+job_setup.postscript_5 = sed -e "s:_DATDIR_:$DATDIR/:" -e "s:_DIRECT_:`pwd`/:" -e "s:_SEED1_:$RANDOM:" -e "s:_SEED2_:$RANDOM:" -e "s:_NSHOW_:3000000:" -e "s:RUNNR   1:RUNNR   ${FIELDS[4]}:" /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/%(release)s/link/Production/JobConfig/cosmic/CORSIKAInput.txt > corsika_conf.txt
 job_setup.postscript_6 = cat corsika_conf.txt
 job_setup.postscript_7 = corsika77400Linux_QGSJET_fluka < corsika_conf.txt > corsika_log.txt
 job_setup.postscript_8 = mv DAT* `cat filename.txt`
-job_setup.postscript_9 = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/MDC2020/POMS/loggedMu2e.sh --sam-data-tier=Output:sim -c `cat torun.txt`
+job_setup.postscript_9 = loggedMu2e.sh -c `cat torun.txt`
 
 job_setup.multifile = False
 job_setup.setup_local = True
@@ -151,22 +147,19 @@ job_output.dest = %(outdir_fcl)s/%(stage_name)s/%(desc)s/fcl/
 job_setup.prescript = samweb list-file-locations --defname=sim.%(submitter)s.CosmicDSStopsCORSIKA.%(desc)s.art --schema=root | cut -f1 > inputs.txt
 job_setup.prescript_2 = echo '#include "JobConfig/common/artcat.fcl"' >> template.fcl
 job_setup.prescript_3 = echo 'outputs.out.fileName: "sim.DSOWNER.CosmicDSStopsCORSIKACat.DSCONF.SEQ.art"' >> template.fcl
-
-executable.name = generate_fcl
-executable.arg_1 = --description=%(stage_name)s
-executable.arg_2 = --dsconf=%(desc)s
-executable.arg_3 = --dsowner=%(submitter)s
-executable.arg_4 = --merge=50
-executable.arg_5 = --inputs=inputs.txt
-executable.arg_6 = --embed
-executable.arg_7 = template.fcl
+executable.name = true
+executable_2.name = generate_fcl
+executable_2.arg_4 = --merge=50
+executable_2.arg_5 = --inputs=inputs.txt
+executable_2.arg_6 = --embed
+executable_2.arg_7 = template.fcl
 
 job_setup.ifdh_art = False
 
 [stage_generation_cat]
 global.stage_name = CosmicDSStopsCORSIKACat
 job_output_2.add_to_dataset = sim.%(submitter)s.CosmicDSStopsCORSIKACat.%(desc)s.art
-job_output_2.dest = %(outdir_sim)s/CosmicDSStopsCORSIKACat/%(desc)s/art
+job_output_2.dest = %(outdir_sim_tape)s/CosmicDSStopsCORSIKACat/%(desc)s/art
 job_output_1.dest = %(logdir_bck)s/%(stage_name)s/%(desc)s/tbz/
 
 global.artRoot_dataset = sim.%(submitter)s.CosmicDSStopsCORSIKACat.%(desc)s.art
@@ -185,23 +178,21 @@ global.stage_name = CosmicCORSIKA
 job_output.dest = %(outdir_fcl)s/%(stage_name)s/%(desc)s/fcl/
 
 job_setup.prescript = samweb list-file-locations --defname=sim.%(submitter)s.CosmicDSStopsCORSIKACat.%(desc)s.art --schema=root | cut -f1 > inputs.txt
-executable.name = generate_fcl
-executable.arg_1 = --description=%(stage_name)s
-executable.arg_2 = --dsconf=%(desc)s
-executable.arg_3 = --dsowner=%(submitter)s
-executable.arg_4 = --aux=1:physics.filters.cosmicResample.fileNames:inputs.txt
-executable.arg_5 = --run-number=1205
-executable.arg_6 = --events-per-job=65000
-executable.arg_7 = --njobs=293
-executable.arg_8 = --embed
-executable.arg_9 = JobConfig/cosmic/S2Resampler.fcl
+executable.name = true
+executable_2.name = generate_fcl
+executable_2.arg_4 = --aux=1:physics.filters.cosmicResample.fileNames:inputs.txt
+executable_2.arg_5 = --run-number=1205
+executable_2.arg_6 = --events-per-job=65000
+executable_2.arg_7 = --njobs=293
+executable_2.arg_8 = --embed
+executable_2.arg_9 = JobConfig/cosmic/S2Resampler.fcl
 job_setup.postscript_5 = sed -i "s/\.Cosmic\./\.CosmicCORSIKA\./g" cnf.*.fcl
 job_setup.ifdh_art = False
 
 [stage_resampler]
 global.stage_name = CosmicCORSIKA
 job_output_2.add_to_dataset = dts.%(submitter)s.CosmicCORSIKA.%(desc)s.art
-job_output_2.dest = %(outdir_dts)s/CosmicCORSIKA/%(desc)s/art
+job_output_2.dest = %(outdir_dts_tape)s/CosmicCORSIKA/%(desc)s/art
 job_output_1.dest = %(logdir_bck)s/%(stage_name)s/%(desc)s/tbz/
 
 global.artRoot_dataset = dts.%(submitter)s.CosmicCORSIKA.%(desc)s.art
@@ -213,9 +204,17 @@ submit.n_files_per_job = 1
 submit.memory = 4000MB
 sam_consumer.limit = 1
 submit.dataset = cnf.%(submitter)s.CosmicCORSIKA.%(desc)s.fcl
-executable.arg_1 = --sam-data-tier=Output:dts
 job_setup.multifile = False
 job_setup.setup_local = True
 
+[stage_recovery]
+submit.dataset=%(override_dataset)s
+
 [executable]
-name = /cvmfs/mu2e.opensciencegrid.org/Musings/SimJob/current/link/Production/MDC2020/POMS/loggedMu2e.sh
+name = loggedMu2e.sh
+
+[executable_2]
+name = true
+arg_1 = --description=%(stage_name)s
+arg_2 = --dsconf=%(desc)s
+arg_3 = --dsowner=%(submitter)s

CampaignConfig/mdc2020_corsika.ini

@@ -1,7 +1,7 @@
 [campaign]
 experiment = mu2e
 poms_role = production
-name = MDC2020_CORSIKAf
+name = MDC2020_CORSIKA
 campaign_stage_list = generation_fcl, generation, generation_cat_fcl, generation_cat, resampler_fcl, resampler
 
 [campaign_defaults]
@@ -15,7 +15,7 @@ param_overrides="[]"
 test_param_overrides="[]"
 merge_overrides=False
 login_setup=mu2epro_login
-job_type=mu2e_production_jobtype
+job_type=mu2e_cosmic_v2_production_jobtype
 stage_type=regular
 output_ancestor_depth=1
 
@@ -67,16 +67,17 @@ file_pattern_1 = %.art
 campaign_stage_1 = resampler_fcl
 file_pattern_1 = %.fcl
 
-[job_type mu2e_production_jobtype]
+[job_type corsika_mdc2020_jobtype]
 launch_script = fife_launch
 parameters = [["-c ", "/mu2e/app/users/srsoleti/workDir/Production/CampaignConfig/mdc2020_corsika.cfg"]]
 output_file_patterns = %.art
 recoveries = [["proj_status",[["-Osubmit.dataset=","%(dataset)s"]]]]
 
-[job_type corsika_mdc2020_jobtype]
+[job_type mu2e_cosmic_v2_production_jobtype]
 launch_script = fife_launch
 parameters = [["-c ", "/mu2e/app/users/srsoleti/workDir/Production/CampaignConfig/mdc2020_corsika.cfg"]]
 output_file_patterns = %.art
+recoveries = [["proj_status",[["-Oglobal.override_dataset=','%(dataset)s"],["--stage ","recovery"]]]]
 
 [job_type generate_fcl_production_jobtype]
 launch_script = fife_launch