From 74f15eb8304532ac1aff377ce259233b6460abc6 Mon Sep 17 00:00:00 2001 From: Alex Austregesilo Date: Wed, 21 Aug 2024 11:36:39 -0400 Subject: [PATCH] current version of official merge scripts --- launch_scripts/merge_trees/cron_exec.sh | 2 +- launch_scripts/merge_trees/cron_merge | 2 +- launch_scripts/merge_trees/jobs_merge.config | 24 ++++++++++---------- launch_scripts/merge_trees/merge_trees.py | 16 ++++++------- launch_scripts/merge_trees/script.sh | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/launch_scripts/merge_trees/cron_exec.sh b/launch_scripts/merge_trees/cron_exec.sh index c96db399..e39b9a2b 100755 --- a/launch_scripts/merge_trees/cron_exec.sh +++ b/launch_scripts/merge_trees/cron_exec.sh @@ -3,5 +3,5 @@ date source ~/env_monitoring_launch.sh export PATH=/site/bin:${PATH} #because .login isn't executed, and need this path for SWIF -python ~/monitoring/merge_trees/merge_trees.py ~/monitoring/merge_trees/jobs_merge.config 0 +python ~/hd_utilities/launch_scripts/merge_trees/merge_trees.py ~/hd_utilities/launch_scripts/merge_trees/jobs_merge.config 0 diff --git a/launch_scripts/merge_trees/cron_merge b/launch_scripts/merge_trees/cron_merge index fdca0393..7cae8eb3 100644 --- a/launch_scripts/merge_trees/cron_merge +++ b/launch_scripts/merge_trees/cron_merge @@ -3,4 +3,4 @@ # Min Hour Day Month Weekday Command #----------------------------------------------------------------------------- MAILTO=aaustreg@jlab.org -*/15 * * * * /bin/bash -c ~/monitoring/merge_trees/cron_exec.sh > /home/gxproj1/merge_trees.log 2>&1 +*/15 * * * * /bin/bash -c ~/hd_utilities/launch_scripts/merge_trees/cron_exec.sh > /home/gxproj6/merge_trees.log 2>&1 diff --git a/launch_scripts/merge_trees/jobs_merge.config b/launch_scripts/merge_trees/jobs_merge.config index 72fe8a9f..33b6b8bc 100644 --- a/launch_scripts/merge_trees/jobs_merge.config +++ b/launch_scripts/merge_trees/jobs_merge.config @@ -6,25 +6,25 @@ # For files & directories: Must specify full path # SCICOMP JOB ACCOUNTING -PROJECT halld # http://scicomp.jlab.org/scicomp/#/projects -TRACK production # https://scicomp.jlab.org/docs/batch_job_tracks -OS general +PROJECT halld-pro # http://scicomp.jlab.org/scicomp/#/projects +TRACK production # https://scicomp.jlab.org/docs/batch_job_tracks +OS el9 # JOB RESOURCES NCORES 1 -DISK 500GB -RAM 8GB -TIMELIMIT 8hrs +DISK 1000GB +RAM 4GB +TIMELIMIT 16hrs -RUNPERIOD 2017-01 -VERSION 01 -BATCH 01 +RUNPERIOD 2018-01 +VERSION 24 +BATCH 04 WORKFLOW analysis_[RUNPERIOD]_ver[VERSION]_batch[BATCH] # Input workflow # JOB, SCRIPT CONTROL -ENVFILE /home/gxproj1/env_monitoring_launch.sh -SCRIPTFILE /home/gxproj1/monitoring/merge_trees/script.sh -CACHE_PIN_DAYS 60 # max is 60, 0 or comment-out for none +ENVFILE /home/gxproj6/env_analysis_launch.sh +SCRIPTFILE /home/gxproj6/hd_utilities/launch_scripts/merge_trees/script.sh +CACHE_PIN_DAYS 0 # max is 60, 0 or comment-out for none # FILE INPUT, OUTPUT BASE DIRECTORIES INDATA_TOPDIR /volatile/halld/analysis/RunPeriod-[RUNPERIOD]/ver[VERSION]/ diff --git a/launch_scripts/merge_trees/merge_trees.py b/launch_scripts/merge_trees/merge_trees.py index d4854b5a..3908d814 100755 --- a/launch_scripts/merge_trees/merge_trees.py +++ b/launch_scripts/merge_trees/merge_trees.py @@ -381,25 +381,25 @@ def main(argv): print("Retry failed jobs in the launch workflow: ") command = "/usr/local/bin/swif2 retry-jobs -workflow " + LAUNCH_WORKFLOW + " -problems SLURM_FAILED SLURM_CANCELLED SLURM_NODE_FAIL SITE_LAUNCH_FAIL SITE_PREP_FAIL SWIF_INPUT_FAIL SWIF_SYSTEM_ERROR" if VERBOSE > 1: - print(command) + print(command) try_command(command) # MODIFY FAILED JOBS - command = "swif2 modify-jobs -ram add 4gb -workflow " + LAUNCH_WORKFLOW + " -problems SLURM_OUT_OF_MEMORY" + command = "swif2 modify-jobs -ram add 4gb -workflow " + LAUNCH_WORKFLOW + " -problems SLURM_OUT_OF_MEMORY" if VERBOSE > 1: - print command - try_command(command) + print(command) + try_command(command) - command = "swif2 modify-jobs -time add 4h -workflow " + LAUNCH_WORKFLOW + " -problems SLURM_TIMEOUT" + command = "swif2 modify-jobs -time add 4h -workflow " + LAUNCH_WORKFLOW + " -problems SLURM_TIMEOUT" if VERBOSE > 1: - print command - try_command(command) + print(command) + try_command(command) print("Retry failed jobs in the merge workflow: ") command = "/usr/local/bin/swif2 retry-jobs -workflow " + MERGE_WORKFLOW + " -problems SLURM_CANCELLED SLURM_TIMEOUT SLURM_NODE_FAIL SITE_LAUNCH_FAIL SITE_PREP_FAIL SWIF_INPUT_FAIL SWIF_SYSTEM_ERROR" if VERBOSE > 1: - print (command) + print (command) try_command(command) diff --git a/launch_scripts/merge_trees/script.sh b/launch_scripts/merge_trees/script.sh index 4e5e76d7..59441214 100755 --- a/launch_scripts/merge_trees/script.sh +++ b/launch_scripts/merge_trees/script.sh @@ -51,7 +51,7 @@ Merge_Files() #cp -v $INPUTDIR/$TYPE/$RUN/${TYPE}_${RUN}_*.root . - LD_PRELOAD=/home/gxproj6/monitoring/merge_trees/startup_C.so hadd $TEMP_FILE $INPUTDIR/$TYPE/$RUN/${TYPE}_${RUN}_*.root + LD_PRELOAD=/home/gxproj6/hd_utilities/launch_scripts/merge_trees/startup_C.so hadd $TEMP_FILE $INPUTDIR/$TYPE/$RUN/${TYPE}_${RUN}_*.root #hadd $TEMP_FILE $INPUTDIR/$TYPE/$RUN/${TYPE}_${RUN}_*.root # RETURN CODE RETURN_CODE=$?