Merge pull request #207 from JeffersonLab/aaust_launch_merge

typo in swif2 output fixed, automatic retry of jobs
JeffersonLab · Nov 11, 2022 · 5af682b · 5af682b
2 parents dd9b19d + 4c8ceba
commit 5af682b
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 2 deletions.
diff --git a/launch_scripts/merge_trees/jobs_merge.config b/launch_scripts/merge_trees/jobs_merge.config
@@ -27,7 +27,7 @@ SCRIPTFILE                    /home/gxproj1/monitoring/merge_trees/script.sh
 CACHE_PIN_DAYS                60                                            # max is 60, 0 or comment-out for none
 
 # FILE INPUT, OUTPUT BASE DIRECTORIES
-INDATA_TOPDIR     /cache/halld/RunPeriod-[RUNPERIOD]/analysis/ver[VERSION]/
+INDATA_TOPDIR     /volatile/halld/analysis/RunPeriod-[RUNPERIOD]/ver[VERSION]/
 
 OUTDIR_LARGE      /cache/halld/RunPeriod-[RUNPERIOD]/analysis/ver[VERSION]/
 OUTDIR_SMALL      /work/halld2/analysis/RunPeriod-[RUNPERIOD]/ver[VERSION]/                  # log files
diff --git a/launch_scripts/merge_trees/merge_trees.py b/launch_scripts/merge_trees/merge_trees.py
@@ -218,7 +218,7 @@ def build_launch_dictionary(WORKFLOW):
                                 job_dictionary[run_string] = 1
 
 
-                elif (field == "slum_exitcode"): 
+                elif (field == "slurm_exitcode"):
 			job_result = line.split()[2]
                         if (job_result == "0"):
                                 run_done = 1
@@ -376,6 +376,13 @@ def main(argv):
 
         print "New runs complete and submitted:" + str(n_submit)
 
+        # RETRY FAILED JOBS
+        command = "swif2 retry-jobs -workflow " + LAUNCH_WORKFLOW + " -problems SLURM_FAILED SLURM_CANCELLED SLURM_TIMEOUT SLURM_NODE_FAIL"
+        if VERBOSE > 1:
+                print command
+        try_command(command)
+
+
 if __name__ == "__main__":
 	main(sys.argv[1:])