From 82d09c3bd3896ca87877894b0b49fd720838bd36 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 24 May 2017 10:26:39 -0400 Subject: [PATCH 1/5] first attempt at condor submit --- MCwrapper/gluex_MC.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/MCwrapper/gluex_MC.py b/MCwrapper/gluex_MC.py index cd5f5bd7..3aca9deb 100755 --- a/MCwrapper/gluex_MC.py +++ b/MCwrapper/gluex_MC.py @@ -107,6 +107,25 @@ def qsub_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, DA status = subprocess.call(mkdircom, shell=True) status = subprocess.call(add_command, shell=True) +def condor_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, DATA_OUTPUT_BASE_DIR, TIMELIMIT, RUNNING_DIR ): + STUBNAME = str(RUNNUM) + "_" + str(FILENUM) + JOBNAME = WORKFLOW + "_" + STUBNAME + + mkdircom="mkdir -p "+DATA_OUTPUT_BASE_DIR+"/log/" + + f=open('MCcondor.submit','w') + f.write("Executable = "+indir+"\n") + f.write("NameAdd = "+JOBNAME+"\n") + f.write("Arguments = "+COMMAND+"\n") + f.write("Error = error_"+JOBNAME+".log\n") + f.write("Log = out_"+JOBNAME+".log\n") + f.write("RequestCpus = "+NCORES+"\n") + f.write("Queue 1\n") + f.close() + + status = subprocess.call(mkdircom, shell=True) + status = subprocess.call("condor_submit MCcondor.submit", shell=True) + def showhelp(): helpstring= "variation=%s where %s is a valid jana_calib_context variation string (default is \"mc\")\n" @@ -148,7 +167,7 @@ def main(argv): #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! print "*********************************" - print "Welcome to v1.6 of the MCwrapper" + print "Welcome to v1.7 of the MCwrapper" print "Thomas Britton 05/19/17" print "*********************************" @@ -412,6 +431,8 @@ def main(argv): swif_add_job(WORKFLOW, RUNNUM, FILENUM,str(indir),COMMAND,VERBOSE,PROJECT,TRACK,NCORES,DISK,RAM,TIMELIMIT,OS,DATA_OUTPUT_BASE_DIR) elif BATCHSYS.upper()=="QSUB": qsub_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, DATA_OUTPUT_BASE_DIR, TIMELIMIT, RUNNING_DIR ) + elif BATCHSYS.upper()=="CONDOR": + condor_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, DATA_OUTPUT_BASE_DIR, TIMELIMIT, RUNNING_DIR ) if BATCHRUN == 1 and BATCHSYS.upper() == "SWIF": From e2eeb20727040810ca4677009736d499e335e001 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 24 May 2017 10:55:44 -0400 Subject: [PATCH 2/5] better cleanup...removed superfluous variable --- MCwrapper/gluex_MC.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/MCwrapper/gluex_MC.py b/MCwrapper/gluex_MC.py index 3aca9deb..ad108335 100755 --- a/MCwrapper/gluex_MC.py +++ b/MCwrapper/gluex_MC.py @@ -115,7 +115,6 @@ def condor_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, f=open('MCcondor.submit','w') f.write("Executable = "+indir+"\n") - f.write("NameAdd = "+JOBNAME+"\n") f.write("Arguments = "+COMMAND+"\n") f.write("Error = error_"+JOBNAME+".log\n") f.write("Log = out_"+JOBNAME+".log\n") @@ -123,8 +122,14 @@ def condor_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, f.write("Queue 1\n") f.close() + add_command="condor_submit -name "+JOBNAME+" MCcondor.submit" + if add_command.find(';')!=-1 or add_command.find('&')!=-1 or mkdircom.find(';')!=-1 or mkdircom.find('&')!=-1 or mkdircom2.find(';')!=-1 or mkdircom2.find('&')!=-1:#THIS CHECK HELPS PROTEXT AGAINST A POTENTIAL HACK VIA CONFIG FILES + print "Nice try.....you cannot use ; or &" + exit(1) + status = subprocess.call(mkdircom, shell=True) - status = subprocess.call("condor_submit MCcondor.submit", shell=True) + status = subprocess.call(add_command, shell=True) + status = subprocess.call("rm MCcondor.submit", shell=True) def showhelp(): From 82a4b098b23fc7c1bc1972301ef159e9f713bf68 Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 24 May 2017 10:56:41 -0400 Subject: [PATCH 3/5] fixed typo --- MCwrapper/gluex_MC.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MCwrapper/gluex_MC.py b/MCwrapper/gluex_MC.py index ad108335..17acd863 100755 --- a/MCwrapper/gluex_MC.py +++ b/MCwrapper/gluex_MC.py @@ -123,7 +123,7 @@ def condor_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, f.close() add_command="condor_submit -name "+JOBNAME+" MCcondor.submit" - if add_command.find(';')!=-1 or add_command.find('&')!=-1 or mkdircom.find(';')!=-1 or mkdircom.find('&')!=-1 or mkdircom2.find(';')!=-1 or mkdircom2.find('&')!=-1:#THIS CHECK HELPS PROTEXT AGAINST A POTENTIAL HACK VIA CONFIG FILES + if add_command.find(';')!=-1 or add_command.find('&')!=-1 or mkdircom.find(';')!=-1 or mkdircom.find('&')!=-1:#THIS CHECK HELPS PROTEXT AGAINST A POTENTIAL HACK VIA CONFIG FILES print "Nice try.....you cannot use ; or &" exit(1) From 0b4a0deba5d7ad351b089f34767c7effe1c926fa Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 24 May 2017 10:59:28 -0400 Subject: [PATCH 4/5] properly directed log files properly directed log files --- MCwrapper/gluex_MC.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MCwrapper/gluex_MC.py b/MCwrapper/gluex_MC.py index 17acd863..f4e34bbd 100755 --- a/MCwrapper/gluex_MC.py +++ b/MCwrapper/gluex_MC.py @@ -116,8 +116,8 @@ def condor_add_job(VERBOSE, WORKFLOW, RUNNUM, FILENUM, indir, COMMAND, NCORES, f=open('MCcondor.submit','w') f.write("Executable = "+indir+"\n") f.write("Arguments = "+COMMAND+"\n") - f.write("Error = error_"+JOBNAME+".log\n") - f.write("Log = out_"+JOBNAME+".log\n") + f.write("Error = "+DATA_OUTPUT_BASE_DIR+"/log/"+"error_"+JOBNAME+".log\n") + f.write("Log = "+DATA_OUTPUT_BASE_DIR+"/log/"+"out_"+JOBNAME+".log\n") f.write("RequestCpus = "+NCORES+"\n") f.write("Queue 1\n") f.close() From 29794f7e41a7c86f45e6d452e473755fb71f5ebe Mon Sep 17 00:00:00 2001 From: Unknown Date: Wed, 24 May 2017 11:44:34 -0400 Subject: [PATCH 5/5] simple condor submit for local clusters --- MCwrapper/gluex_MC.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MCwrapper/gluex_MC.py b/MCwrapper/gluex_MC.py index f4e34bbd..827c65df 100755 --- a/MCwrapper/gluex_MC.py +++ b/MCwrapper/gluex_MC.py @@ -173,7 +173,7 @@ def main(argv): print "*********************************" print "Welcome to v1.7 of the MCwrapper" - print "Thomas Britton 05/19/17" + print "Thomas Britton 05/24/17" print "*********************************" @@ -312,7 +312,7 @@ def main(argv): TAGSTR=rm_comments[0].strip() elif str(parts[0]).upper()=="CUSTOM_PLUGINS" : CUSTOM_PLUGINS=rm_comments[0].strip() - elif str(parts[0]).upper()=="BATCH_SYS" : + elif str(parts[0]).upper()=="BATCH_SYSTEM" : BATCHSYS=rm_comments[0].strip() elif str(parts[0]).upper()=="RUNNING_DIRECTORY" : RUNNING_DIR=rm_comments[0].strip()