full set to run on nodes

summittests/envvars-template.sh

@@ -5,4 +5,6 @@ export S3_ENDPOINT_URL="https://s3dfrgw.slac.stanford.edu"
 export AWS_ACCESS_KEY_ID=""
 export AWS_SECRET_ACCESS_KEY=""
 export bucket="rubin-summit-users"
-export prefix="testx-lhn"
+export prefix="test-lhn"
+export nodes="lsstcam-dc03 lsstcam-dc04 lsstcam-dc05 lsstcam-dc06 lsstcam-dc07 lsstcam-dc08 lsstcam-dc09 lsstcam-dc10"
+

summittests/gatherLogs.sh

@@ -0,0 +1,11 @@
+# put the scripts on all the nodes
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+
+
+for n in $nodes; do 
+   scp ${n}:/home/womullan/s3daemon/summittests/*log logs
+   done

summittests/pushToNodes.sh

@@ -0,0 +1,11 @@
+# put the scripts on all the nodes
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+
+
+for n in $nodes; do 
+   rsync -a --exclude '/home/womullan/s3daemon/logs' /home/womullan/s3daemon ${n}:
+   done

summittests/runOnNodes.sh

@@ -0,0 +1,14 @@
+# use ssh to run given scrip on all the nodes in $nodes
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+
+cmd="./$1"
+
+for n in $nodes; do 
+   exec="cd s3daemon/summittests; ${cmd}" 
+   echo $exec
+   ssh $n "${exec}" &
+   done

summittests/runs3daemon.sh

@@ -1,12 +1,17 @@
 # Simple config for s3daemon
+# eventually this shoudl be installed and excuted by puppe on every node
 # assumes s3daemon has been installed or pip install -r of its requirements.txt
 # see https://github.com/lsst-dm/s3daemon/blob/main/README.md
 
 # Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
 
 
 source  envvars.sh
+source venv/bin/activate
+
+host=`echo $HOSTNAME | cut -d'.' -f1`
+logfile="${host}-s3daemon.log"
 
 echo "S3_ENDPOINT_URL is $S3_ENDPOINT_URL"
 
-python ../python/s3daemon/s3daemon.py &
+python ../python/s3daemon/s3daemon.py > $logfile  2>&1 &

summittests/sendTest.sh

@@ -4,14 +4,22 @@
 
 
 source  envvars.sh
+source venv/bin/activate
 
 echo "S3_ENDPOINT_URL is $S3_ENDPOINT_URL"
 count=20
 
-files=`ls ~tonyj/Data/MC_C_20200822_000054/MC_C_20200822* | head -${count}`
-echo $files
+files=`ls  ~tonyj/Test/MC*/* | head -${count}`
+
+fcount=0
 for f in $files; do 
    key=`echo $f | cut -d'/' -f6 | sed 's/.fits//g'`
    echo Sending $f $key
    python ../python/s3daemon/send.py $f “/${bucket}/${prefix}/${key}” & 
-   done
+   fcount=$((fcount + 1))
+   if [ $((fcount % 20)) ==  0 ]
+   then
+       sleep 20s
+       echo "Sleeping"
+   fi 
+done

summittests/stops3daemon.sh

@@ -0,0 +1,5 @@
+# find the process and kill
+
+proc=`ps -ealf | grep s3daem | grep -v grep | cut -d' ' -f5`
+
+kill -9 $proc