Merge pull request #4 from lsst-dm/tickets/PREOPS-3881

PREOPS-3881: add test scripts and reporting

.gitignore

@@ -127,3 +127,4 @@ dmypy.json
 
 # Pyre type checker
 .pyre/
+envvars.sh

summittests/Makefile

@@ -0,0 +1,18 @@
+VENVDIR = venv
+
+venv:
+	python -m venv $(VENVDIR)
+	( \
+		. $(VENVDIR)/bin/activate; \
+		pip install -r ../requirements.txt; \
+		pip install -r requirements.txt; \
+	)
+
+report: venv
+	( \
+		. $(VENVDIR)/bin/activate; \
+		python report.py logs/*; \
+	)
+
+clean:
+	rm -rf $(VENVDIR)

summittests/cpFilesfromS3.sh

@@ -0,0 +1,7 @@
+#assumes you already logged in to mycloud with  mc where mycloud is the s3 endpoint URL
+
+for f in `mc ls  mycloud/rubin-summit-users/test-lhn/ | cut -d' '  -f 7` ; do 
+   s3="mycloud/rubin-summit-users/test-lhn/$f"
+   mc cp  $s3 /tmp/Test/$f.fits
+done
+

summittests/envvars-template.sh

@@ -0,0 +1,11 @@
+# Source this 
+# fill in the correct vals and call this envvars.sh
+
+export S3_ENDPOINT_URL="https://s3dfrgw.slac.stanford.edu"
+export AWS_ACCESS_KEY_ID=""
+export AWS_SECRET_ACCESS_KEY=""
+export bucket="rubin-summit-users"
+export prefix="test-lhn"
+export nodes="lsstcam-dc03 lsstcam-dc04 lsstcam-dc05 lsstcam-dc06 lsstcam-dc07 lsstcam-dc08 lsstcam-dc09 lsstcam-dc10"
+export datadir="~tonyj/Test/"
+

summittests/gatherLogs.sh

@@ -0,0 +1,13 @@
+# pull all logs back to main node
+# Assumes you are in the simmit tests directory
+#
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+
+pwd=`pwd`
+for n in $nodes; do 
+   scp ${n}:${pwd}/*log logs
+   done

summittests/pushToNodes.sh

@@ -0,0 +1,11 @@
+# put the scripts on all the nodes
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+
+
+for n in $nodes; do 
+   rsync -a --exclude '/home/womullan/s3daemon/logs' --exclude '*.log' /home/womullan/s3daemon ${n}:
+   done

summittests/readFile.sh

@@ -0,0 +1,19 @@
+# just read the file  noop
+# see https://github.com/lsst-dm/s3daemon/blob/main/README.md
+
+
+if [ $1 ]
+then
+  f="$1"
+else
+  echo "Specify file "
+  exit 1
+fi
+
+start=`date +%s`
+while IFS= read -r line
+do
+   : # nothing
+done <"$f"
+end=`date +%s`
+echo Reading $f `expr $end - $start` seconds 

summittests/readTest.sh

@@ -0,0 +1,36 @@
+# Simple read 20 files each in own thread for a node for camera
+# this is just to see the times for the read as opposed to send
+# see https://github.com/lsst-dm/s3daemon/blob/main/README.md
+
+
+source  envvars.sh
+
+totalfiles=20
+if [ $1 ] 
+then
+  repeat="$1"
+fi
+echo "Just read total $totalfiles"
+
+files=`ls   ${datadir}/MC*/* `
+
+fcount=0
+
+while [[ fcount -le totalfiles ]] 
+do
+for f in $files; do 
+
+   ./readFile.sh  $f  &
+
+   fcount=$((fcount + 1))
+   if [[ fcount -ge totalfiles ]]
+   then
+      break
+   fi
+   if [ $((fcount % 20)) ==  0 ]
+   then
+       sleep 20s
+       echo "Sleeping after $fcount"
+   fi 
+done
+done

summittests/report.py

@@ -0,0 +1,81 @@
+import re
+import sys
+from typing import Dict, TextIO
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+FIELDRE = re.compile(r'INFO (.+) __main__ - ([0-9]+.[0-9]+) ([0-9]+.[0-9]+)')
+
+
+def oneplot(host, times, filecount):
+    """Plot a histogram of the times for one host"""
+    print(f"{host} {filecount} files Mean:{np.mean(times):.2f} max:{np.max(times):.2f} "
+          f"min: {np.min(times):.2f} std:{np.std(times):.2f}, var:{np.var(times):.2f} seconds")
+    bins = np.arange(2, 40.0, 1)
+    plt.clf()
+    plt.hist(times, bins=bins)
+    plt.title(f"{host} {filecount} files")
+    plt.ylabel("Number of files")
+    plt.xlabel("Time (seconds)")
+    plt.axvline(x=7, color='r', label='requirement')
+
+    plt.savefig(f"plots/{host}.png")
+    pass
+
+
+def plot(results):
+    """Plot a histogram of the times for all hosts"""
+    alltimes = []
+    filecount = 0
+    hosts = sorted(results.keys())
+    for host in hosts:
+        times_from_log = results[host]
+        times = [t[2] for t in times_from_log]
+        alltimes.append(times)
+        oneplot(host, times, len(times))
+        filecount += len(times)
+    if len(results.keys()) > 1:
+        oneplot("all", alltimes, filecount)
+
+
+def load(host: str, fin: TextIO, results: Dict) -> Dict:
+    """Load the times from one file"""
+    times = []
+    for line in fin:
+        if 'Starting' in line:
+            continue
+        fields = FIELDRE.match(line)
+        if not fields:
+            print(f"Could not parse {line} for {host}", sys.stderr)
+            continue
+        time = [fields.group(1), float(fields.group(2)), float(fields.group(3))]
+        times.append(time)
+    results[host] = times
+    return results
+
+
+def process(files: list) -> Dict:
+    """Process all the files make a dict of times"""
+    results = {}
+    name1 = re.compile(r'.*/*(lsstcam-dc[0,1][0-9])-.*')
+    name2 = re.compile(r'(.+[0,1][0-9])-.')
+    for f in files:
+        match = name1.match(f)
+        if match is None:
+            match = name2.match(f)
+        if match is None:
+            print(f"Could not get host from {f}", sys.stderr)
+            host = f.replace('.log', '')
+        else:
+            host = match.group(1)
+        with open(f, 'r') as fin:
+            load(host, fin, results)
+
+    return results
+
+
+if __name__ == '__main__':
+    files = sys.argv[1:]
+    results = process(files)
+    plot(results)

summittests/requirements.txt

@@ -0,0 +1,4 @@
+matplotlib
+numpy
+pandas
+

summittests/runOnNodes.sh

@@ -0,0 +1,14 @@
+# use ssh to run given scrip on all the nodes in $nodes
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+
+cmd="$@"
+
+for n in $nodes; do 
+   exec="cd s3daemon/summittests; ${cmd}" 
+   echo $exec
+   ssh $n "${exec}" &
+   done

summittests/runs3daemon.sh

@@ -0,0 +1,18 @@
+# Simple config for s3daemon
+# eventually this shoudl be installed and excuted by puppe on every node
+# assumes s3daemon has been installed or pip install -r of its requirements.txt
+# see https://github.com/lsst-dm/s3daemon/blob/main/README.md
+
+# Note to jump between lsstcam nodes use 'kinit' since ipa controls it all
+
+
+source  envvars.sh
+source venv/bin/activate
+
+host=`echo $HOSTNAME | cut -d'.' -f1`
+logfile="${host}-s3daemon.log"
+export S3DAEMON_LOG="$logfile"
+
+echo "S3_ENDPOINT_URL is $S3_ENDPOINT_URL log $logfile"
+
+python ../python/s3daemon/s3daemon.py  &

summittests/sendTest.sh

@@ -0,0 +1,40 @@
+# Simple send 20 files each in own thrread like a node for camera
+# assumes s3daemon has been installed or pip install -r of its requirements.txt
+# see https://github.com/lsst-dm/s3daemon/blob/main/README.md
+
+
+source  envvars.sh
+source venv/bin/activate
+
+
+
+total=2
+if [ $1 ] 
+then
+  total="$1"
+fi
+echo "$HOSTNAME S3_ENDPOINT_URL is $S3_ENDPOINT_URL  total $total $datadir"
+
+files=`ls  ${datadir}/MC*/* `
+
+fcount=0
+
+while [[ fcount -le total ]] 
+do
+for f in $files; do 
+   key=`echo $f | cut -d'/' -f6`
+   python ../python/s3daemon/send.py $f /${bucket}/${prefix}/${key} & 
+   fcount=$((fcount + 1))
+   if [[ fcount -ge total0 ]]
+   then
+       break
+   fi
+   if [ $((fcount % 20)) ==  0 ]
+   then
+       echo Sent $key total $fcount
+       echo "Sleeping"
+       sleep 20s
+   fi 
+done
+done
+echo Sent  total $fcount

summittests/stops3daemon.sh

@@ -0,0 +1,5 @@
+# find the process and kill
+
+#proc=`ps -ealf | grep s3daem | grep -v grep | cut -d' ' -f5`
+#kill -9 $proc
+pkill -if '.*s3daemon.*'