From 0ccd5817e16ea33f0d21d623effa6d4a0932eff6 Mon Sep 17 00:00:00 2001
From: Chris Burr <christopher.burr@cern.ch>
Date: Wed, 18 Oct 2023 14:16:25 +0900
Subject: [PATCH] Add monitoring of available space in kind when running the
 demo

---
 run_demo.sh | 33 +++++++++++++++++++++++++--------
 1 file changed, 25 insertions(+), 8 deletions(-)

diff --git a/run_demo.sh b/run_demo.sh
index 9ae1076..b42f88a 100755
--- a/run_demo.sh
+++ b/run_demo.sh
@@ -25,6 +25,28 @@ function cleanup(){
   rm -rf "${tmp_dir}"
 }
 
+function space_monitor(){
+  # Continiously monitor if the cluster is low on space
+  sleep 15
+  while true; do
+    percent_free=$(docker exec diracx-demo-control-plane df / | awk 'NR == 2 { print substr($5, 1, length($5)-1) }')
+    cluster_free_gb=$(docker exec diracx-demo-control-plane df -BG / | awk 'NR == 2 { print substr($4, 1, length($4)-1) }')
+    if [ "${cluster_free_gb}" -lt 5 ]; then
+      printf "%b Cluster is low on space (%sGB free, %s%%)\n" "${WARN_EMOJI}" "${cluster_free_gb}" "${percent_free}"
+    fi
+
+    if [ ${mount_containerd} -eq 1 ]; then
+      containerd_volume_size="$(docker exec diracx-demo-control-plane du -s -BG /var/lib/containerd | cut -d 'G' -f 1)"
+      if [[ "${containerd_volume_size}" -gt 10 ]]; then
+        printf "%b Volume for containerd is %s GB, if you want to save space " "${WARN_EMOJI}" "${containerd_volume_size}"
+        printf "shutdown the demo and run \"docker volume rm diracx-demo-containerd\"\n"
+      fi
+    fi
+
+    sleep 60
+  done
+}
+
 function check_hostname(){
   # Check that the hostname resolves to an IP address
   # dig doesn't consider the effect of /etc/hosts so we use ping instead
@@ -317,6 +339,9 @@ if grep '{{' "${demo_dir}/values.yaml"; then
   exit 1
 fi
 
+# Start background task to monitor the cluster space
+space_monitor &
+
 # Create the cluster itself
 printf "%b Starting Kind cluster...\n" ${UNICORN_EMOJI}
 "${demo_dir}/kind" create cluster \
@@ -383,12 +408,4 @@ while true; do
     printf "%b No need to restart! %b\n" "${PARTY_EMOJI}" "${PARTY_EMOJI}"
     machine_hostname_has_changed=0
   fi
-  # Check the size of the containerd storage
-  if [ ${mount_containerd} -eq 1 ]; then
-    containerd_volume_size="$(docker exec diracx-demo-control-plane du -s -BG /var/lib/containerd | cut -d 'G' -f 1)"
-    if [[ "${containerd_volume_size}" -gt 10 ]]; then
-      printf "%b Volume for containerd is %s GB, if you want to save space " "${WARN_EMOJI}" "${containerd_volume_size}"
-      printf "shutdown the demo and run \"docker volume rm diracx-demo-containerd\"\n"
-    fi
-  fi
 done