From 427c046d8e8a755e52804d7b472e3394852e5eee Mon Sep 17 00:00:00 2001
From: welpaolo <paolo.sottovia@canonical.com>
Date: Thu, 16 Nov 2023 11:25:51 +0200
Subject: [PATCH] [DPE-3017] Patch Pebble issue on shutdown on failure (#54)

---
 files/spark/bin/charmed-spark-entrypoint.sh | 12 ++++-
 tests/integration/integration-tests.sh      | 54 +++++++++++++++++++++
 2 files changed, 65 insertions(+), 1 deletion(-)

diff --git a/files/spark/bin/charmed-spark-entrypoint.sh b/files/spark/bin/charmed-spark-entrypoint.sh
index 1fcb332a..57214b73 100644
--- a/files/spark/bin/charmed-spark-entrypoint.sh
+++ b/files/spark/bin/charmed-spark-entrypoint.sh
@@ -1,5 +1,15 @@
 #!/bin/bash
 
+function finish {
+  if [ $? -ne 0 ]
+  then
+    kill -1 1
+    sleep 1
+  fi
+}
+trap finish EXIT
+
+
 FLAVOUR=$1
 
 echo "Running script with ${FLAVOUR} flavour"
@@ -17,4 +27,4 @@ case "${FLAVOUR}" in
     echo "Component \"${FLAVOUR}\" unknown"
     exit 1
     ;;
-esac
\ No newline at end of file
+esac
diff --git a/tests/integration/integration-tests.sh b/tests/integration/integration-tests.sh
index 312810a3..a9267613 100755
--- a/tests/integration/integration-tests.sh
+++ b/tests/integration/integration-tests.sh
@@ -259,6 +259,54 @@ run_example_job_in_pod_with_metrics() {
 }
 
 
+run_example_job_with_error_in_pod() {
+  SPARK_EXAMPLES_JAR_NAME="spark-examples_2.12-$(get_spark_version).jar"
+
+  PREVIOUS_JOB=$(kubectl get pods | grep driver | tail -n 1 | cut -d' ' -f1)
+  NAMESPACE=$1
+  USERNAME=$2
+
+  kubectl exec testpod -- env UU="$USERNAME" NN="$NAMESPACE" JJ="$SPARK_EXAMPLES_JAR_NAME" IM="$(spark_image)" \
+                  /bin/bash -c 'spark-client.spark-submit \
+                  --username $UU --namespace $NN \
+                  --conf spark.kubernetes.driver.request.cores=100m \
+                  --conf spark.kubernetes.executor.request.cores=100m \
+                  --conf spark.kubernetes.container.image=$IM \
+                  --class org.apache.spark.examples.SparkPi \
+                  local:///opt/spark/examples/jars/$JJ -1'
+
+  # kubectl --kubeconfig=${KUBE_CONFIG} get pods
+  DRIVER_JOB=$(kubectl get pods -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1)
+
+  if [[ "${DRIVER_JOB}" == "${PREVIOUS_JOB}" ]]
+  then
+    echo "ERROR: Sample job has not run!"
+    exit 1
+  fi
+
+  # Check job output
+  res=$(kubectl logs $(kubectl get pods -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1) -n ${NAMESPACE} | grep 'Exception in thread' | wc -l)
+  echo -e "Number of errors: \n ${res}"
+  if [ "${res}" != "1" ]; then
+      echo "ERROR: Error is not captured."
+      exit 1
+  fi
+  status=$(kubectl get pod $(kubectl get pods -n ${NAMESPACE} | grep driver | tail -n 1 | cut -d' ' -f1) -n ${NAMESPACE} | tail -1 | cut -d " " -f 9)
+  if [ "${status}" = "Completed" ]; then
+      echo "ERROR: Status should not be set to Completed."
+      exit 1
+  fi
+  if [ "${status}" = "Error" ]; then
+      echo "Status is correctly set to ERROR!"
+  fi
+
+}
+
+test_example_job_in_pod_with_errors() {
+  run_example_job_with_error_in_pod tests spark
+}
+
+
 test_example_job_in_pod_with_templates() {
   run_example_job_in_pod_with_pod_templates tests spark
 }
@@ -371,6 +419,12 @@ echo -e "########################################"
 
 (setup_user_admin_context && test_example_job_in_pod_with_metrics && cleanup_user_success) || cleanup_user_failure_in_pod
 
+
+echo -e "########################################"
+echo -e "RUN EXAMPLE JOB WITH ERRORS"
+echo -e "########################################"
+
+(setup_user_admin_context && test_example_job_in_pod_with_errors && cleanup_user_success) || cleanup_user_failure_in_pod
 echo -e "##################################"
 echo -e "TEARDOWN TEST POD"
 echo -e "##################################"