Add better management of SIGTERM for the elasticsearch child process of run.sh

Dockerfile

@@ -62,6 +62,7 @@ ENV DISCOVERY_SERVICE elasticsearch-discovery
 # Volume for Elasticsearch data
 VOLUME ["/data"]
 
+# Run elasticsearch as unprivileged
 RUN chown elasticsearch:elasticsearch -R /usr/share/elasticsearch /data && \
     chown elasticsearch:elasticsearch -R /opt/jdk-10.0.2/conf
 USER elasticsearch

run.sh

@@ -2,7 +2,22 @@
 
 set -ex
 
-export NODE_NAME=${NODE_NAME:-${HOSTNAME}}
+export POST_TERM_WAIT=${POST_TERM_WAIT:-15}
+
+# SIGTERM-handler
+term_handler() {
+  if [ $PID -ne 0 ]; then
+    set +e
+    kill -15 "$PID" # SIGTERM
+    wait "$PID"
+    echo "Sleeping $POST_TERM_WAIT Seconds before exiting the term_handler"
+    sleep $POST_TERM_WAIT
+    set -e
+  fi
+  exit 0;
+  #exit 143; # 128 + 15 -- SIGTERM
+}
+
 
 BASE=/usr/share/elasticsearch
 
@@ -11,9 +26,11 @@ if [ "$MEMORY_LOCK" == "true" ]; then
     ulimit -l unlimited
 fi
 
+NODE_NAME=${NODE_NAME:-${HOSTNAME}}
+
 # Set a random node name if not set.
 if [ -z "${NODE_NAME}" ]; then
-	NODE_NAME=${HOSTNAME}
+	NODE_NAME=$(uuidgen)
 fi
 export NODE_NAME=${NODE_NAME}
 
@@ -38,6 +55,8 @@ if [ ! -z "${ES_PLUGINS_INSTALL}" ]; then
    IFS=$OLDIFS
 fi
 
+# Configure Shard Allocation Awareness
+# XXX: If runnig kubernetes and kubernetes is runnign in the cloud -> Fetch zone from node 
 if [ ! -z "${SHARD_ALLOCATION_AWARENESS_ATTR}" ]; then
     # this will map to a file like  /etc/hostname => /dockerhostname so reading that file will get the
     #  container hostname
@@ -52,30 +71,24 @@ if [ ! -z "${SHARD_ALLOCATION_AWARENESS_ATTR}" ]; then
 fi
 
 # configuration overrides
+# CONF directory and files need to be writable by the user running the container
+
+## DNS Timers
 if [ ! -z "${NETWORK_ADDRESS_CACHE_TTL}" ]; then
-    sed -i -e "s/#networkaddress.cache.ttl=-1/networkaddress.cache.ttl=${NETWORK_ADDRESS_CACHE_TTL}/" /opt/jdk-10.0.2/conf/security/java.security
+    sed -i -e "s/#networkaddress.cache.ttl=.*/networkaddress.cache.ttl=${NETWORK_ADDRESS_CACHE_TTL}/" /opt/jdk-10.0.2/conf/security/java.security
 fi
 
 if [ ! -z "${NETWORK_ADDRESS_CACHE_NEGATIVE_TTL}" ]; then
-    sed -i -e ""s/networkaddress.cache.negative.ttl=10/networkaddress.cache.negative.ttl=${NETWORK_ADDRESS_CACHE_NEGATIVE_TTL}/"" /opt/jdk-10.0.2/conf/security/java.security
+    sed -i -e ""s/networkaddress.cache.negative.ttl=.*/networkaddress.cache.negative.ttl=${NETWORK_ADDRESS_CACHE_NEGATIVE_TTL}/"" /opt/jdk-10.0.2/conf/security/java.security
 fi
 
-# run
-if [[ $(whoami) == "root" ]]; then
-    chown -R elasticsearch:elasticsearch $BASE
-    chown -R elasticsearch:elasticsearch /data
-    exec su-exec elasticsearch $BASE/bin/elasticsearch $ES_EXTRA_ARGS
-else
-    # the container's first process is not running as 'root',
-    # it does not have the rights to chown. however, we may
-    # assume that it is being ran as 'elasticsearch', and that
-    # the volumes already have the right permissions. this is
-    # the case for kubernetes for example, when 'runAsUser: 1000'
-    # and 'fsGroup:1000' are defined in the pod's security context.
-    $BASE/bin/elasticsearch $ES_EXTRA_ARGS
-
-    # Adding additional sleep, once ES recieves a SIGTERM, run.sh will continue to
-    # this point and wait for 5 seconds (stopping k8s from killing the POD before
-    # ES leader election is complete)
-    sleep 5
-fi
+# Trap the TERM Signals
+trap 'kill ${!}; term_handler' SIGTERM
+
+# run Elasticsearch in the background
+$BASE/bin/elasticsearch $ES_EXTRA_ARGS &
+PID="$!"
+
+while true ; do
+   tail -f /dev/null & wait ${!}
+done

scripts/pre-stop-master.sh

@@ -1,2 +1,4 @@
 #!/bin/bash
-pkill -SIGTERM java
+# Disable since now the run.sh entrypoint will deal with SIGTERMing java
+#pkill -SIGTERM java
+exit 0

scripts/stop-local-routing.sh

@@ -17,9 +17,10 @@ while true ; do
   echo -e "Wait for node ${NODE_NAME} to become empty"
   SHARDS_ALLOCATION=$(curl --retry 3 -s -XGET 'http://localhost:9200/_cat/shards')
   if ! echo "${SHARDS_ALLOCATION}" | grep -E "${NODE_NAME}"; then
-    # Send Sigterm to elasticsearch once the relocation is finished
-    sleep 2
-    pkill -SIGTERM -P 1
+    # Send Sigterm to elasticsearch once the relocation is finished 
+    # Disabled since now the run.sh entrypoint will deal with sigterming the right process
+    # sleep 2
+    # pkill -SIGTERM -P 1
     break
   fi
   sleep 2