From 349f37aa0acc5be60267eab48fe1521cb1c93847 Mon Sep 17 00:00:00 2001 From: Robert Gildein Date: Mon, 14 Oct 2024 20:05:59 +0200 Subject: [PATCH] Fix log forwarding for driver flavour Signed-off-by: Robert Gildein --- images/charmed-spark/bin/sparkd.sh | 44 ++++++++++++++-------- images/charmed-spark/pebble/log-layer.yaml | 3 +- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/images/charmed-spark/bin/sparkd.sh b/images/charmed-spark/bin/sparkd.sh index 336203fb..dc8b3195 100644 --- a/images/charmed-spark/bin/sparkd.sh +++ b/images/charmed-spark/bin/sparkd.sh @@ -1,13 +1,35 @@ #!/bin/bash +function get_loki_url { + if [ ! -z "$LOKI_URL" ]; then + echo "$(<<< "$LOKI_URL" sed -e 's`[][\\/.*^$]`\\&`g')" + elif [ -f "$SPARK_CONF_DIR/spark.properties" ]; then + loki_url="$(grep -oP '(?<=spark.executorEnv.LOKI_URL=).*' $SPARK_CONF_DIR/spark.properties)" + echo "$(<<< "$loki_url" sed -e 's/\\:/:/g' -e 's`[][\\/.*^$]`\\&`g')" + fi +} + function get_log_layer { - LOG_LAYER_FILE="/opt/pebble/log-layer.yaml" - ESCAPED_LOKI_URL="$(<<< "$LOKI_URL" sed -e 's`[][\\/.*^$]`\\&`g')" - sed -e "s/\$LOKI_URL/$ESCAPED_LOKI_URL/g" \ + local loki_url=$1 + local log_layer_file=${2-"/opt/pebble/log-layer.yaml"} + sed -e "s/\$LOKI_URL/$loki_url/g" \ + -e "s/\$FLAVOUR/$FLAVOUR/g" \ -e "s/\$SPARK_APPLICATION_ID/$SPARK_APPLICATION_ID/g" \ -e "s/\$SPARK_USER/$SPARK_USER/g" \ - -e "s/\$SPARK_EXECUTOR_POD_NAME/$SPARK_EXECUTOR_POD_NAME/g" \ - $LOG_LAYER_FILE + -e "s/\$HOSTNAME/$HOSTNAME/g" \ + $log_layer_file +} + +function log_forwarding { + local loki_url=$(get_loki_url) + if [ ! -z "$loki_url" ]; then + echo "Log-forwarding to Loki is enabled" + local rendered_log_layer=$(get_log_layer $loki_url) + echo "$rendered_log_layer" | tee /tmp/rendered_log_layer.yaml + pebble add logging /tmp/rendered_log_layer.yaml + else + echo "Log-forwarding to Loki is disabled." + fi } function finish { @@ -19,26 +41,18 @@ function finish { } trap finish EXIT -if [ ! -z "${LOKI_URL}" ] -then - echo "Configuring log-forwarding to Loki." - RENDERED_LOG_LAYER=$(get_log_layer) - echo "$RENDERED_LOG_LAYER" | tee /tmp/rendered_log_layer.yaml - pebble add logging /tmp/rendered_log_layer.yaml -else - echo "Log-forwarding to Loki is disabled." -fi - FLAVOUR=$1 echo "Running script with ${FLAVOUR} flavour" case "${FLAVOUR}" in driver|executor) + log_forwarding pushd /opt/spark ./entrypoint.sh "$@" ;; "") + log_forwarding # Infinite sleep to allow pebble to be running indefinitely sleep inf ;; diff --git a/images/charmed-spark/pebble/log-layer.yaml b/images/charmed-spark/pebble/log-layer.yaml index 27345ab6..231ece4a 100644 --- a/images/charmed-spark/pebble/log-layer.yaml +++ b/images/charmed-spark/pebble/log-layer.yaml @@ -6,7 +6,8 @@ log-targets: services: [all] labels: product: charmed-spark + flavour: $FLAVOUR app: spark app_id: $SPARK_APPLICATION_ID user: $SPARK_USER - pod: $SPARK_EXECUTOR_POD_NAME + hostname: $HOSTNAME