feat(scripts): add opensearch index mapping script (reanahub#827)

jlemesh · Aug 30, 2024 · 5c35771 · 5c35771
1 parent a68a304
commit 5c35771
Show file tree

Hide file tree

Showing 4 changed files with 243 additions and 0 deletions.
diff --git a/MANIFEST.in b/MANIFEST.in
@@ -27,6 +27,7 @@ recursive-include helm *.helmignore
 recursive-include helm *.lock
 recursive-include scripts *.sh
 recursive-include scripts *.py
+recursive-include scripts *.json
 recursive-include etc *.yaml
 
 # Helm

diff --git a/scripts/opensearch/job_log_mapping.json b/scripts/opensearch/job_log_mapping.json
@@ -0,0 +1,65 @@
+{
+    "dynamic": "strict",
+    "properties": {
+        "@timestamp": {
+            "type": "date"
+        },
+        "kubernetes": {
+            "properties": {
+                "container_image": {
+                    "type": "text"
+                },
+                "container_name": {
+                    "type": "text"
+                },
+                "docker_id": {
+                    "type": "text"
+                },
+                "host": {
+                    "type": "text"
+                },
+                "labels": {
+                    "properties": {
+                        "batch.kubernetes.io/controller-uid": { "type": "text" },
+                        "batch.kubernetes.io/job-name": { "type": "text" },
+                        "controller-uid": { "type": "text" },
+                        "job-name": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword"
+                                }
+                            }
+                        },
+                        "reana-run-job-workflow-uuid": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword"
+                                }
+                            }
+                        }
+                    }
+                },
+                "namespace_name": {
+                    "type": "text"
+                },
+                "pod_id": {
+                    "type": "text"
+                },
+                "pod_name": {
+                    "type": "text"
+                }
+            }
+        },
+        "log": {
+            "type": "text"
+        },
+        "stream": {
+            "type": "text"
+        },
+        "time": {
+            "type": "date"
+        }
+    }
+}
diff --git a/scripts/opensearch/opensearch.sh b/scripts/opensearch/opensearch.sh
@@ -0,0 +1,111 @@
+#!/bin/bash
+#
+# This file is part of REANA.
+# Copyright (C) 2024 CERN.
+#
+# REANA is free software; you can redistribute it and/or modify it
+# under the terms of the MIT License; see LICENSE file for more details.
+
+set -euo pipefail
+
+# Usage examples:
+#
+# Local environment with OpenSearch security plugin off:
+# ./opensearch.sh
+#
+# Local environment with OpenSearch security plugin on (kubectl should be installed):
+# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_PROTOCOL=https KUBECTL_CERTS=true ./opensearch.sh
+#
+# Local environment to remote OpenSearch, with OpenSearch security plugin on (kubectl should be installed):
+# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_HOST=opensearch.cern.ch OPENSEARCH_PROTOCOL=https KUBECTL_CERTS=true ./opensearch.sh
+#
+# From inside OpenSearch pod, with OpenSearch security plugin on:
+# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_PROTOCOL=https CERTS_FROM_FILE=true ./opensearch.sh
+#
+# Local environment to remote OpenSearch, with OpenSearch security plugin on:
+# OPENSEARCH_HOST=opensearch.cern.ch OPENSEARCH_PROTOCOL=https CERTS_FROM_FILE=true CERT_DIR=. ./opensearch.sh
+
+# OpenSearch address configuration
+OPENSEARCH_PROTOCOL="${OPENSEARCH_PROTOCOL:-http}" # http or https
+OPENSEARCH_HOST="${OPENSEARCH_HOST:-localhost}"
+OPENSEARCH_PORT="${OPENSEARCH_PORT:-9200}"
+OPENSEARCH_CN="${OPENSEARCH_CN:-$OPENSEARCH_HOST}" # OpenSearch common name in node's TLS certificate, defaults to OPENSEARCH_HOST
+OPENSEARCH_ADDRESS=$OPENSEARCH_PROTOCOL://$OPENSEARCH_CN:$OPENSEARCH_PORT
+
+# Index mapping configuration
+JOB_LOG_IDX="${JOB_LOG_IDX:-fluentbit-job_log}"
+WORKFLOW_LOG_IDX="${WORKFLOW_LOG_IDX:-fluentbit-workflow_log}"
+JOB_LOG_MAPPING_FILE_PATH="${JOB_LOG_MAPPING_FILE_PATH:-job_log_mapping.json}"
+WORKFLOW_LOG_MAPPING_FILE_PATH="${WORKFLOW_LOG_MAPPING_FILE_PATH:-workflow_log_mapping.json}"
+
+# TLS configuration
+KUBECTL_CERTS="${KUBECTL_CERTS:-false}" # Use admin certificates from Kubernetes secret; makes cURL use TLS while connecting to OpenSearch
+CERTS_FROM_FILE="${CERTS_FROM_FILE:-false}" # Use admin certificates from files; makes cURL use TLS while connecting to OpenSearch
+CERT_DIR="${CERT_DIR:-config/certs}" # If CERTS_FROM_FILE is true, the directory where the certificates are stored (files should be named ca.crt, admin.crt and admin.key)
+KUBECTL_SECRET_NAME="${KUBECTL_SECRET_NAME:-reana-opensearch-tls-secret}" # If KUBECTL_CERTS is true, the secret name to use
+
+# Execute cURL when TLS is enabled
+curl_exec_secure() {
+  curl --connect-to "$OPENSEARCH_CN:$OPENSEARCH_PORT:$OPENSEARCH_HOST" \
+    --cacert <(echo "$CACERT") --cert <(echo "$CERT") --key <(echo "$KEY") \
+    -H 'Content-Type: application/json' "$@"
+  echo ""
+}
+
+# Execute cURL when TLS is disabled
+curl_exec() {
+  curl -H 'Content-Type: application/json' "$@"
+  echo ""
+}
+
+# Setup indices and mappings
+run() {
+  echo "OpenSearch adderess: $OPENSEARCH_ADDRESS."
+  echo "Create indices (ignore errors if already created):"
+  echo -n "$JOB_LOG_IDX response: "
+  $1 -XPUT "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX"
+
+  echo -n "$WORKFLOW_LOG_IDX response: "
+  $1 -XPUT "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX"
+
+
+  echo "Create mappings:"
+  echo -n "$JOB_LOG_IDX response: "
+  $1 -XPUT "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX/_mapping" \
+    --data "@$JOB_LOG_MAPPING_FILE_PATH"
+
+  echo -n "$WORKFLOW_LOG_IDX response: "
+  $1 -XPUT "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX/_mapping" \
+    --data "@$WORKFLOW_LOG_MAPPING_FILE_PATH"
+
+
+  echo "Update mappings:"
+  echo -n "$JOB_LOG_IDX response: "
+  $1 -XPOST "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX/_update_by_query"
+
+  echo -n "$WORKFLOW_LOG_IDX response: "
+  $1 -XPOST "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX/_update_by_query"
+
+  echo "Done."
+}
+
+# Execute script
+if [[ $KUBECTL_CERTS = true ]]; then
+    echo "Using certificates from Kubernetes secret."
+    CACERT=$(kubectl get secret "$KUBECTL_SECRET_NAME" \
+      -ogo-template='{{ index .data "ca.crt" | base64decode }}')
+    CERT=$(kubectl get secret "$KUBECTL_SECRET_NAME" \
+      -ogo-template='{{ index .data "admin.crt" | base64decode }}')
+    KEY=$(kubectl get secret "$KUBECTL_SECRET_NAME" \
+      -ogo-template='{{ index .data "admin.key" | base64decode }}')
+    run curl_exec_secure
+elif [[ $CERTS_FROM_FILE = true ]]; then
+    echo "Using certificates from files."
+    CACERT=$(cat "$CERT_DIR"/ca.crt)
+    CERT=$(cat "$CERT_DIR"/admin.crt)
+    KEY=$(cat "$CERT_DIR"/admin.key)
+    run curl_exec_secure
+else
+    echo "Not using TLS."
+    run curl_exec
+fi
diff --git a/scripts/opensearch/workflow_log_mapping.json b/scripts/opensearch/workflow_log_mapping.json
@@ -0,0 +1,66 @@
+{
+    "dynamic": "strict",
+    "properties": {
+        "@timestamp": {
+            "type": "date"
+        },
+        "kubernetes": {
+            "properties": {
+                "container_image": {
+                    "type": "text"
+                },
+                "container_name": {
+                    "type": "text"
+                },
+                "docker_id": {
+                    "type": "text"
+                },
+                "host": {
+                    "type": "text"
+                },
+                "labels": {
+                    "properties": {
+                        "batch.kubernetes.io/controller-uid": { "type": "text" },
+                        "batch.kubernetes.io/job-name": { "type": "text" },
+                        "controller-uid": { "type": "text" },
+                        "job-name": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword"
+                                }
+                            }
+                        },
+                        "reana-run-batch-workflow-uuid": {
+                            "type": "text",
+                            "fields": {
+                                "keyword": {
+                                    "type": "keyword"
+                                }
+                            }
+                        },
+                        "reana_workflow_mode": { "type": "text" }
+                    }
+                },
+                "namespace_name": {
+                    "type": "text"
+                },
+                "pod_id": {
+                    "type": "text"
+                },
+                "pod_name": {
+                    "type": "text"
+                }
+            }
+        },
+        "log": {
+            "type": "text"
+        },
+        "stream": {
+            "type": "text"
+        },
+        "time": {
+            "type": "date"
+        }
+    }
+}