From 5c357716f58d7cea49c142076c735a7df4880908 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jelizaveta=20Leme=C5=A1eva?= Date: Fri, 30 Aug 2024 09:21:15 +0200 Subject: [PATCH] feat(scripts): add opensearch index mapping script (#827) --- MANIFEST.in | 1 + scripts/opensearch/job_log_mapping.json | 65 +++++++++++ scripts/opensearch/opensearch.sh | 111 +++++++++++++++++++ scripts/opensearch/workflow_log_mapping.json | 66 +++++++++++ 4 files changed, 243 insertions(+) create mode 100644 scripts/opensearch/job_log_mapping.json create mode 100755 scripts/opensearch/opensearch.sh create mode 100644 scripts/opensearch/workflow_log_mapping.json diff --git a/MANIFEST.in b/MANIFEST.in index c831a72e..8aede757 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -27,6 +27,7 @@ recursive-include helm *.helmignore recursive-include helm *.lock recursive-include scripts *.sh recursive-include scripts *.py +recursive-include scripts *.json recursive-include etc *.yaml # Helm diff --git a/scripts/opensearch/job_log_mapping.json b/scripts/opensearch/job_log_mapping.json new file mode 100644 index 00000000..10f47f9a --- /dev/null +++ b/scripts/opensearch/job_log_mapping.json @@ -0,0 +1,65 @@ +{ + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "kubernetes": { + "properties": { + "container_image": { + "type": "text" + }, + "container_name": { + "type": "text" + }, + "docker_id": { + "type": "text" + }, + "host": { + "type": "text" + }, + "labels": { + "properties": { + "batch.kubernetes.io/controller-uid": { "type": "text" }, + "batch.kubernetes.io/job-name": { "type": "text" }, + "controller-uid": { "type": "text" }, + "job-name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "reana-run-job-workflow-uuid": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + } + } + }, + "namespace_name": { + "type": "text" + }, + "pod_id": { + "type": "text" + }, + "pod_name": { + "type": "text" + } + } + }, + "log": { + "type": "text" + }, + "stream": { + "type": "text" + }, + "time": { + "type": "date" + } + } +} diff --git a/scripts/opensearch/opensearch.sh b/scripts/opensearch/opensearch.sh new file mode 100755 index 00000000..21be13b6 --- /dev/null +++ b/scripts/opensearch/opensearch.sh @@ -0,0 +1,111 @@ +#!/bin/bash +# +# This file is part of REANA. +# Copyright (C) 2024 CERN. +# +# REANA is free software; you can redistribute it and/or modify it +# under the terms of the MIT License; see LICENSE file for more details. + +set -euo pipefail + +# Usage examples: +# +# Local environment with OpenSearch security plugin off: +# ./opensearch.sh +# +# Local environment with OpenSearch security plugin on (kubectl should be installed): +# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_PROTOCOL=https KUBECTL_CERTS=true ./opensearch.sh +# +# Local environment to remote OpenSearch, with OpenSearch security plugin on (kubectl should be installed): +# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_HOST=opensearch.cern.ch OPENSEARCH_PROTOCOL=https KUBECTL_CERTS=true ./opensearch.sh +# +# From inside OpenSearch pod, with OpenSearch security plugin on: +# OPENSEARCH_CN=opensearch-cluster-master.default.svc.cluster.local OPENSEARCH_PROTOCOL=https CERTS_FROM_FILE=true ./opensearch.sh +# +# Local environment to remote OpenSearch, with OpenSearch security plugin on: +# OPENSEARCH_HOST=opensearch.cern.ch OPENSEARCH_PROTOCOL=https CERTS_FROM_FILE=true CERT_DIR=. ./opensearch.sh + +# OpenSearch address configuration +OPENSEARCH_PROTOCOL="${OPENSEARCH_PROTOCOL:-http}" # http or https +OPENSEARCH_HOST="${OPENSEARCH_HOST:-localhost}" +OPENSEARCH_PORT="${OPENSEARCH_PORT:-9200}" +OPENSEARCH_CN="${OPENSEARCH_CN:-$OPENSEARCH_HOST}" # OpenSearch common name in node's TLS certificate, defaults to OPENSEARCH_HOST +OPENSEARCH_ADDRESS=$OPENSEARCH_PROTOCOL://$OPENSEARCH_CN:$OPENSEARCH_PORT + +# Index mapping configuration +JOB_LOG_IDX="${JOB_LOG_IDX:-fluentbit-job_log}" +WORKFLOW_LOG_IDX="${WORKFLOW_LOG_IDX:-fluentbit-workflow_log}" +JOB_LOG_MAPPING_FILE_PATH="${JOB_LOG_MAPPING_FILE_PATH:-job_log_mapping.json}" +WORKFLOW_LOG_MAPPING_FILE_PATH="${WORKFLOW_LOG_MAPPING_FILE_PATH:-workflow_log_mapping.json}" + +# TLS configuration +KUBECTL_CERTS="${KUBECTL_CERTS:-false}" # Use admin certificates from Kubernetes secret; makes cURL use TLS while connecting to OpenSearch +CERTS_FROM_FILE="${CERTS_FROM_FILE:-false}" # Use admin certificates from files; makes cURL use TLS while connecting to OpenSearch +CERT_DIR="${CERT_DIR:-config/certs}" # If CERTS_FROM_FILE is true, the directory where the certificates are stored (files should be named ca.crt, admin.crt and admin.key) +KUBECTL_SECRET_NAME="${KUBECTL_SECRET_NAME:-reana-opensearch-tls-secret}" # If KUBECTL_CERTS is true, the secret name to use + +# Execute cURL when TLS is enabled +curl_exec_secure() { + curl --connect-to "$OPENSEARCH_CN:$OPENSEARCH_PORT:$OPENSEARCH_HOST" \ + --cacert <(echo "$CACERT") --cert <(echo "$CERT") --key <(echo "$KEY") \ + -H 'Content-Type: application/json' "$@" + echo "" +} + +# Execute cURL when TLS is disabled +curl_exec() { + curl -H 'Content-Type: application/json' "$@" + echo "" +} + +# Setup indices and mappings +run() { + echo "OpenSearch adderess: $OPENSEARCH_ADDRESS." + echo "Create indices (ignore errors if already created):" + echo -n "$JOB_LOG_IDX response: " + $1 -XPUT "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX" + + echo -n "$WORKFLOW_LOG_IDX response: " + $1 -XPUT "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX" + + + echo "Create mappings:" + echo -n "$JOB_LOG_IDX response: " + $1 -XPUT "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX/_mapping" \ + --data "@$JOB_LOG_MAPPING_FILE_PATH" + + echo -n "$WORKFLOW_LOG_IDX response: " + $1 -XPUT "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX/_mapping" \ + --data "@$WORKFLOW_LOG_MAPPING_FILE_PATH" + + + echo "Update mappings:" + echo -n "$JOB_LOG_IDX response: " + $1 -XPOST "$OPENSEARCH_ADDRESS/$JOB_LOG_IDX/_update_by_query" + + echo -n "$WORKFLOW_LOG_IDX response: " + $1 -XPOST "$OPENSEARCH_ADDRESS/$WORKFLOW_LOG_IDX/_update_by_query" + + echo "Done." +} + +# Execute script +if [[ $KUBECTL_CERTS = true ]]; then + echo "Using certificates from Kubernetes secret." + CACERT=$(kubectl get secret "$KUBECTL_SECRET_NAME" \ + -ogo-template='{{ index .data "ca.crt" | base64decode }}') + CERT=$(kubectl get secret "$KUBECTL_SECRET_NAME" \ + -ogo-template='{{ index .data "admin.crt" | base64decode }}') + KEY=$(kubectl get secret "$KUBECTL_SECRET_NAME" \ + -ogo-template='{{ index .data "admin.key" | base64decode }}') + run curl_exec_secure +elif [[ $CERTS_FROM_FILE = true ]]; then + echo "Using certificates from files." + CACERT=$(cat "$CERT_DIR"/ca.crt) + CERT=$(cat "$CERT_DIR"/admin.crt) + KEY=$(cat "$CERT_DIR"/admin.key) + run curl_exec_secure +else + echo "Not using TLS." + run curl_exec +fi diff --git a/scripts/opensearch/workflow_log_mapping.json b/scripts/opensearch/workflow_log_mapping.json new file mode 100644 index 00000000..ad37ed10 --- /dev/null +++ b/scripts/opensearch/workflow_log_mapping.json @@ -0,0 +1,66 @@ +{ + "dynamic": "strict", + "properties": { + "@timestamp": { + "type": "date" + }, + "kubernetes": { + "properties": { + "container_image": { + "type": "text" + }, + "container_name": { + "type": "text" + }, + "docker_id": { + "type": "text" + }, + "host": { + "type": "text" + }, + "labels": { + "properties": { + "batch.kubernetes.io/controller-uid": { "type": "text" }, + "batch.kubernetes.io/job-name": { "type": "text" }, + "controller-uid": { "type": "text" }, + "job-name": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "reana-run-batch-workflow-uuid": { + "type": "text", + "fields": { + "keyword": { + "type": "keyword" + } + } + }, + "reana_workflow_mode": { "type": "text" } + } + }, + "namespace_name": { + "type": "text" + }, + "pod_id": { + "type": "text" + }, + "pod_name": { + "type": "text" + } + } + }, + "log": { + "type": "text" + }, + "stream": { + "type": "text" + }, + "time": { + "type": "date" + } + } +}