From 66781d3e6dc299fe3876ca79a709f9ebca645b3c Mon Sep 17 00:00:00 2001 From: ns-sjadeja <140080339+ns-sjadeja@users.noreply.github.com> Date: Wed, 13 Mar 2024 03:43:58 +0530 Subject: [PATCH] Release 5.0.1 GA (#41) --- diagnose | 111 ++++++++++++--------- replica_set | 38 ++++++- setup | 280 ++++++++++++++++++++++++++++++++++++++++++++-------- start | 4 + stop | 6 +- 5 files changed, 344 insertions(+), 95 deletions(-) diff --git a/diagnose b/diagnose index efa2eef..536f5b6 100755 --- a/diagnose +++ b/diagnose @@ -7,6 +7,7 @@ green='\033[0;32m' yellow='\033[0;33m' SUBJECT=some-unique-id +DISCARD_MONGO_STRING_REGX="\\x1b]0;mongosh mongodb:\/\/@127\.0\.0\.1:27017\/cte?directConnection=true&serverSelectionTimeoutMS=2000\\x07\\x1b\[1G \\x1b\[1G" # --- Locks ------------------------------------------------------- LOCK_FILE=/tmp/$SUBJECT.lock @@ -262,7 +263,7 @@ out_docker_stats() { out_plugin_version(){ # Execute the command and filter the output - command_output=$($2 exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.$1.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d') + command_output=$($2 exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.$1.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") # Store the plugin values in a variable plugin_values=$(echo "$command_output" | grep -Po "(?<=plugin: ')[^']*") @@ -295,46 +296,52 @@ out_platform_details() { return 1 fi if [[ $RHEL: == *"Red Hat"* ]]; then - SETTINGS=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find()") - CE_VERSION=$(echo $SETTINGS | grep -Po "(?<=databaseVersion: ')[^']*") + SETTINGS=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{'databaseVersion': 1, 'uid':1, _id:0})" | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + CE_VERSION=$(echo "$SETTINGS" | grep -Po "(?<=databaseVersion: ')[^']*") echo "------- CE Platform Details -------" >&$OUT echo "> CE-Version: $CE_VERSION" >>$OUT - CE_UID=$(echo $SETTINGS | grep -Po "(?<=uid: ')[^']*") + CE_UID=$(echo "$SETTINGS" | grep -Po "(?<=uid: ')[^']*") echo "> CE-INSTALLATION ID: $CE_UID" >>$OUT echo -e "\n------- Proxy Details -------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find({},{proxy:{scheme: 1, server:1, username:1}, _id:0})" | sed -e '/podman/d' -e '/exit code/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{proxy:{scheme: 1, server:1, username:1}, _id:0})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- Settings Details -------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find({},{proxy:{scheme: 1, server:1, username:1}, _id:0, logLevel:1, logsCleanup:1, ssoEnable:1, forceAuth:1, secretsManagerSettings:{enabled:1, params:{authMethod:1, provider:1}}})" | sed -e '/podman/d' -e '/exit code/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{_id:0, logLevel:1, logsCleanup:1, ssoEnable:1, forceAuth:1, secretsManagerSettings:{enabled:1, params:{authMethod:1, provider:1}}})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- Module Details -------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find({},{platforms: 1, _id:0})" | sed -e '/podman/d' -e '/exit code/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{platforms: 1, _id:0})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- Plugin Repo Details -------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.repos.find({}, {name: 1, url:1, isDefault:1, _id:0}).toArray()" | sed -e '/podman/d' -e '/exit code/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.repos.find({}, {name: 1, url:1, isDefault:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- CTE Plugin Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/podman/d' -e '/exit code/d' >>$OUT - echo "Active Indicator(s): $($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.indicators.countDocuments({active:true})" | sed -e '/podman/d' -e '/exit code/d')" >> $OUT - echo "Not Active Indicator(s): $($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.indicators.countDocuments({active:false})" | sed -e '/podman/d' -e '/exit code/d')" >> $OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + active_indicators=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:true})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Active Indicator(s): $active_indicators" >> $OUT + not_active_indicators=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:false})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Not Active Indicator(s): $not_active_indicators" >> $OUT echo -e "\n------- CTO Plugin Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.itsm_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/podman/d' -e '/exit code/d' >>$OUT - echo "Alert(s): $($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.itsm_alerts.countDocuments({})" | sed -e '/podman/d' -e '/exit code/d')" >> $OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + alerts=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.itsm_alerts.countDocuments({})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Alert(s): $alerts" >> $OUT echo -e "\n------- URE Plugin Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/podman/d' -e '/exit code/d' >>$OUT - echo "User(s): $($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_users.countDocuments({type:'user'})" | sed -e '/podman/d' -e '/exit code/d')" >> $OUT - echo "Host(s): $($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_users.countDocuments({type:'host'})" | sed -e '/podman/d' -e '/exit code/d')" >> $OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + users=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'user'})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "User(s): $users" >> $OUT + hosts=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'host'})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Host(s): $hosts" >> $OUT echo -e "\n------- ARE Plugin Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.grc_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/podman/d' -e '/exit code/d' >>$OUT - echo "Application(s): $($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.grc_applications.countDocuments({})" | sed -e '/podman/d' -e '/exit code/d')" >> $OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + applications=$($podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.grc_applications.countDocuments({})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Application(s): $applications" >> $OUT echo -e "\n------- CLS Plugin Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cls_configurations.find({}, {name: 1, plugin:1, attributeMapping: 1, parameters: {transformData: 1}, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e 's/\"transformData\"/\"MappingTransformation\"/g' -e '/podman/d' -e '/exit code/d' >> $OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_configurations.find({}, {name: 1, plugin:1, attributeMapping: 1, parameters: {transformData: 1}, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e 's/\"transformData\"/\"MappingTransformation\"/g' -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >> $OUT echo -e "\n------- Configured Plugin Version --------\n" >>$OUT out_plugin_version "configurations" $podman_compose @@ -344,61 +351,67 @@ out_platform_details() { out_plugin_version "itsm_configurations" $podman_compose echo -e "\n------- CTE Business Rule Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cte_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cte_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- CTO Business Rule Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.itsm_business_rules.find({}, {name: 1, muted:1, alertFilters:1, dedupeRules:1, muteRules:1, queues:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_business_rules.find({}, {name: 1, muted:1, alertFilters:1, dedupeRules:1, muteRules:1, queues:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- URE Business Rule Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_business_rules.find({}, {name: 1, muted:1, userFilters:1, muteRules:1, actions:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_business_rules.find({}, {name: 1, muted:1, userFilters:1, muteRules:1, actions:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- ARE Business Rule Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.grc_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- CLS Business Rule Details --------" >>$OUT - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cls_business_rules.find({}, {name: 1, muted:1, filters:1, siemMappings:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_business_rules.find({}, {name: 1, muted:1, filters:1, siemMappings:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT else - SETTINGS=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find()") - CE_VERSION=$(echo $SETTINGS | grep -Po "(?<=databaseVersion: ')[^']*") + SETTINGS=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{'databaseVersion': 1, 'uid':1, _id:0})" | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + CE_VERSION=$(echo "$SETTINGS" | grep -Po "(?<=databaseVersion: ')[^']*") echo "------- CE Platform Details -------" >&$OUT echo "> CE-Version: $CE_VERSION" >>$OUT - CE_UID=$(echo $SETTINGS | grep -Po "(?<=uid: ')[^']*") + CE_UID=$(echo "$SETTINGS" | grep -Po "(?<=uid: ')[^']*") echo "> CE-INSTALLATION ID: $CE_UID" >>$OUT echo -e "\n------- Proxy Details -------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find({},{proxy:{scheme: 1, server:1, username:1}, _id:0})" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{proxy:{scheme: 1, server:1, username:1}, _id:0})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- Settings Details -------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find({},{proxy:{scheme: 1, server:1, username:1}, _id:0, logLevel:1, logsCleanup:1, ssoEnable:1, forceAuth:1, secretsManagerSettings:{enabled:1, params:{authMethod:1, provider:1}}})" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{_id:0, logLevel:1, logsCleanup:1, ssoEnable:1, forceAuth:1, secretsManagerSettings:{enabled:1, params:{authMethod:1, provider:1}}})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- Module Details -------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.settings.find({},{platforms: 1, _id:0})" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.settings.find({},{platforms: 1, _id:0})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- Plugin Repo Details -------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.repos.find({}, {name: 1, url:1, isDefault:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.repos.find({}, {name: 1, url:1, isDefault:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- CTE Plugin Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT - echo "Active Indicator(s): $($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.indicators.countDocuments({active:true})" | sed -e '/\"t\"/d')" >> $OUT - echo "Not Active Indicator(s): $($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.indicators.countDocuments({active:false})" | sed -e '/\"t\"/d')" >> $OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + active_indicators=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:true})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Active Indicator(s): $active_indicators" >> $OUT + not_active_indicators=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.indicators.countDocuments({active:false})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Not Active Indicator(s): $not_active_indicators" >> $OUT echo -e "\n------- CTO Plugin Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.itsm_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT - echo "Alert(s): $($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.itsm_alerts.countDocuments({})" | sed -e '/\"t\"/d')" >> $OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + alerts=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.itsm_alerts.countDocuments({})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Alert(s): $alerts" >> $OUT echo -e "\n------- URE Plugin Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT - echo "User(s): $($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_users.countDocuments({type:'user'})" | sed -e '/\"t\"/d')" >> $OUT - echo "Host(s): $($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_users.countDocuments({type:'host'})" | sed -e '/\"t\"/d')" >> $OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + users=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'user'})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "User(s): $users" >> $OUT + hosts=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.cre_users.countDocuments({type:'host'})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Host(s): $hosts" >> $OUT echo -e "\n------- ARE Plugin Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.grc_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT - echo "Application(s): $($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.grc_applications.countDocuments({})" | sed -e '/\"t\"/d')" >> $OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_configurations.find({}, {name: 1, plugin:1, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT + applications=$($docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.grc_applications.countDocuments({})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g") + echo "Application(s): $applications" >> $OUT echo -e "\n------- CLS Plugin Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cls_configurations.find({}, {name: 1, plugin:1, attributeMapping: 1, parameters: {transformData: 1}, lastRunAt:1, active:1, _id:0}).toArray()" | sed -e '/\"t\"/d' -e 's/\"transformData\"/\"MappingTransformation\"/g' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_configurations.find({}, {name: 1, plugin:1, attributeMapping: 1, parameters: {transformData: 1}, lastRunAt:1, active:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" -e 's/\"transformData\"/\"MappingTransformation\"/g' >>$OUT echo -e "\n------- Configured Plugin Version --------\n" >>$OUT out_plugin_version "configurations" $docker_compose @@ -408,19 +421,19 @@ out_platform_details() { out_plugin_version "itsm_configurations" $docker_compose echo -e "\n------- CTE Business Rule Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cte_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cte_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- CTO Business Rule Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.itsm_business_rules.find({}, {name: 1, muted:1, alertFilters:1, dedupeRules:1, muteRules:1, queues:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.itsm_business_rules.find({}, {name: 1, muted:1, alertFilters:1, dedupeRules:1, muteRules:1, queues:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- URE Business Rule Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cre_business_rules.find({}, {name: 1, muted:1, userFilters:1, muteRules:1, actions:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cre_business_rules.find({}, {name: 1, muted:1, userFilters:1, muteRules:1, actions:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- ARE Business Rule Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.grc_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.grc_business_rules.find({}, {name: 1, muted:1, filters:1, exceptions:1, sharedWith:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT echo -e "\n------- CLS Business Rule Details --------" >>$OUT - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.cls_business_rules.find({}, {name: 1, muted:1, filters:1, siemMappings:1, _id:0}).toArray()" | sed -e '/\"t\"/d' >>$OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); JSON.stringify(db.cls_business_rules.find({}, {name: 1, muted:1, filters:1, siemMappings:1, _id:0}).toArray(), null, 2)" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >>$OUT fi stop_spin "CE Details\t\t" $pid "success" echo $OUT @@ -502,9 +515,9 @@ out_tenant_names() { return 1 fi if [[ $RHEL: == *"Red Hat"* ]]; then - $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.tenants.find({},{_id:0, name:1, tenantName:1, initialRange:1, alert_types:1})" | sed -e '/podman/d' -e '/exit code/d' >> $OUT + $podman_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.tenants.find({},{_id:0, name:1, tenantName:1, initialRange:1, alert_types:1})" | sed -e '/podman/d' -e '/exit code/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >> $OUT else - $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.tenants.find({},{_id:0, name:1, tenantName:1, initialRange:1, alert_types:1})" | sed -e '/\"t\"/d' >> $OUT + $docker_compose exec mongodb-primary mongosh --quiet --username cteadmin --password $MAINTENANCE_PASSWORD cte --eval "db.getMongo().setReadPref('secondary'); db.tenants.find({},{_id:0, name:1, tenantName:1, initialRange:1, alert_types:1})" | sed -e '/\"t\"/d' | sed -r 's/\x1B\[(([0-9]{1,2})?(;)?([0-9]{1,2})?)?[m,K,H,f,J]//g' | sed -e "s/$DISCARD_MONGO_STRING_REGX//g" >> $OUT fi stop_spin "Tenant Names\t" $pid "success" echo $OUT diff --git a/replica_set b/replica_set index 9066dec..4c6bb85 100755 --- a/replica_set +++ b/replica_set @@ -8,8 +8,8 @@ from pymongo.errors import OperationFailure, ServerSelectionTimeoutError from dotenv import load_dotenv load_dotenv() -path = os.environ.get("LOCATION") -load_dotenv(path) +load_dotenv(os.environ.get("LOCATION")) +load_dotenv(f'{os.environ.get("HA_NFS_DATA_DIRECTORY")}/config/.env', override=True) MAINTENANCE_PASSWORD_ESCAPED = os.getenv("MAINTENANCE_PASSWORD_ESCAPED") HA_CURRENT_NODE = os.environ["HA_CURRENT_NODE"] @@ -50,6 +50,24 @@ def create_replica_set(connection_string): ] } + print("Validating if replica set is already initialized...") + retry_count = 0 + while retry_count < REPLICA_SET_INITIALIZE_RETRY_COUNT: + retry_count += 1 + try: + replica_status = client.admin.command("replSetGetStatus") + if replica_status.get("ok") == 1.0: + print("The MongoDB Replica set is already initialized.") + exit(1) + except ServerSelectionTimeoutError as ex: + if retry_count == REPLICA_SET_INITIALIZE_RETRY_COUNT: + raise ex + time.sleep(30) + except OperationFailure as error: + if 'NotYetInitialized' in str(error): + break + raise error + print("Initializing MongoDB replica set...") retry_count = 0 while retry_count < REPLICA_SET_INITIALIZE_RETRY_COUNT: @@ -65,6 +83,7 @@ def create_replica_set(connection_string): retry = 30 wait_time = 30 + settings_found = False while True: print("Waiting for migrations to be completed...") time.sleep(5) @@ -77,6 +96,9 @@ def create_replica_set(connection_string): retry -= 1 time.sleep(wait_time) continue + if not settings_found: + print("Settings collection found in database.") + settings_found = True db_version = dict(response).get("databaseVersion", "") if db_version == os.environ["INSTALL_VERSION"]: print("Migrations completed.") @@ -86,7 +108,6 @@ def create_replica_set(connection_string): print("Execute this command in remaining nodes to form a cluster:\n > ./start") break except OperationFailure as ex: - print("The MongoDB Replica set is already initialized.") raise ex finally: client.close() @@ -125,6 +146,17 @@ def update_connection_info(old_ip, new_ip): with open(".env", "w+") as env_file: env_file.write(content) + # Update shared .env file + env_location = f'{os.environ.get("HA_NFS_DATA_DIRECTORY")}/config/.env' + with open(env_location, "r") as env_file: + content = env_file.read() + content = content.replace( + f"HA_PRIMARY_NODE_IP={old_ip}", + f"HA_PRIMARY_NODE_IP={new_ip}" + ) + with open(env_location, "w+") as env_file: + env_file.write(content) + def remove_replica_set(connection_string): print("Removing node from the MongoDB replica set...") diff --git a/setup b/setup index 9f5b4c6..3aaa485 100755 --- a/setup +++ b/setup @@ -82,6 +82,7 @@ DEFAULT_INPUTS = { "REQUESTS_TIMEOUT": {"default": 300, "skip": True, "help": ""}, "POPEN_TIMEOUT": {"default": 1800, "skip": True, "help": ""}, "IS_MPASS_CONFIGURED": {"default": True, "skip": True, "help": ""}, + "MONGO_COMPATIBILITY": {"default": False, "skip": True, "help": ""}, "RABBITMQ_COOKIE": {"default": "", "skip": True, "help": ""}, "LOCATION": {"default": "", "skip": True, "help": ""}, "COMPOSE_HTTP_TIMEOUT": {"default": 600, "skip": True, "help": ""}, @@ -99,7 +100,7 @@ MIN_PODMAN_VERSION = "3.4.2" MIN_PODMAN_COMPOSE_VERSION = "1.0.3" N = 5 -CURRENT_VERSION = "5.0.0" +CURRENT_VERSION = "5.0.1" PREVIOUS_VERSION = "4.2.0" CURRENT_BETA_VERSION = "5.0.1" CE_VERSION_1 = f"CE v{N}-{CURRENT_VERSION}" @@ -115,17 +116,27 @@ CORE_VERSION_2_TAG = ( UI_VERSION_2_TAG = ( f"netskopetechnicalalliances/cloudexchange:ui{N-1}-{PREVIOUS_VERSION}" ) -CORE_VERSION_BETA_TAG = f"netskopetechnicalalliances/cloudexchange:core-{CURRENT_BETA_VERSION}-beta" -UI_VERSION_BETA_TAG = f"netskopetechnicalalliances/cloudexchange:ui-{CURRENT_BETA_VERSION}-beta" +CORE_VERSION_BETA_TAG = ( + f"netskopetechnicalalliances/cloudexchange:core-{CURRENT_BETA_VERSION}-beta" +) +UI_VERSION_BETA_TAG = ( + f"netskopetechnicalalliances/cloudexchange:ui-{CURRENT_BETA_VERSION}-beta" +) CORE_VERSION_1_LATEST_TAG = f"netskopetechnicalalliances/cloudexchange:core{N}-latest" UI_VERSION_1_LATEST_TAG = f"netskopetechnicalalliances/cloudexchange:ui{N}-latest" CORE_VERSION_2_LATEST_TAG = f"netskopetechnicalalliances/cloudexchange:core{N-1}-latest" UI_VERSION_2_LATEST_TAG = f"netskopetechnicalalliances/cloudexchange:ui{N-1}-latest" +RECOMMENDED_DOCKER_VERSION = "25.0.3" +RECOMMENDED_DOCKER_COMPOSE_VERSION = "2.24.6" +RECOMMENDED_PODMAN_VERSION = "4.6.1" +RECOMMENDED_PODMAN_COMPOSE_VERSION = "1.0.6" + GIT_PLUGIN_REPO = "https://github.com/netskopeoss/ta_cloud_exchange_plugins.git" MONGO_USER_ID = 999 +MONGO_MIGRATION_TIMEOUT = 60 is_ui_running = False is_rabbitmq_running = False @@ -207,14 +218,7 @@ def ce_as_vm_check(): def fetch_container_info(): global is_ui_running, is_rabbitmq_running, is_mongodb_running try: - p = subprocess.Popen( - ["docker", "ps", "--format", "json"], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE, - ) - out, err = p.communicate() - if err: + if isRedHat(): p = subprocess.Popen( ["podman", "ps", "--format", "json"], stderr=subprocess.PIPE, @@ -224,11 +228,17 @@ def fetch_container_info(): out, err = p.communicate() containers = json.loads(out.decode("utf-8")) if err: - print_warning( - f"Unable to fetch container information. {err.decode('utf-8')}\n" - ) - return + raise Exception("Unable to fetch container information.") else: + p = subprocess.Popen( + ["docker", "ps", "--format", "json"], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE, + ) + out, err = p.communicate() + if err: + raise Exception("Unable to fetch container information.") containers = [json.loads(line) for line in out.decode("utf-8").splitlines()] for container_info in containers: @@ -356,11 +366,61 @@ def create_env_if_not_exist(location=".env"): raise Exception(f"Error occurred while creating file: {e}") -def get_all_existed_env_variable(location=".env"): - AVAILABLE_INPUTS["CORE_HTTP_PROXY"] = "" - AVAILABLE_INPUTS["CORE_HTTPS_PROXY"] = "" - AVAILABLE_INPUTS["DESTINATION_PATH"] = "" +def execute_shell_command(command): + """Function to exectute shell command using python script.""" + p = None + try: + p = subprocess.check_output(command, shell=True) + except Exception as e: + if p: + p.kill() + raise Exception(f"Error occurred while executing command. Error: {e}") + +def run_temp_mongo_container_for_migration( + maintenance_password, http_proxy, https_proxy +): + """Migrate mongo data by running temp mongo container and update featurecompatibilityversion variable from mongo.""" + deployment = None + MONGO_MIGRATE_IMAGE = "mongo:5.0.21" + temp_mongo_container_name = "mongo-migration" + + if isRedHat(): + deployment = "podman" + else: + deployment = "docker" + + # If from previous temp container is still running then kill that container and start again. + remove_container = f"{deployment} rm -f {temp_mongo_container_name} >/dev/null 2>&1" + execute_shell_command(remove_container) + time.sleep(5) + + create_container = f"{deployment} run -d -t -e MONGODB_ADVERTISED_HOSTNAME=mongodb-primary \ + -e MONGO_INITDB_ROOT_USERNAME=root \ + -e MONGO_INITDB_ROOT_PASSWORD={maintenance_password} \ + -e MONGO_INITDB_DATABASE=cte \ + -e MONGODB_USERNAME=cteadmin \ + -e MONGODB_PASSWORD={maintenance_password} \ + -e HTTP_PROXY=${http_proxy} \ + -e HTTPS_PROXY=${https_proxy} \ + -v ./data/mongo-data/data/db:/data/db:z \ + --name {temp_mongo_container_name} \ + index.docker.io/{MONGO_MIGRATE_IMAGE} >/dev/null 2>&1" + execute_shell_command(create_container) + time.sleep(MONGO_MIGRATION_TIMEOUT // 2) + + eval_command = """'db.adminCommand({setFeatureCompatibilityVersion: "5.0"})'""" + compatibility_command = f"{deployment} exec {temp_mongo_container_name} mongo -u root --password {maintenance_password} admin --eval {eval_command}" + execute_shell_command(compatibility_command) + + time.sleep(MONGO_MIGRATION_TIMEOUT // 2) + remove_container = f"{deployment} rm -f {temp_mongo_container_name} >/dev/null 2>&1" + execute_shell_command(remove_container) + + print_pass("The migration of database has been successfully completed.") + + +def get_all_existed_env_variable(location=".env", override=True): try: with open(location, "r") as f: if os.stat(location).st_size > 0: @@ -368,7 +428,8 @@ def get_all_existed_env_variable(location=".env"): for line in f.readlines(): backup.write(line) key, value = line.split("=", 1) - AVAILABLE_INPUTS[key] = value.strip() + if override or key not in AVAILABLE_INPUTS: + AVAILABLE_INPUTS[key] = value.strip() if AVAILABLE_INPUTS.get("HTTPS_PROXY"): AVAILABLE_INPUTS["CORE_HTTP_PROXY"] = AVAILABLE_INPUTS["HTTPS_PROXY"] @@ -514,7 +575,7 @@ def check_machine_specs(): if is_ha: try: port_flag = True - for port in [4369, 5672, 15672, 25672]: + for port in [4369, 5672, 15672, 25672, 35672]: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) result = sock.connect_ex(("127.0.0.1", port)) if result == 0: @@ -569,6 +630,8 @@ def check_docker_versions(): f"Docker Version {docker_version} (Minimum {MIN_DOCKER_VERSION} is required)" ) CHECKS["Docker"] = Status.FAIL + if docker_version != RECOMMENDED_DOCKER_VERSION: + print_warning(f"The recommended docker version is {RECOMMENDED_DOCKER_VERSION}") command = "docker-compose --version" p = subprocess.check_output(command, shell=True) @@ -581,6 +644,9 @@ def check_docker_versions(): f"Docker Compose Version {docker_compose_version} (Minimum {MIN_DOCKER_COMPOSE_VERSION} is required)" ) CHECKS["Docker-Compose"] = Status.FAIL + if docker_compose_version != RECOMMENDED_DOCKER_COMPOSE_VERSION: + print_warning(f"The recommended docker-compose version is {RECOMMENDED_DOCKER_COMPOSE_VERSION}") + except Exception as e: CHECKS["Docker-Compose"] = Status.NOT_VERIFIED CHECKS["Docker"] = Status.NOT_VERIFIED @@ -610,6 +676,8 @@ def check_podman_versions(): f"Podman Version {podman_version} (Minimum {MIN_PODMAN_VERSION} is required)" ) CHECKS["Podman"] = Status.FAIL + if podman_version != RECOMMENDED_PODMAN_VERSION: + print_warning(f"The recommended podman version is {RECOMMENDED_PODMAN_VERSION}") podman_compose_version = "" for line in lines: @@ -627,6 +695,8 @@ def check_podman_versions(): f"Podman Compose Version {podman_compose_version} (Minimum {MIN_PODMAN_COMPOSE_VERSION} is required)" ) CHECKS["Podman-Compose"] = Status.FAIL + if podman_compose_version != RECOMMENDED_PODMAN_COMPOSE_VERSION: + print_warning(f"The recommended podman-compose version is {RECOMMENDED_PODMAN_COMPOSE_VERSION}") command = "rpm -qa" p = subprocess.check_output(command, shell=True) @@ -777,7 +847,7 @@ def ask_for_initial_inputs(): if isProxy == "y" or isProxy == "yes": # HTTPS_PROXY - cur_https_proxy = parse_proxy(AVAILABLE_INPUTS["CORE_HTTPS_PROXY"]) + cur_https_proxy = parse_proxy(AVAILABLE_INPUTS.get("CORE_HTTPS_PROXY", None)) print( f"\nEnter Proxy URL (e.g. http(s)://10.0.0.0:3128) {('(Current: ' + cur_https_proxy + ')') if cur_https_proxy is not None else ''}" @@ -1386,7 +1456,7 @@ def migrate_mongo(): def copy_env(inputs): while True: - if AVAILABLE_INPUTS["DESTINATION_PATH"] == "": + if AVAILABLE_INPUTS.get("DESTINATION_PATH", "") == "": destination = input( "> Enter the destination path (e.g. './backup_env'): " ).strip() @@ -1395,7 +1465,9 @@ def copy_env(inputs): f"\n> Enter the destination path for backup of .env (Current: \"{AVAILABLE_INPUTS['DESTINATION_PATH']}\"): " ).strip() destination = ( - AVAILABLE_INPUTS["DESTINATION_PATH"] if destination == "" else destination + AVAILABLE_INPUTS.get("DESTINATION_PATH", "") + if destination == "" + else destination ) if destination.strip() == get_secret_location(inputs).strip(): print_warning( @@ -1418,7 +1490,7 @@ def copy_env(inputs): print("backup completed...") break else: - AVAILABLE_INPUTS["DESTINATION_PATH"] == "" + AVAILABLE_INPUTS.get("DESTINATION_PATH", "") == "" raise Exception(err.decode("utf-8")) except Exception as e: if p: @@ -1428,7 +1500,7 @@ def copy_env(inputs): def take_env_backup(inputs): - if AVAILABLE_INPUTS["DESTINATION_PATH"] == "": + if AVAILABLE_INPUTS.get("DESTINATION_PATH", "") == "": isBackup = ( input("\n\n> Do you want to take backup of .env file? [y/n]: ") .strip() @@ -1462,11 +1534,11 @@ def assign_memory_to_rabbitmq_container(): mem_bytes = get_total_physical_memory() if not mem_bytes: return - mem_mib = mem_bytes/(1024**2) + mem_mib = mem_bytes / (1024**2) - AVAILABLE_INPUTS["RABBITMQ_RAM_RESOURCES"] = round(mem_mib * (6/32)) - AVAILABLE_INPUTS["MONGODB_RAM_RESOURCES"] = round(mem_mib * (4/32)) - AVAILABLE_INPUTS["CORE_RAM_RESOURCES"] = round(mem_mib * (18/32)) + AVAILABLE_INPUTS["RABBITMQ_RAM_RESOURCES"] = round(mem_mib * (6 / 32)) + AVAILABLE_INPUTS["MONGODB_RAM_RESOURCES"] = round(mem_mib * (4 / 32)) + AVAILABLE_INPUTS["CORE_RAM_RESOURCES"] = round(mem_mib * (18 / 32)) AVAILABLE_INPUTS["UI_RAM_RESOURCES"] = 256 except Exception: pass @@ -1795,12 +1867,17 @@ def update_yaml_file(filename): "RABBITMQ_USE_LONGNAME=true" ) + for service in compose_data["services"]: + compose_data["services"][service]["env_file"] = [ + f"{AVAILABLE_INPUTS['HA_NFS_DATA_DIRECTORY']}/config/.env", "${LOCATION}"] + # Add Ports compose_data["services"]["rabbitmq-stats"]["ports"] = [ "4369:4369", "5672:5672", "15672:15672", "25672:25672", + "35672:35672" ] compose_data["services"]["mongodb-primary"]["ports"] = ["27017:27017"] @@ -1904,7 +1981,44 @@ def ask_for_ha_values(primary=False): ) # Keep the static value for now. - AVAILABLE_INPUTS["SYNC_STRATEGY"] = "2" + AVAILABLE_INPUTS["SYNC_STRATEGY"] = "1" + + +def generate_dependency_message(version_dict): + required_versions = [] + for dep, versions in version_dict.items(): + version_string = [] + if "min_version" in versions: + version_string.append(f">={versions['min_version']}") + if "max_version" in versions: + version_string.append(f"<={versions['max_version']}") + required_versions.append(dep + ", ".join(version_string)) + return " | ".join(required_versions) + + +def check_python_prerequisites(): + print("\nVerifying the Python prerequisites...") + try: + version_dict = { + "pyyaml": {"min_version": "6.0.0"}, + "python-dotenv": {"min_version": "0.20.0", "max_version": "1.0.0"}, + "pymongo": {"min_version": "4.1.1", "max_version": "4.3.3"}, + } + import pkg_resources + + for package, version in version_dict.items(): + module_version = pkg_resources.get_distribution(package).version + if "min_version" in version: + if not compare_versions(module_version, version["min_version"]): + raise Exception(f"Invalid version of the '{package}' package") + if "max_version" in version: + if not compare_versions(version["max_version"], module_version): + raise Exception(f"Invalid version of the '{package}' package") + except Exception as err: + print_fail(f"{err}. Please make sure the dependencies ({generate_dependency_message(version_dict)}) " + "are satisfied before running ./setup script.") + exit(1) + print_pass("Python prerequisites verified successfully.") def set_up_ha_values(): @@ -1927,19 +2041,49 @@ def set_up_ha_values(): AVAILABLE_INPUTS.pop("HA_IP_LIST", None) AVAILABLE_INPUTS.pop("HA_CURRENT_NODE", None) AVAILABLE_INPUTS.pop("HA_PRIMARY_NODE_IP", None) + # previously we have HA and now we want to setup on-prem/OVA. + if ( + "IS_MPASS_CONFIGURED" in AVAILABLE_INPUTS.keys() + and AVAILABLE_INPUTS.get("LOCATION", None) + and AVAILABLE_INPUTS.get("LOCATION", None) + not in [".env.keys", "/etc/.env.keys"] + ): + dest_location = None + env_file_location = None + if ce_as_vm_check(): + dest_location = "/etc/" + env_file_location = "/etc/.env.keys" + else: + dest_location = "." + env_file_location = ".env.keys" + command = ( + f"sudo mv {AVAILABLE_INPUTS.get('LOCATION', None)} {dest_location}" + ) + execute_shell_command(command) + AVAILABLE_INPUTS["LOCATION"] = env_file_location return break + check_python_prerequisites() + while True: location = input( f"> Please provide the location of the NFS mount directory (Current: '{AVAILABLE_INPUTS.get('HA_NFS_DATA_DIRECTORY', '')}'): " ).strip().rstrip("/") or AVAILABLE_INPUTS.get("HA_NFS_DATA_DIRECTORY", "") if os.path.isdir(location): + # previously we have on-prem / OVA and now we want to setup HA. + if is_ha and AVAILABLE_INPUTS.get("LOCATION", None) in [ + ".env.keys", + "/etc/.env.keys", + ]: + command = f"sudo mv {AVAILABLE_INPUTS.get('LOCATION')} {location}/" + execute_shell_command(command) + AVAILABLE_INPUTS["LOCATION"] = f"{location}/.env.keys" break print("The provided path doesn't exist. Please provide a valid path.") try: get_all_existed_env_variable( - f"{location}/config/.env" + f"{location}/config/.env", override=False ) # Get env variables from shared location except Exception: pass # For fresh setup, shared .env doesn't exist @@ -1952,29 +2096,70 @@ def set_up_ha_values(): if not ce_as_vm_check(): update_yaml_file("podman-compose") + def parse_selinux_status(output): selinux_info = {} - lines = output.splitlines() for line in lines: - key, value = map(str.strip, line.split(':')) + key, value = map(str.strip, line.split(":")) selinux_info[key] = value - return selinux_info + +def mongo_feature_compatibility(maintenance_password): + """ + Make mongo data compatibility when migrating from older to newer version. + """ + is_mongo_compatible = ( + True + if get_existing_variable_value(".env", "MONGO_COMPATIBILITY") == "True" + else False + ) + mongo_data_path = "data/mongo-data/data/db/" + if not is_mongo_compatible and len(os.listdir(mongo_data_path)) > 1: + print() + print_warning( + "The migration process for database is currently underway. Kindly wait for a few minutes." + ) + run_temp_mongo_container_for_migration( + maintenance_password, + AVAILABLE_INPUTS.get("CORE_HTTP_PROXY", ""), + AVAILABLE_INPUTS.get("CORE_HTTPS_PROXY", ""), + ) + AVAILABLE_INPUTS["MONGO_COMPATIBILITY"] = True + + def is_selinux_enabled(): try: - process = subprocess.Popen(['sestatus'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process = subprocess.Popen( + ["sestatus"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) stdout, _ = process.communicate() if process.returncode == 0: status_output = stdout.decode("utf-8").strip() selinux_info = parse_selinux_status(status_output) - return selinux_info.get("SELinux status", "").strip().lower() == "enabled" + return selinux_info.get("SELinux status", "").strip().lower() == "enabled" else: return False except Exception: return False + +def get_existing_variable_value(filepath, key_name): + """Retrieve the value of an environment variable from a specific file path.""" + try: + with open(filepath, "r") as f: + if os.stat(filepath).st_size > 0: + for line in f.readlines(): + key, value = line.split("=", 1) + if key == key_name: + return value.strip() + except Exception as e: + raise Exception( + f"Error occured while getting maintenance password. Error: {e}." + ) + + def run(): try: parser = argparse.ArgumentParser( @@ -2005,13 +2190,18 @@ def run(): pull_from_github() if is_selinux_enabled(): if not should_ignore: - print_fail("SE Linux is enabled. Please DISABLE SE Linux for successful deployment of Netskope Cloud Exchange.") + print_fail( + "SE Linux is enabled. Please DISABLE SE Linux for successful deployment of Netskope Cloud Exchange." + ) exit(1) else: - print_warning("Ignoring the SE linux... '--ignore-failures' flag applied.") + print_warning( + "Ignoring the SE linux... '--ignore-failures' flag applied." + ) fetch_container_info() create_env_if_not_exist() if location: + check_python_prerequisites() get_all_existed_env_variable() # Get local env variables get_all_existed_env_variable( f"{location}/config/.env" @@ -2090,9 +2280,19 @@ def run(): f"CA certificate successfully found in the {custom_cert_path} folder." ) assign_memory_to_rabbitmq_container() + mpass = None + if AVAILABLE_INPUTS.get("MAINTENANCE_PASSWORD", ""): + mpass = AVAILABLE_INPUTS.get("MAINTENANCE_PASSWORD", "") + elif AVAILABLE_INPUTS.get("LOCATION", "") and os.path.exists( + AVAILABLE_INPUTS["LOCATION"] + ): + mpass = get_existing_variable_value( + AVAILABLE_INPUTS["LOCATION"], "MAINTENANCE_PASSWORD" + ) + mongo_feature_compatibility(mpass) + put_env_variable(AVAILABLE_INPUTS) take_env_backup(AVAILABLE_INPUTS) - message = "\nSetup completed successfully...\n\nExecute this command to start the CE:\n >" if location: put_env_variable(AVAILABLE_INPUTS, f"{location}/config/.env") diff --git a/start b/start index d2ebd7b..891f622 100755 --- a/start +++ b/start @@ -3,6 +3,10 @@ . ./.env set -e +if [ -n "${HA_NFS_DATA_DIRECTORY}" ]; then + . ${HA_NFS_DATA_DIRECTORY}/config/.env +fi + if [ -f "/.cloud_exchange_vm.marker" ]; then file_path="/opt/cloudexchange/containers/ce_containers.tar" if [ -f "$file_path" ] && [ -z $SKIP_IMPORT ]; then diff --git a/stop b/stop index a177b4c..1e4676f 100755 --- a/stop +++ b/stop @@ -35,15 +35,15 @@ if [ $? -eq 127 ]; then exit 1; else if [ -z "${HA_IP_LIST}" ]; then - podman-compose -f podman-compose.yml down + podman-compose -f podman-compose.yml down -v else podman-compose -f podman-compose-ha.yml down -v fi fi else if [ -z "${HA_IP_LIST}" ]; then - docker-compose stop + docker-compose down -v else - docker-compose -f docker-compose-ha.yml stop + docker-compose -f docker-compose-ha.yml down -v fi fi