From eac5fea8dfe6ac1e5c544b9a7cf3dbc9375430dd Mon Sep 17 00:00:00 2001 From: Viktor Ribchev Date: Tue, 23 Jan 2024 15:50:03 +0200 Subject: [PATCH] Added installation of Telegraf. Added provisioning of GraphDB backup script. Added keepalive and file max size settings. Improved logging of the GraphDB install script. Added script descriptions Removed dependency on application config resource from backup script Added support for single instance GDB to the backup script Changed use_azure_cli_auth to true --- CHANGELOG.md | 13 ++--- README.md | 24 ++++----- azure.pkr.hcl | 17 ++++--- build.pkr.hcl | 3 +- create_image_definition.sh | 23 ++++++--- files/graphdb_backup | 72 ++++++++++++++++++++++++++ files/install_graphdb.sh | 102 ++++++++++++++++++++++++++++++++----- variables.pkr.hcl | 91 ++++++++++++++++++++++----------- 8 files changed, 270 insertions(+), 75 deletions(-) create mode 100644 files/graphdb_backup diff --git a/CHANGELOG.md b/CHANGELOG.md index ec74ab9..578bb63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,15 +2,16 @@ All notable changes to the Packer template for creating GraphDB Azure VM images will be documented in this file. -## 1.2.0 - -- Removed features preventing build with Standard_B1ls VM -- Fixed create_image_definition.sh -- Added support for specifying subscription in create_image_definition.sh - ## 1.1.0 - Installed Azure CLI +- Removed features preventing build with Standard_B1ls VM. +- Fixed create_image_definition.sh +- Added subscription as an attribute in the [create_image_definition.sh](create_image_definition.sh) helper script. +- Added installation of Telegraf. +- Added provisioning of GraphDB backup script. +- Added keepalive and file max size settings. +- Improved logging of the GraphDB install script. ## 1.0.0 diff --git a/README.md b/README.md index a077ff1..0003275 100644 --- a/README.md +++ b/README.md @@ -34,17 +34,17 @@ Follow these steps to build an AMI for GraphDB using Packer: replication regions, subscription, client and tennat IDs. To do so, create a variables file `variables.pkrvars.hcl`, example file: ```bash - subscription_id = "" - client_id = "" - client_secret = "" - tenant_id = "" - primary_location = "East US" - replication_regions = ["North Europe", "UK South"] - image_definition_name = "10.4.0-x86_64" - gdb_version = "10.4.0" - gallery_resource_group = "Packer-RG" - gallery_name = "GraphDB" - my_ip_address = "" + subscription_id = "" + client_id = "" + client_secret = "" + tenant_id = "" + primary_location = "East US" + replication_regions = ["North Europe", "UK South"] + image_definition_name = "10.4.0-x86_64" + gdb_version = "10.4.0" + gallery_resource_group = "Packer-RG" + gallery_name = "GraphDB" + allowed_inbound_ip_addresses = "" ``` 4. **Build the AMI**: @@ -105,7 +105,7 @@ The following points can be customized in a packer variables file `variables.pkr **Networking Configuration** -* my_ip_address (string): Your IP address for network security settings. +* allowed_inbound_ip_addresses (string): Specify the list of IP addresses and CIDR blocks that should be allowed access to the VM. **OS and Image Defaults** diff --git a/azure.pkr.hcl b/azure.pkr.hcl index cde77f8..a7befe2 100644 --- a/azure.pkr.hcl +++ b/azure.pkr.hcl @@ -1,15 +1,16 @@ locals { timestamp = regex_replace(timestamp(), "[- TZ:]", "") - # https://github.com/hashicorp/packer-plugin-azure/issues/65 + # See https://github.com/hashicorp/packer-plugin-azure/issues/65 version_timestamp = formatdate("YYYY.MM.DD", timestamp()) } source azure-arm ubuntu-x86-64 { - client_id = var.client_id - client_secret = var.client_secret - subscription_id = var.subscription_id - tenant_id = var.tenant_id - location = var.primary_location + use_azure_cli_auth = var.use_azure_cli_auth + client_id = var.client_id + client_secret = var.client_secret + subscription_id = var.subscription_id + tenant_id = var.tenant_id + location = var.primary_location shared_image_gallery_destination { subscription = var.subscription_id @@ -38,7 +39,7 @@ source azure-arm ubuntu-x86-64 { image_sku = var.image_sku vm_size = var.vm_size - allowed_inbound_ip_addresses = [var.my_ip_address] + allowed_inbound_ip_addresses = var.allowed_inbound_ip_addresses } #TODO Not yet supported by Packer https://github.com/hashicorp/packer/issues/12188#issuecomment-1380736642 @@ -72,5 +73,5 @@ source azure-arm ubuntu-x86-64 { # # vm_size = "Standard_D2ps_v5" # -# allowed_inbound_ip_addresses = [var.my_ip_address] +# allowed_inbound_ip_addresses = [var.allowed_inbound_ip_addresses] #} diff --git a/build.pkr.hcl b/build.pkr.hcl index 361c84e..154d876 100644 --- a/build.pkr.hcl +++ b/build.pkr.hcl @@ -8,7 +8,8 @@ build { "./files/graphdb.env", "./files/graphdb.service", "./files/graphdb-cluster-proxy.service", - "./files/install_graphdb.sh" + "./files/install_graphdb.sh", + "./files/graphdb_backup" ] destination = "/tmp/" } diff --git a/create_image_definition.sh b/create_image_definition.sh index 8429ad5..fe3c675 100644 --- a/create_image_definition.sh +++ b/create_image_definition.sh @@ -1,5 +1,10 @@ #!/bin/bash +# This script sets up Azure Shared Image Gallery (SIG) image definition and builds an image using Packer. +# It checks for a variables file, extracts necessary parameters, constructs an Azure CLI command for image definition creation, +# executes the command, waits for creation completion, +# and finally uses Packer to build the image based on the specified variables. + set -euxo pipefail # Required when running the script on Windows @@ -10,22 +15,25 @@ variables_file="variables.pkrvars.hcl" # Check if the file exists if [ -f "$variables_file" ]; then - # Use grep to extract the variable values - + # Extracts required variables from the variables_file subscription_id=$(grep 'subscription_id' "$variables_file" | cut -d '"' -f 2) + echo "Subscription ID: $subscription_id" image_definition_name=$(grep 'image_definition_name' "$variables_file" | cut -d '"' -f 2) + echo "Image definition name: $image_definition_name" gdb_version=$(grep 'gdb_version' "$variables_file" | cut -d '"' -f 2) + echo "GraphDB version: $gdb_version" gallery_resource_group=$(grep 'gallery_resource_group' "$variables_file" | cut -d '"' -f 2) + echo "Resource group: $gallery_resource_group" gallery_name=$(grep 'gallery_name' "$variables_file" | cut -d '"' -f 2) + echo "Gallery: $gallery_name" - # Check if any of the required variables is empty - if [ -z "$image_definition_name" ] || [ -z "$gdb_version" ] || [ -z "$gallery_resource_group" ] || [ -z "$gallery_name" ]; then + # Checks if any of the required variables is empty + if [ -z "$image_definition_name" ] || [ -z "$gdb_version" ] || [ -z "$gallery_resource_group" ] || [ -z "$gallery_name" ] || [ -z "$subscription_id" ]; then echo "One or more required variables are not defined in $variables_file." exit 1 fi - # Construct the az sig image-definition create command -# TODO extend command with --features 'IsAcceleratedNetworkSupported=true DiskControllerTypes=SCSI,NVMe' when we have quota for Standard_B2pts_v2 + # Constructs the az sig image-definition create command az_command="az sig image-definition create \ --subscription $subscription_id \ -g $gallery_resource_group \ @@ -42,8 +50,11 @@ if [ -f "$variables_file" ]; then --maximum-memory 128 " echo "Extracted variables and constructed Azure CLI command:" + echo "Creating SIG" eval "$az_command" + # Waits for the Shared Image Gallery to be created az sig image-definition wait -i "$image_definition_name" -r "$gallery_name" -g "$gallery_resource_group" --created --subscription $subscription_id + echo "Begin building of the Azure VM image" packer build -var-file="variables.pkrvars.hcl" . else echo "The variables file $variables_file does not exist." diff --git a/files/graphdb_backup b/files/graphdb_backup new file mode 100644 index 0000000..4dbd197 --- /dev/null +++ b/files/graphdb_backup @@ -0,0 +1,72 @@ +#!/bin/bash + +# This script utilizes the cloud backup functionality of GraphDB. +# The script accepts four variables in the following order: graphdb_user, graphdb_password, backup_storage_account_name, backup_storage_container_name + +set -euo pipefail + +az login --identity + +GRAPHDB_USER="${1}" +GRAPHDB_PASSWORD="${2}" +NODE_STATE="$(curl --silent --fail --user "$GRAPHDB_USER:$GRAPHDB_PASSWORD" localhost:7200/rest/cluster/node/status | jq -r .nodeState)" +BACKUP_NAME="$(date +'%Y-%m-%d_%H-%M-%S').tar" +BACKUP_STORAGE_ACCOUNT_NAME="${3}" +BACKUP_STORAGE_CONTAINER_NAME="${4}" +max_retries=3 +retry_count=0 + +perform_backup() { + while [ "$retry_count" -lt "$max_retries" ]; do + start_time=$(date +%s) + + response_code=$(curl -X POST --write-out %{http_code} --silent --output /dev/null \ + --header 'Content-Type: application/json' \ + -u "$GRAPHDB_USER:$GRAPHDB_PASSWORD" \ + --header 'Accept: application/json' \ + -d "{\"bucketUri\": \"az://${BACKUP_STORAGE_CONTAINER_NAME}/${BACKUP_NAME}?blob_storage_account=${BACKUP_STORAGE_ACCOUNT_NAME}\", \"backupOptions\": {\"backupSystemData\": true}}" \ + 'http://localhost:7200/rest/recovery/cloud-backup' + ) + + end_time=$(date +%s) + elapsed_time=$((end_time - start_time)) + + if [ "$response_code" -eq 200 ]; then + echo "Backup uploaded successfully to ${BACKUP_STORAGE_ACCOUNT_NAME} in $elapsed_time seconds." + break + else + echo "Failed to complete the backup and upload. HTTP Response Code: $response_code" + echo "Request took: $elapsed_time" + + if [ "$retry_count" -eq "$max_retries" ]; then + echo "Max retries reached. Backup could not be created. Exiting..." + return 1 + else + echo "Retrying..." + fi + + ((retry_count=retry_count + 1)) + sleep 5 + fi + done +} + +# Checks if GraphDB is running in cluster +IS_CLUSTER=$( + curl -s -o /dev/null \ + -u "$GRAPHDB_USER:$GRAPHDB_PASSWORD" \ + -w "%{http_code}" \ + http://localhost:7200/rest/monitor/cluster +) + +if [ "$IS_CLUSTER" == 200 ]; then + # Checks if the current GraphDB instance is Leader, otherwise exits. + if [ "$NODE_STATE" != "LEADER" ]; then + echo "The current node is not the leader, but $NODE_STATE" + exit 0 + fi + perform_backup +elif [ "$IS_CLUSTER" == 503 ]; then + perform_backup +fi + diff --git a/files/install_graphdb.sh b/files/install_graphdb.sh index e3b1b54..b25eb79 100644 --- a/files/install_graphdb.sh +++ b/files/install_graphdb.sh @@ -1,35 +1,70 @@ #!/bin/bash -set -euxo pipefail +# This script performs the following tasks: +# * Sets the timezone to UTC. +# * Installs necessary tools such as bash-completion, jq, nvme-cli, openjdk-11-jdk, and unzip. +# * Installs the Azure CLI. +# * Creates a system user "graphdb" for GraphDB service. +# * Creates GraphDB directories and sets up the necessary permissions. +# * Downloads and installs GraphDB, configuring systemd for GraphDB and GraphDB proxy. +# * Installs Telegraf for monitoring. +# * Adjusts system settings for keepalive and file max size. +# * Provisions a backup script. +# * Clears authorized_keys files for security. + +set -euo pipefail + +echo "##############################" +echo "# Begin Image Creation #" +echo "##############################" until ping -c 1 google.com &>/dev/null; do - echo "waiting for outbound connectivity" + echo "Waiting for outbound connectivity" sleep 5 done timedatectl set-timezone UTC -# Install Tools -apt-get -o DPkg::Lock::Timeout=300 update -y -apt-get -o DPkg::Lock::Timeout=300 install -y bash-completion jq nvme-cli openjdk-11-jdk unzip +echo "##########################" +echo "# Installing Tools #" +echo "##########################" + +apt-get -qq -o DPkg::Lock::Timeout=300 update -y +apt-get -qq -o DPkg::Lock::Timeout=300 install -y bash-completion jq nvme-cli openjdk-11-jdk unzip + +echo "##############################" +echo "# Installing Azure CLI #" +echo "##############################" -# Install Azure CLI curl -sL https://aka.ms/InstallAzureCLIDeb | bash +echo "Azure CLI installed" + +echo "###################################" +echo "# Creating the GraphDB user #" +echo "###################################" -# Create the GraphDB user useradd --comment "GraphDB Service User" --create-home --system --shell /bin/bash --user-group graphdb +echo "User created" + +echo "######################################" +echo "# Creating GraphDB directories #" +echo "######################################" -# Create GraphDB directories mkdir -p /etc/graphdb \ /etc/graphdb-cluster-proxy \ /var/opt/graphdb/node \ /var/opt/graphdb/cluster-proxy -# Download and install GraphDB +echo "Directories created" + +echo "############################################" +echo "# Downloading and installing GraphDB #" +echo "############################################" + cd /tmp curl -O https://maven.ontotext.com/repository/owlim-releases/com/ontotext/graphdb/graphdb/"${GRAPHDB_VERSION}"/graphdb-"${GRAPHDB_VERSION}"-dist.zip -unzip graphdb-"${GRAPHDB_VERSION}"-dist.zip +unzip -qq graphdb-"${GRAPHDB_VERSION}"-dist.zip rm graphdb-"${GRAPHDB_VERSION}"-dist.zip mv graphdb-"${GRAPHDB_VERSION}" /opt/graphdb-"${GRAPHDB_VERSION}" ln -s /opt/graphdb-"${GRAPHDB_VERSION}" /opt/graphdb @@ -42,7 +77,12 @@ chown -R graphdb:graphdb /etc/graphdb \ /opt/graphdb-${GRAPHDB_VERSION} \ /var/opt/graphdb -# Configure systemd for GraphDB and GraphDB proxy +echo "GraphDB installed" + +echo "###########################################################" +echo "# Configuring systemd for GraphDB and GraphDB proxy #" +echo "###########################################################" + mv /tmp/graphdb-cluster-proxy.service /lib/systemd/system/graphdb-cluster-proxy.service mv /tmp/graphdb.service /lib/systemd/system/graphdb.service @@ -50,5 +90,43 @@ systemctl daemon-reload systemctl enable graphdb.service systemctl start graphdb.service -# Shred authorized_keys +echo "systemd configured" + +echo "#############################" +echo "# Installing Telegraf #" +echo "#############################" + +curl -s https://repos.influxdata.com/influxdata-archive.key > influxdata-archive.key +echo '943666881a1b8d9b849b74caebf02d3465d6beb716510d86a39f6c8e8dac7515 influxdata-archive.key' | sha256sum -c && cat influxdata-archive.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive.gpg > /dev/null +echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list +apt-get update && apt-get install telegraf + +echo "Telegraf installed" + +echo "#############################################" +echo "# Setting keepalive and file max size #" +echo "#############################################" + +echo 'net.ipv4.tcp_keepalive_time = 120' | tee -a /etc/sysctl.conf +echo 'fs.file-max = 262144' | tee -a /etc/sysctl.conf + +sysctl -p + +echo "###################################" +echo "# Provisioning Backup Script #" +echo "###################################" + +mv /tmp/graphdb_backup /usr/bin/graphdb_backup +chmod +x /usr/bin/graphdb_backup +echo "Backup script is provisioned" + +echo "###################################" +echo "# Shredding authorized_keys #" +echo "###################################" + shred -u /root/.ssh/authorized_keys /home/packer/.ssh/authorized_keys || true +echo "Authorized keys are shredded" + +echo "#################################" +echo "# Image Creation Complete #" +echo "#################################" diff --git a/variables.pkr.hcl b/variables.pkr.hcl index 471dd62..19ec100 100644 --- a/variables.pkr.hcl +++ b/variables.pkr.hcl @@ -1,85 +1,116 @@ +# The following 4 variables are for access via Service Principal + variable subscription_id { - type = string + description = "ID of the subscription the VM image would be build in" + type = string + default = null } variable tenant_id { - type = string + description = "Microsoft Entra tenant ID" + type = string + default = null } variable client_id { - type = string + description = "Client ID of the service principle" + type = string + default = null } variable client_secret { - type = string + description = "Client secret of the service principal" + type = string + default = null +} + +variable use_azure_cli_auth { + description = "CLI auth will use the information from an active az login session to connect to Azure and set the subscription id and tenant id associated to the signed in account." + type = bool + default = true } +# Generic variables + variable primary_location { - type = string + description = "Azure datacenter in which your VM will build" + type = string } variable image_definition_name { - type = string + description = "Image Name" + type = string } variable gdb_version { - type = string + description = "Version of GraphDB which will be installed" + type = string } variable gallery_resource_group { - type = string + description = "Name of the resource group where the Shared Image Gallery is located" + type = string } variable gallery_name { - type = string + description = "Name of the Shared Image Gallery" + type = string } -variable my_ip_address { - type = string +variable allowed_inbound_ip_addresses { + description = "Specify the list of IP addresses and CIDR blocks that should be allowed access to the VM" + type = list(string) } variable replication_regions { - type = list(string) + description = "A list of regions to replicate the image version in" + type = list(string) } variable os_type { - type = string - default = "Linux" + description = "Type of the OS, this configures an SSH authorized key" + type = string + default = "Linux" } variable image_offer { - type = string - default = "0001-com-ubuntu-server-jammy" + description = "Name of the publisher's offer to use for your base image" + type = string + default = "0001-com-ubuntu-server-jammy" } variable image_publisher { - type = string - default = "canonical" + description = "Name of the publisher to use for your base image (Azure Marketplace Images only)" + type = string + default = "canonical" } variable image_sku { - type = string - default = "22_04-lts-gen2" + description = "SKU of the image offer to use for your base image (Azure Marketplace Images only)." + type = string + default = "22_04-lts-gen2" } variable vm_size { - type = string - default = "Standard_B1ls" - # TODO replace with B2pts_v2 when quota is available in order to build with IsAcceleratedNetworkSupported=true DiskControllerTypes=SCSI,NVMe features - # default = "Standard_B2ats_v2" + description = "Size of the VM used for building" + type = string + default = "Standard_B1ls" } variable image_replica_count { - type = number - default = 1 + description = "The number of replicas of the Image Version to be created per region. Replica count must be between 1 and 100" + type = number + default = 1 } variable shared_gallery_image_version_exclude_from_latest { - type = bool - default = false + description = "If set to true, Virtual Machines deployed from the latest version of the Image Definition won't use this Image Version" + type = bool + default = false } variable os_disk_size_gb { - type = number - default = 30 + description = "Specify the size of the OS disk in GB (gigabytes). Values of zero or less than zero are ignored." + type = number + default = 30 }