Added installation of Telegraf.

Added provisioning of GraphDB backup script.
Added keepalive and file max size settings.
Improved logging of the GraphDB install script.
Added script descriptions
Removed dependency on application config resource from backup script
Added support for single instance GDB to the backup script
Changed use_azure_cli_auth to true

CHANGELOG.md

@@ -2,15 +2,16 @@
 
 All notable changes to the Packer template for creating GraphDB Azure VM images will be documented in this file.
 
-## 1.2.0
-
-- Removed features preventing build with Standard_B1ls VM
-- Fixed create_image_definition.sh
-- Added support for specifying subscription in create_image_definition.sh
-
 ## 1.1.0
 
 - Installed Azure CLI
+- Removed features preventing build with Standard_B1ls VM.
+- Fixed create_image_definition.sh
+- Added subscription as an attribute in the [create_image_definition.sh](create_image_definition.sh) helper script.
+- Added installation of Telegraf.
+- Added provisioning of GraphDB backup script.
+- Added keepalive and file max size settings.
+- Improved logging of the GraphDB install script.
 
 ## 1.0.0
 

README.md

@@ -34,17 +34,17 @@ Follow these steps to build an AMI for GraphDB using Packer:
    replication regions, subscription, client and tennat IDs. To do so, create a variables file `variables.pkrvars.hcl`,
    example file:
       ```bash
-      subscription_id        = "<your_azure_subscription_id>"
-      client_id              = "<your_azure_service_principal_id>"
-      client_secret          = "<your_azure_service_principal_secret>"
-      tenant_id              = "<your_azure_tenant_id>"
-      primary_location       = "East US"
-      replication_regions    = ["North Europe", "UK South"]
-      image_definition_name  = "10.4.0-x86_64"
-      gdb_version            = "10.4.0"
-      gallery_resource_group = "Packer-RG"
-      gallery_name           = "GraphDB"
-      my_ip_address          = "<your_public_IP_address>"
+      subscription_id              = "<your_azure_subscription_id>"
+      client_id                    = "<your_azure_service_principal_id>"
+      client_secret                = "<your_azure_service_principal_secret>"
+      tenant_id                    = "<your_azure_tenant_id>"
+      primary_location             = "East US"
+      replication_regions          = ["North Europe", "UK South"]
+      image_definition_name        = "10.4.0-x86_64"
+      gdb_version                  = "10.4.0"
+      gallery_resource_group       = "Packer-RG"
+      gallery_name                 = "GraphDB"
+      allowed_inbound_ip_addresses = "<your_public_IP_address>"
       ```
 
 4. **Build the AMI**:
@@ -105,7 +105,7 @@ The following points can be customized in a packer variables file `variables.pkr
 
 **Networking Configuration**
 
-* my_ip_address (string): Your IP address for network security settings.
+* allowed_inbound_ip_addresses (string): Specify the list of IP addresses and CIDR blocks that should be allowed access to the VM.
 
 **OS and Image Defaults**
 

azure.pkr.hcl

@@ -1,15 +1,16 @@
 locals {
   timestamp = regex_replace(timestamp(), "[- TZ:]", "")
-  #  https://github.com/hashicorp/packer-plugin-azure/issues/65
+  # See https://github.com/hashicorp/packer-plugin-azure/issues/65
   version_timestamp = formatdate("YYYY.MM.DD", timestamp())
 }
 
 source azure-arm ubuntu-x86-64 {
-  client_id       = var.client_id
-  client_secret   = var.client_secret
-  subscription_id = var.subscription_id
-  tenant_id       = var.tenant_id
-  location        = var.primary_location
+  use_azure_cli_auth = var.use_azure_cli_auth
+  client_id          = var.client_id
+  client_secret      = var.client_secret
+  subscription_id    = var.subscription_id
+  tenant_id          = var.tenant_id
+  location           = var.primary_location
 
   shared_image_gallery_destination {
     subscription        = var.subscription_id
@@ -38,7 +39,7 @@ source azure-arm ubuntu-x86-64 {
   image_sku                 = var.image_sku
 
   vm_size                      = var.vm_size
-  allowed_inbound_ip_addresses = [var.my_ip_address]
+  allowed_inbound_ip_addresses = var.allowed_inbound_ip_addresses
 }
 
 #TODO Not yet supported by Packer https://github.com/hashicorp/packer/issues/12188#issuecomment-1380736642
@@ -72,5 +73,5 @@ source azure-arm ubuntu-x86-64 {
 #
 #  vm_size = "Standard_D2ps_v5"
 #
-#  allowed_inbound_ip_addresses = [var.my_ip_address]
+#  allowed_inbound_ip_addresses = [var.allowed_inbound_ip_addresses]
 #}

build.pkr.hcl

@@ -8,7 +8,8 @@ build {
       "./files/graphdb.env",
       "./files/graphdb.service",
       "./files/graphdb-cluster-proxy.service",
-      "./files/install_graphdb.sh"
+      "./files/install_graphdb.sh",
+      "./files/graphdb_backup"
     ]
     destination = "/tmp/"
   }

create_image_definition.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# This script sets up Azure Shared Image Gallery (SIG) image definition and builds an image using Packer.
+# It checks for a variables file, extracts necessary parameters, constructs an Azure CLI command for image definition creation,
+# executes the command, waits for creation completion,
+# and finally uses Packer to build the image based on the specified variables.
+
 set -euxo pipefail
 
 # Required when running the script on Windows
@@ -10,22 +15,25 @@ variables_file="variables.pkrvars.hcl"
 
 # Check if the file exists
 if [ -f "$variables_file" ]; then
-  # Use grep to extract the variable values
-
+  # Extracts required variables from the variables_file
   subscription_id=$(grep 'subscription_id' "$variables_file" | cut -d '"' -f 2)
+  echo "Subscription ID: $subscription_id"
   image_definition_name=$(grep 'image_definition_name' "$variables_file" | cut -d '"' -f 2)
+  echo "Image definition name: $image_definition_name"
   gdb_version=$(grep 'gdb_version' "$variables_file" | cut -d '"' -f 2)
+  echo "GraphDB version: $gdb_version"
   gallery_resource_group=$(grep 'gallery_resource_group' "$variables_file" | cut -d '"' -f 2)
+  echo "Resource group: $gallery_resource_group"
   gallery_name=$(grep 'gallery_name' "$variables_file" | cut -d '"' -f 2)
+  echo "Gallery: $gallery_name"
 
-  # Check if any of the required variables is empty
-  if [ -z "$image_definition_name" ] || [ -z "$gdb_version" ] || [ -z "$gallery_resource_group" ] || [ -z "$gallery_name" ]; then
+  # Checks if any of the required variables is empty
+  if [ -z "$image_definition_name" ] || [ -z "$gdb_version" ] || [ -z "$gallery_resource_group" ] || [ -z "$gallery_name" ] || [ -z "$subscription_id" ]; then
     echo "One or more required variables are not defined in $variables_file."
     exit 1
   fi
-  # Construct the az sig image-definition create command
 
-# TODO extend command with --features 'IsAcceleratedNetworkSupported=true DiskControllerTypes=SCSI,NVMe' when we have quota for Standard_B2pts_v2
+  # Constructs the az sig image-definition create command
   az_command="az sig image-definition create \
        --subscription $subscription_id \
        -g $gallery_resource_group \
@@ -42,8 +50,11 @@ if [ -f "$variables_file" ]; then
        --maximum-memory 128 "
 
   echo "Extracted variables and constructed Azure CLI command:"
+  echo "Creating SIG"
   eval "$az_command"
+  # Waits for the Shared Image Gallery to be created
   az sig image-definition wait -i "$image_definition_name" -r "$gallery_name" -g "$gallery_resource_group" --created --subscription $subscription_id
+  echo "Begin building of the Azure VM image"
   packer build -var-file="variables.pkrvars.hcl" .
 else
   echo "The variables file $variables_file does not exist."

files/graphdb_backup

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# This script utilizes the cloud backup functionality of GraphDB.
+# The script accepts four variables in the following order: graphdb_user, graphdb_password, backup_storage_account_name, backup_storage_container_name
+
+set -euo pipefail
+
+az login --identity
+
+GRAPHDB_USER="${1}"
+GRAPHDB_PASSWORD="${2}"
+NODE_STATE="$(curl --silent --fail --user "$GRAPHDB_USER:$GRAPHDB_PASSWORD" localhost:7200/rest/cluster/node/status | jq -r .nodeState)"
+BACKUP_NAME="$(date +'%Y-%m-%d_%H-%M-%S').tar"
+BACKUP_STORAGE_ACCOUNT_NAME="${3}"
+BACKUP_STORAGE_CONTAINER_NAME="${4}"
+max_retries=3
+retry_count=0
+
+perform_backup() {
+  while [ "$retry_count" -lt "$max_retries" ]; do
+    start_time=$(date +%s)
+
+    response_code=$(curl -X POST --write-out %{http_code} --silent --output /dev/null \
+      --header 'Content-Type: application/json' \
+      -u "$GRAPHDB_USER:$GRAPHDB_PASSWORD" \
+      --header 'Accept: application/json' \
+      -d "{\"bucketUri\": \"az://${BACKUP_STORAGE_CONTAINER_NAME}/${BACKUP_NAME}?blob_storage_account=${BACKUP_STORAGE_ACCOUNT_NAME}\", \"backupOptions\": {\"backupSystemData\": true}}" \
+      'http://localhost:7200/rest/recovery/cloud-backup'
+    )
+
+    end_time=$(date +%s)
+    elapsed_time=$((end_time - start_time))
+
+    if [ "$response_code" -eq 200 ]; then
+      echo "Backup uploaded successfully to ${BACKUP_STORAGE_ACCOUNT_NAME} in $elapsed_time seconds."
+      break
+    else
+      echo "Failed to complete the backup and upload. HTTP Response Code: $response_code"
+      echo "Request took: $elapsed_time"
+
+      if [ "$retry_count" -eq "$max_retries" ]; then
+        echo "Max retries reached. Backup could not be created. Exiting..."
+        return 1
+      else
+        echo "Retrying..."
+      fi
+
+      ((retry_count=retry_count + 1))
+      sleep 5
+    fi
+  done
+}
+
+# Checks if GraphDB is running in cluster
+IS_CLUSTER=$(
+  curl -s -o /dev/null \
+    -u "$GRAPHDB_USER:$GRAPHDB_PASSWORD" \
+    -w "%{http_code}" \
+    http://localhost:7200/rest/monitor/cluster
+)
+
+if [ "$IS_CLUSTER" == 200 ]; then
+  # Checks if the current GraphDB instance is Leader, otherwise exits.
+  if [ "$NODE_STATE" != "LEADER" ]; then
+    echo "The current node is not the leader, but $NODE_STATE"
+    exit 0
+  fi
+  perform_backup
+elif [ "$IS_CLUSTER" == 503 ]; then
+  perform_backup
+fi
+

files/install_graphdb.sh

@@ -1,35 +1,70 @@
 #!/bin/bash
 
-set -euxo pipefail
+# This script performs the following tasks:
+# * Sets the timezone to UTC.
+# * Installs necessary tools such as bash-completion, jq, nvme-cli, openjdk-11-jdk, and unzip.
+# * Installs the Azure CLI.
+# * Creates a system user "graphdb" for GraphDB service.
+# * Creates GraphDB directories and sets up the necessary permissions.
+# * Downloads and installs GraphDB, configuring systemd for GraphDB and GraphDB proxy.
+# * Installs Telegraf for monitoring.
+# * Adjusts system settings for keepalive and file max size.
+# * Provisions a backup script.
+# * Clears authorized_keys files for security.
+
+set -euo pipefail
+
+echo "##############################"
+echo "#    Begin Image Creation    #"
+echo "##############################"
 
 until ping -c 1 google.com &>/dev/null; do
-  echo "waiting for outbound connectivity"
+  echo "Waiting for outbound connectivity"
   sleep 5
 done
 
 timedatectl set-timezone UTC
 
-# Install Tools
-apt-get -o DPkg::Lock::Timeout=300 update -y
-apt-get -o DPkg::Lock::Timeout=300 install -y bash-completion jq nvme-cli openjdk-11-jdk unzip
+echo "##########################"
+echo "#    Installing Tools    #"
+echo "##########################"
+
+apt-get -qq -o DPkg::Lock::Timeout=300 update -y
+apt-get -qq -o DPkg::Lock::Timeout=300 install -y bash-completion jq nvme-cli openjdk-11-jdk unzip
+
+echo "##############################"
+echo "#    Installing Azure CLI    #"
+echo "##############################"
 
-# Install Azure CLI
 curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+echo "Azure CLI installed"
+
+echo "###################################"
+echo "#    Creating the GraphDB user    #"
+echo "###################################"
 
-# Create the GraphDB user
 useradd --comment "GraphDB Service User" --create-home --system --shell /bin/bash --user-group graphdb
+echo "User created"
+
+echo "######################################"
+echo "#    Creating GraphDB directories    #"
+echo "######################################"
 
-# Create GraphDB directories
 mkdir -p /etc/graphdb \
          /etc/graphdb-cluster-proxy \
          /var/opt/graphdb/node \
          /var/opt/graphdb/cluster-proxy
 
-# Download and install GraphDB
+echo "Directories created"
+
+echo "############################################"
+echo "#    Downloading and installing GraphDB    #"
+echo "############################################"
+
 cd /tmp
 curl -O https://maven.ontotext.com/repository/owlim-releases/com/ontotext/graphdb/graphdb/"${GRAPHDB_VERSION}"/graphdb-"${GRAPHDB_VERSION}"-dist.zip
 
-unzip graphdb-"${GRAPHDB_VERSION}"-dist.zip
+unzip -qq graphdb-"${GRAPHDB_VERSION}"-dist.zip
 rm graphdb-"${GRAPHDB_VERSION}"-dist.zip
 mv graphdb-"${GRAPHDB_VERSION}" /opt/graphdb-"${GRAPHDB_VERSION}"
 ln -s /opt/graphdb-"${GRAPHDB_VERSION}" /opt/graphdb
@@ -42,13 +77,56 @@ chown -R graphdb:graphdb /etc/graphdb \
                          /opt/graphdb-${GRAPHDB_VERSION} \
                          /var/opt/graphdb
 
-# Configure systemd for GraphDB and GraphDB proxy
+echo "GraphDB installed"
+
+echo "###########################################################"
+echo "#    Configuring systemd for GraphDB and GraphDB proxy    #"
+echo "###########################################################"
+
 mv /tmp/graphdb-cluster-proxy.service /lib/systemd/system/graphdb-cluster-proxy.service
 mv /tmp/graphdb.service /lib/systemd/system/graphdb.service
 
 systemctl daemon-reload
 systemctl enable graphdb.service
 systemctl start graphdb.service
 
-# Shred authorized_keys
+echo "systemd configured"
+
+echo "#############################"
+echo "#    Installing Telegraf    #"
+echo "#############################"
+
+curl -s https://repos.influxdata.com/influxdata-archive.key > influxdata-archive.key
+echo '943666881a1b8d9b849b74caebf02d3465d6beb716510d86a39f6c8e8dac7515 influxdata-archive.key' | sha256sum -c && cat influxdata-archive.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive.gpg > /dev/null
+echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
+apt-get update && apt-get install telegraf
+
+echo "Telegraf installed"
+
+echo "#############################################"
+echo "#    Setting keepalive and file max size    #"
+echo "#############################################"
+
+echo 'net.ipv4.tcp_keepalive_time = 120' | tee -a /etc/sysctl.conf
+echo 'fs.file-max = 262144' | tee -a /etc/sysctl.conf
+
+sysctl -p
+
+echo "###################################"
+echo "#    Provisioning Backup Script   #"
+echo "###################################"
+
+mv /tmp/graphdb_backup /usr/bin/graphdb_backup
+chmod +x /usr/bin/graphdb_backup
+echo "Backup script is provisioned"
+
+echo "###################################"
+echo "#    Shredding authorized_keys    #"
+echo "###################################"
+
 shred -u /root/.ssh/authorized_keys /home/packer/.ssh/authorized_keys || true
+echo "Authorized keys are shredded"
+
+echo "#################################"
+echo "#    Image Creation Complete    #"
+echo "#################################"