Merge pull request #11 from Ontotext-AD/TES-381

TES-381 Added provisioning of graphdb backup script

CHANGELOG.md

@@ -2,15 +2,16 @@
 
 All notable changes to the Packer template for creating GraphDB Azure VM images will be documented in this file.
 
-## 1.2.0
-
-- Removed features preventing build with Standard_B1ls VM
-- Fixed create_image_definition.sh
-- Added support for specifying subscription in create_image_definition.sh
-
 ## 1.1.0
 
 - Installed Azure CLI
+- Removed features preventing build with Standard_B1ls VM.
+- Fixed create_image_definition.sh
+- Added subscription as an attribute in the [create_image_definition.sh](create_image_definition.sh) helper script.
+- Added installation of Telegraf.
+- Added provisioning of GraphDB backup script.
+- Added keepalive and file max size settings.
+- Improved logging of the GraphDB install script.
 
 ## 1.0.0
 

README.md

@@ -34,17 +34,17 @@ Follow these steps to build an AMI for GraphDB using Packer:
    replication regions, subscription, client and tennat IDs. To do so, create a variables file `variables.pkrvars.hcl`,
    example file:
       ```bash
-      subscription_id        = "<your_azure_subscription_id>"
-      client_id              = "<your_azure_service_principal_id>"
-      client_secret          = "<your_azure_service_principal_secret>"
-      tenant_id              = "<your_azure_tenant_id>"
-      primary_location       = "East US"
-      replication_regions    = ["North Europe", "UK South"]
-      image_definition_name  = "10.4.0-x86_64"
-      gdb_version            = "10.4.0"
-      gallery_resource_group = "Packer-RG"
-      gallery_name           = "GraphDB"
-      my_ip_address          = "<your_public_IP_address>"
+      subscription_id              = "<your_azure_subscription_id>"
+      client_id                    = "<your_azure_service_principal_id>"
+      client_secret                = "<your_azure_service_principal_secret>"
+      tenant_id                    = "<your_azure_tenant_id>"
+      primary_location             = "East US"
+      replication_regions          = ["North Europe", "UK South"]
+      image_definition_name        = "10.4.0-x86_64"
+      gdb_version                  = "10.4.0"
+      gallery_resource_group       = "Packer-RG"
+      gallery_name                 = "GraphDB"
+      allowed_inbound_ip_addresses = "<your_public_IP_address>"
       ```
 
 4. **Build the AMI**:
@@ -105,7 +105,7 @@ The following points can be customized in a packer variables file `variables.pkr
 
 **Networking Configuration**
 
-* my_ip_address (string): Your IP address for network security settings.
+* allowed_inbound_ip_addresses (string): Specify the list of IP addresses and CIDR blocks that should be allowed access to the VM.
 
 **OS and Image Defaults**
 

azure.pkr.hcl

@@ -1,15 +1,16 @@
 locals {
   timestamp = regex_replace(timestamp(), "[- TZ:]", "")
-  #  https://github.com/hashicorp/packer-plugin-azure/issues/65
+  # See https://github.com/hashicorp/packer-plugin-azure/issues/65
   version_timestamp = formatdate("YYYY.MM.DD", timestamp())
 }
 
 source azure-arm ubuntu-x86-64 {
-  client_id       = var.client_id
-  client_secret   = var.client_secret
-  subscription_id = var.subscription_id
-  tenant_id       = var.tenant_id
-  location        = var.primary_location
+  use_azure_cli_auth = var.use_azure_cli_auth
+  client_id          = var.client_id
+  client_secret      = var.client_secret
+  subscription_id    = var.subscription_id
+  tenant_id          = var.tenant_id
+  location           = var.primary_location
 
   shared_image_gallery_destination {
     subscription        = var.subscription_id
@@ -38,7 +39,7 @@ source azure-arm ubuntu-x86-64 {
   image_sku                 = var.image_sku
 
   vm_size                      = var.vm_size
-  allowed_inbound_ip_addresses = [var.my_ip_address]
+  allowed_inbound_ip_addresses = var.allowed_inbound_ip_addresses
 }
 
 #TODO Not yet supported by Packer https://github.com/hashicorp/packer/issues/12188#issuecomment-1380736642
@@ -72,5 +73,5 @@ source azure-arm ubuntu-x86-64 {
 #
 #  vm_size = "Standard_D2ps_v5"
 #
-#  allowed_inbound_ip_addresses = [var.my_ip_address]
+#  allowed_inbound_ip_addresses = [var.allowed_inbound_ip_addresses]
 #}

build.pkr.hcl

@@ -8,7 +8,8 @@ build {
       "./files/graphdb.env",
       "./files/graphdb.service",
       "./files/graphdb-cluster-proxy.service",
-      "./files/install_graphdb.sh"
+      "./files/install_graphdb.sh",
+      "./files/graphdb_backup"
     ]
     destination = "/tmp/"
   }

create_image_definition.sh

@@ -1,5 +1,10 @@
 #!/bin/bash
 
+# This script sets up Azure Shared Image Gallery (SIG) image definition and builds an image using Packer.
+# It checks for a variables file, extracts necessary parameters, constructs an Azure CLI command for image definition creation,
+# executes the command, waits for creation completion,
+# and finally uses Packer to build the image based on the specified variables.
+
 set -euxo pipefail
 
 # Required when running the script on Windows
@@ -10,22 +15,25 @@ variables_file="variables.pkrvars.hcl"
 
 # Check if the file exists
 if [ -f "$variables_file" ]; then
-  # Use grep to extract the variable values
-
+  # Extracts required variables from the variables_file
   subscription_id=$(grep 'subscription_id' "$variables_file" | cut -d '"' -f 2)
+  echo "Subscription ID: $subscription_id"
   image_definition_name=$(grep 'image_definition_name' "$variables_file" | cut -d '"' -f 2)
+  echo "Image definition name: $image_definition_name"
   gdb_version=$(grep 'gdb_version' "$variables_file" | cut -d '"' -f 2)
+  echo "GraphDB version: $gdb_version"
   gallery_resource_group=$(grep 'gallery_resource_group' "$variables_file" | cut -d '"' -f 2)
+  echo "Resource group: $gallery_resource_group"
   gallery_name=$(grep 'gallery_name' "$variables_file" | cut -d '"' -f 2)
+  echo "Gallery: $gallery_name"
 
-  # Check if any of the required variables is empty
-  if [ -z "$image_definition_name" ] || [ -z "$gdb_version" ] || [ -z "$gallery_resource_group" ] || [ -z "$gallery_name" ]; then
+  # Checks if any of the required variables is empty
+  if [ -z "$image_definition_name" ] || [ -z "$gdb_version" ] || [ -z "$gallery_resource_group" ] || [ -z "$gallery_name" ] || [ -z "$subscription_id" ]; then
     echo "One or more required variables are not defined in $variables_file."
     exit 1
   fi
-  # Construct the az sig image-definition create command
 
-# TODO extend command with --features 'IsAcceleratedNetworkSupported=true DiskControllerTypes=SCSI,NVMe' when we have quota for Standard_B2pts_v2
+  # Constructs the az sig image-definition create command
   az_command="az sig image-definition create \
        --subscription $subscription_id \
        -g $gallery_resource_group \
@@ -42,8 +50,11 @@ if [ -f "$variables_file" ]; then
        --maximum-memory 128 "
 
   echo "Extracted variables and constructed Azure CLI command:"
+  echo "Creating SIG"
   eval "$az_command"
+  # Waits for the Shared Image Gallery to be created
   az sig image-definition wait -i "$image_definition_name" -r "$gallery_name" -g "$gallery_resource_group" --created --subscription $subscription_id
+  echo "Begin building of the Azure VM image"
   packer build -var-file="variables.pkrvars.hcl" .
 else
   echo "The variables file $variables_file does not exist."

files/graphdb_backup

@@ -0,0 +1,72 @@
+#!/bin/bash
+
+# This script utilizes the cloud backup functionality of GraphDB.
+# The script accepts four variables in the following order: graphdb_user, graphdb_password, backup_storage_account_name, backup_storage_container_name
+
+set -euo pipefail
+
+az login --identity
+
+GRAPHDB_USER="${1}"
+GRAPHDB_PASSWORD="${2}"
+NODE_STATE="$(curl --silent --fail --user "$GRAPHDB_USER:$GRAPHDB_PASSWORD" localhost:7200/rest/cluster/node/status | jq -r .nodeState)"
+BACKUP_NAME="$(date +'%Y-%m-%d_%H-%M-%S').tar"
+BACKUP_STORAGE_ACCOUNT_NAME="${3}"
+BACKUP_STORAGE_CONTAINER_NAME="${4}"
+max_retries=3
+retry_count=0
+
+perform_backup() {
+  while [ "$retry_count" -lt "$max_retries" ]; do
+    start_time=$(date +%s)
+
+    response_code=$(curl -X POST --write-out %{http_code} --silent --output /dev/null \
+      --header 'Content-Type: application/json' \
+      -u "$GRAPHDB_USER:$GRAPHDB_PASSWORD" \
+      --header 'Accept: application/json' \
+      -d "{\"bucketUri\": \"az://${BACKUP_STORAGE_CONTAINER_NAME}/${BACKUP_NAME}?blob_storage_account=${BACKUP_STORAGE_ACCOUNT_NAME}\", \"backupOptions\": {\"backupSystemData\": true}}" \
+      'http://localhost:7200/rest/recovery/cloud-backup'
+    )
+
+    end_time=$(date +%s)
+    elapsed_time=$((end_time - start_time))
+
+    if [ "$response_code" -eq 200 ]; then
+      echo "Backup uploaded successfully to ${BACKUP_STORAGE_ACCOUNT_NAME} in $elapsed_time seconds."
+      break
+    else
+      echo "Failed to complete the backup and upload. HTTP Response Code: $response_code"
+      echo "Request took: $elapsed_time"
+
+      if [ "$retry_count" -eq "$max_retries" ]; then
+        echo "Max retries reached. Backup could not be created. Exiting..."
+        return 1
+      else
+        echo "Retrying..."
+      fi
+
+      ((retry_count=retry_count + 1))
+      sleep 5
+    fi
+  done
+}
+
+# Checks if GraphDB is running in cluster
+IS_CLUSTER=$(
+  curl -s -o /dev/null \
+    -u "$GRAPHDB_USER:$GRAPHDB_PASSWORD" \
+    -w "%{http_code}" \
+    http://localhost:7200/rest/monitor/cluster
+)
+
+if [ "$IS_CLUSTER" == 200 ]; then
+  # Checks if the current GraphDB instance is Leader, otherwise exits.
+  if [ "$NODE_STATE" != "LEADER" ]; then
+    echo "The current node is not the leader, but $NODE_STATE"
+    exit 0
+  fi
+  perform_backup
+elif [ "$IS_CLUSTER" == 503 ]; then
+  perform_backup
+fi
+

files/install_graphdb.sh

@@ -1,35 +1,70 @@
 #!/bin/bash
 
-set -euxo pipefail
+# This script performs the following tasks:
+# * Sets the timezone to UTC.
+# * Installs necessary tools such as bash-completion, jq, nvme-cli, openjdk-11-jdk, and unzip.
+# * Installs the Azure CLI.
+# * Creates a system user "graphdb" for GraphDB service.
+# * Creates GraphDB directories and sets up the necessary permissions.
+# * Downloads and installs GraphDB, configuring systemd for GraphDB and GraphDB proxy.
+# * Installs Telegraf for monitoring.
+# * Adjusts system settings for keepalive and file max size.
+# * Provisions a backup script.
+# * Clears authorized_keys files for security.
+
+set -euo pipefail
+
+echo "##############################"
+echo "#    Begin Image Creation    #"
+echo "##############################"
 
 until ping -c 1 google.com &>/dev/null; do
-  echo "waiting for outbound connectivity"
+  echo "Waiting for outbound connectivity"
   sleep 5
 done
 
 timedatectl set-timezone UTC
 
-# Install Tools
-apt-get -o DPkg::Lock::Timeout=300 update -y
-apt-get -o DPkg::Lock::Timeout=300 install -y bash-completion jq nvme-cli openjdk-11-jdk unzip
+echo "##########################"
+echo "#    Installing Tools    #"
+echo "##########################"
+
+apt-get -qq -o DPkg::Lock::Timeout=300 update -y
+apt-get -qq -o DPkg::Lock::Timeout=300 install -y bash-completion jq nvme-cli openjdk-11-jdk unzip
+
+echo "##############################"
+echo "#    Installing Azure CLI    #"
+echo "##############################"
 
-# Install Azure CLI
 curl -sL https://aka.ms/InstallAzureCLIDeb | bash
+echo "Azure CLI installed"
+
+echo "###################################"
+echo "#    Creating the GraphDB user    #"
+echo "###################################"
 
-# Create the GraphDB user
 useradd --comment "GraphDB Service User" --create-home --system --shell /bin/bash --user-group graphdb
+echo "User created"
+
+echo "######################################"
+echo "#    Creating GraphDB directories    #"
+echo "######################################"
 
-# Create GraphDB directories
 mkdir -p /etc/graphdb \
          /etc/graphdb-cluster-proxy \
          /var/opt/graphdb/node \
          /var/opt/graphdb/cluster-proxy
 
-# Download and install GraphDB
+echo "Directories created"
+
+echo "############################################"
+echo "#    Downloading and installing GraphDB    #"
+echo "############################################"
+
 cd /tmp
 curl -O https://maven.ontotext.com/repository/owlim-releases/com/ontotext/graphdb/graphdb/"${GRAPHDB_VERSION}"/graphdb-"${GRAPHDB_VERSION}"-dist.zip
 
-unzip graphdb-"${GRAPHDB_VERSION}"-dist.zip
+unzip -qq graphdb-"${GRAPHDB_VERSION}"-dist.zip
 rm graphdb-"${GRAPHDB_VERSION}"-dist.zip
 mv graphdb-"${GRAPHDB_VERSION}" /opt/graphdb-"${GRAPHDB_VERSION}"
 ln -s /opt/graphdb-"${GRAPHDB_VERSION}" /opt/graphdb
@@ -42,13 +77,56 @@ chown -R graphdb:graphdb /etc/graphdb \
                          /opt/graphdb-${GRAPHDB_VERSION} \
                          /var/opt/graphdb
 
-# Configure systemd for GraphDB and GraphDB proxy
+echo "GraphDB installed"
+
+echo "###########################################################"
+echo "#    Configuring systemd for GraphDB and GraphDB proxy    #"
+echo "###########################################################"
+
 mv /tmp/graphdb-cluster-proxy.service /lib/systemd/system/graphdb-cluster-proxy.service
 mv /tmp/graphdb.service /lib/systemd/system/graphdb.service
 
 systemctl daemon-reload
 systemctl enable graphdb.service
 systemctl start graphdb.service
 
-# Shred authorized_keys
+echo "systemd configured"
+
+echo "#############################"
+echo "#    Installing Telegraf    #"
+echo "#############################"
+
+curl -s https://repos.influxdata.com/influxdata-archive.key > influxdata-archive.key
+echo '943666881a1b8d9b849b74caebf02d3465d6beb716510d86a39f6c8e8dac7515 influxdata-archive.key' | sha256sum -c && cat influxdata-archive.key | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/influxdata-archive.gpg > /dev/null
+echo 'deb [signed-by=/etc/apt/trusted.gpg.d/influxdata-archive.gpg] https://repos.influxdata.com/debian stable main' | sudo tee /etc/apt/sources.list.d/influxdata.list
+apt-get update && apt-get install telegraf
+
+echo "Telegraf installed"
+
+echo "#############################################"
+echo "#    Setting keepalive and file max size    #"
+echo "#############################################"
+
+echo 'net.ipv4.tcp_keepalive_time = 120' | tee -a /etc/sysctl.conf
+echo 'fs.file-max = 262144' | tee -a /etc/sysctl.conf
+
+sysctl -p
+
+echo "###################################"
+echo "#    Provisioning Backup Script   #"
+echo "###################################"
+
+mv /tmp/graphdb_backup /usr/bin/graphdb_backup
+chmod +x /usr/bin/graphdb_backup
+echo "Backup script is provisioned"
+
+echo "###################################"
+echo "#    Shredding authorized_keys    #"
+echo "###################################"
+
 shred -u /root/.ssh/authorized_keys /home/packer/.ssh/authorized_keys || true
+echo "Authorized keys are shredded"
+
+echo "#################################"
+echo "#    Image Creation Complete    #"
+echo "#################################"