diff --git a/.gitignore b/.gitignore index 7f46f04..55b944c 100644 --- a/.gitignore +++ b/.gitignore @@ -10,8 +10,8 @@ crash.log crash.*.log # Exclude all .tfvars files, which are likely to contain sensitive data, such as -# password, private keys, and other secrets. These should not be part of version -# control as they are data points which are potentially sensitive and subject +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject # to change depending on the environment. *.tfvars *.tfvars.json @@ -35,3 +35,11 @@ terraform.rc # IDEs .idea/ + +#Certificates +*.pem +*.p12 +*.pub + +#Licenses +*.license diff --git a/main.tf b/main.tf index 3a4d559..a56e59e 100644 --- a/main.tf +++ b/main.tf @@ -182,6 +182,7 @@ module "roles" { identity_principal_id = module.identity.identity_principal_id key_vault_id = module.vault.key_vault_id backups_storage_container_id = module.backup.storage_account_id + private_dns_zone = module.dns.private_dns_zone_id } # Managed GraphDB configurations in the Key Vault @@ -314,5 +315,21 @@ module "vm" { tags = local.tags # Wait for configurations to be created in the key vault and roles to be assigned - depends_on = [module.configuration, module.roles] + depends_on = [module.configuration, module.roles, module.dns] +} + +module "dns" { + source = "./modules/dns" + + resource_name_prefix = var.resource_name_prefix + resource_group_name = azurerm_resource_group.graphdb.name + identity_name = module.identity.identity_name + identity_principal_id = module.identity.identity_principal_id + virtual_network_id = azurerm_virtual_network.graphdb.id + + tags = local.tags + + depends_on = [ + module.identity + ] } diff --git a/modules/dns/main.tf b/modules/dns/main.tf new file mode 100644 index 0000000..f681373 --- /dev/null +++ b/modules/dns/main.tf @@ -0,0 +1,13 @@ +resource "azurerm_private_dns_zone" "zone" { + name = "${var.resource_name_prefix}.dns.zone" + resource_group_name = var.resource_group_name + tags = var.tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "zone_link" { + name = "${var.resource_name_prefix}-dns-link" + resource_group_name = var.resource_group_name + private_dns_zone_name = azurerm_private_dns_zone.zone.name + virtual_network_id = var.virtual_network_id + tags = var.tags +} diff --git a/modules/dns/outputs.tf b/modules/dns/outputs.tf new file mode 100644 index 0000000..7559989 --- /dev/null +++ b/modules/dns/outputs.tf @@ -0,0 +1,4 @@ +output "private_dns_zone_id" { + description = "ID of the private DNS zone for Azure DNS resolving" + value = azurerm_private_dns_zone.zone.id +} diff --git a/modules/dns/variables.tf b/modules/dns/variables.tf new file mode 100644 index 0000000..f807ac9 --- /dev/null +++ b/modules/dns/variables.tf @@ -0,0 +1,30 @@ +variable "resource_group_name" { + description = "Resource group name where the DNS zone will be created" + type = string +} + +variable "resource_name_prefix" { + description = "Resource name prefix used for tagging and naming Azure resources" + type = string +} + +variable "identity_name" { + description = "Name of a user assigned identity with permissions" + type = string +} + +variable "virtual_network_id" { + description = "Virtual network the DNS will be linked to" + type = string +} + +variable "tags" { + description = "Common resource tags." + type = map(string) + default = {} +} + +variable "identity_principal_id" { + description = "Principal identifier of a user assigned identity with permissions" + type = string +} diff --git a/modules/roles/main.tf b/modules/roles/main.tf index 9b13fdd..ddc3c85 100644 --- a/modules/roles/main.tf +++ b/modules/roles/main.tf @@ -43,3 +43,9 @@ resource "azurerm_role_assignment" "rg-contributor-role" { depends_on = [azurerm_role_definition.managed_disk_manager] } + +resource "azurerm_role_assignment" "dns_zone_role_assignment" { + principal_id = var.identity_principal_id + role_definition_name = "Private DNS Zone Contributor" + scope = var.private_dns_zone +} diff --git a/modules/roles/variables.tf b/modules/roles/variables.tf index f9da504..68b578f 100644 --- a/modules/roles/variables.tf +++ b/modules/roles/variables.tf @@ -30,3 +30,10 @@ variable "backups_storage_container_id" { description = "Identifier of the storage container for GraphDB backups" type = string } + +# DNS + +variable "private_dns_zone" { + description = "Identifier of a Private DNS zone" + type = string +} diff --git a/modules/vm/templates/entrypoint.sh.tpl b/modules/vm/templates/entrypoint.sh.tpl index ba69e4c..a7656e7 100644 --- a/modules/vm/templates/entrypoint.sh.tpl +++ b/modules/vm/templates/entrypoint.sh.tpl @@ -12,78 +12,79 @@ az login --identity # Find/create/attach volumes INSTANCE_HOSTNAME=\'$(hostname)\' -SUBSCRIPTION_ID=$(az account show --query "id" --output tsv) -RESOURSE_GROUP=$(az vmss list --query "[0].resourceGroup" --output tsv) -VMSS_NAME=$(az vmss list --query "[0].name" --output tsv) -INSTANCE_ID=$(az vmss list-instances --resource-group $RESOURSE_GROUP --name $VMSS_NAME --query "[?contains(osProfile.computerName, $${INSTANCE_HOSTNAME})].instanceId" --output tsv) -ZONE_ID=$(az vmss list-instances --resource-group $RESOURSE_GROUP --name $VMSS_NAME --query "[?contains(osProfile.computerName, $${INSTANCE_HOSTNAME})].zones" --output tsv) -REGION_ID=$(az vmss list-instances --resource-group $RESOURSE_GROUP --name $VMSS_NAME --query "[?contains(osProfile.computerName, $${INSTANCE_HOSTNAME})].location" --output tsv) +RESOURCE_GROUP=$(curl -H Metadata:true "http://169.254.169.254/metadata/instance/compute/resourceGroupName?api-version=2021-01-01&format=text") +VMSS_NAME=$(curl -H Metadata:true "http://169.254.169.254/metadata/instance/compute/vmScaleSetName?api-version=2021-01-01&format=text") +INSTANCE_ID=$(basename $(curl -H Metadata:true "http://169.254.169.254/metadata/instance/compute/resourceId?api-version=2021-01-01&format=text")) +ZONE_ID=$(curl -H Metadata:true "http://169.254.169.254/metadata/instance/compute/zone?api-version=2021-01-01&format=text") +REGION_ID=$(curl -H Metadata:true "http://169.254.169.254/metadata/instance/compute/location?api-version=2021-01-01&format=text") # Do NOT change the LUN. Based on this we find and mount the disk in the VM LUN=2 - DISK_IOPS=${disk_iops_read_write} DISK_THROUGHPUT=${disk_mbps_read_write} DISK_SIZE_GB=${disk_size_gb} - -# TODO Define the disk name based on the hostname ?? -diskName="Disk_$${VMSS_NAME}_$${INSTANCE_ID}" - -for i in $(seq 1 6); do -# Wait for existing disks in the VMSS which are unattached -existingUnattachedDisk=$( - az disk list --resource-group $RESOURSE_GROUP \ - --query "[?diskState=='Unattached' && starts_with(name, 'Disk_$${VMSS_NAME}')].{Name:name}" \ - --output tsv - ) - - if [ -z "$${existingUnattachedDisk:-}" ]; then - echo 'Disk not yet available' - sleep 10 - else - break +ATTACHED_DISK=$(az vmss list-instances --resource-group "$RESOURCE_GROUP" --name "$VMSS_NAME" --query "[?instanceId=='$INSTANCE_ID'].storageProfile.dataDisks[].name" --output tsv) + +# Checks if a disk is attached (handles terraform apply updates to the userdata script) +if [ -z "$ATTACHED_DISK" ]; then + + for i in $(seq 1 6); do + # Wait for existing disks in the VMSS which are unattached + existingUnattachedDisk=$( + az disk list --resource-group $RESOURCE_GROUP \ + --query "[?diskState=='Unattached' && starts_with(name, 'Disk-$${RESOURCE_GROUP}') && zones[0]=='$${ZONE_ID}'].{Name:name}" \ + --output tsv + ) + + if [ -z "$${existingUnattachedDisk:-}" ]; then + echo 'Disk not yet available' + sleep 10 + else + break + fi + done + + if [ -z "$existingUnattachedDisk" ]; then + echo "Creating a new managed disk" + # Fetch the number of elements + DISKS_IN_ZONE=$(az disk list --query "length([?zones[0]=='$${ZONE_ID}'])" --output tsv) + + # Increment the number for the new name + DISK_ORDER=$((DISKS_IN_ZONE + 1)) + DISK_NAME="Disk-$${RESOURCE_GROUP}-$${ZONE_ID}-$${DISK_ORDER}" + + az disk create --resource-group $RESOURCE_GROUP \ + --name $DISK_NAME \ + --size-gb $DISK_SIZE_GB \ + --location $REGION_ID \ + --sku PremiumV2_LRS \ + --zone $ZONE_ID \ + --os-type Linux \ + --disk-iops-read-write $DISK_IOPS \ + --disk-mbps-read-write $DISK_THROUGHPUT \ + --tags createdBy=$INSTANCE_HOSTNAME \ + --public-network-access Disabled \ + --network-access-policy DenyAll fi -done -if [ -z "$existingUnattachedDisk" ]; then - echo "Creating a new managed disk" - az disk create --resource-group $RESOURSE_GROUP \ - --name $diskName \ - --size-gb $DISK_SIZE_GB \ - --location $REGION_ID \ - --sku PremiumV2_LRS \ - --zone $ZONE_ID \ - --os-type Linux \ - --disk-iops-read-write $DISK_IOPS \ - --disk-mbps-read-write $DISK_THROUGHPUT \ - --public-network-access Disabled \ - --network-access-policy DenyAll + # Try to attach an existing managed disk + availableDisks=$(az disk list --resource-group $RESOURCE_GROUP --query "[?diskState=='Unattached' && starts_with(name, 'Disk-$${RESOURCE_GROUP}') && zones[0]=='$${ZONE_ID}'].{Name:name}" --output tsv) + echo "Attaching available disk $availableDisks." + # Set Internal Field Separator to newline to handle spaces in names + IFS=$'\n' + # Would iterate through all available disks and attempt to attach them + for availableDisk in $availableDisks; do + az vmss disk attach --vmss-name $VMSS_NAME --resource-group $RESOURCE_GROUP --instance-id $INSTANCE_ID --lun $LUN --disk "$availableDisk" || true + done fi - -# Checks if a managed disk is attached to the instance -attachedDisk=$(az vmss list-instances --resource-group "$RESOURSE_GROUP" --name "$VMSS_NAME" --query "[?instanceId==\"$INSTANCE_ID\"].storageProfile.dataDisks[].name" --output tsv) - -if [ -z "$attachedDisk" ]; then - echo "No data disks attached for instance ID $INSTANCE_ID in VMSS $VMSS_NAME." - # Try to attach an existing managed disk - availableDisks=$(az disk list --resource-group $RESOURSE_GROUP --query "[?diskState=='Unattached' && starts_with(name, 'Disk_$${VMSS_NAME}') && zones[0]=='$${ZONE_ID}'].{Name:name}" --output tsv) - echo "Attaching available disk $availableDisks." - # Set Internal Field Separator to newline to handle spaces in names - IFS=$'\n' - # Would iterate through all available disks and attempt to attach them - for availableDisk in $availableDisks; do - az vmss disk attach --vmss-name $VMSS_NAME --resource-group $RESOURSE_GROUP --instance-id $INSTANCE_ID --lun $LUN --disk "$availableDisk" || true - done -fi - # Gets device name based on LUN graphdb_device=$(lsscsi --scsi --size | awk '/\[1:.*:0:2\]/ {print $7}') # Check if the device is present after attaching the disk if [ -b "$graphdb_device" ]; then - echo "Device $graphdb_device is available." + echo "Device $graphdb_device is available." else - echo "Device $graphdb_device is not available. Something went wrong." - exit 1 + echo "Device $graphdb_device is not available. Something went wrong." + exit 1 fi # create a file system if there isn't any @@ -100,7 +101,7 @@ if ! mount | grep -q "$graphdb_device"; then # Add an entry to the fstab file to automatically mount the disk if ! grep -q "$graphdb_device" /etc/fstab; then - echo "$graphdb_device $disk_mount_point ext4 defaults 0 2" >> /etc/fstab + echo "$graphdb_device $disk_mount_point ext4 defaults 0 2" >>/etc/fstab fi # Mount the disk @@ -118,10 +119,71 @@ chown -R graphdb:graphdb /var/opt/graphdb # # DNS hack +# This provides stable network addresses for GDB instances in Azure VMSS # +IP_ADDRESS=$(hostname -I | awk '{print $1}') + +for i in $(seq 1 6); do + # Waits for DNS zone to be created and role assigned + DNS_ZONE_NAME=$(az network private-dns zone list --query "[].name" --output tsv) + if [ -z "$${DNS_ZONE_NAME:-}" ]; then + echo 'Zone not available yet' + sleep 10 + else + break + fi +done + +# Get all FQDN records from the private DNS zone containing "node" +ALL_FQDN_RECORDS=($(az network private-dns record-set list -z $DNS_ZONE_NAME --resource-group $RESOURCE_GROUP --query "[?contains(name, 'node')].fqdn" --output tsv)) +# Get all instance IDs for a specific VMSS +INSTANCE_IDS=($(az vmss list-instances --resource-group $RESOURCE_GROUP --name $VMSS_NAME --query "[].instanceId" --output tsv)) +# Sort instance IDs +SORTED_INSTANCE_IDs=($(echo "$${INSTANCE_IDS[@]}" | tr ' ' '\n' | sort)) +# Find the lowest, middle and highest instance IDs +LOWEST_INSTANCE_ID=$${SORTED_INSTANCE_IDs[0]} +MIDDLE_INSTANCE_ID=$${SORTED_INSTANCE_IDs[1]} +HIGHEST_INSTANCE_ID=$${SORTED_INSTANCE_IDs[2]} + +# Will ping a DNS record, if no response is returned, will update the DNS record with the IP of the instance +ping_and_set_dns_record() { + local dns_record="$1" + echo "Pinging $dns_record" + if ping -c 5 "$dns_record"; then + echo "Ping successful" + else + echo "Ping failed for $dns_record" + # Extracts the record name + RECORD_NAME=$(echo "$dns_record" | awk -F'.' '{print $1}') + az network private-dns record-set a update --resource-group $RESOURCE_GROUP --zone-name $DNS_ZONE_NAME --name $RECORD_NAME --set ARecords[0].ipv4Address="$IP_ADDRESS" + fi +} + +# assign DNS record name based on instanceId +for i in "$${!SORTED_INSTANCE_IDs[@]}"; do + if [ "$INSTANCE_ID" == "$${LOWEST_INSTANCE_ID}" ]; then + RECORD_NAME="node-1" + elif [ "$INSTANCE_ID" == "$${MIDDLE_INSTANCE_ID}" ]; then + RECORD_NAME="node-2" + elif [ "$INSTANCE_ID" == "$${HIGHEST_INSTANCE_ID}" ]; then + RECORD_NAME="node-3" + fi + # Get the FQDN for the current instance + FQDN=$(az network private-dns record-set list -z $DNS_ZONE_NAME --resource-group $RESOURCE_GROUP --query "[?contains(name, '$RECORD_NAME')].fqdn" --output tsv) + + if [ -z "$${FQDN:-}" ]; then + az network private-dns record-set a add-record --resource-group $RESOURCE_GROUP --zone-name $DNS_ZONE_NAME --record-set-name $RECORD_NAME --ipv4-address "$IP_ADDRESS" + else + for record in "$${ALL_FQDN_RECORDS[@]}"; do + ping_and_set_dns_record "$record" + done + fi + + break +done -# TODO: Should be based on something stable, e.g. volume id -node_dns=$(hostname) +# Gets the full DNS record for the current instance +node_dns=$(az network private-dns record-set a show --resource-group $RESOURCE_GROUP --zone-name $DNS_ZONE_NAME --name $RECORD_NAME --output tsv --query "fqdn" | rev | cut -c 2- | rev) # # GraphDB configuration overrides @@ -136,14 +198,14 @@ az keyvault secret download --vault-name ${key_vault_name} --name graphdb-licens graphdb_cluster_token=$(az keyvault secret show --vault-name ${key_vault_name} --name graphdb-cluster-token | jq -rj .value | base64 -d) # TODO: where is the vhost here? -cat << EOF > /etc/graphdb/graphdb.properties +cat < /etc/graphdb/graphdb.properties graphdb.auth.token.secret=$graphdb_cluster_token graphdb.connector.port=7200 graphdb.external-url=http://$${node_dns}:7200/ graphdb.rpc.address=$${node_dns}:7300 EOF -cat << EOF > /etc/graphdb-cluster-proxy/graphdb.properties +cat < /etc/graphdb-cluster-proxy/graphdb.properties graphdb.auth.token.secret=$graphdb_cluster_token graphdb.connector.port=7201 graphdb.external-url=https://${graphdb_external_address_fqdn} @@ -163,7 +225,7 @@ jvm_max_memory=$(echo "$total_memory_gb * 0.85" | bc | cut -d'.' -f1) mkdir -p /etc/systemd/system/graphdb.service.d/ -cat << EOF > /etc/systemd/system/graphdb.service.d/overrides.conf +cat < /etc/systemd/system/graphdb.service.d/overrides.conf [Service] Environment="GDB_HEAP_SIZE=$${jvm_max_memory}g" EOF @@ -172,7 +234,7 @@ EOF # Appends configuration overrides to graphdb.properties if [[ $secrets == *"graphdb-properties"* ]]; then echo "Using graphdb.properties overrides" - az keyvault secret show --vault-name ${key_vault_name} --name graphdb-properties | jq -rj .value | base64 -d >> /etc/graphdb/graphdb.properties + az keyvault secret show --vault-name ${key_vault_name} --name graphdb-properties | jq -rj .value | base64 -d >>/etc/graphdb/graphdb.properties fi # Appends environment overrides to GDB_JAVA_OPTS @@ -244,3 +306,72 @@ systemctl enable graphdb-cluster-proxy.service systemctl start graphdb-cluster-proxy.service echo "Finished GraphDB instance configuration" + +# +# Cluster creation +# +GRAPHDB_ADMIN_PASSWORD="$(az keyvault secret show --vault-name ${key_vault_name} --name graphdb-password --query "value" --output tsv)" + +check_gdb() { + local gdb_address="$1:7200/protocol" + if curl -s --head -u "admin:$${GRAPHDB_ADMIN_PASSWORD}" --fail $gdb_address >/dev/null; then + return 0 # Success, endpoint is available + else + return 1 # Endpoint not available yet + fi +} + +# Waits for 3 DNS records to be available +wait_dns_records() { + ALL_FQDN_RECORDS=($(az network private-dns record-set list -z $DNS_ZONE_NAME --resource-group $RESOURCE_GROUP --query "[?contains(name, 'node')].fqdn" --output tsv)) + if [ "$${ALL_FQDN_RECORDS[@]}" -ne 3 ]; then + sleep 5 + wait_dns_records + fi +} + +wait_dns_records + +if [ "$INSTANCE_ID" == "$${LOWEST_INSTANCE_ID}" ]; then + for record in "$${ALL_FQDN_RECORDS[@]}"; do + echo $record + # Removes the '.' at the end of the DNS address + cleanedAddress=$${record%?} + while ! check_gdb $cleanedAddress; do + echo "Waiting for GDB $record to start" + sleep 5 + done + done + + echo "All GDB instances are available. Creating cluster" + # Checks if the cluster already exists + is_cluster=$(curl -s -o /dev/null -u "admin:$${GRAPHDB_ADMIN_PASSWORD}" -w "%%{http_code}" http://localhost:7200/rest/monitor/cluster) + + if [ "$is_cluster" != 200 ]; then + curl -X POST http://localhost:7200/rest/cluster/config \ + -H 'Content-type: application/json' \ + -u "admin:$${GRAPHDB_ADMIN_PASSWORD}" \ + -d "{\"nodes\": [\"node-1.$${DNS_ZONE_NAME}:7300\",\"node-2.$${DNS_ZONE_NAME}:7300\",\"node-3.$${DNS_ZONE_NAME}:7300\"]}" + else + echo "Cluster exists" + fi +fi + +# +# Change admin user password and enable security +# +security_enabled=$(curl -s -X GET --header 'Accept: application/json' -u "admin:$${GRAPHDB_ADMIN_PASSWORD}" 'http://localhost:7200/rest/security') + +# Check if GDB security is enabled +if [[ $security_enabled == "true" ]]; then + echo "Security is enabled" +else + # Set the admin password + curl --location --request PATCH 'http://localhost:7200/rest/security/users/admin' \ + --header 'Content-Type: application/json' \ + --data "{ \"password\": \"$${GRAPHDB_ADMIN_PASSWORD}\" }" + # Enable the security + curl -X POST --header 'Content-Type: application/json' --header 'Accept: */*' -d 'true' 'http://localhost:7200/rest/security' +fi + +echo "Script completed." diff --git a/variables.tf b/variables.tf index 7e83cb9..e740b41 100644 --- a/variables.tf +++ b/variables.tf @@ -3,6 +3,11 @@ variable "resource_name_prefix" { description = "Resource name prefix used for tagging and naming Azure resources" type = string + + validation { + condition = can(regex("^[a-zA-Z0-9-]+$", var.resource_name_prefix)) && !can(regex("^-", var.resource_name_prefix)) + error_message = "Resource name prefix cannot start with a hyphen and can only contain letters, numbers, and hyphens." + } } variable "location" {