Component updates 04/24 and host encryption

.ado/pipelines/config/configuration.yaml

@@ -5,13 +5,13 @@ variables:
   value: 'foundational-online'
 
 - name: 'terraformVersion'    # Terraform Version
-  value: '1.6.2-*'
+  value: '1.8.1-*'
 - name: 'kubernetesVersion'   # kubernetes version used for aks clusters
-  value: '1.28.5'
+  value: '1.29.2'
 - name: 'helmVersion'         # helm package manager version
-  value: 'v3.14.0'
+  value: 'v3.14.4'
 - name: 'ingressNginxVersion' # nginx ingress controller helm chart version
-  value: '4.7.3'
+  value: '4.10.0'
 - name: 'certManagerVersion'  # cert-manager helm chart version
   value: 'v1.13.3'
 - name: 'dotnetSdkVersion'    # dotnet sdk version

.ado/pipelines/config/variables-values-e2e.yaml

@@ -6,6 +6,11 @@ variables:
 - name:  'prefix'
   value: 'mce2e'          # <===== CHANGE THIS! Must not be longer than 6 characters! Needs to be a unique prefix
 
+# Host-encryption for compute resources (needs to be enabled on subscription-level)
+# https://learn.microsoft.com/en-us/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli
+- name: 'enableHostEncryption'
+  value: 'true'           # <===== CHANGE THIS! Set to 'true' to enable host encryption
+
 # The first value in 'stampLocations' is the primary region used for global services.
 # IMPORTANT! Changing the primary region (first value) is a BREAKING change and will destroy CosmosDB and Front Door.
 # Check which regions are valid. There is a list in /src/infra/README.md

.ado/pipelines/config/variables-values-int.yaml

@@ -6,6 +6,11 @@ variables:
 - name:  'prefix'
   value: 'mcint'          # <===== CHANGE THIS! Must not be longer than 6 characters! Needs to be a unique prefix
 
+# Host-encryption for compute resources (needs to be enabled on subscription-level)
+# https://learn.microsoft.com/en-us/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli
+- name: 'enableHostEncryption'
+  value: 'true'           # <===== CHANGE THIS! Set to 'true' to enable host encryption
+
 # The first value in 'stampLocations' is the primary region used for global services.
 # IMPORTANT! Changing the primary region (first value) is a BREAKING change and will destroy CosmosDB and Front Door.
 - name: 'stampLocations'

.ado/pipelines/config/variables-values-prod.yaml

@@ -6,7 +6,11 @@ variables:
 - name:  'prefix'
   value: 'afprod'          # <===== CHANGE THIS! Must not be longer than 6 characters! Needs to be a unique prefix
 
-
+# Host-encryption for compute resources (needs to be enabled on subscription-level)
+# https://learn.microsoft.com/en-us/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli
+- name: 'enableHostEncryption'
+  value: 'true'           # <===== CHANGE THIS! Set to 'true' to enable host encryption
+
 # The first value in 'stampLocations' is the primary region used for global services.
 # IMPORTANT! Changing the primary region (first value) is a BREAKING change and will destroy CosmosDB and Front Door.
 - name: 'stampLocations'

.ado/pipelines/templates/steps-terraform-apply.yaml

@@ -33,6 +33,7 @@ steps:
       terraform plan -input=false -out=tf_plan \
                      -var=environment="$(environment)" \
                      -var=prefix="${{ parameters.customPrefix }}" \
+                     -var=aks_enable_host_encryption="$(enableHostEncryption)" \
                      -var-file="variables-$(environment).tfvars" \
                      ${{ parameters.customAttributes }}
 

docs/reference-implementation/Getting-Started-CLI.md

@@ -137,6 +137,8 @@ All pipelines require an Azure DevOps service connection to access the target Az
 
 > **Important!** The AAD Service Principal needs **subscription-level owner permissions** as the pipeline will create various role assignments.
 
+> **Important!** Azure Mission-Critical is using host-level encryption compute. This feature needs to be manually registered in each subscription. See [Use the Azure CLI to enable end-to-end encryption using encryption at host](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli) for more.
+
 You need to repeat these steps for each of the environments that you want to create. But you can also only start with one for now. If so, we recommend to start with `e2e`.
 
 ```powershell
@@ -204,12 +206,13 @@ Modify the respective file for the environment which you want to deploy. At leas
 
 | Required to modify | Key | Description | Sample value |
 | --- | --- | --- | --- |
-| **YES** | prefix | Custom prefix used for Azure resources. **Must not be longer than 6 characters!** | mye2e |
-| **YES** | contactEmail | E-mail alias used for alerting. **Be careful which address you put in here as it will potentially receive a lot of notification emails** | alwaysonappnet@example.com |
-| NO | terraformResourceGroup | Resource Group where the Terraform state Storage account will be deployed | terraformstate-rg |
-| NO | stampLocations | List of locations (Azure Regions) where this environment will be deployed into. You can keep the default to start with.  | ["northeurope", "eastus2"] |
-| NO | envDnsZoneRG | OPTIONAL: Name of the Azure Resource group which holds the Azure DNS Zone for your custom domain. Not required if you do not plan to use a custom DNS name | mydns-rg |
-| NO | envDomainName | OPTIONAL: Name of the Azure DNS Zone. Not required if you do not plan to use a custom DNS name | example.com |
+| **YES** | `prefix` | Custom prefix used for Azure resources. **Must not be longer than 6 characters!** | `mye2e` |
+| **YES** | `contactEmail` | E-mail alias used for alerting. **Be careful which address you put in here as it will potentially receive a lot of notification emails** | `alwaysonappnet@example.com` |
+| NO | `terraformResourceGroup` | Resource Group where the Terraform state Storage account will be deployed | `terraformstate-rg` |
+| NO | `stampLocations` | List of locations (Azure Regions) where this environment will be deployed into. You can keep the default to start with.  | `["northeurope", "eastus2"]` |
+| NO | `envDnsZoneRG` | OPTIONAL: Name of the Azure Resource group which holds the Azure DNS Zone for your custom domain. Not required if you do not plan to use a custom DNS name | `mydns-rg` |
+| NO | `envDomainName` | OPTIONAL: Name of the Azure DNS Zone. Not required if you do not plan to use a custom DNS name | `example.com` |
+| NO | `enableHostEncryption` | Enable or disable host-encryption for compute resources (needs to be enabled per-subscription) | `true` |
 
 **After modifying the file, make sure to commit and push the changes to your Git repository.**
 

docs/reference-implementation/Getting-Started.md

@@ -46,7 +46,6 @@ To deploy the Azure Mission-Critical reference implementation, you need to creat
 
 - [Create an organization or project collection](https://learn.microsoft.com/azure/devops/organizations/accounts/create-organization?view=azure-devops)
 
-
 #### Create a new Azure DevOps project
 
 Once you have created an Azure DevOps organization, you can create a new project in that organization. Go to the Azure DevOps portal, select the desired Organization and Click on "+ New Project" in the upper right hand corner.
@@ -123,6 +122,8 @@ All pipelines require an Azure DevOps service connection to access the target Az
 
 > **Important!** The AAD Service Principal needs **subscription-level owner permissions** as the pipeline will create various role assignments.
 
+> **Important!** Azure Mission-Critical is using host-level encryption compute. This feature needs to be manually registered in each subscription. See [Use the Azure CLI to enable end-to-end encryption using encryption at host](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli) for more.
+
 You need to repeat these steps for each of the environments that you want to create. But you can also only start with one for now. If so, we recommend to start with `e2e`.
 
 ```powershell
@@ -184,18 +185,18 @@ Modify the respective file for the environment which you want to deploy. At leas
 
 | Required to modify | Key | Description | Sample value |
 | --- | --- | --- | --- |
-| **YES** | prefix | Custom prefix used for Azure resources. **Must not be longer than 6 characters!** | mye2e |
-| **YES** | contactEmail | E-mail alias used for alerting. **Be careful which address you put in here as it will potentially receive a lot of notification emails** | alwaysonappnet@example.com |
-| NO | terraformResourceGroup | Resource Group where the Terraform state Storage account will be deployed | terraformstate-rg |
-| NO | stampLocations | List of locations (Azure Regions) where this environment will be deployed into. You can keep the default to start with.  | ["northeurope", "eastus2"] |
-| NO | envDnsZoneRG | OPTIONAL: Name of the Azure Resource group which holds the Azure DNS Zone for your custom domain. Not required if you do not plan to use a custom DNS name | mydns-rg |
-| NO | envDomainName | OPTIONAL: Name of the Azure DNS Zone. Not required if you do not plan to use a custom DNS name | example.com |
+| **YES** | `prefix` | Custom prefix used for Azure resources. **Must not be longer than 6 characters!** | `mye2e` |
+| **YES** | `contactEmail` | E-mail alias used for alerting. **Be careful which address you put in here as it will potentially receive a lot of notification emails** | `alwaysonappnet@example.com` |
+| NO | `terraformResourceGroup` | Resource Group where the Terraform state Storage account will be deployed | `terraformstate-rg` |
+| NO | `stampLocations` | List of locations (Azure Regions) where this environment will be deployed into. You can keep the default to start with.  | `["northeurope", "eastus2"]` |
+| NO | `envDnsZoneRG` | OPTIONAL: Name of the Azure Resource group which holds the Azure DNS Zone for your custom domain. Not required if you do not plan to use a custom DNS name | `mydns-rg` |
+| NO | `envDomainName` | OPTIONAL: Name of the Azure DNS Zone. Not required if you do not plan to use a custom DNS name | example.com |
+| NO | `enableHostEncryption` | Enable or disable host-encryption for compute resources (needs to be enabled per-subscription) | `true` |
 
 **After modifying the file, make sure to commit and push the changes to your Git repository.**
 
 For more details on the variables, you can consult [this guide](/.ado/pipelines/README.md#configuration-files).
 
-
 ### 7) Execute the first deployment
 
 After completing all previous steps in this guide, you can start executing the pipelines to spin up the infrastructure.

docs/reference-implementation/Troubleshooting.md

@@ -74,6 +74,28 @@ Location: SwedenCentral, Current Limit: 100, Current Usage: 96, Additional Requi
 
 **Error:**
 
+```console
+│ Agent Pool Name: "workloadpool"): performing CreateOrUpdate: unexpected status 400 (400 Bad Request) with response: {
+│   "code": "SubscriptionNotEnabledEncryptionAtHost",
+│   "details": null,
+│   "message": "Subscription does not enable EncryptionAtHost.",
+│   "subcode": ""
+│  }
+```
+
+**Description:** Host encryption needs to be enabled at subscription-level when `enableHostEncryption` is set to `true` in `variables-values-<env>.yaml` in `.ado/pipelines/config`.
+
+**Solution:**
+
+Host encryption needs to be enabled at subscription-level: [Use the Azure CLI to enable end-to-end encryption using encryption at host
+](https://learn.microsoft.com/en-us/azure/virtual-machines/linux/disks-enable-host-based-encryption-cli)
+
+If you don't want to use Host-encryption, you can disable it by setting `enableHostEncryption` to `false` in `variables-values-<env>.yaml` in `.ado/pipelines/config`.
+
+---
+
+**Error:**
+
 ```console
 Error: deleting Front Door (Subscription: "xxxxx-8cbd-46f2-a146-yyyyyyyyyy"
 │ Resource Group Name: "xxxxx-global-rg"

src/app/AlwaysOn.BackgroundProcessor/Dockerfile

@@ -1,4 +1,4 @@
-FROM mcr.microsoft.com/dotnet/sdk:8.0.101 AS build-env
+FROM mcr.microsoft.com/dotnet/sdk:8.0.203 AS build-env
 WORKDIR /app
 
 COPY . ./

src/app/AlwaysOn.CatalogService/Dockerfile

@@ -1,5 +1,5 @@
 # Create build environment
-FROM mcr.microsoft.com/dotnet/sdk:8.0.101 AS build-env
+FROM mcr.microsoft.com/dotnet/sdk:8.0.203 AS build-env
 WORKDIR /app
 COPY . ./
 RUN dotnet publish AlwaysOn.CatalogService -c Release -o AlwaysOn.CatalogService/out

src/app/AlwaysOn.HealthService/Dockerfile

@@ -1,5 +1,5 @@
 # Create build environment
-FROM mcr.microsoft.com/dotnet/sdk:8.0.101 AS build-env
+FROM mcr.microsoft.com/dotnet/sdk:8.0.203 AS build-env
 WORKDIR /app
 COPY . ./
 RUN dotnet publish AlwaysOn.HealthService -c Release -o AlwaysOn.HealthService/out

src/app/AlwaysOn.Shared/AlwaysOn.Shared.csproj

@@ -10,7 +10,7 @@
     <PackageReference Include="Microsoft.ApplicationInsights.AspNetCore" Version="2.22.0" />
     <PackageReference Include="Microsoft.Azure.Cosmos" Version="3.38.1" />
     <PackageReference Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.2" />
+    <PackageReference Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="8.0.3" />
   </ItemGroup>
 
 </Project>

src/app/AlwaysOn.UI/Dockerfile

@@ -1,5 +1,5 @@
 # Create build environment
-FROM node:21.6.2 as build-env
+FROM node:21.7.1 as build-env
 
 WORKDIR /app
 COPY . ./

src/config/ingress-nginx/values.helm.yaml

@@ -3,6 +3,7 @@
 # https://github.com/kubernetes/ingress-nginx/blob/master/charts/ingress-nginx/values.yaml
 
 controller:
+  allowSnippetAnnotations: true
   securityContext:
     readOnlyRootFilesystem: true # If a container does not need to write files, it should be run with a read-only filesystem. 
     privileged: false # Running a container as privileged gives all capabilities to the container, and it also lifts all the limitations enforced by the device cgroup controller.

src/infra/monitoring/grafana/Dockerfile

@@ -16,7 +16,7 @@ RUN chmod +x insertqueries.sh && ./insertqueries.sh
 WORKDIR /app/healthmodelpanel
 RUN npm install
 
-FROM grafana/grafana:9.5.16
+FROM grafana/grafana:9.5.18
 
 # In order to run unsigned plugins such as our health model panel, we need to explicitly allow them in an env variable. 
 ENV GF_PLUGINS_ALLOW_LOADING_UNSIGNED_PLUGINS="healthmodelpanel"

src/infra/monitoring/grafana/terraform/globalresources/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.97.1"
     }
   }
 

src/infra/monitoring/grafana/terraform/stamps/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.97.1"
     }
   }
 

src/infra/workload/globalresources/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.97.1"
     }
     azapi = {
       source  = "Azure/azapi"

src/infra/workload/releaseunit/main.tf

@@ -2,11 +2,11 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.101.0"
     }
     azapi = {
       source  = "Azure/azapi"
-      version = "1.12.1"
+      version = "1.13.1"
     }
   }
 
@@ -21,6 +21,7 @@ provider "azurerm" {
       prevent_deletion_if_contains_resources = false
     }
   }
+  skip_provider_registration = true
 }
 
 provider "azapi" {}

src/infra/workload/releaseunit/modules/stamp/kubernetes.tf

@@ -87,6 +87,8 @@ resource "azurerm_kubernetes_cluster_node_pool" "workload" {
   os_disk_type          = "Ephemeral"
   orchestrator_version  = var.aks_kubernetes_version
 
+  enable_host_encryption = var.aks_enable_host_encryption # host encryption needs to be enabled per-subscription
+
   mode  = "User" # Define this node pool as a "user" aka workload node pool
   zones = [1, 2, 3]
 

src/infra/workload/releaseunit/modules/stamp/main.tf

@@ -2,11 +2,11 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.101.0"
     }
     azapi = {
       source  = "Azure/azapi"
-      version = "1.12.1"
+      version = "1.13.1"
     }
   }
 }

src/infra/workload/releaseunit/modules/stamp/variables.tf

@@ -96,6 +96,12 @@ variable "aks_user_node_pool_autoscale_maximum" {
   type        = number
 }
 
+variable "aks_enable_host_encryption" {
+  default     = true
+  type        = bool
+  description = "Enable host encryption for AKS nodes"
+}
+
 variable "event_hub_thoughput_units" {
   description = "Number of Throughput Units for Event Hub Namespace"
   type        = number

src/infra/workload/releaseunit/stamp.tf

@@ -23,8 +23,6 @@ module "stamp" {
 
   vnet_address_space = module.stamp_addresses.network_cidr_blocks[each.value]
 
-  aks_kubernetes_version = var.aks_kubernetes_version # kubernetes version
-
   prefix       = local.prefix       # handing over the resource prefix
   default_tags = local.default_tags # handing over the resource tags
   queued_by    = var.queued_by
@@ -39,6 +37,10 @@ module "stamp" {
   frontdoor_id_header                    = var.frontdoor_id_header
   acr_name                               = var.acr_name
 
+  aks_kubernetes_version = var.aks_kubernetes_version # Defined kubernetes version
+
+  aks_enable_host_encryption  = var.aks_enable_host_encryption # Enable host encryption
+
   aks_system_node_pool_sku_size          = var.aks_system_node_pool_sku_size
   aks_system_node_pool_autoscale_minimum = var.aks_system_node_pool_autoscale_minimum
   aks_system_node_pool_autoscale_maximum = var.aks_system_node_pool_autoscale_maximum

src/infra/workload/releaseunit/variables.tf

@@ -101,6 +101,12 @@ variable "aks_kubernetes_version" {
   type        = string
 }
 
+variable "aks_enable_host_encryption" {
+  default     = true
+  type        = bool
+  description = "Enable host encryption for AKS nodes"
+}
+
 variable "aks_system_node_pool_sku_size" {
   description = "VM SKU of the AKS system nodes"
   type        = string

src/testing/loadtest-azure/infra/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.97.1"
     }
   }
 

src/testing/loadtest-locust/infra/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.97.1"
     }
   }
 

src/testing/userload-generator/infra/main.tf

@@ -2,7 +2,7 @@ terraform {
   required_providers {
     azurerm = {
       source  = "hashicorp/azurerm"
-      version = "3.94.0"
+      version = "3.97.1"
     }
   }