POL-1402 Azure Rightsizing Compute: Consider Managed Disk (#2808)

* update * fix * update * fix * update * fix * update * update * update * update * update
flexera-public · Nov 7, 2024 · 6908c0b · 6908c0b
1 parent 108d5c0
commit 6908c0b
Show file tree

Hide file tree

Showing 6 changed files with 105 additions and 19 deletions.
diff --git a/cost/azure/rightsize_compute_instances/CHANGELOG.md b/cost/azure/rightsize_compute_instances/CHANGELOG.md
@@ -1,5 +1,10 @@
 # Changelog
 
+## v6.0.0
+
+- Recommendations now consider number of attached data disks. Sizes that would not support the current number of attached disks for an instance will not be recommended.
+- Azure credential now requires `Microsoft.Compute/locations/vmSizes/read` permission to support the above.
+
 ## v5.4.0
 
 - Added support for downsizing multiple sizes where appropriate

diff --git a/cost/azure/rightsize_compute_instances/README.md b/cost/azure/rightsize_compute_instances/README.md
@@ -2,7 +2,7 @@
 
 ## What It Does
 
-This policy checks all the instances in Azure Subscriptions for the average or maximum CPU and/or memory usage over a user-specified number of days. If the usage is less than the user provided Idle Instance CPU and/or memory percentage threshold then the Virtual Machine is recommended for deletion. If the usage is less than the user provided Underutilized Instance CPU and/or Memory percentage threshold then the Virtual Machine is recommended for downsizing. Both sets of Virtual Machines returned from this policy are emailed to the user.
+This policy checks all the instances in Azure Subscriptions for the average or maximum CPU and/or memory usage over a user-specified number of days. If the usage is less than the user provided Idle Instance CPU and/or memory percentage threshold then the Virtual Machine is recommended for deletion. If the usage is less than the user provided Underutilized Instance CPU and/or Memory percentage threshold, and the total number of currently attached disks is supported with a smaller instance size, then the Virtual Machine is recommended for downsizing. Both sets of Virtual Machines returned from this policy are emailed to the user.
 
 ## How It Works
 
@@ -74,6 +74,7 @@ For administrators [creating and managing credentials](https://docs.flexera.com/
 - [**Azure Resource Manager Credential**](https://docs.flexera.com/flexera/EN/Automation/ProviderCredentials.htm#automationadmin_109256743_1124668) (*provider=azure_rm*) which has the following permissions:
   - `Microsoft.Insights/metrics/read`
   - `Microsoft.Compute/skus/read`
+  - `Microsoft.Compute/locations/vmSizes/read`
   - `Microsoft.Compute/virtualMachines/read`
   - `Microsoft.Compute/virtualMachines/write`*
   - `Microsoft.Compute/virtualMachines/powerOff/action`*

diff --git a/cost/azure/rightsize_compute_instances/azure_compute_rightsizing.pt b/cost/azure/rightsize_compute_instances/azure_compute_rightsizing.pt
@@ -7,7 +7,7 @@ category "Cost"
 severity "low"
 default_frequency "weekly"
 info(
-  version: "5.4.0",
+  version: "6.0.0",
   provider: "Azure",
   service: "Compute",
   policy_set: "Rightsize Compute Instances",
@@ -435,7 +435,7 @@ datasource "ds_azure_instances" do
     pagination $pagination_azure
     host $param_azure_endpoint
     path join(["/subscriptions/", val(iter_item, "id"), "/providers/Microsoft.Compute/virtualMachines"])
-    query "api-version", "2024-03-01"
+    query "api-version", "2024-07-01"
     header "User-Agent", "RS Policies"
     # Ignore status 400, 403, and 404 which can be returned in certain (legacy) types of Azure Subscriptions
     ignore_status [400, 403, 404]
@@ -452,6 +452,7 @@ datasource "ds_azure_instances" do
       field "imageOffer", jmes_path(col_item, "properties.storageProfile.imageReference.offer")
       field "imageSKU", jmes_path(col_item, "properties.storageProfile.imageReference.sku")
       field "osType", jmes_path(col_item, "properties.storageProfile.osDisk.osType")
+      field "dataDisks", jmes_path(col_item, "properties.storageProfile.dataDisks")
       field "resourceType", jmes_path(col_item, "properties.hardwareProfile.vmSize")
       field "tags", jmes_path(col_item, "tags")
       field "subscriptionId", val(iter_item, "id")
@@ -497,6 +498,9 @@ script "js_azure_instances_with_status", type: "javascript" do
   })
 
   result = _.map(ds_azure_instances, function(vm) {
+    dataDiskCount = 0
+    if (vm['dataDisks']) { dataDiskCount = vm['dataDisks'].length }
+
     return {
       resourceID: vm["resourceID"],
       resourceGroup: vm["resourceGroup"],
@@ -507,6 +511,7 @@ script "js_azure_instances_with_status", type: "javascript" do
       imageOffer: vm["imageOffer"],
       imageSKU: vm["imageSKU"],
       osType: vm["osType"],
+      dataDiskCount: dataDiskCount,
       resourceType: vm["resourceType"],
       tags: vm["tags"],
       subscriptionId: vm["subscriptionId"],
@@ -605,11 +610,7 @@ script "js_azure_instances_region_filtered", type: "javascript" do
   if (param_regions_list.length > 0) {
     result = _.filter(ds_azure_instances_tag_filtered, function(vm) {
       include_vm = _.contains(param_regions_list, vm['region'])
-
-      if (param_regions_allow_or_deny == "Deny") {
-        include_vm = !include_vm
-      }
-
+      if (param_regions_allow_or_deny == "Deny") { include_vm = !include_vm }
       return include_vm
     })
   } else {
@@ -728,6 +729,7 @@ datasource "ds_azure_instances_metrics" do
     field "imageOffer", val(iter_item, "imageOffer")
     field "imageSKU", val(iter_item, "imageSKU")
     field "osType", val(iter_item, "osType")
+    field "dataDiskCount", val(iter_item, "dataDiskCount")
     field "resourceType", val(iter_item, "resourceType")
     field "subscriptionId", val(iter_item, "subscriptionId")
     field "subscriptionName", val(iter_item, "subscriptionName")
@@ -922,6 +924,7 @@ script "js_azure_instances_metrics_organized", type: "javascript" do
       imageOffer: vm['imageOffer'],
       imageSKU: vm['imageSKU'],
       osType: vm['osType'],
+      dataDiskCount: vm['dataDiskCount'],
       resourceType: vm['resourceType'],
       accountID: vm['subscriptionId'],
       accountName: vm['subscriptionName'],
@@ -1063,12 +1066,69 @@ datasource "ds_azure_instance_size_map" do
   end
 end
 
+datasource "ds_azure_regions_for_instance_sizes" do
+  run_script $js_azure_regions_for_instance_sizes, $ds_azure_instances_metrics_organized
+end
+
+script "js_azure_regions_for_instance_sizes", type: "javascript" do
+  parameters "ds_azure_instances_metrics_organized"
+  result "result"
+  code <<-EOS
+  result = []
+
+  if (ds_azure_instances_metrics_organized.length > 0) {
+    regions = _.uniq(_.compact(_.pluck(ds_azure_instances_metrics_organized, "region")))
+
+    result = _.map(regions, function(region) {
+      return {
+        subscriptionId: ds_azure_instances_metrics_organized[0]['accountID'],
+        region: region
+      }
+    })
+  }
+EOS
+end
+
+datasource "ds_azure_instance_size_details" do
+  iterate $ds_azure_regions_for_instance_sizes
+  request do
+    auth $auth_azure
+    pagination $pagination_azure
+    host $param_azure_endpoint
+    path join(["/subscriptions/", val(iter_item, "subscriptionId"), "/providers/Microsoft.Compute/locations/", val(iter_item, "region"), "/vmSizes"])
+    query "api-version", "2024-07-01"
+    header "User-Agent", "RS Policies"
+    # Ignore status 400, 403, and 404 which can be returned in certain (legacy) types of Azure Subscriptions
+    ignore_status [400, 403, 404]
+  end
+  result do
+    encoding "json"
+    collect jmes_path(response, "value[*]") do
+      field "name", jmes_path(col_item, "name")
+      field "maxDataDiskCount", jmes_path(col_item, "maxDataDiskCount")
+    end
+  end
+end
+
+datasource "ds_azure_disk_count_table" do
+  run_script $js_azure_disk_count_table, $ds_azure_instance_size_details
+end
+
+script "js_azure_disk_count_table", type: "javascript" do
+  parameters "ds_azure_instance_size_details"
+  result "result"
+  code <<-EOS
+  result = {}
+  _.each(ds_azure_instance_size_details, function(size) { result[size['name']] = size['maxDataDiskCount'] })
+EOS
+end
+
 datasource "ds_idle_and_underutil_instances" do
-  run_script $js_idle_and_underutil_instances, $ds_azure_instances_region_filtered, $ds_azure_instances_metrics_organized, $ds_instance_costs_grouped, $ds_azure_instance_size_map, $ds_stats_interval, $ds_currency, $ds_applied_policy, $param_stats_idle_threshold_cpu_value, $param_stats_idle_threshold_mem_value, $param_stats_underutil_threshold_cpu_value, $param_stats_underutil_threshold_mem_value, $param_stats_check_both, $param_stats_threshold, $param_min_savings, $param_stats_lookback, $param_exclude_databricks, $param_downsize_multiple
+  run_script $js_idle_and_underutil_instances, $ds_azure_instances_region_filtered, $ds_azure_instances_metrics_organized, $ds_instance_costs_grouped, $ds_azure_instance_size_map, $ds_stats_interval, $ds_azure_disk_count_table, $ds_currency, $ds_applied_policy, $param_stats_idle_threshold_cpu_value, $param_stats_idle_threshold_mem_value, $param_stats_underutil_threshold_cpu_value, $param_stats_underutil_threshold_mem_value, $param_stats_check_both, $param_stats_threshold, $param_min_savings, $param_stats_lookback, $param_exclude_databricks, $param_downsize_multiple
 end
 
 script "js_idle_and_underutil_instances", type:"javascript" do
-  parameters "ds_azure_instances_region_filtered", "ds_azure_instances_metrics_organized", "ds_instance_costs_grouped", "ds_azure_instance_size_map", "ds_stats_interval", "ds_currency", "ds_applied_policy", "param_stats_idle_threshold_cpu_value", "param_stats_idle_threshold_mem_value", "param_stats_underutil_threshold_cpu_value", "param_stats_underutil_threshold_mem_value", "param_stats_check_both", "param_stats_threshold", "param_min_savings", "param_stats_lookback", "param_exclude_databricks", "param_downsize_multiple"
+  parameters "ds_azure_instances_region_filtered", "ds_azure_instances_metrics_organized", "ds_instance_costs_grouped", "ds_azure_instance_size_map", "ds_stats_interval", "ds_azure_disk_count_table", "ds_currency", "ds_applied_policy", "param_stats_idle_threshold_cpu_value", "param_stats_idle_threshold_mem_value", "param_stats_underutil_threshold_cpu_value", "param_stats_underutil_threshold_mem_value", "param_stats_check_both", "param_stats_threshold", "param_min_savings", "param_stats_lookback", "param_exclude_databricks", "param_downsize_multiple"
   result "result"
   code <<-'EOS'
   // Used for formatting numbers to look pretty
@@ -1182,13 +1242,13 @@ script "js_idle_and_underutil_instances", type:"javascript" do
         instance["threshold"] = param_stats_underutil_threshold_cpu_value
         instance["memoryThreshold"] = param_stats_underutil_threshold_mem_value
 
-        if (ds_azure_instance_size_map[instance['resourceType']]) {
+        if (ds_azure_instance_size_map[instance['resourceType']] && ds_azure_instance_size_map[instance['resourceType']]['down'] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['resourceType']]['down']] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['resourceType']]['down']] >= instance["dataDiskCount"]) {
           instance["newResourceType"] = ds_azure_instance_size_map[instance['resourceType']]['down']
         }
 
-        if (param_downsize_multiple == "Yes") {
+        if (instance["newResourceType"] && param_downsize_multiple == "Yes") {
           if (checking_cpu && checking_mem && param_stats_check_both == "Both CPU and Memory") {
-            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && cpu_value * 2 < param_stats_underutil_threshold_cpu_value && mem_value * 2 < param_stats_underutil_threshold_mem_value) {
+            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] >= instance["dataDiskCount"] && cpu_value * 2 < param_stats_underutil_threshold_cpu_value && mem_value * 2 < param_stats_underutil_threshold_mem_value) {
               cpu_value = cpu_value * 2
               mem_value = mem_value * 2
               instance["newResourceType"] = ds_azure_instance_size_map[instance['newResourceType']]['down']
@@ -1197,7 +1257,7 @@ script "js_idle_and_underutil_instances", type:"javascript" do
           }
 
           if (checking_cpu && checking_mem && param_stats_check_both == "Either CPU or Memory") {
-            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && (cpu_value * 2 < param_stats_underutil_threshold_cpu_value || mem_value * 2 < param_stats_underutil_threshold_mem_value)) {
+            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] >= instance["dataDiskCount"] && (cpu_value * 2 < param_stats_underutil_threshold_cpu_value || mem_value * 2 < param_stats_underutil_threshold_mem_value)) {
               cpu_value = cpu_value * 2
               mem_value = mem_value * 2
               instance["newResourceType"] = ds_azure_instance_size_map[instance['newResourceType']]['down']
@@ -1206,15 +1266,15 @@ script "js_idle_and_underutil_instances", type:"javascript" do
           }
 
           if (checking_cpu && !checking_mem) {
-            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && cpu_value * 2 < param_stats_underutil_threshold_cpu_value) {
+            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] >= instance["dataDiskCount"] && cpu_value * 2 < param_stats_underutil_threshold_cpu_value) {
               cpu_value = cpu_value * 2
               instance["newResourceType"] = ds_azure_instance_size_map[instance['newResourceType']]['down']
               savingsMultiplier += 1
             }
           }
 
           if (!checking_cpu && checking_mem) {
-            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && mem_value * 2 < param_stats_underutil_threshold_mem_value) {
+            while (ds_azure_instance_size_map[instance['newResourceType']]['down'] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] && ds_azure_disk_count_table[ds_azure_instance_size_map[instance['newResourceType']]['down']] >= instance["dataDiskCount"] && mem_value * 2 < param_stats_underutil_threshold_mem_value) {
               mem_value = mem_value * 2
               instance["newResourceType"] = ds_azure_instance_size_map[instance['newResourceType']]['down']
               savingsMultiplier += 1
@@ -1508,6 +1568,9 @@ policy "pol_utilization" do
       field "osType" do
         label "Operating System"
       end
+      field "dataDiskCount" do
+        label "Attached Disks (#)"
+      end
       field "imagePublisher" do
         label "Image Publisher"
       end
@@ -1646,6 +1709,9 @@ policy "pol_utilization" do
       field "osType" do
         label "Operating System"
       end
+      field "dataDiskCount" do
+        label "Attached Disks (#)"
+      end
       field "imagePublisher" do
         label "Image Publisher"
       end

diff --git a/cost/azure/rightsize_compute_instances/azure_compute_rightsizing_meta_parent.pt b/cost/azure/rightsize_compute_instances/azure_compute_rightsizing_meta_parent.pt
@@ -7,7 +7,7 @@ category "Meta"
 default_frequency "15 minutes"
 info(
   provider: "Azure",
-  version: "5.4.0", # This version of the Meta Parent Policy Template should match the version of the Child Policy Template as it appears in the Catalog for best reliability
+  version: "6.0.0", # This version of the Meta Parent Policy Template should match the version of the Child Policy Template as it appears in the Catalog for best reliability
   publish: "true",
   deprecated: "false"
 )
@@ -1116,6 +1116,9 @@ policy "policy_scheduled_report" do
       field "osType" do
         label "Operating System"
       end
+      field "dataDiskCount" do
+        label "Attached Disks (#)"
+      end
       field "imagePublisher" do
         label "Image Publisher"
       end
@@ -1239,6 +1242,9 @@ policy "policy_scheduled_report" do
       field "osType" do
         label "Operating System"
       end
+      field "dataDiskCount" do
+        label "Attached Disks (#)"
+      end
       field "imagePublisher" do
         label "Image Publisher"
       end

diff --git a/data/policy_permissions_list/master_policy_permissions_list.json b/data/policy_permissions_list/master_policy_permissions_list.json
@@ -3667,7 +3667,7 @@
     {
       "id": "./cost/azure/rightsize_compute_instances/azure_compute_rightsizing.pt",
       "name": "Azure Rightsize Compute Instances",
-      "version": "5.4.0",
+      "version": "6.0.0",
       "providers": [
         {
           "name": "azure_rm",
@@ -3682,6 +3682,11 @@
               "read_only": true,
               "required": true
             },
+            {
+              "name": "Microsoft.Compute/locations/vmSizes/read",
+              "read_only": true,
+              "required": true
+            },
             {
               "name": "Microsoft.Compute/virtualMachines/read",
               "read_only": true,

diff --git a/data/policy_permissions_list/master_policy_permissions_list.yaml b/data/policy_permissions_list/master_policy_permissions_list.yaml
@@ -2121,7 +2121,7 @@
       required: true
 - id: "./cost/azure/rightsize_compute_instances/azure_compute_rightsizing.pt"
   name: Azure Rightsize Compute Instances
-  version: 5.4.0
+  version: 6.0.0
   :providers:
   - :name: azure_rm
     :permissions:
@@ -2131,6 +2131,9 @@
     - name: Microsoft.Compute/skus/read
       read_only: true
       required: true
+    - name: Microsoft.Compute/locations/vmSizes/read
+      read_only: true
+      required: true
     - name: Microsoft.Compute/virtualMachines/read
       read_only: true
       required: true