From 40674321485ae39e8f04b7a5596f6c73c7eaf8c6 Mon Sep 17 00:00:00 2001 From: Shawn Huckabay Date: Fri, 11 Oct 2024 05:44:40 -0500 Subject: [PATCH] update --- .../azure_missing_subscriptions/CHANGELOG.md | 4 + .../azure_missing_subscriptions/README.md | 2 +- .../azure_missing_subscriptions.pt | 4 +- .../google_missing_projects/CHANGELOG.md | 5 + .../google/google_missing_projects/README.md | 39 ++ .../google_missing_projects.pt | 336 ++++++++++++++++++ .../master_policy_permissions_list.json | 29 +- .../master_policy_permissions_list.yaml | 16 +- .../validated_policy_templates.yaml | 1 + 9 files changed, 431 insertions(+), 5 deletions(-) create mode 100644 automation/google/google_missing_projects/CHANGELOG.md create mode 100644 automation/google/google_missing_projects/README.md create mode 100644 automation/google/google_missing_projects/google_missing_projects.pt diff --git a/automation/azure/azure_missing_subscriptions/CHANGELOG.md b/automation/azure/azure_missing_subscriptions/CHANGELOG.md index 1ad604f8d4..59ea7ed41b 100644 --- a/automation/azure/azure_missing_subscriptions/CHANGELOG.md +++ b/automation/azure/azure_missing_subscriptions/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v0.1.1 + +- Removed outdated reference to 'Optima' from policy metadata. Functionality unchanged. + ## v0.1 - initial release diff --git a/automation/azure/azure_missing_subscriptions/README.md b/automation/azure/azure_missing_subscriptions/README.md index 5f33caf7f3..716d93e8f3 100644 --- a/automation/azure/azure_missing_subscriptions/README.md +++ b/automation/azure/azure_missing_subscriptions/README.md @@ -37,4 +37,4 @@ The [Provider-Specific Credentials](https://docs.flexera.com/flexera/EN/Automati ## Cost -This Policy Template does not incur any cloud costs. +This policy template does not incur any cloud costs. diff --git a/automation/azure/azure_missing_subscriptions/azure_missing_subscriptions.pt b/automation/azure/azure_missing_subscriptions/azure_missing_subscriptions.pt index 4cdf0a2821..359c514393 100644 --- a/automation/azure/azure_missing_subscriptions/azure_missing_subscriptions.pt +++ b/automation/azure/azure_missing_subscriptions/azure_missing_subscriptions.pt @@ -7,9 +7,9 @@ severity "low" category "Cost" default_frequency "weekly" info( - version: "0.1", + version: "0.1.1", provider: "Flexera", - service: "Optima", + service: "Cloud Cost Optimization", policy_set: "Automation", publish: "false" ) diff --git a/automation/google/google_missing_projects/CHANGELOG.md b/automation/google/google_missing_projects/CHANGELOG.md new file mode 100644 index 0000000000..a1ed544621 --- /dev/null +++ b/automation/google/google_missing_projects/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## v0.1.0 + +- Initial release diff --git a/automation/google/google_missing_projects/README.md b/automation/google/google_missing_projects/README.md new file mode 100644 index 0000000000..493de45bb4 --- /dev/null +++ b/automation/google/google_missing_projects/README.md @@ -0,0 +1,39 @@ +# Google Missing Projects + +## What It Does + +This policy checks the stored Flexera CCO billing data for Google from 3 days ago to obtain a list of Google Projects that we have billing data for and compares that to the list of Google Projects returned by the Google Cloud Resource Manager API. An incident is raised and email sent containing any projects present in Flexera CCO but not returned by the Google Cloud Resource Manager API, as well as projects returned by the Google Cloud Resource Manager API but not present in Flexera CCO. The user can select which of those two reports they'd like to produce. + +## Input Parameters + +This policy has the following input parameters required when launching the policy. + +- *Email Addresses* - Email addresses of the recipients you wish to notify when new incidents are created. +- *Report Selection* - Whether to report Projects missing in the Google Cloud Resource Manager API but present in CCO data, the opposite, or both. +- *Projects Ignore List* - A list of Project IDs/names to never include in the results. Leave blank to not filter results. + +## Policy Actions + +The following policy actions are taken on any resources found to be out of compliance. + +- Send an email report + +## Prerequisites + +This Policy Template uses [Credentials](https://docs.flexera.com/flexera/EN/Automation/ManagingCredentialsExternal.htm) for authenticating to datasources -- in order to apply this policy you must have a Credential registered in the system that is compatible with this policy. If there are no Credentials listed when you apply the policy, please contact your Flexera Org Admin and ask them to register a Credential that is compatible with this policy. The information below should be consulted when creating the credential(s). + +- [**Google Cloud Credential**](https://docs.flexera.com/flexera/EN/Automation/ProviderCredentials.htm#automationadmin_4083446696_1121577) (*provider=gce*) which has the following: + - `resourcemanager.projects.get` + +- [**Flexera Credential**](https://docs.flexera.com/flexera/EN/Automation/ProviderCredentials.htm) (*provider=flexera*) which has the following roles: + - `billing_center_viewer` + +The [Provider-Specific Credentials](https://docs.flexera.com/flexera/EN/Automation/ProviderCredentials.htm) page in the docs has detailed instructions for setting up Credentials for the most common providers. + +## Supported Clouds + +- Google + +## Cost + +This policy template does not incur any cloud costs. diff --git a/automation/google/google_missing_projects/google_missing_projects.pt b/automation/google/google_missing_projects/google_missing_projects.pt new file mode 100644 index 0000000000..5d53b3b585 --- /dev/null +++ b/automation/google/google_missing_projects/google_missing_projects.pt @@ -0,0 +1,336 @@ +name "Google Missing Projects" +rs_pt_ver 20180301 +type "policy" +short_description "Reports any Google Projects present in Flexera One that are not accessible via the Google automation credential. See the [README](https://github.com/flexera-public/policy_templates/tree/master/automation/google/google_missing_projects) and [docs.flexera.com/flexera/EN/Automation](https://docs.flexera.com/flexera/EN/Automation/AutomationGS.htm) to learn more." +long_description "" +category "Cost" +severity "low" +default_frequency "weekly" +info( + version: "0.1.0", + provider: "Flexera", + service: "Cloud Cost Optimization", + policy_set: "Automation", + publish: "false" +) + +############################################################################### +# Parameters +############################################################################### + +parameter "param_email" do + type "list" + category "Policy Settings" + label "Email Addresses" + description "A list of email addresses to notify." + default [] +end + +parameter "param_report_selection" do + type "string" + category "Policy Settings" + label "Report Selection" + description "Whether to report Projects missing in the Google Cloud Resource Manager API but present in CCO data, the opposite, or both." + allowed_values "Missing in Google API", "Missing in CCO", "Both" + default "Missing in Google API" +end + +parameter "param_projects_list" do + type "list" + category "Filters" + label "Projects Ignore List" + description "A list of Project IDs/names to never include in the results. Leave blank to not filter results." + default [] +end + +############################################################################### +# Authentication +############################################################################### + +credentials "auth_google" do + schemes "oauth2" + label "Google" + description "Select the Google Cloud Credential from the list." + tags "provider=gce" +end + +credentials "auth_flexera" do + schemes "oauth2" + label "Flexera" + description "Select Flexera One OAuth2 credentials" + tags "provider=flexera" +end + +############################################################################### +# Pagination +############################################################################### + +pagination "pagination_google" do + get_page_marker do + body_path "nextPageToken" + end + set_page_marker do + query "pageToken" + end +end + +############################################################################### +# Datasources & Scripts +############################################################################### + +# Get applied policy metadata for use later +datasource "ds_applied_policy" do + request do + auth $auth_flexera + host rs_governance_host + path join(["/api/governance/projects/", rs_project_id, "/applied_policies/", policy_id]) + header "Api-Version", "1.0" + end +end + +datasource "ds_billing_centers" do + request do + auth $auth_flexera + host rs_optima_host + path join(["/analytics/orgs/", rs_org_id, "/billing_centers"]) + header "Api-Version", "1.0" + header "User-Agent", "RS Policies" + query "view", "allocation_table" + ignore_status [403] + end + result do + encoding "json" + collect jmes_path(response, "[*]") do + field "href", jmes_path(col_item, "href") + field "id", jmes_path(col_item, "id") + field "name", jmes_path(col_item, "name") + field "parent_id", jmes_path(col_item, "parent_id") + end + end +end + +# Gather top level billing center IDs for when we pull cost data +datasource "ds_top_level_bcs" do + run_script $js_top_level_bcs, $ds_billing_centers +end + +script "js_top_level_bcs", type: "javascript" do + parameters "ds_billing_centers" + result "result" + code <<-EOS + filtered_bcs = _.filter(ds_billing_centers, function(bc) { + return bc['parent_id'] == null || bc['parent_id'] == undefined + }) + + result = _.compact(_.pluck(filtered_bcs, 'id')) +EOS +end + +datasource "ds_flexera_cco_data" do + request do + run_script $js_flexera_cco_data, $ds_top_level_bcs, rs_org_id, rs_optima_host + end + result do + encoding "json" + collect jmes_path(response, "rows[*]") do + field "bill_source", jmes_path(col_item, "dimensions.bill_source") + field "vendor_account", jmes_path(col_item, "dimensions.vendor_account") + field "vendor_account_name", jmes_path(col_item, "dimensions.vendor_account_name") + end + end +end + +script "js_flexera_cco_data", type: "javascript" do + parameters "ds_top_level_bcs", "rs_org_id", "rs_optima_host" + result "request" + code <<-EOS + end_date = new Date() + end_date.setDate(end_date.getDate() - 2) + end_date = end_date.toISOString().split('T')[0] + + start_date = new Date() + start_date.setDate(start_date.getDate() - 3) + start_date = start_date.toISOString().split('T')[0] + + var request = { + auth: "auth_flexera", + host: rs_optima_host, + verb: "POST", + path: "/bill-analysis/orgs/" + rs_org_id + "/costs/aggregated", + body_fields: { + dimensions: ["bill_source", "vendor_account", "vendor_account_name"], + granularity: "day", + start_at: start_date, + end_at: end_date, + metrics: ["cost_amortized_unblended_adj"], + billing_center_ids: ds_top_level_bcs, + limit: 100000, + filter: { + "type": "or", + "expressions": [ + { "dimension": "vendor", "type": "equal", "value": "Google" }, + { "dimension": "vendor", "type": "equal", "value": "google" }, + { "dimension": "vendor", "type": "equal", "value": "Google Cloud" }, + { "dimension": "vendor", "type": "equal", "value": "google cloud" }, + { "dimension": "vendor", "type": "equal", "value": "Google Cloud Platform" }, + { "dimension": "vendor", "type": "equal", "value": "google cloud platform" }, + { "dimension": "vendor", "type": "equal", "value": "GCP" }, + { "dimension": "vendor", "type": "equal", "value": "gcp" } + ] + } + }, + headers: { + 'User-Agent': "RS Policies", + 'Api-Version': "1.0" + }, + ignore_status: [400] + } +EOS +end + +datasource "ds_google_projects" do + request do + auth $auth_google + pagination $pagination_google + host "cloudresourcemanager.googleapis.com" + path "/v1/projects/" + query "filter", "lifecycleState=ACTIVE" + end + result do + encoding "json" + collect jmes_path(response, "projects[*]") do + field "id", jmes_path(col_item, "projectId") + field "number", jmes_path(col_item, "projectNumber") + field "name", jmes_path(col_item, "name") + field "parent", jmes_path(col_item, "parent") + field "createTime", jmes_path(col_item, "createTime") + field "lifecycleState", jmes_path(col_item, "lifecycleState") + field "tags", jmes_path(col_item, "labels") + end + end +end + +datasource "ds_missing_projects_api" do + run_script $js_missing_projects_api, $ds_flexera_cco_data, $ds_google_projects, $ds_applied_policy, $param_projects_list, $param_report_selection +end + +script "js_missing_projects_api", type: "javascript" do + parameters "ds_flexera_cco_data", "ds_google_projects", "ds_applied_policy", "param_projects_list", "param_report_selection" + result "result" + code <<-'EOS' + result = [] + + if (param_report_selection != 'Missing in CCO') { + automation_ids = _.map(ds_google_projects, function(project) { + return project['id'].toLowerCase().trim() + }) + + missing_projects = _.reject(ds_flexera_cco_data, function(project) { + return _.contains(automation_ids, project['vendor_account'].toLowerCase().trim()) + }) + + result = _.map(missing_projects, function(project) { + return { + accountID: project['vendor_account'].toLowerCase().trim(), + accountName: project['vendor_account_name'], + bill_source: project['bill_source'], + policy_name: ds_applied_policy['name'] + } + }) + + // Remove filtered results + result = _.reject(result, function(project) { + return _.contains(param_projects_list, project['accountID']) || _.contains(param_projects_list, project['accountName']) + }) + + result = _.sortBy(result, 'accountID') + result = _.sortBy(result, 'bill_source') + } +EOS +end + +datasource "ds_missing_projects_cco" do + run_script $js_missing_projects_cco, $ds_flexera_cco_data, $ds_google_projects, $ds_applied_policy, $param_projects_list, $param_report_selection +end + +script "js_missing_projects_cco", type: "javascript" do + parameters "ds_flexera_cco_data", "ds_google_projects", "ds_applied_policy", "param_projects_list", "param_report_selection" + result "result" + code <<-'EOS' + result = [] + + if (param_report_selection != 'Missing in Google API') { + cco_ids = _.map(ds_flexera_cco_data, function(project) { + return project['vendor_account'].toLowerCase().trim() + }) + + missing_projects = _.reject(ds_google_projects, function(project) { + return _.contains(cco_ids, project['id'].toLowerCase().trim()) + }) + + result = _.map(missing_projects, function(project) { + return { + accountID: project['id'].toLowerCase().trim(), + accountName: project['name'], + policy_name: ds_applied_policy['name'] + } + }) + + // Remove filtered results + result = _.reject(result, function(project) { + return _.contains(param_projects_list, project['accountID']) || _.contains(param_projects_list, project['accountName']) + }) + + result = _.sortBy(result, 'accountID') + } +EOS +end + +############################################################################### +# Policy +############################################################################### + +policy "pol_missing_projects" do + validate_each $ds_missing_projects_api do + summary_template "{{ with index data 0 }}{{ .policy_name }}{{ end }}: {{ len data }} Google API Missing Projects" + check eq(val(item, "accountID"), "") + escalate $esc_email + export do + resource_level false + field "accountID" do + label "Project ID" + end + field "accountName" do + label "Project Name" + end + field "bill_source" do + label "Bill Connection" + end + end + end + validate_each $ds_missing_projects_cco do + summary_template "{{ with index data 0 }}{{ .policy_name }}{{ end }}: {{ len data }} Google CCO Missing Projects" + check eq(val(item, "accountID"), "") + escalate $esc_email + export do + resource_level false + field "accountID" do + label "Project ID" + end + field "accountName" do + label "Project Name" + end + end + end +end + +############################################################################### +# Escalations +############################################################################### + +escalation "esc_email" do + automatic true + label "Send Email" + description "Send incident email" + email $param_email +end diff --git a/data/policy_permissions_list/master_policy_permissions_list.json b/data/policy_permissions_list/master_policy_permissions_list.json index 2bc3284a2a..343f24128c 100644 --- a/data/policy_permissions_list/master_policy_permissions_list.json +++ b/data/policy_permissions_list/master_policy_permissions_list.json @@ -94,7 +94,7 @@ { "id": "./automation/azure/azure_missing_subscriptions/azure_missing_subscriptions.pt", "name": "Azure Missing Subscriptions", - "version": "0.1", + "version": "0.1.1", "providers": [ { "name": "azure_rm", @@ -257,6 +257,33 @@ } ] }, + { + "id": "./automation/google/google_missing_projects/google_missing_projects.pt", + "name": "Google Missing Projects", + "version": "0.1.0", + "providers": [ + { + "name": "gce", + "permissions": [ + { + "name": "resourcemanager.projects.get", + "read_only": true, + "required": true + } + ] + }, + { + "name": "flexera", + "permissions": [ + { + "name": "billing_center_viewer", + "read_only": true, + "required": true + } + ] + } + ] + }, { "id": "./automation/google/google_rbd_from_label/google_rbd_from_label.pt", "name": "Google Rule-Based Dimension From Project Labels", diff --git a/data/policy_permissions_list/master_policy_permissions_list.yaml b/data/policy_permissions_list/master_policy_permissions_list.yaml index 7e6594e6d6..9632573d71 100644 --- a/data/policy_permissions_list/master_policy_permissions_list.yaml +++ b/data/policy_permissions_list/master_policy_permissions_list.yaml @@ -52,7 +52,7 @@ required: true - id: "./automation/azure/azure_missing_subscriptions/azure_missing_subscriptions.pt" name: Azure Missing Subscriptions - version: '0.1' + version: 0.1.1 :providers: - :name: azure_rm :permissions: @@ -142,6 +142,20 @@ required: false description: Only required for taking action (updating applied policies); the policy will still function in a read-only capacity without these permissions. +- id: "./automation/google/google_missing_projects/google_missing_projects.pt" + name: Google Missing Projects + version: 0.1.0 + :providers: + - :name: gce + :permissions: + - name: resourcemanager.projects.get + read_only: true + required: true + - :name: flexera + :permissions: + - name: billing_center_viewer + read_only: true + required: true - id: "./automation/google/google_rbd_from_label/google_rbd_from_label.pt" name: Google Rule-Based Dimension From Project Labels version: 1.3.1 diff --git a/tools/policy_master_permission_generation/validated_policy_templates.yaml b/tools/policy_master_permission_generation/validated_policy_templates.yaml index 9e11c76db6..7eb52fc78d 100644 --- a/tools/policy_master_permission_generation/validated_policy_templates.yaml +++ b/tools/policy_master_permission_generation/validated_policy_templates.yaml @@ -202,6 +202,7 @@ validated_policy_templates: - "./security/azure/table_storage_logging/table_storage_logging.pt" - "./security/azure/webapp_tls_version_support/azure_webapp_min_tls_version.pt" # Google +- "./automation/google/google_missing_projects/google_missing_projects.pt" - "./automation/google/google_rbd_from_label/google_rbd_from_label.pt" - "./compliance/google/long_stopped_instances/google_long_stopped_instances.pt" - "./compliance/google/unlabeled_resources/unlabeled_resources.pt"