From 7fa6cab354e65963954d7eccc108d0cd1809514a Mon Sep 17 00:00:00 2001 From: Gabriel Pop <94497545+gpop63@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:38:14 +0200 Subject: [PATCH] [GCP] Add billing additional fields (#8343) * add additional fields * build docs * use sum instead of max * bump package version * update sample event * change label from max to sum * add tags in sample event * add documentation * bump kibana version feature is available in 8.12.0 --- packages/gcp/_dev/build/docs/billing.md | 4 +++ packages/gcp/changelog.yml | 5 ++++ .../gcp/data_stream/billing/fields/fields.yml | 24 ++++++++++++++++ packages/gcp/data_stream/billing/manifest.yml | 2 +- .../gcp/data_stream/billing/sample_event.json | 17 ++++++++++- packages/gcp/docs/README.md | 24 +++++++++++++++- packages/gcp/docs/billing.md | 28 ++++++++++++++++++- ...-76c9e920-e890-11ea-bf8c-d13ebf358a78.json | 8 +++--- packages/gcp/manifest.yml | 4 +-- 9 files changed, 106 insertions(+), 10 deletions(-) diff --git a/packages/gcp/_dev/build/docs/billing.md b/packages/gcp/_dev/build/docs/billing.md index f9cff298c3a..2eb3ac126e3 100644 --- a/packages/gcp/_dev/build/docs/billing.md +++ b/packages/gcp/_dev/build/docs/billing.md @@ -8,6 +8,10 @@ Please see [export cloud billing data to BigQuery](https://cloud.google.com/bill In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data table named `gcp_billing_export_v1_`. There is a defined schema for Google Cloud daily cost data that is exported to BigQuery. Please see [daily cost detail data schema](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema) for more details. +For standard usage cost data, set the table pattern format to `gcp_billing_export_v1`. This table pattern is set as the default when no other is specified. + +For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. Detailed tables include the standard fields and additional fields, such as `effective_price`, enabling a more granular view of expenses. + ## Sample Event {{event "billing"}} diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index 22b7242cc51..2d1bfddc385 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.32.0" + changes: + - description: Add new billing data stream fields. + type: enhancement + link: https://github.com/elastic/integrations/pull/8343 - version: "2.31.2" changes: - description: Fix pipeline error parsing DNS logs with empty rdata field. diff --git a/packages/gcp/data_stream/billing/fields/fields.yml b/packages/gcp/data_stream/billing/fields/fields.yml index 01a7e615bf3..42105539dda 100644 --- a/packages/gcp/data_stream/billing/fields/fields.yml +++ b/packages/gcp/data_stream/billing/fields/fields.yml @@ -20,3 +20,27 @@ - name: billing_account_id type: keyword description: Project Billing Account ID. + - name: sku_id + type: keyword + description: The ID of the resource used by the service. + - name: sku_description + type: keyword + description: A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. + - name: service_id + type: keyword + description: The ID of the service that the usage is associated with. + - name: service_description + type: keyword + description: The Google Cloud service that reported the Cloud Billing data. + - name: effective_price + type: float + description: The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. + - name: tags + type: nested + description: A collection of key-value pairs that provide additional metadata. + fields: + - name: key + type: keyword + - name: value + type: keyword + diff --git a/packages/gcp/data_stream/billing/manifest.yml b/packages/gcp/data_stream/billing/manifest.yml index 0b2342e9492..a3e05353891 100644 --- a/packages/gcp/data_stream/billing/manifest.yml +++ b/packages/gcp/data_stream/billing/manifest.yml @@ -22,7 +22,7 @@ streams: multi: false required: true show_user: true - description: "Daily cost detail billing table name prefix." + description: "Daily cost detail billing table name prefix. Use gcp_billing_export_resource_v1 for detailed billing tables." default: gcp_billing_export_v1 - name: cost_type type: text diff --git a/packages/gcp/data_stream/billing/sample_event.json b/packages/gcp/data_stream/billing/sample_event.json index 2acd0b43085..6e5fbf64c45 100644 --- a/packages/gcp/data_stream/billing/sample_event.json +++ b/packages/gcp/data_stream/billing/sample_event.json @@ -22,7 +22,22 @@ "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": [ + { + "key": "stage", + "value": "prod" + }, + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index 5c844cb468b..c88246b5051 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -1493,9 +1493,16 @@ The `billing` dataset collects GCP Billing information from Google Cloud BigQuer | event.module | Event module | constant_keyword | | gcp.billing.billing_account_id | Project Billing Account ID. | keyword | | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword | +| gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float | | gcp.billing.invoice_month | Billing report month. | keyword | | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword | | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword | +| gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword | +| gcp.billing.service_id | The ID of the service that the usage is associated with. | keyword | +| gcp.billing.sku_description | A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. | keyword | +| gcp.billing.sku_id | The ID of the resource used by the service. | keyword | +| gcp.billing.tags.key | | keyword | +| gcp.billing.tags.value | | keyword | | gcp.billing.total | Total billing amount. | float | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | @@ -1544,7 +1551,22 @@ An example event for `billing` looks as following: "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": [ + { + "key": "stage", + "value": "prod" + }, + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { diff --git a/packages/gcp/docs/billing.md b/packages/gcp/docs/billing.md index 06128132194..66e98cfae67 100644 --- a/packages/gcp/docs/billing.md +++ b/packages/gcp/docs/billing.md @@ -8,6 +8,10 @@ Please see [export cloud billing data to BigQuery](https://cloud.google.com/bill In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data table named `gcp_billing_export_v1_`. There is a defined schema for Google Cloud daily cost data that is exported to BigQuery. Please see [daily cost detail data schema](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema) for more details. +For standard usage cost data, set the table pattern format to `gcp_billing_export_v1`. This table pattern is set as the default when no other is specified. + +For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. Detailed tables include the standard fields and additional fields, such as `effective_price`, enabling a more granular view of expenses. + ## Sample Event An example event for `billing` looks as following: @@ -37,7 +41,22 @@ An example event for `billing` looks as following: "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": [ + { + "key": "stage", + "value": "prod" + }, + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { @@ -82,9 +101,16 @@ An example event for `billing` looks as following: | event.module | Event module | constant_keyword | | gcp.billing.billing_account_id | Project Billing Account ID. | keyword | | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword | +| gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float | | gcp.billing.invoice_month | Billing report month. | keyword | | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword | | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword | +| gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword | +| gcp.billing.service_id | The ID of the service that the usage is associated with. | keyword | +| gcp.billing.sku_description | A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. | keyword | +| gcp.billing.sku_id | The ID of the resource used by the service. | keyword | +| gcp.billing.tags.key | | keyword | +| gcp.billing.tags.value | | keyword | | gcp.billing.total | Total billing amount. | float | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | diff --git a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json index 72a276661d7..101c0255ab6 100644 --- a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json +++ b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json @@ -143,8 +143,8 @@ "10b91492-efef-490d-bc7a-c2074b2eae84": { "dataType": "number", "isBucketed": false, - "label": "Maximum of gcp.billing.total", - "operationType": "max", + "label": "Sum of gcp.billing.total", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, @@ -337,7 +337,7 @@ "dataType": "number", "isBucketed": false, "label": "Total Billing Cost", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, @@ -459,7 +459,7 @@ "dataType": "number", "isBucketed": false, "label": "Total Billing", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index f24425e827e..b7ae8b24cb0 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.31.2" +version: "2.32.0" description: Collect logs and metrics from Google Cloud Platform with Elastic Agent. type: integration icons: @@ -13,7 +13,7 @@ categories: - google_cloud conditions: kibana: - version: ^8.7.1 + version: ^8.12.0 screenshots: - src: /img/filebeat-gcp-audit.png title: filebeat gcp audit