From 13039966fd6f5d59135f6524d034128f9e1fe734 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 01:42:01 +0200 Subject: [PATCH 1/9] add additional fields --- .../gcp/data_stream/billing/fields/fields.yml | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/packages/gcp/data_stream/billing/fields/fields.yml b/packages/gcp/data_stream/billing/fields/fields.yml index 01a7e615bf3..42105539dda 100644 --- a/packages/gcp/data_stream/billing/fields/fields.yml +++ b/packages/gcp/data_stream/billing/fields/fields.yml @@ -20,3 +20,27 @@ - name: billing_account_id type: keyword description: Project Billing Account ID. + - name: sku_id + type: keyword + description: The ID of the resource used by the service. + - name: sku_description + type: keyword + description: A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. + - name: service_id + type: keyword + description: The ID of the service that the usage is associated with. + - name: service_description + type: keyword + description: The Google Cloud service that reported the Cloud Billing data. + - name: effective_price + type: float + description: The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. + - name: tags + type: nested + description: A collection of key-value pairs that provide additional metadata. + fields: + - name: key + type: keyword + - name: value + type: keyword + From c2b1f027fe52ed57e768af9f6844cfd4a27c2920 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 01:42:50 +0200 Subject: [PATCH 2/9] build docs --- packages/gcp/docs/README.md | 7 +++++++ packages/gcp/docs/billing.md | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index 577421df002..2da73a4f767 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -1486,9 +1486,16 @@ The `billing` dataset collects GCP Billing information from Google Cloud BigQuer | event.module | Event module | constant_keyword | | gcp.billing.billing_account_id | Project Billing Account ID. | keyword | | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword | +| gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float | | gcp.billing.invoice_month | Billing report month. | keyword | | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword | | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword | +| gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword | +| gcp.billing.service_id | The ID of the service that the usage is associated with. | keyword | +| gcp.billing.sku_description | A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. | keyword | +| gcp.billing.sku_id | The ID of the resource used by the service. | keyword | +| gcp.billing.tags.key | | keyword | +| gcp.billing.tags.value | | keyword | | gcp.billing.total | Total billing amount. | float | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | diff --git a/packages/gcp/docs/billing.md b/packages/gcp/docs/billing.md index 06128132194..01084651a42 100644 --- a/packages/gcp/docs/billing.md +++ b/packages/gcp/docs/billing.md @@ -82,9 +82,16 @@ An example event for `billing` looks as following: | event.module | Event module | constant_keyword | | gcp.billing.billing_account_id | Project Billing Account ID. | keyword | | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword | +| gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float | | gcp.billing.invoice_month | Billing report month. | keyword | | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword | | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword | +| gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword | +| gcp.billing.service_id | The ID of the service that the usage is associated with. | keyword | +| gcp.billing.sku_description | A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. | keyword | +| gcp.billing.sku_id | The ID of the resource used by the service. | keyword | +| gcp.billing.tags.key | | keyword | +| gcp.billing.tags.value | | keyword | | gcp.billing.total | Total billing amount. | float | | host.architecture | Operating system architecture. | keyword | | host.containerized | If the host is a container. | boolean | From ad386ef1b360c86eb046d2b3dfae3984615b07ef Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 01:43:11 +0200 Subject: [PATCH 3/9] use sum instead of max --- .../dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json index 72a276661d7..420768db6af 100644 --- a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json +++ b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json @@ -144,7 +144,7 @@ "dataType": "number", "isBucketed": false, "label": "Maximum of gcp.billing.total", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, @@ -337,7 +337,7 @@ "dataType": "number", "isBucketed": false, "label": "Total Billing Cost", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, @@ -459,7 +459,7 @@ "dataType": "number", "isBucketed": false, "label": "Total Billing", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, From 5c83775cfa873d60c98b1ab459e9ac8634000a82 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 02:08:46 +0200 Subject: [PATCH 4/9] bump package version --- packages/gcp/changelog.yml | 5 +++++ packages/gcp/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml index bfc21d79851..65b02399ade 100644 --- a/packages/gcp/changelog.yml +++ b/packages/gcp/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.31.0" + changes: + - description: Add new sku, service and effective_price billing fields. + type: enhancement + link: https://github.com/elastic/integrations/pull/8343 - version: "2.30.1" changes: - description: Fix mappings of group fields diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 09ad056f47c..648e8da3912 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -1,6 +1,6 @@ name: gcp title: Google Cloud Platform -version: "2.30.1" +version: "2.31.0" description: Collect logs and metrics from Google Cloud Platform with Elastic Agent. type: integration icons: From 1a81089d708306f0eb85ad3296e93fbffd78d174 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 15 Nov 2023 22:57:04 +0200 Subject: [PATCH 5/9] update sample event --- packages/gcp/data_stream/billing/sample_event.json | 8 +++++++- packages/gcp/docs/README.md | 8 +++++++- packages/gcp/docs/billing.md | 8 +++++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/packages/gcp/data_stream/billing/sample_event.json b/packages/gcp/data_stream/billing/sample_event.json index 2acd0b43085..9fd0c424e1e 100644 --- a/packages/gcp/data_stream/billing/sample_event.json +++ b/packages/gcp/data_stream/billing/sample_event.json @@ -22,7 +22,13 @@ "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": {} } }, "metricset": { diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index 2da73a4f767..9b688d548f6 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -1544,7 +1544,13 @@ An example event for `billing` looks as following: "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": {} } }, "metricset": { diff --git a/packages/gcp/docs/billing.md b/packages/gcp/docs/billing.md index 01084651a42..b7d1b5f70ef 100644 --- a/packages/gcp/docs/billing.md +++ b/packages/gcp/docs/billing.md @@ -37,7 +37,13 @@ An example event for `billing` looks as following: "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": {} } }, "metricset": { From 212d1412e23d2bf4cf87537f0c1545fdaab01a4d Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 21 Nov 2023 22:25:12 +0200 Subject: [PATCH 6/9] change label from max to sum --- .../dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json index 420768db6af..101c0255ab6 100644 --- a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json +++ b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json @@ -143,7 +143,7 @@ "10b91492-efef-490d-bc7a-c2074b2eae84": { "dataType": "number", "isBucketed": false, - "label": "Maximum of gcp.billing.total", + "label": "Sum of gcp.billing.total", "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" From d6b3f30ba8b56f155f1e6517463d2716eeac12a9 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 22 Nov 2023 13:14:04 +0200 Subject: [PATCH 7/9] add tags in sample event --- packages/gcp/data_stream/billing/sample_event.json | 11 ++++++++++- packages/gcp/docs/README.md | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/gcp/data_stream/billing/sample_event.json b/packages/gcp/data_stream/billing/sample_event.json index 9fd0c424e1e..6e5fbf64c45 100644 --- a/packages/gcp/data_stream/billing/sample_event.json +++ b/packages/gcp/data_stream/billing/sample_event.json @@ -28,7 +28,16 @@ "sku_description": "Network Inter Region Ingress from Jakarta to Americas", "service_description": "Compute Engine", "effective_price": 0.00292353, - "tags": {} + "tags": [ + { + "key": "stage", + "value": "prod" + }, + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md index abb2ffd73c1..c88246b5051 100644 --- a/packages/gcp/docs/README.md +++ b/packages/gcp/docs/README.md @@ -1557,7 +1557,16 @@ An example event for `billing` looks as following: "sku_description": "Network Inter Region Ingress from Jakarta to Americas", "service_description": "Compute Engine", "effective_price": 0.00292353, - "tags": {} + "tags": [ + { + "key": "stage", + "value": "prod" + }, + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { From 7f5d426d759df5115272e9a3ed2bcec448140e4f Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 22 Nov 2023 13:14:58 +0200 Subject: [PATCH 8/9] add documentation --- packages/gcp/_dev/build/docs/billing.md | 4 ++++ packages/gcp/data_stream/billing/manifest.yml | 2 +- packages/gcp/docs/billing.md | 15 ++++++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/gcp/_dev/build/docs/billing.md b/packages/gcp/_dev/build/docs/billing.md index f9cff298c3a..2eb3ac126e3 100644 --- a/packages/gcp/_dev/build/docs/billing.md +++ b/packages/gcp/_dev/build/docs/billing.md @@ -8,6 +8,10 @@ Please see [export cloud billing data to BigQuery](https://cloud.google.com/bill In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data table named `gcp_billing_export_v1_`. There is a defined schema for Google Cloud daily cost data that is exported to BigQuery. Please see [daily cost detail data schema](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema) for more details. +For standard usage cost data, set the table pattern format to `gcp_billing_export_v1`. This table pattern is set as the default when no other is specified. + +For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. Detailed tables include the standard fields and additional fields, such as `effective_price`, enabling a more granular view of expenses. + ## Sample Event {{event "billing"}} diff --git a/packages/gcp/data_stream/billing/manifest.yml b/packages/gcp/data_stream/billing/manifest.yml index 0b2342e9492..a3e05353891 100644 --- a/packages/gcp/data_stream/billing/manifest.yml +++ b/packages/gcp/data_stream/billing/manifest.yml @@ -22,7 +22,7 @@ streams: multi: false required: true show_user: true - description: "Daily cost detail billing table name prefix." + description: "Daily cost detail billing table name prefix. Use gcp_billing_export_resource_v1 for detailed billing tables." default: gcp_billing_export_v1 - name: cost_type type: text diff --git a/packages/gcp/docs/billing.md b/packages/gcp/docs/billing.md index b7d1b5f70ef..66e98cfae67 100644 --- a/packages/gcp/docs/billing.md +++ b/packages/gcp/docs/billing.md @@ -8,6 +8,10 @@ Please see [export cloud billing data to BigQuery](https://cloud.google.com/bill In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data table named `gcp_billing_export_v1_`. There is a defined schema for Google Cloud daily cost data that is exported to BigQuery. Please see [daily cost detail data schema](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema) for more details. +For standard usage cost data, set the table pattern format to `gcp_billing_export_v1`. This table pattern is set as the default when no other is specified. + +For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. Detailed tables include the standard fields and additional fields, such as `effective_price`, enabling a more granular view of expenses. + ## Sample Event An example event for `billing` looks as following: @@ -43,7 +47,16 @@ An example event for `billing` looks as following: "sku_description": "Network Inter Region Ingress from Jakarta to Americas", "service_description": "Compute Engine", "effective_price": 0.00292353, - "tags": {} + "tags": [ + { + "key": "stage", + "value": "prod" + }, + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { From aff916890f11c591f7e90cdca99b41bf7704f97f Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Wed, 22 Nov 2023 13:31:28 +0200 Subject: [PATCH 9/9] bump kibana version feature is available in 8.12.0 --- packages/gcp/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml index 4bf62b782b4..b7ae8b24cb0 100644 --- a/packages/gcp/manifest.yml +++ b/packages/gcp/manifest.yml @@ -13,7 +13,7 @@ categories: - google_cloud conditions: kibana: - version: ^8.7.1 + version: ^8.12.0 screenshots: - src: /img/filebeat-gcp-audit.png title: filebeat gcp audit