[GCP] Add billing additional fields (#8343)

* add additional fields * build docs * use sum instead of max * bump package version * update sample event * change label from max to sum * add tags in sample event * add documentation * bump kibana version feature is available in 8.12.0
elastic · Nov 22, 2023 · 7fa6cab · 7fa6cab
1 parent 4337929
commit 7fa6cab
Show file tree

Hide file tree

Showing 9 changed files with 106 additions and 10 deletions.
diff --git a/packages/gcp/_dev/build/docs/billing.md b/packages/gcp/_dev/build/docs/billing.md
@@ -8,6 +8,10 @@ Please see [export cloud billing data to BigQuery](https://cloud.google.com/bill
 
 In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data table named `gcp_billing_export_v1_<BILLING_ACCOUNT_ID>`. There is a defined schema for Google Cloud daily cost data that is exported to BigQuery. Please see [daily cost detail data schema](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema) for more details.
 
+For standard usage cost data, set the table pattern format to `gcp_billing_export_v1`. This table pattern is set as the default when no other is specified.
+
+For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. Detailed tables include the standard fields and additional fields, such as `effective_price`, enabling a more granular view of expenses.
+
 ## Sample Event
 
 {{event "billing"}}

diff --git a/packages/gcp/changelog.yml b/packages/gcp/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "2.32.0"
+  changes:
+    - description: Add new billing data stream fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/8343
 - version: "2.31.2"
   changes:
     - description: Fix pipeline error parsing DNS logs with empty rdata field.

diff --git a/packages/gcp/data_stream/billing/fields/fields.yml b/packages/gcp/data_stream/billing/fields/fields.yml
@@ -20,3 +20,27 @@
     - name: billing_account_id
       type: keyword
       description: Project Billing Account ID.
+    - name: sku_id
+      type: keyword
+      description: The ID of the resource used by the service.
+    - name: sku_description
+      type: keyword
+      description: A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US.
+    - name: service_id
+      type: keyword
+      description: The ID of the service that the usage is associated with.
+    - name: service_description
+      type: keyword
+      description: The Google Cloud service that reported the Cloud Billing data.
+    - name: effective_price
+      type: float
+      description: The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price.
+    - name: tags
+      type: nested
+      description: A collection of key-value pairs that provide additional metadata.
+      fields:
+        - name: key
+          type: keyword
+        - name: value
+          type: keyword
+
diff --git a/packages/gcp/data_stream/billing/manifest.yml b/packages/gcp/data_stream/billing/manifest.yml
@@ -22,7 +22,7 @@ streams:
         multi: false
         required: true
         show_user: true
-        description: "Daily cost detail billing table name prefix."
+        description: "Daily cost detail billing table name prefix. Use gcp_billing_export_resource_v1 for detailed billing tables."
         default: gcp_billing_export_v1
       - name: cost_type
         type: text

diff --git a/packages/gcp/data_stream/billing/sample_event.json b/packages/gcp/data_stream/billing/sample_event.json
@@ -22,7 +22,22 @@
             "invoice_month": "202106",
             "project_id": "containerlib-prod-12763",
             "project_name": "elastic-containerlib-prod",
-            "total": 4717.170681
+            "total": 4717.170681,
+            "sku_id": "0D56-2F80-52A5",
+            "service_id": "6F81-5844-456A",
+            "sku_description": "Network Inter Region Ingress from Jakarta to Americas",
+            "service_description": "Compute Engine",
+            "effective_price": 0.00292353,
+            "tags": [
+                {
+                    "key": "stage",
+                    "value": "prod"
+                },
+                {
+                    "key": "size",
+                    "value": "standard"
+                }
+            ]
         }
     },
     "metricset": {

diff --git a/packages/gcp/docs/README.md b/packages/gcp/docs/README.md
@@ -1493,9 +1493,16 @@ The `billing` dataset collects GCP Billing information from Google Cloud BigQuer
 | event.module | Event module | constant_keyword |
 | gcp.billing.billing_account_id | Project Billing Account ID. | keyword |
 | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword |
+| gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float |
 | gcp.billing.invoice_month | Billing report month. | keyword |
 | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword |
 | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword |
+| gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword |
+| gcp.billing.service_id | The ID of the service that the usage is associated with. | keyword |
+| gcp.billing.sku_description | A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. | keyword |
+| gcp.billing.sku_id | The ID of the resource used by the service. | keyword |
+| gcp.billing.tags.key |  | keyword |
+| gcp.billing.tags.value |  | keyword |
 | gcp.billing.total | Total billing amount. | float |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |
@@ -1544,7 +1551,22 @@ An example event for `billing` looks as following:
             "invoice_month": "202106",
             "project_id": "containerlib-prod-12763",
             "project_name": "elastic-containerlib-prod",
-            "total": 4717.170681
+            "total": 4717.170681,
+            "sku_id": "0D56-2F80-52A5",
+            "service_id": "6F81-5844-456A",
+            "sku_description": "Network Inter Region Ingress from Jakarta to Americas",
+            "service_description": "Compute Engine",
+            "effective_price": 0.00292353,
+            "tags": [
+                {
+                    "key": "stage",
+                    "value": "prod"
+                },
+                {
+                    "key": "size",
+                    "value": "standard"
+                }
+            ]
         }
     },
     "metricset": {

diff --git a/packages/gcp/docs/billing.md b/packages/gcp/docs/billing.md
@@ -8,6 +8,10 @@ Please see [export cloud billing data to BigQuery](https://cloud.google.com/bill
 
 In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data table named `gcp_billing_export_v1_<BILLING_ACCOUNT_ID>`. There is a defined schema for Google Cloud daily cost data that is exported to BigQuery. Please see [daily cost detail data schema](https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema) for more details.
 
+For standard usage cost data, set the table pattern format to `gcp_billing_export_v1`. This table pattern is set as the default when no other is specified.
+
+For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. Detailed tables include the standard fields and additional fields, such as `effective_price`, enabling a more granular view of expenses.
+
 ## Sample Event
 
 An example event for `billing` looks as following:
@@ -37,7 +41,22 @@ An example event for `billing` looks as following:
             "invoice_month": "202106",
             "project_id": "containerlib-prod-12763",
             "project_name": "elastic-containerlib-prod",
-            "total": 4717.170681
+            "total": 4717.170681,
+            "sku_id": "0D56-2F80-52A5",
+            "service_id": "6F81-5844-456A",
+            "sku_description": "Network Inter Region Ingress from Jakarta to Americas",
+            "service_description": "Compute Engine",
+            "effective_price": 0.00292353,
+            "tags": [
+                {
+                    "key": "stage",
+                    "value": "prod"
+                },
+                {
+                    "key": "size",
+                    "value": "standard"
+                }
+            ]
         }
     },
     "metricset": {
@@ -82,9 +101,16 @@ An example event for `billing` looks as following:
 | event.module | Event module | constant_keyword |
 | gcp.billing.billing_account_id | Project Billing Account ID. | keyword |
 | gcp.billing.cost_type | Cost types include regular, tax, adjustment, and rounding_error. | keyword |
+| gcp.billing.effective_price | The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. | float |
 | gcp.billing.invoice_month | Billing report month. | keyword |
 | gcp.billing.project_id | Project ID of the billing report belongs to. | keyword |
 | gcp.billing.project_name | Project Name of the billing report belongs to. | keyword |
+| gcp.billing.service_description | The Google Cloud service that reported the Cloud Billing data. | keyword |
+| gcp.billing.service_id | The ID of the service that the usage is associated with. | keyword |
+| gcp.billing.sku_description | A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. | keyword |
+| gcp.billing.sku_id | The ID of the resource used by the service. | keyword |
+| gcp.billing.tags.key |  | keyword |
+| gcp.billing.tags.value |  | keyword |
 | gcp.billing.total | Total billing amount. | float |
 | host.architecture | Operating system architecture. | keyword |
 | host.containerized | If the host is a container. | boolean |

diff --git a/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json b/packages/gcp/kibana/dashboard/gcp-76c9e920-e890-11ea-bf8c-d13ebf358a78.json
@@ -143,8 +143,8 @@
                                                 "10b91492-efef-490d-bc7a-c2074b2eae84": {
                                                     "dataType": "number",
                                                     "isBucketed": false,
-                                                    "label": "Maximum of gcp.billing.total",
-                                                    "operationType": "max",
+                                                    "label": "Sum of gcp.billing.total",
+                                                    "operationType": "sum",
                                                     "scale": "ratio",
                                                     "sourceField": "gcp.billing.total"
                                                 },
@@ -337,7 +337,7 @@
                                                     "dataType": "number",
                                                     "isBucketed": false,
                                                     "label": "Total Billing Cost",
-                                                    "operationType": "max",
+                                                    "operationType": "sum",
                                                     "scale": "ratio",
                                                     "sourceField": "gcp.billing.total"
                                                 },
@@ -459,7 +459,7 @@
                                                     "dataType": "number",
                                                     "isBucketed": false,
                                                     "label": "Total Billing",
-                                                    "operationType": "max",
+                                                    "operationType": "sum",
                                                     "scale": "ratio",
                                                     "sourceField": "gcp.billing.total"
                                                 },

diff --git a/packages/gcp/manifest.yml b/packages/gcp/manifest.yml
@@ -1,6 +1,6 @@
 name: gcp
 title: Google Cloud Platform
-version: "2.31.2"
+version: "2.32.0"
 description: Collect logs and metrics from Google Cloud Platform with Elastic Agent.
 type: integration
 icons:
@@ -13,7 +13,7 @@ categories:
   - google_cloud
 conditions:
   kibana:
-    version: ^8.7.1
+    version: ^8.12.0
 screenshots:
   - src: /img/filebeat-gcp-audit.png
     title: filebeat gcp audit