From f6cf21833c71402c194ab95031bddaafd516cbfa Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sun, 22 Oct 2023 22:43:59 +0300 Subject: [PATCH 01/11] GCP Billing Enhancement Implemented a check to differentiate between standard and detailed cost usage tables Refactoring the 'row' structure used for storing BigQuery results, optimizing data handling Added support for tags Introduced new cost usage data fields --- .../metricbeat/module/gcp/billing/billing.go | 231 +++++++++++++++--- 1 file changed, 202 insertions(+), 29 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 9ab3bbee959a..2c1cc5d5ea94 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -161,6 +161,25 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (err erro return nil } +const detailedTablePrefix = "gcp_billing_export_resource_v1" + +// isDetailedTable checks if the table pattern matches the naming convention for +// detailed cost usage tables, which includes the term "gcp_billing_export_resource_v1". +// +// Standard tables pattern: +// - "gcp_billing_export_v1_" +// Detailed tables pattern: +// - "gcp_billing_export_resource_v1_" +// +// Full table names example: +// Standard: +// - "a-project-123456.dataset.gcp_billing_export_v1_011702_58A742_BQB4E8" +// Detailed: +// - "a-project-123456.dataset.gcp_billing_export_resource_v1_011702_58A742_BQB4E8" +func isDetailedTable(tableName string) bool { + return strings.Contains(strings.ToLower(tableName), detailedTablePrefix) +} + func getCurrentMonth() string { currentTime := time.Now() return fmt.Sprintf("%04d%02d", currentTime.Year(), int(currentTime.Month())) @@ -216,8 +235,24 @@ func getTables(ctx context.Context, client *bigquery.Client, datasetID string, t return tables, nil } +// row represents a structure for storing data obtained from a BigQuery standard or detailed cost usage result. +type row struct { + InvoiceMonth string `bigquery:"invoice_month"` + ProjectId string `bigquery:"project_id"` + ProjectName string `bigquery:"project_name"` + BillingAccountId string `bigquery:"billing_account_id"` + CostType string `bigquery:"cost_type"` + SkuId string `bigquery:"sku_id"` + SkuDescription string `bigquery:"sku_description"` + ServiceId string `bigquery:"service_id"` + ServiceDescription string `bigquery:"service_description"` + Tags string `bigquery:"tags_string"` + TotalExact float64 `bigquery:"total_exact"` + EffectivePrice float64 `bigquery:"effective_price"` +} + func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, tableMeta tableMeta, month string, costType string) ([]mb.Event, error) { - var events []mb.Event + events := make([]mb.Event, 0) query := generateQuery(tableMeta.tableFullID, month, costType) m.logger.Debug("bigquery query = ", query) @@ -254,7 +289,8 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, } for { - var row []bigquery.Value + var row row + err := it.Next(&row) if errors.Is(err, iterator.Done) { break @@ -266,34 +302,63 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, return events, err } - if len(row) == 6 { - events = append(events, createEvents(row, m.config.ProjectID)) - } + events = append(events, createEvents(row, tableMeta.tableFullID, m.config.ProjectID)) } return events, nil } -func createEvents(rowItems []bigquery.Value, projectID string) mb.Event { +// createTagsMap converts a comma-separated string of key-value pairs into a flattened map. +// It is used to convert the tags string generated by the SQL query to a map. +// The SQL query generates tags string by concatenating key-value pairs from an array of structs (tags). +func createTagsMap(tagsItem bigquery.Value) mapstr.M { + tagsMap := mapstr.M{} + + if tags, ok := tagsItem.(string); ok { + pairs := strings.Split(tags, ",") + for _, pair := range pairs { + kv := strings.Split(pair, ":") + if len(kv) == 2 { + _, _ = tagsMap.Put(kv[0], kv[1]) + } + } + } + + return tagsMap.Flatten() +} + +func createEvents(row row, tableName, projectID string) mb.Event { event := mb.Event{} + + tagsMap := createTagsMap(row.Tags) + event.MetricSetFields = mapstr.M{ - "invoice_month": rowItems[0], - "project_id": rowItems[1], - "project_name": rowItems[2], - "billing_account_id": rowItems[3], - "cost_type": rowItems[4], - "total": rowItems[5], + "invoice_month": row.InvoiceMonth, + "project_id": row.ProjectId, + "project_name": row.ProjectName, + "billing_account_id": row.BillingAccountId, + "cost_type": row.CostType, + "total": row.TotalExact, + "sku_id": row.SkuId, + "sku_description": row.SkuDescription, + "service_id": row.ServiceId, + "service_description": row.ServiceDescription, + "tags": tagsMap, + } + + if isDetailedTable(tableName) { + _, _ = event.MetricSetFields.Put("effective_price", row.EffectivePrice) } event.RootFields = mapstr.M{ "cloud.provider": "gcp", "cloud.project.id": projectID, - "cloud.project.name": rowItems[2], - "cloud.account.id": rowItems[3], + "cloud.project.name": row.ProjectName, + "cloud.account.id": row.BillingAccountId, } // create eventID for each current_date + invoice_month + project_id + cost_type currentDate := getCurrentDate() - event.ID = generateEventID(currentDate, rowItems) + event.ID = generateEventID(currentDate, row) return event } @@ -302,10 +367,10 @@ func getCurrentDate() string { return fmt.Sprintf("%04d%02d%02d", currentTime.Year(), int(currentTime.Month()), currentTime.Day()) } -func generateEventID(currentDate string, rowItems []bigquery.Value) string { +func generateEventID(currentDate string, row row) string { // create eventID using hash of current_date + invoice.month + project.id + project.name // This will prevent more than one billing metric getting collected in the same day. - eventID := currentDate + rowItems[0].(string) + rowItems[1].(string) + rowItems[2].(string) + eventID := currentDate + row.InvoiceMonth + row.ProjectId + row.ProjectName h := sha256.New() h.Write([]byte(eventID)) prefix := hex.EncodeToString(h.Sum(nil)) @@ -315,26 +380,134 @@ func generateEventID(currentDate string, rowItems []bigquery.Value) string { // generateQuery returns the query to be used by the BigQuery client to retrieve monthly // cost types breakdown. func generateQuery(tableName, month, costType string) string { + if isDetailedTable(tableName) { + return createDetailedQuery(tableName, month, costType) + } + + return createStandardQuery(tableName, month, costType) +} + +func createStandardQuery(tableName, month, costType string) string { // The table name is user provided, so it may contains special characters. // In order to allow any character in the table identifier, use the Quoted identifier format. // See https://github.com/elastic/beats/issues/26855 // NOTE: is not possible to escape backtics (`) in a multiline string escapedTableName := fmt.Sprintf("`%s`", tableName) + + query := fmt.Sprintf(` +SELECT + invoice.month AS invoice_month, + project.id AS project_id, + project.name AS project_name, + billing_account_id, + cost_type, + IFNULL(sku.id, '') AS sku_id, + IFNULL(sku.description, '') AS sku_description, + IFNULL(service.id, '') AS service_id, + IFNULL(service.description, '') AS service_description, + ARRAY_TO_STRING(ARRAY( + SELECT + CONCAT(t.key, ':', t.value) + FROM + UNNEST(tags) AS t), ',') AS tags_string, + (SUM(CAST(cost * 1000000 AS int64)) + SUM(IFNULL(( + SELECT + SUM(CAST(c.amount * 1000000 AS int64)) + FROM + UNNEST(credits) c), 0))) / 1000000 AS total_exact +FROM + %s +WHERE + project.id IS NOT NULL + AND invoice.month = '%s' + AND cost_type = '%s' +GROUP BY + invoice_month, + project_id, + project_name, + billing_account_id, + cost_type, + sku_id, + sku_description, + service_id, + service_description, + tags_string +ORDER BY + invoice_month ASC, + project_id ASC, + project_name ASC, + billing_account_id ASC, + cost_type ASC, + sku_id ASC, + sku_description ASC, + service_id ASC, + service_description ASC, + tags_string ASC;`, + escapedTableName, month, costType) + + return query +} + +func createDetailedQuery(tableName, month, costType string) string { + // The table name is user provided, so it may contains special characters. + // In order to allow any character in the table identifier, use the Quoted identifier format. + // See https://github.com/elastic/beats/issues/26855 + // NOTE: is not possible to escape backtics (`) in a multiline string + escapedTableName := fmt.Sprintf("`%s`", tableName) + query := fmt.Sprintf(` SELECT - invoice.month, - project.id, - project.name, + invoice.month AS invoice_month, + project.id AS project_id, + project.name AS project_name, billing_account_id, cost_type, - (SUM(CAST(cost * 1000000 AS int64)) - + SUM(IFNULL((SELECT SUM(CAST(c.amount * 1000000 as int64)) FROM UNNEST(credits) c), 0))) / 1000000 - AS total_exact -FROM %s -WHERE project.id IS NOT NULL -AND invoice.month = '%s' -AND cost_type = '%s' -GROUP BY 1, 2, 3, 4, 5 -ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC;`, escapedTableName, month, costType) + IFNULL(sku.id, '') AS sku_id, + IFNULL(sku.description, '') AS sku_description, + IFNULL(service.id, '') AS service_id, + IFNULL(service.description, '') AS service_description, + IFNULL(SAFE_CAST(price.effective_price AS float64), 0) AS effective_price, + ARRAY_TO_STRING(ARRAY( + SELECT + CONCAT(t.key, ':', t.value) + FROM + UNNEST(tags) AS t), ',') AS tags_string, + (SUM(CAST(cost * 1000000 AS int64)) + SUM(IFNULL(( + SELECT + SUM(CAST(c.amount * 1000000 AS int64)) + FROM + UNNEST(credits) c), 0))) / 1000000 AS total_exact +FROM + %s +WHERE + project.id IS NOT NULL + AND invoice.month = '%s' + AND cost_type = '%s' +GROUP BY + invoice_month, + project_id, + project_name, + billing_account_id, + cost_type, + sku_id, + sku_description, + service_id, + service_description, + effective_price, + tags_string +ORDER BY + invoice_month ASC, + project_id ASC, + project_name ASC, + billing_account_id ASC, + cost_type ASC, + sku_id ASC, + sku_description ASC, + service_id ASC, + service_description ASC, + effective_price ASC, + tags_string ASC;`, + escapedTableName, month, costType) + return query } From f1ac4a3b0afc43c55a582c3c61e784cbb4311fe6 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sun, 22 Oct 2023 22:44:17 +0300 Subject: [PATCH 02/11] add tests fix older tests --- .../module/gcp/billing/billing_test.go | 178 +++++++++++++++++- 1 file changed, 174 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing_test.go b/x-pack/metricbeat/module/gcp/billing/billing_test.go index 0b257b3a6ebb..f44fbae2d76e 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing_test.go +++ b/x-pack/metricbeat/module/gcp/billing/billing_test.go @@ -5,11 +5,16 @@ package billing import ( - "io/ioutil" + "io" "log" "strconv" "testing" + "cloud.google.com/go/bigquery" + + "github.com/elastic/beats/v7/metricbeat/mb" + "github.com/elastic/elastic-agent-libs/mapstr" + "github.com/stretchr/testify/assert" ) @@ -20,7 +25,7 @@ func TestGetCurrentMonth(t *testing.T) { } func TestGenerateQuery(t *testing.T) { - log.SetOutput(ioutil.Discard) + log.SetOutput(io.Discard) query := generateQuery("my-table", "jan", "cost") log.Println(query) @@ -28,7 +33,172 @@ func TestGenerateQuery(t *testing.T) { // verify that table name quoting is in effect assert.Contains(t, query, "`my-table`") // verify the group by is preserved - assert.Contains(t, query, "GROUP BY 1, 2, 3, 4, 5") + assert.Contains(t, query, "GROUP BY\n\tinvoice_month,\n\tproject_id,\n\tproject_name,\n\tbilling_account_id,\n\tcost_type") // verify the order by is preserved - assert.Contains(t, query, "ORDER BY 1 ASC, 2 ASC, 3 ASC, 4 ASC, 5 ASC") + assert.Contains(t, query, "ORDER BY\n\tinvoice_month ASC,\n\tproject_id ASC,\n\tproject_name ASC,\n\tbilling_account_id ASC,\n\tcost_type ASC") +} + +func TestCreateTagsMap(t *testing.T) { + assert := assert.New(t) + + testCases := []struct { + name string + tagsItem bigquery.Value + want mapstr.M + }{ + { + name: "valid tags", + tagsItem: "tag1.a:value1,tag2.b:value2", + want: mapstr.M{"tag1.a": "value1", "tag2.b": "value2"}, + }, + { + name: "valid tags no values", + tagsItem: "tag1:,tag2:", + want: mapstr.M{"tag1": "", "tag2": ""}, + }, + { + name: "no tags", + tagsItem: "", + want: mapstr.M{}, + }, + { + name: "invalid format", + tagsItem: "tag1 value1,tag2 value2", + want: mapstr.M{}, + }, + } + + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + tags := createTagsMap(tt.tagsItem) + assert.Equal(tt.want, tags) + }) + } +} + +func TestIsDetailedTable(t *testing.T) { + assert := assert.New(t) + + testCases := []struct { + tableName string + expected bool + }{ + // Positive test cases + {"project-name-123456.dataset.gcp_billing_export_resource_v1_011702_58A742_BQB4E8", true}, + {"GCP_BILLING_EXPORT_RESOURCE_V1", true}, + {"prefix_gcp_billing_export_resource_v1_suffix", true}, + + // Negative test cases + {"project-name-123456.dataset.gcp_billing_export_v1_011702_58A742_BQB4E8", false}, + {"gcp_billing_export", false}, + {"random_table_name", false}, + {"", false}, + } + + for _, tc := range testCases { + result := isDetailedTable(tc.tableName) + assert.Equal(tc.expected, result, "Expected output for table name '%s' to be '%v', got '%v'", tc.tableName, tc.expected, result) + } +} + +func TestCreateEvents(t *testing.T) { + assert := assert.New(t) + + t.Run("standard table", func(t *testing.T) { + row := row{ + InvoiceMonth: "202001", + ProjectId: "project-123456", + ProjectName: "My Project 12345", + BillingAccountId: "011702_58A742_BQB4E8", + CostType: "regular", + SkuId: "F449-33EC-A5EF", + SkuDescription: "E2 Instance Ram running in Americas", + ServiceId: "6F81-5844-456A", + ServiceDescription: "Compute Engine", + Tags: "tag1:value1,tag2.a.b/c:value2,tag3:", + TotalExact: 123.45, + } + + date := getCurrentDate() + id := generateEventID(date, row) + + expected := mb.Event{ + ID: id, + RootFields: mapstr.M{ + "cloud.provider": "gcp", + "cloud.project.id": "project-123456", + "cloud.project.name": "My Project 12345", + "cloud.account.id": "011702_58A742_BQB4E8", + }, + MetricSetFields: mapstr.M{ + "invoice_month": "202001", + "project_id": "project-123456", + "project_name": "My Project 12345", + "billing_account_id": "011702_58A742_BQB4E8", + "cost_type": "regular", + "total": 123.45, + "sku_id": "F449-33EC-A5EF", + "sku_description": "E2 Instance Ram running in Americas", + "service_id": "6F81-5844-456A", + "service_description": "Compute Engine", + "tags": mapstr.M{ + "tag1": "value1", + "tag2.a.b/c": "value2", + "tag3": "", + }, + }, + } + event := createEvents(row, "project-123456.dataset.gcp_billing_export_v1_011702_58A742_BQB4E8", "project-123456") + assert.Equal(expected, event) + }) + + t.Run("detailed table", func(t *testing.T) { + row := row{ + InvoiceMonth: "202001", + ProjectId: "project-123456", + ProjectName: "My Project 12345", + BillingAccountId: "011702_58A742_BQB4E8", + CostType: "regular", + SkuId: "F449-33EC-A5EF", + SkuDescription: "E2 Instance Ram running in Americas", + ServiceId: "6F81-5844-456A", + ServiceDescription: "Compute Engine", + Tags: "tag1:value1,tag2.a.b/c:value2,tag3:", + TotalExact: 123.45, + EffectivePrice: 123.45, + } + + date := getCurrentDate() + id := generateEventID(date, row) + + expected := mb.Event{ + ID: id, + RootFields: mapstr.M{ + "cloud.provider": "gcp", + "cloud.project.id": "project-123456", + "cloud.project.name": "My Project 12345", + "cloud.account.id": "011702_58A742_BQB4E8", + }, + MetricSetFields: mapstr.M{ + "invoice_month": "202001", + "project_id": "project-123456", + "project_name": "My Project 12345", + "billing_account_id": "011702_58A742_BQB4E8", + "cost_type": "regular", + "total": 123.45, + "sku_id": "F449-33EC-A5EF", + "sku_description": "E2 Instance Ram running in Americas", + "service_id": "6F81-5844-456A", + "service_description": "Compute Engine", + "effective_price": 123.45, + "tags": mapstr.M{ + "tag1": "value1", + "tag2.a.b/c": "value2", + "tag3": "", + }, + }, + } + event := createEvents(row, "project-123456.dataset.gcp_billing_export_resource_v1_011702_58A742_BQB4E8", "project-123456") + assert.Equal(expected, event) + }) } From 61e984ac49abb059fdd0198de38e979b8ab4e2eb Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Sun, 22 Oct 2023 22:44:28 +0300 Subject: [PATCH 03/11] add changelog entry --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 21e7fa23daf7..20d79a66fa5b 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -255,7 +255,7 @@ is collected by it. - Add GCP Carbon Footprint metricbeat data {pull}34820[34820] - Add event loop utilization metric to Kibana module {pull}35020[35020] - Align on the algorithm used to transform Prometheus histograms into Elasticsearch histograms {pull}36647[36647] - +- Enhance GCP billing with detailed tables identification, additional fields, and optimized data handling. {pull}36902[36902] *Osquerybeat* From 8ae9e9f36f0c970b2c83f3802be59a021e36d75d Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Mon, 23 Oct 2023 23:05:50 +0300 Subject: [PATCH 04/11] update docs and fields --- metricbeat/docs/fields.asciidoc | 54 +++++++++++++++++++ .../module/gcp/billing/_meta/data.json | 10 +++- .../module/gcp/billing/_meta/docs.asciidoc | 22 +++++--- .../module/gcp/billing/_meta/fields.yml | 19 +++++++ x-pack/metricbeat/module/gcp/fields.go | 2 +- 5 files changed, 99 insertions(+), 8 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index b3f93a420a24..c4f5dc8d84a7 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35365,6 +35365,60 @@ type: float -- +*`gcp.billing.sku_id`*:: ++ +-- +The ID of the resource used by the service. + +type: keyword + +-- + +*`gcp.billing.sku_description`*:: ++ +-- +A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. + +type: text + +-- + +*`gcp.billing.service_id`*:: ++ +-- +The ID of the service that the usage is associated with. + +type: keyword + +-- + +*`gcp.billing.service_description`*:: ++ +-- +The Google Cloud service that reported the Cloud Billing data. + +type: text + +-- + +*`gcp.billing.tags`*:: ++ +-- +A collection of key-value pairs that provide additional metadata. + +type: object + +-- + +*`gcp.billing.effective_price`*:: ++ +-- +The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. + +type: float + +-- + [float] === carbon diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/data.json b/x-pack/metricbeat/module/gcp/billing/_meta/data.json index efbb1088bec9..67c624ef2ad0 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/data.json +++ b/x-pack/metricbeat/module/gcp/billing/_meta/data.json @@ -16,7 +16,15 @@ "invoice_month": "202106", "project_id": "containerlib-prod-12763", "project_name": "elastic-containerlib-prod", - "total": 4717.170681 + "total": 4717.170681, + "sku_id": "0D56-2F80-52A5", + "service_id": "6F81-5844-456A", + "sku_description": "Network Inter Region Ingress from Jakarta to Americas", + "service_description": "Compute Engine", + "effective_price": 0.00292353, + "tags": { + "size": "medium" + } } }, "metricset": { diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc index d66330f927f6..d1d3ac691a11 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc @@ -1,18 +1,28 @@ `billing` metricset is designed for collecting billing metrics from Google Cloud BigQuery daily cost detail table. BigQuery is a fully-managed, serverless data warehouse. -Cloud Billing export to BigQuery enables you to export detailed Google Cloud +Cloud Billing export to BigQuery enables you to export standard and detailed Google Cloud billing data (such as usage, cost estimates, and pricing data) automatically throughout the day to a BigQuery dataset that you specify. Then you can access your Cloud Billing data from BigQuery for detailed analysis using Metricbeat. Please see https://cloud.google.com/billing/docs/how-to/export-data-bigquery[export cloud billing data to BigQuery] for more details on how to export billing data. -In BigQuery dataset, detailed Google Cloud daily cost data is loaded into a data -table named `gcp_billing_export_v1_`. There is a defined -schema for Google Cloud daily cost data that is exported to BigQuery. Please see -https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables#data-schema[ -daily cost detail data schema] for more details. +In BigQuery, Google Cloud daily cost data is categorized into two formats: +standard and detailed. Each format is stored within a designated dataset and +follows a structured schema for precise cost analysis. For a comprehensive +understanding of these formats, consult the https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/standard-usage#standard-usage-cost-data-schema[ +standard] and https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/detailed-usage#detailed-usage-cost-data-schema[ +detailed] data schema documentation. + +For regular cost monitoring and analysis, standard tables utilize the +`gcp_billing_export_v1_` format, presenting a comprehensive +view of Google Cloud expenses. + +Alternatively, detailed tables, identifiable by the `gcp_billing_export_resource_v1_` +format, offer richer data. They include additional fields, such as `effective_price`, +enabling a more granular view of expenses. + [float] === Metricset-specific configuration notes diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml b/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml index d760059e7ab8..08bad0971fb8 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml @@ -15,3 +15,22 @@ - name: total type: float description: Total billing amount. + - name: sku_id + type: keyword + description: The ID of the resource used by the service. + - name: sku_description + type: text + description: A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. + - name: service_id + type: keyword + description: The ID of the service that the usage is associated with. + - name: service_description + type: text + description: The Google Cloud service that reported the Cloud Billing data. + - name: tags + type: object + object_type: keyword + description: A collection of key-value pairs that provide additional metadata. + - name: effective_price + type: float + description: The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index fa6cdabb470d..cd8542efc7b6 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzsXVtz3LaSfvevQOXF8ZYy2Thb++DaOlWKfJzjOlaOKpJTtU88INgzAw8IMLhInvz6LdxIcIYzw+FlFKXWeYkkEv31Bd2NRgP8Dm1g+w6tSPUKIU01g3fo9c9CrBigGyZMge4Y1kshy9evEJLAACt4h1b4FUIFKCJppang79DfXiGE0M83d6gUhWHwCqElBVaod+4P3yGOS4iE7D+9rezPUpj4m/T59B2Gc2Cq/nV8VeRfgOjk1x144j+Pi1MtJOUrVIKWlKj9kXchpDCMArn4j9afDkKx//wvM//EBrZPQhadA5egcYE1nmtwy+osY6ut0lDOMrQEJYwkMNngceBvaoH4/745bVetcQthcmfdHX/NSlxVlK/Co9+0Bj9inbfBHPUaayRBG8mhQEspStSaitd3H9HvBuR2scdWThmjfHWIXmuYn/yz0TSSd9rzuy2WdKaizqkSsRChvDxe7SuuS+ctpDdCafesQpQTZgpAElaGYXmFNP56hXDxxShdAtdXCPMCSWF4YYUOUgq56MBD+aOgBLJScL0egikKTEIlpEZunC5ClRTOFmgxhMqdfxt9fI/EEuk1RKVGujkwwVcKadFFXAuNWQfdJRNYH6b6YF+rKeFSGK73zYtgmQvey7pu3KPogxC6kpTrY2aWgx5jaJPJ2wrbs4iWNe5eUo8Q7E9jQNj3R8BQIB+tiQ+TxL1/20piKWQ3DKPwCo6RTkYcgyH55QAwElYD6f/q3uwgeYJiDWqhiKjgh/Nn4L19D/2wx2UPam8XTBB8QOS9yL7tEq71BHHg73KsoLCTeC0KwcRq2wtXieUG9OSo/LCDMP04FMyPZ2pGLJcKtBrqigOxMEqHKxZlZTT088X+2dki/ZJKeMKMLQopqgqKRb7V0MW59V2HGf/IiSitet3rKAyG8q2bipFID/pZhckGtMqIC2OPmJkur9wTTRjsDDyUK405gQWpzEKCdY1QZERIUAfB7CWUO3B+MWUO0qYEbhwUh0WCOzhrmzCFhCHSPwXNebNM0xIWCsgAUJ/tAM5ZYsYCMMqRAiJ4oXqRX1Skyz+covxgNSAxcQEisI2Z81ZQoJu7zz6LpgoRIyVwzbYWmVEQBdZHSAVVm4UEPNSib6z9WXjeou1IPp+3A/cinIlqnBnXEOyQHsHHfyFRgXR+/aiSHIonSTVMw78dSgNHWvQTgCM9sQTcmP1FUEIp5HaRW9sSfCFxmSn6BwyEYq3WJeghw7aoPAVrnNYqf7tdoIc1VcFXWwMWnG0RfsSU4Zz52fbbbVgl+nTQCtS+DG/REpeUdcbBYywZBcVAlm49/GaW2bGejxv1hKuM8oH2avWzpxk3ZygPqFYGlPaTmGqFxBNHliZSFSbwLNwKo6dkN05Sx2LDsRbPwS8H/STkZkH5SoJSU7lhAvQxllYsmEDmHCQhK1g4zzQcUUwuRmGCCYWjwP70CHIgiInlcjYcUx3JZ44Tb1KskMAEH4bWWKEcgCNpOKd8ddRkPYDMuflBMP7OcGV9qB0GKcoJRBxPWCGlsdRQXCV51gLd47JiUCB4BLlF//2fzV+ulxokUvbvlK+uUIE1thOVC40eqaJxlprKTswf3jav7q047KuVFKTXkuN9eHjyNUeyAmJGaZCLdbFUCwuOiwK6DO+g2Pcy/oLa/NHrndfW8I/3H+4dQ79YAt6ZYQnRGqzqaQEI14hOw1VaSJt7E1xhQvW2I/geyYQP4o7D1ah9ab5GKniEYc3n55/OQGo0ZfQPlzCNAmtjTgWSANd27VAD9WR28oge+AxfA2Z6vc1yJshmDv3XJJAnERXuCkR9VP5F5IslpgyKGdB9EXmwyTV+BOTpWE33tEcLLvq1OdGlM+Y8dMrkpc1JLiA95+VretYl9sRZrybmU3SzYBmj7gbofErfRTpM9Q3SOQ2gU6qDzWCLJV/UdYjMp9RZ4+8mdp3/e/3rL/UCUjUFkD4gq2oOX4mJpo/ggeGqYtSXj1UPRERwjSkHOQcuByihcBpO0N2B1f6Q6GwRpRqzQ/eJww6OTW5KzPFqRvnY/OY20NjNbvqbfwV+I3YqAVqzD2lBsyT/3S5Kr7z0rupaXyBtJ2wOaGnYkjLWlGwVWUNhWC8uHqnUBrNQtZ1e4GH8pmhqNXA6FhJhM307tK/bHhbs6d6INVVarCQuT0nfrUR8pBRiY6UbYECzXHXP2P8xyi44nO9UCNvX4pLfPVIvDPyGpkLUV2nDgCdclxVBYeSpNHQW3l1kUJU1Q2qD2Io+2iihsYa+gez5tJeGubN1WL88oSYboTyDPhNp9NJqutFkVyr9tt0+xKdnXAQXgpgSuF4UYIU+wlE9tNyTMoSAUkvDahLIkzgQO2sgbrtkThiWgPJG+7sBSUEhIRETYmOqU+D8Zsac6ByFjurJanPQaF7/e7WBf8fsxAeKMu0Fe7KhS+Kq7ga7uUP3GpNNIekjSNcQFt52u+P7HY5LId1bP//z768ns8JmKzqkVW5Xzwa0bPTO4o0pDcMul7y5++z7Ltzaod5ndPsMIbDXCPpVxU7ywGhJ9cgNWwvbI3WjxU3KhtBofGlJZrpdVIvbI+67gxp2AZygEMGcC43gKwEo0A8Iq6C89h/s+47KdMXLt/91hgRDBjnZpnyj7TByh3HOXKc9xeo85tKwe86m+67J2HR9JQFrV+jHfMdwUqsJBJ/BbqBaQwkSsyxUK/08HLjv8kkQzFA9Zl0B9XOPcr8fM9RL7GONcpsWbRh1Bryu/jstWG9PY6GGPdgxug+b5t2BYVKAM836MmXghcWJ3s4zyLGyqcwSG6YHbmo2UaJy7VJ2KHWFcik2wFEhnrgLFdsKrlCJvwjpGtpLyrv72PcAjpvZt62SyiBrnFkBlwpgwaj/0jEsiHSEcw32Mo0vtQtkLMdPLLvSUDuGu8auS9Lvlz+D9R7tRDhpqbSEZDHjjXKfv1ZXwlk64KLwXZhh0wDnDCbLhpMxk8w4zB9HeTzWebzBLvL+zuBCU7mWxUXX2YnqBi+xa+SuNWakrfkmed6x/hpnZft5aGIRE6ejvuEqtbipsVPXEpMtJQztWP0gIa14+QEtTtbN1JSwfQvVwE7bHfu4DHCPeGAzpkN8yEqmtowZVlceaFqWG4g25CjjJ17i3EJuNyPGCwSkkQlqn/VWB7nLZv+pdMdMpyYZCROobnCezuGOT6ZntNDYlxs7izOvx4F9uQ1OviPYunM5gbvXsTsCvwKu58KuJeYqNPlMDL+iReYLF8OPipT4K7rz58D/dT/SVi2eEcc82htfsTekkoKAUvG4x1iQWYGhFPzZtpGc7hk8AouNsx7QqJQ3MjXxyju2tOziHLAcr0Sx5y3G+7NTfqISxch5luK2XmJ6zB3+YRrYj4KZcop8sUHsTreFJUV9ACf0a1iSlwrgCXsjbP2XLr7SuXqcpRO4ZsnU/OAd2VkO1lem4I9laMeriDul8XCYsNZ3wBAOZ13kWAgTuMgxw5z0vXjmk8AF+im+Mtup9LXWlVrkmGyAF9m4Ink7/CUnpXC94RtaUv7x8HD3/b2TCvJisYoUKODoNM1upMOynZ1AHbGFw+L5tgbijl50gO0DUFWCq6GrsuOy9EMHYdZYvxUSEUzW8MbKEr5qkBwzh//b+zd9GbiUDRBGgWtloZ4n4ZlVfy6YS6n50JwJcuyCyH5cRCsYd/zxIEyHrLa0h5u77z+/v4sRfwdrsNMGc4wJSyaeFuiDkHYA95NCVL9WyGm4uevCpqRNtzxSWgIu3Wnafsxn445dtoXQOns5nRhOMDLuiO9xNQY0qevoxcm8qgscz6y7wbx3o6d8/ln38dNPHbb07XKELt70Y2dmXXQzdnqS1CgvM0lSmPNK/dJTIOGs8/I8UmVKsayS4ut2QZhQ7t4ezoH4U0zDiifp3T31WLG1VwLSIEvK3c01bmlp5+f9/SfkYZzEOWom7tZJG6n9dpvYqL93qyegcVZ6GFGjx99uz0PE4ekCeiRu3TZMiaICPgHEm3BQKjngaLRdbLpzUW3YUpjV2rmek1jbCwCGNXBCj2zfdhzY6HFco8XJtV35a0lzky6yPektIpgRw5y0nWk8rYGnrTX+rgQriVgvsKzFekh0btaNseTP+7dw1N7a0Vba2aRPr0N2fFBadZz3txFi9v/yGyk/3zjwUqS3V/B0ItoV2UpodH3zz5aDE9zLKsrICe2woJZScG3tynoUqQ/vOMwml18fHlAJWBlpJSIkAkzWibdBOegnAB4ZxLw45WvqFOElT5rIRPcKd7YphU5L9WVPpT5ynWGivWyhtapmlxBautaVWj+n4Nzt4EhLWvkDoEGQV43XijlbK0eK1wDXoutcMZ9a5Dwj7y3HfJDFejrFwkDbNNzK73QG+/xhSJVC6DUUju1vKUeletOwn4bh18oJQmlMNlc+WpWUGw2tlSzDW5BhFVJhFaqVtdv28yHdEeneuHCf1tjf5ejezxhyi++hL2ZUJlcm7zX4ncnvTT7bXoziuFJroV00Z2I1as/TxTt3eUZ9GkUpvHJ7zK43uvBHuWuiPQD5DpMs32Y+27wowL2zIF4nAckx9ETwJV1lpirwJA0yJN6/5wcOJ/MRWWO+AnXlNe7nUnIpwLYC/w0GUIYdFzc3ZRalMXoVPkbvKZBE7TNDGq5pwQpQOkLO8GrYHYfXK3CeMWznvokW6oeP8IcIdB9gItaLQz1f0CavaS8w2URGJpxTtW1gsuHiiUGx8lPpuvm53rZrzbUCGHV7+Jb0SfSz+FjDW6hrXr7Fi80CL1AgWv/hTdBHCuwk8K2GjIjBO7QtofuzbM1NH0mu17RJufYLGjpLtIiPoN+N0BilfSKnsF/WEzeL7BTECP+cslIALjIGWoOccxZUJmdUrb3gLU3kaSItKkpavPQEXoois1PXDsYohznRP62FAhQpuQWX170DfCsKutxek837+MAE8/oQe1l6wdBUjO5zEB1uOqUm0NG4nod+0DuF3xOwTRSSGvoM2UuwgnjRYEr9tZ0mao2AF5Wg3IY1o12X1hZ0K4704sPwmtZ0fFwiMoTkIsZ6a0ENC3MlQ11s7aYbkzIxOg+dlqFxOVTg1lnH/LrqNsJDNjjCBDv4mVttB3ibWF+VYSxLEt9ZokrCSM944jpwMCpgSTmNxZ6uVxUktYzvm2JGi8vvT8fQ8K2R3uKaPn7hU8uBnpHLQZxTm5bAX0GN06vQSWac7tR6FlxqjbDWUFbduNBnzugGHAPqyhdM7TuuiVQiWlYMSuDarywKAb5nPMearP21kXVege6FX6LUV4twtm1uuRMcWi8s/P3GCTFpFe/7i9xHPa11iPqKxvRdd78GrirAEpWGaVoxf9NjZ9m6Jeh2NJ52zdydF43IIHZSh1lrlueCHxd9FMAcrtQOe9TeT+Pies5lZNwtbrs8LSIKm2OHNugRK0ffkmdXLxeJ8veR3J11g3+dmN8hxhlMNhJpBxH0RPUaccG/s7a8bUmVFsNsu83OnBaxw9SlzOB/iCjgb4OM4VzhXbI2055f++WOcRWaU5w9m8nvM+o3HCdgsm6dmZGpcORlCNq46Jw14B+q44yP9K6gO766P66knxTzXfXclfi72xoc3JeyterRRndjFXq48XqCzov3O30XoYpf+2UHwFUzHIg3BxG/hGpY2IkYZO5/znrRGI7+bKuPMbwo4MXMtZH2vHgxGW8inznqD14qZ5ZGAqY/e8z2MJ8/WA+bGHsdEBfpWdyNJ5hsUE3YslNSxmh9ccGD9w11JyxVzQdJYpO564ZtTxLlTpHjekqlG23NEttF2o63QyetQph39WqENlFc96ScVd17HjG7OlvE38jbNU1SIkUMLUctv+bN3wD36gDtVqPhfbgtbq5OQ1zRkcfo3wPTuPEJ7uMa2FVDE1FYo7N/KUGvReFoxzzNN0XblWeXHWCj138sMGFZjhUUWVAkdl8SmQZxvX5yNumV5hwDD2YTbiRZScw1FMjTRkowYFtUGDc9wpPXN586c+SGjSY6DUT/WYWv2V3ffEq/zNNxP/ZhJEGMqgJCl5RkFllp9JioviPV2HlT4iIVUKR4UFIT3ay0g2bnTqXdG4mmsdUJLlfqhN35UedpIMf7Lb12hp4n3rlayQ+mUAUS5YZsQLfA1t+uZVip1pU+7tvosfueE/f5RlTg7ZVjxl0aFJ+TUPnDo1iHxq5wOY/vzn/ELJ7YFMbfBF3gzuNTrbtIXRKSxej5cixvugtVY9KEGav1GK6r+5Op8v8CAAD//8Sus/0=" + return "eJzsXduP27aaf89fQZyXNIupu00X+xAsDjCdnPQETXoG9aTAPulQ5GebMUWqJDUT969f8CZRtmzLuniaYpOnmZHI33fhd+NH6lu0hd0btCblC4QMMxzeoJc/SbnmgO64rCi659ispCpevkBIAQes4Q1a4xcIUdBEsdIwKd6gv79ACKGf7u5RIWnF4QVCKwac6jfuD98igQuIE9l/Zlfan5Ws4m/S59N3OM6B6/rX8VWZfwZikl934In/PC7BjFRMrFEBRjGiD0feh5DCqDSoxX+0/nQUiv3nf5n5J7awe5KKdg5cgMEUGzzX4JbUWcbWO22gmGVoBVpWisBkg8eB/1YzxP//23m9ao1LZZU77e74a1bgsmRiHR79W2vwE9r5Maij2WCDFJhKCaBopWSBWkvx9v49+r0CtVsckJUzzplYH5uvNcyP/tmoGsk77fXdZku6UlHnUolYiNSeHy8OBdcl8xbSO6mNe1YjJgivKCAF64pjdYMM/nKDMP1caVOAMDcIC4qUrAS1TAelpFp04GHiUTICWSGF2QzBFBmmoJTKIDdO10Slkk4XGB0yy71/G71/i+QKmQ1EocZ5c+BSrDUysmtyIw3mHfOuuMTm+KwP9rV6JlzISpiu4fW2GkjXwwYSmuLCtuaUonznfqlBPTICx+ZNhusAYODLCfpu0x8PQNgROpGgd1Ih+IKLksMNwntvrKQKa2lppMJrQEyjpcGCYtX87tOykyA/wyTMDGN5w2F/UekABmstCcMGKHpi3doagYzhroXTsi0tRF5vgTpobdvj/F2XFuO17kDRYe/PupI9NSCScyBRC7aw+/YR8wpQiZkKprdU8pFRQJhSZh/EvPHNHVhhtbIDPkJWKka6rN25tbcBRDZYrYEiN4RTLC/DIOEWc5c/f9LO5i1//oQMA6UX6FdYWbI0IlIYhYlxI1kWsxXCZckZwbnVYWk2oJ6YhhvEzEvtRudM++fh0KMQrPKWRhx3KHfuUfROSlMqJswpz5KDGeNbJjOxlnxPIlrVuHsZ2gjB/jQGhH1/BIxxdmQZ1un7t07nOmE4RRxuOXpjSA305WAUrAfO/6t7s2PKMzPWoBaayBK+v3zhL+176PsDKnvM9nrBJcFHWN5r2tddzLUGIw78bY6tRyzAbCSVXK53vXAVWG3BTI7KDzsI0w9DwfxwoWTkaqXBdDmuXtFXmCyM0mGKZVFWBvrZYv/sbMH9iil4wpwvqJJlCXSR7wx0UW5t13HC3wsiCyte9zoKg8UoLE7SY/6sxGQLRmfERa7Opw9HEwa7AA8T2mBBYEHKaqHAmkagGZEK9FEwBznkHpxfqiIHZUMANw6KwyIpHJyNzZFCgBDnPwfNWbPMsAIWGsgAUJ9cWGKNJeY8AGMCaSBSUN1r+kVJuuzDuZltmLSyoU0SwWPurBVQdHf/yUdvTCNSKQXC8J1FVmmIDOvDJMr0dqEAD9XoO6t/Fp7XaDuST+HtwL0mzmQ5To1rCHZIj+D9v5AsQTm7flJIDsWTYgamod8OZUAgI/sxwE09MQfcmP1ZUEAh1W6RW92SYqFwkWn2BwyEYrXW5eQhqbao/AxWOa1W/vZxgR42TAdbbRVYCr5D+BEzboN2t9p++xiyEx8OWobal+E1WuGC8U4/eIokm+0OJOmjh9+sMpc5Pxs1+gmXGRMD9dXK50Aybs0wEVCtK9DGL2JmNJJPAtk5kS4xgWehVlZmSnLjInUkNhQb+Rz0CjBPUm0XTKwVaD2VGSbAHmM11YIJ01yCJEQFC2eZhiOKwcUoTDAhczTYnx5BDQQxMV8uhlOVJ+KZ05M3IVYIYIINQxusUQ4gkKqEYGJ9UmU9gMyZ+UEw/sFxaW2oHQZpJghEHE9YI22wMkBvkjhrgZauJkkRPILaof/+z+YvtysDCmn7dybWN67GZheqkAY9Ms3iKq1KuzC/f928epBx2FdLJUmvlONteHjynCPJgHilDajFhq70woITkkKX4h1l+0HET5mNH73cRa0N/3z7bukI+sVO4I0ZVhC1wYreVQhrROfhal8OzgguMWFm1+F8T0TCR3HH4WrUfjeuRipFhGHV56cfL0BaGcbZHy5gGgXW+pwSFAFhQknTAw318XYc0QNfJTaAudnsspxLsp1D/vUUyE8RBe4KRH1E/lnmixVmHOgM6D7LPOjkBj8C8vNYSffURwsu2rU50aUr5jJ0usoLG5NcgXvOytfzWZPYE2edTcwn6CZhGSPuBuh8Qt9HOkz0DdI5FaCTq4PVYIeVWNR1iMyH1Flj7yY2nf97++svdQKpmwJIH5BlOYetxG7zywMLO02OuT0QESkMZgLUHLgcoGSG83CC7I5k+0O8s0WUSswO3ccPOzg2uCmwwOsZ+WPjm49hjv3opr/6l+B7L6ZioNsR9WFBk5L/bpPSG8+9m7rWF6a2CzYHtKr4inGebOSTDdCK96LikSlTYR6qttMzPIzfFE2tBM77QiJtpG+H9nXb44w93w61YdrItcLFOe67TMR7Sim3lrsBBjTpqnvGtxvYhMPZTo2wfS2m/O6ROjHwG5oaMV+lDQOeMV2WBbRS58LQWWh3nkGXVg2ZdWJr9mi9hMEG+jqy55Ne6uYulmH98oSSbJjyDPJMuNFLqulGk81U+m27vYtPz5gEU0mqAoRZULBMH2GoHlrmSVeEgNaritdTID/FEd9ZA3HbJXPCsBNor7S/V6AYaCQV4lJuq/IcOL+ZMSc6N0NH9WS9Pao0L/+93sK/Y3TiHUWRtn8+WdelcFk3gN7do6XBZEsVewTlekDD2253/LCpeSWVe+unn//xcjItbLaiQ1jldvWsQ8tG7yzeVUXFsYsl7+4/xf4nkewzph16NYJ+VbGzNHBWMDNyw9bC9kjdaHGTsploNL60JDPdLqrF7RH33UENuwC+dY5gIaRB8IUAUPQ9wjoIr/0H32lWMDNd8fL1f13AwRBBTrYp30g7jNyhnDPXac+ROo+6NOResum+rzI2XF8rwMYV+rHYU5xUa8KEz6A3UG6gAIV5FqqVfh0O3Hf5IAnmqB6zroD6tceE348ZaiUOsUa+TYs2jDoDXlf/nRZs6IQeCTXswY6Rfdg073YMkwKcadUXKQFfmZ/obTwDH0sbyqxwxc3ATc3GS5SuXcoOpW9QruQWBKLySThXsSvhBhX4s1Sun7tgovvoygHAcSv7Y6ukMkgbZxbAtRxYUOq/tA8LLB1hXIO+TGNLbYKM1fiFZTMNvae4G+y6JP1++TNo78lOhLOaygpIkpnmNE+bvlZXwkUyEJL6LsywaYBzDpNFw8mYSWQc1o+beTzWeazBPvL+xuBKS7nmxVXz7ER0g1PsGrlrjRmpa75JXnTkX+O07DAOTTRi4nDUN1ylGjc1duZaYrKVgqEdq+8UpBUvP6DFybuJmhK2b6Ea2Gm7px/XAe4RD2zGdIiPacnUmjFDduWBpmW5gWhDjDJ+4SXGLcR2M2K8gkMaGaD2ybc6prtu9J9yd8xyaoKRsIDqBufpDO74YHpGDY19ubGzOPNyHNiX2+AUe4ytO5cTuAcduyPwaxBmLuxGYaFDk8/E8EtGM1+4GH5UpMBf0L0/2f+v5UhdtXhGHPNob3zF3pBSSQJax+MeY0FmFEMhxbNtIznZc3gEHhtnPaBRIW8kauLMO7a07OMckI6Xkh5Yi/H27JydKCUduc5S3NZKTI+5wz5MA/tR8qqYIl5sELvTbSGlqA/ghH4NO+W1HHhC3ghd/6WLrnStnibpDK5ZIjU/eEd0loO1lSn4UxHa6SriXmk8HCas5R0whMNZVzkWwiWmOeZYkL53TX2QmKIf4yuznUrfGFPqRY7JFgTNxhXJ2+4vOSmF6w3f0JLyz4eH+++WjivIs8UKUqKAo1M1u5EOi3b2HHXEFg6L57saiDt60QG2D0BdSqGHZmWneemHDsyssX4jFSKYbOCV5SV8MaAE5g7/N8tXfQm4lg4QzkAYbaFexuGZRX8pmGuJ+diaCXzsgsh/WEQtGHf88ShMh6zWtIe7++8+vb2PHn8Pa9DTBnP0CSsun/y9YQ939+4n7W9achJu7rqwIWnTLY+0UYALd5q2H/HZuGOXbSa0zl5Ox4YzhIw74ntajAFNajp6UTKv6ALFM8tuMO3d6JmYf9W9//Bjhy59sxohi1f9yJlZFt2EnV8kNcrrLJIU5rxcv/YSSCjrvGmQlJnWPCuV/LJbEC61u7dHCH9J4PHtqr7JTDJWbO1VgAyoggl3c41LLe36XC4/IA/jLM5RK3G/Ttpw7bePiY76e7d6AhqnpccRNXL87eNliAQ8XUGOxOVtw4QoSxATQLwLB6WSA46VscmmOxfVhq1ktd4403MWazsB4NiAIOzE9u25uzm7j2vsX9LKtFEsr9Ik20+9QwRzUnHHbacaTxsQaWuNvyvBciLWCyxpsR4SjZs1Yzz58+EtHLW1dnNr43TSh9chOj7KrdrP+9sIMf9//o3kn28c+Fq4d1DwdCzaZ9laGnR793PLwEnheRV55Jh2nFErJYWxemUtijLHdxxm48uvDw+oAKwrZTkiFQJMNom1QTmYJwARCcSCnrM1dYjwNS+aSER3hjvbkkLnufp1L6U+fJ1hoX3dTGtVza7BtDTXVcY8J+PcBwGQUaz0B0ADI28aqxVjtlaMFK8BrlnXmTGfS3KekfaWYT5KYr2cYmGgrRou8zsfwT6/G9KFlGYD1JH9DROo0K8a8lM3/FI7RmiDyfbGe6uCicpAK5PleAcqZCEl1qFaWZttvx7SHZHujQv3NZ3DXY7u/Ywht/ge+0hOWeW6ynsNfl/lyyqfbS9GC1zqjTTOm3O5HrXn6fyduzyjPo2iNV67PWbXG039Ue560h6AfIdJlu8yH21eFeDBWRAvk4DkFHoixYqts6qkeJIGGRLv3/MDh5P5iGywWIO+8RL3aym5FGBXgv/sCuiKn2a3qIoscmN0Fj5G7imQROwzQxouackpaBMhZ3g97I7D2zU4yxi2c19FDfXDR/hDGHoIMGHr1aFezugqr+deYLKNhEy4pmrdwGQr5BMHuvZL6bb5ud62a601Cpy5PXw79Vn0s9jYSrRQ17R8gxfbBV6gMGn9h1dBHimws8B3BjIiB+/Qtpjuz7I1N30ksV7TJuXaL1joLDEyPoJ+r6TBKO0TOYf9upa4SbJTECPsc0oKBUwzDsaAmnMVlFXOmd54xts5kZ8TGVky0qKlJ/BC0swuXTsYZwLmRP+0kRpQnMklXF72DvBHSdlqd0u2b+MDE6zrY+Rl6QVDUxF6SEE0uOmSmkBG43oe+kHvZH5PwDZQSGroM0QvQQviRYPp7C/tMtEbBIKWkgnr1irjurR2YFp+pBcdlajnmo6Oa3iGEFxEX281qCFhrmCoi6z9cGNSIkbHodMSNC6GCtQ67ZhfVt1KeEwHR6hgBz1zi+0IbRPLq6w4z5LAdxavkhDS05+4DhyMKKyYYLHY0/WqhqSW8V1TzGhR+d15Hxq+NdKbXdP7L3wuHejpuRzEOaVpJ/griHF6ETrOjJOd3syCS28QNgaKshsX+iQ424IjQN/4gql9xzWRKsSKkkMBwvjMgkrwPeM5NmTjr42s4wq0lD5Fqa8WEXzX3HInBbReWPj7jZPJlBW87y9y3/G12iHrKxrTd939GrgsAStUVNywkvubHjvL1i1Gt73xtDlzd1w0IoLYCx1mrVleCn6c99EAc5hSO+xJfT+PS5g508i4W9w2eUZGFDbGDm3QIzJH35Jns5erePllnO7emsG/js/vYOMMKhsnaTsR98lmJKT41uryrsVVRofpdpucOTVij6hrqcH/EEnh74OU4VLmXbM2015fh+WOcRWac5Q9m8ofEuo3HCcgsm6dmZGocORlCNqYdM7q8I/VccZ7elfQHV/dH1fST4r5rnruSvzdbQ0O7teyterRRnNjBXq88XqCzou3e30XoYpf22UHwFUzHIhXRxF/DdWwsBMxSN3/nPWiMRT92bKPMbRoEHTm2kh7XXw1EW/CnznqD54rF5ZGAqY/u8/2MJ/fWQ9bGAcdEFfpWdz3J5hsUT2xJadgnLP64oIHbxvqTlimmw+SxCZz1w3bXiTanSLH9ZJKN9qaFNt52o63QyetRlh09WqENlFc96RcVN17Hja7OlvE3/DbNU0yomR0LSc1v6bN3wD34sjcrUbDZbgtbq5OQ1yykcfo3wI3uLEJ7uMa2FVDE1ZYpbN/KcBsJHVzxzjNN0XbzLNLD3BlNn8sMOFZjjXQLAgSuy+JTIO4zp+cTnqhOcMggtqEG0nWCgsDFPm5kZYc+A7Ryi2P8OTt3YfOGLkho/FOA9F/0uFrdrd3H9Iv83Tcj30cSWCjLoGwFSOZRVZUZoxX3+Nq7LwpME0ZFGc8yqmJblbaQ7N3p9L+jUTT6OoElyt1wu78qPM0kOP9ll46Q88T712t5AfTqASF8opswbTA1t+u5Vjr1pU+7tvosfteEPf5RkTx7sYR4y4Nis8pKP3hUWxCY1e4nMd35z9iHk9sysrfBE1x5/Gp1l2kLgjJovf8ejRvugtVY9CEOa/lGK6r+5OJ8v8CAAD//1na61c=" } From a0f7d14c9cfcd0add4a9a242cb4e9a89f35f1b41 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Mon, 23 Oct 2023 23:06:04 +0300 Subject: [PATCH 05/11] add sku and service ids in hash --- x-pack/metricbeat/module/gcp/billing/billing.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 2c1cc5d5ea94..457d50988ee8 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -370,7 +370,7 @@ func getCurrentDate() string { func generateEventID(currentDate string, row row) string { // create eventID using hash of current_date + invoice.month + project.id + project.name // This will prevent more than one billing metric getting collected in the same day. - eventID := currentDate + row.InvoiceMonth + row.ProjectId + row.ProjectName + eventID := currentDate + row.InvoiceMonth + row.ProjectId + row.ProjectName + row.SkuId + row.ServiceId h := sha256.New() h.Write([]byte(eventID)) prefix := hex.EncodeToString(h.Sum(nil)) From eb995a74312567d613871f3a354ed067322749fa Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 24 Oct 2023 22:25:15 +0300 Subject: [PATCH 06/11] improve docs --- .../module/gcp/billing/_meta/docs.asciidoc | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc index d1d3ac691a11..90a2766a2f45 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/billing/_meta/docs.asciidoc @@ -15,14 +15,13 @@ understanding of these formats, consult the https://cloud.google.com/billing/doc standard] and https://cloud.google.com/billing/docs/how-to/export-data-bigquery-tables/detailed-usage#detailed-usage-cost-data-schema[ detailed] data schema documentation. -For regular cost monitoring and analysis, standard tables utilize the -`gcp_billing_export_v1_` format, presenting a comprehensive -view of Google Cloud expenses. - -Alternatively, detailed tables, identifiable by the `gcp_billing_export_resource_v1_` -format, offer richer data. They include additional fields, such as `effective_price`, -enabling a more granular view of expenses. +For standard usage cost data, set the table pattern format to +`gcp_billing_export_v1`. This table pattern is set as the default when no other +is specified. +For detailed usage cost data, set the table pattern to `gcp_billing_export_resource_v1`. +Detailed tables include the standard fields and additional fields, such as +`effective_price`, enabling a more granular view of expenses. [float] === Metricset-specific configuration notes From b59f3c2764bae988948ce5d26952052892a79a4e Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 10:20:10 +0200 Subject: [PATCH 07/11] update fields types --- metricbeat/docs/fields.asciidoc | 6 +++--- .../metricbeat/module/gcp/billing/_meta/fields.yml | 12 ++++++++---- x-pack/metricbeat/module/gcp/fields.go | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/metricbeat/docs/fields.asciidoc b/metricbeat/docs/fields.asciidoc index c4f5dc8d84a7..41f7f08b7485 100644 --- a/metricbeat/docs/fields.asciidoc +++ b/metricbeat/docs/fields.asciidoc @@ -35379,7 +35379,7 @@ type: keyword -- A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. -type: text +type: keyword -- @@ -35397,7 +35397,7 @@ type: keyword -- The Google Cloud service that reported the Cloud Billing data. -type: text +type: keyword -- @@ -35406,7 +35406,7 @@ type: text -- A collection of key-value pairs that provide additional metadata. -type: object +type: nested -- diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml b/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml index 08bad0971fb8..31ff04b153ab 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml +++ b/x-pack/metricbeat/module/gcp/billing/_meta/fields.yml @@ -19,18 +19,22 @@ type: keyword description: The ID of the resource used by the service. - name: sku_description - type: text + type: keyword description: A description of the resource type used by the service. For example, a resource type for Cloud Storage is Standard Storage US. - name: service_id type: keyword description: The ID of the service that the usage is associated with. - name: service_description - type: text + type: keyword description: The Google Cloud service that reported the Cloud Billing data. - name: tags - type: object - object_type: keyword + type: nested description: A collection of key-value pairs that provide additional metadata. + fields: + - name: key + type: keyword + - name: value + type: keyword - name: effective_price type: float description: The charged price for usage of the Google Cloud SKUs and SKU tiers. Reflects contract pricing if applicable, otherwise, it's the list price. \ No newline at end of file diff --git a/x-pack/metricbeat/module/gcp/fields.go b/x-pack/metricbeat/module/gcp/fields.go index cd8542efc7b6..04b919fd0a6c 100644 --- a/x-pack/metricbeat/module/gcp/fields.go +++ b/x-pack/metricbeat/module/gcp/fields.go @@ -19,5 +19,5 @@ func init() { // AssetGcp returns asset data. // This is the base64 encoded zlib format compressed contents of module/gcp. func AssetGcp() string { - return "eJzsXduP27aaf89fQZyXNIupu00X+xAsDjCdnPQETXoG9aTAPulQ5GebMUWqJDUT969f8CZRtmzLuniaYpOnmZHI33fhd+NH6lu0hd0btCblC4QMMxzeoJc/SbnmgO64rCi659ispCpevkBIAQes4Q1a4xcIUdBEsdIwKd6gv79ACKGf7u5RIWnF4QVCKwac6jfuD98igQuIE9l/Zlfan5Ws4m/S59N3OM6B6/rX8VWZfwZikl934In/PC7BjFRMrFEBRjGiD0feh5DCqDSoxX+0/nQUiv3nf5n5J7awe5KKdg5cgMEUGzzX4JbUWcbWO22gmGVoBVpWisBkg8eB/1YzxP//23m9ao1LZZU77e74a1bgsmRiHR79W2vwE9r5Maij2WCDFJhKCaBopWSBWkvx9v49+r0CtVsckJUzzplYH5uvNcyP/tmoGsk77fXdZku6UlHnUolYiNSeHy8OBdcl8xbSO6mNe1YjJgivKCAF64pjdYMM/nKDMP1caVOAMDcIC4qUrAS1TAelpFp04GHiUTICWSGF2QzBFBmmoJTKIDdO10Slkk4XGB0yy71/G71/i+QKmQ1EocZ5c+BSrDUysmtyIw3mHfOuuMTm+KwP9rV6JlzISpiu4fW2GkjXwwYSmuLCtuaUonznfqlBPTICx+ZNhusAYODLCfpu0x8PQNgROpGgd1Ih+IKLksMNwntvrKQKa2lppMJrQEyjpcGCYtX87tOykyA/wyTMDGN5w2F/UekABmstCcMGKHpi3doagYzhroXTsi0tRF5vgTpobdvj/F2XFuO17kDRYe/PupI9NSCScyBRC7aw+/YR8wpQiZkKprdU8pFRQJhSZh/EvPHNHVhhtbIDPkJWKka6rN25tbcBRDZYrYEiN4RTLC/DIOEWc5c/f9LO5i1//oQMA6UX6FdYWbI0IlIYhYlxI1kWsxXCZckZwbnVYWk2oJ6YhhvEzEvtRudM++fh0KMQrPKWRhx3KHfuUfROSlMqJswpz5KDGeNbJjOxlnxPIlrVuHsZ2gjB/jQGhH1/BIxxdmQZ1un7t07nOmE4RRxuOXpjSA305WAUrAfO/6t7s2PKMzPWoBaayBK+v3zhL+176PsDKnvM9nrBJcFHWN5r2tddzLUGIw78bY6tRyzAbCSVXK53vXAVWG3BTI7KDzsI0w9DwfxwoWTkaqXBdDmuXtFXmCyM0mGKZVFWBvrZYv/sbMH9iil4wpwvqJJlCXSR7wx0UW5t13HC3wsiCyte9zoKg8UoLE7SY/6sxGQLRmfERa7Opw9HEwa7AA8T2mBBYEHKaqHAmkagGZEK9FEwBznkHpxfqiIHZUMANw6KwyIpHJyNzZFCgBDnPwfNWbPMsAIWGsgAUJ9cWGKNJeY8AGMCaSBSUN1r+kVJuuzDuZltmLSyoU0SwWPurBVQdHf/yUdvTCNSKQXC8J1FVmmIDOvDJMr0dqEAD9XoO6t/Fp7XaDuST+HtwL0mzmQ5To1rCHZIj+D9v5AsQTm7flJIDsWTYgamod8OZUAgI/sxwE09MQfcmP1ZUEAh1W6RW92SYqFwkWn2BwyEYrXW5eQhqbao/AxWOa1W/vZxgR42TAdbbRVYCr5D+BEzboN2t9p++xiyEx8OWobal+E1WuGC8U4/eIokm+0OJOmjh9+sMpc5Pxs1+gmXGRMD9dXK50Aybs0wEVCtK9DGL2JmNJJPAtk5kS4xgWehVlZmSnLjInUkNhQb+Rz0CjBPUm0XTKwVaD2VGSbAHmM11YIJ01yCJEQFC2eZhiOKwcUoTDAhczTYnx5BDQQxMV8uhlOVJ+KZ05M3IVYIYIINQxusUQ4gkKqEYGJ9UmU9gMyZ+UEw/sFxaW2oHQZpJghEHE9YI22wMkBvkjhrgZauJkkRPILaof/+z+YvtysDCmn7dybWN67GZheqkAY9Ms3iKq1KuzC/f928epBx2FdLJUmvlONteHjynCPJgHilDajFhq70woITkkKX4h1l+0HET5mNH73cRa0N/3z7bukI+sVO4I0ZVhC1wYreVQhrROfhal8OzgguMWFm1+F8T0TCR3HH4WrUfjeuRipFhGHV56cfL0BaGcbZHy5gGgXW+pwSFAFhQknTAw318XYc0QNfJTaAudnsspxLsp1D/vUUyE8RBe4KRH1E/lnmixVmHOgM6D7LPOjkBj8C8vNYSffURwsu2rU50aUr5jJ0usoLG5NcgXvOytfzWZPYE2edTcwn6CZhGSPuBuh8Qt9HOkz0DdI5FaCTq4PVYIeVWNR1iMyH1Flj7yY2nf97++svdQKpmwJIH5BlOYetxG7zywMLO02OuT0QESkMZgLUHLgcoGSG83CC7I5k+0O8s0WUSswO3ccPOzg2uCmwwOsZ+WPjm49hjv3opr/6l+B7L6ZioNsR9WFBk5L/bpPSG8+9m7rWF6a2CzYHtKr4inGebOSTDdCK96LikSlTYR6qttMzPIzfFE2tBM77QiJtpG+H9nXb44w93w61YdrItcLFOe67TMR7Sim3lrsBBjTpqnvGtxvYhMPZTo2wfS2m/O6ROjHwG5oaMV+lDQOeMV2WBbRS58LQWWh3nkGXVg2ZdWJr9mi9hMEG+jqy55Ne6uYulmH98oSSbJjyDPJMuNFLqulGk81U+m27vYtPz5gEU0mqAoRZULBMH2GoHlrmSVeEgNaritdTID/FEd9ZA3HbJXPCsBNor7S/V6AYaCQV4lJuq/IcOL+ZMSc6N0NH9WS9Pao0L/+93sK/Y3TiHUWRtn8+WdelcFk3gN7do6XBZEsVewTlekDD2253/LCpeSWVe+unn//xcjItbLaiQ1jldvWsQ8tG7yzeVUXFsYsl7+4/xf4nkewzph16NYJ+VbGzNHBWMDNyw9bC9kjdaHGTsploNL60JDPdLqrF7RH33UENuwC+dY5gIaRB8IUAUPQ9wjoIr/0H32lWMDNd8fL1f13AwRBBTrYp30g7jNyhnDPXac+ROo+6NOResum+rzI2XF8rwMYV+rHYU5xUa8KEz6A3UG6gAIV5FqqVfh0O3Hf5IAnmqB6zroD6tceE348ZaiUOsUa+TYs2jDoDXlf/nRZs6IQeCTXswY6Rfdg073YMkwKcadUXKQFfmZ/obTwDH0sbyqxwxc3ATc3GS5SuXcoOpW9QruQWBKLySThXsSvhBhX4s1Sun7tgovvoygHAcSv7Y6ukMkgbZxbAtRxYUOq/tA8LLB1hXIO+TGNLbYKM1fiFZTMNvae4G+y6JP1++TNo78lOhLOaygpIkpnmNE+bvlZXwkUyEJL6LsywaYBzDpNFw8mYSWQc1o+beTzWeazBPvL+xuBKS7nmxVXz7ER0g1PsGrlrjRmpa75JXnTkX+O07DAOTTRi4nDUN1ylGjc1duZaYrKVgqEdq+8UpBUvP6DFybuJmhK2b6Ea2Gm7px/XAe4RD2zGdIiPacnUmjFDduWBpmW5gWhDjDJ+4SXGLcR2M2K8gkMaGaD2ybc6prtu9J9yd8xyaoKRsIDqBufpDO74YHpGDY19ubGzOPNyHNiX2+AUe4ytO5cTuAcduyPwaxBmLuxGYaFDk8/E8EtGM1+4GH5UpMBf0L0/2f+v5UhdtXhGHPNob3zF3pBSSQJax+MeY0FmFEMhxbNtIznZc3gEHhtnPaBRIW8kauLMO7a07OMckI6Xkh5Yi/H27JydKCUduc5S3NZKTI+5wz5MA/tR8qqYIl5sELvTbSGlqA/ghH4NO+W1HHhC3ghd/6WLrnStnibpDK5ZIjU/eEd0loO1lSn4UxHa6SriXmk8HCas5R0whMNZVzkWwiWmOeZYkL53TX2QmKIf4yuznUrfGFPqRY7JFgTNxhXJ2+4vOSmF6w3f0JLyz4eH+++WjivIs8UKUqKAo1M1u5EOi3b2HHXEFg6L57saiDt60QG2D0BdSqGHZmWneemHDsyssX4jFSKYbOCV5SV8MaAE5g7/N8tXfQm4lg4QzkAYbaFexuGZRX8pmGuJ+diaCXzsgsh/WEQtGHf88ShMh6zWtIe7++8+vb2PHn8Pa9DTBnP0CSsun/y9YQ939+4n7W9achJu7rqwIWnTLY+0UYALd5q2H/HZuGOXbSa0zl5Ox4YzhIw74ntajAFNajp6UTKv6ALFM8tuMO3d6JmYf9W9//Bjhy59sxohi1f9yJlZFt2EnV8kNcrrLJIU5rxcv/YSSCjrvGmQlJnWPCuV/LJbEC61u7dHCH9J4PHtqr7JTDJWbO1VgAyoggl3c41LLe36XC4/IA/jLM5RK3G/Ttpw7bePiY76e7d6AhqnpccRNXL87eNliAQ8XUGOxOVtw4QoSxATQLwLB6WSA46VscmmOxfVhq1ktd4403MWazsB4NiAIOzE9u25uzm7j2vsX9LKtFEsr9Ik20+9QwRzUnHHbacaTxsQaWuNvyvBciLWCyxpsR4SjZs1Yzz58+EtHLW1dnNr43TSh9chOj7KrdrP+9sIMf9//o3kn28c+Fq4d1DwdCzaZ9laGnR793PLwEnheRV55Jh2nFErJYWxemUtijLHdxxm48uvDw+oAKwrZTkiFQJMNom1QTmYJwARCcSCnrM1dYjwNS+aSER3hjvbkkLnufp1L6U+fJ1hoX3dTGtVza7BtDTXVcY8J+PcBwGQUaz0B0ADI28aqxVjtlaMFK8BrlnXmTGfS3KekfaWYT5KYr2cYmGgrRou8zsfwT6/G9KFlGYD1JH9DROo0K8a8lM3/FI7RmiDyfbGe6uCicpAK5PleAcqZCEl1qFaWZttvx7SHZHujQv3NZ3DXY7u/Ywht/ge+0hOWeW6ynsNfl/lyyqfbS9GC1zqjTTOm3O5HrXn6fyduzyjPo2iNV67PWbXG039Ue560h6AfIdJlu8yH21eFeDBWRAvk4DkFHoixYqts6qkeJIGGRLv3/MDh5P5iGywWIO+8RL3aym5FGBXgv/sCuiKn2a3qIoscmN0Fj5G7imQROwzQxouackpaBMhZ3g97I7D2zU4yxi2c19FDfXDR/hDGHoIMGHr1aFezugqr+deYLKNhEy4pmrdwGQr5BMHuvZL6bb5ud62a601Cpy5PXw79Vn0s9jYSrRQ17R8gxfbBV6gMGn9h1dBHimws8B3BjIiB+/Qtpjuz7I1N30ksV7TJuXaL1joLDEyPoJ+r6TBKO0TOYf9upa4SbJTECPsc0oKBUwzDsaAmnMVlFXOmd54xts5kZ8TGVky0qKlJ/BC0swuXTsYZwLmRP+0kRpQnMklXF72DvBHSdlqd0u2b+MDE6zrY+Rl6QVDUxF6SEE0uOmSmkBG43oe+kHvZH5PwDZQSGroM0QvQQviRYPp7C/tMtEbBIKWkgnr1irjurR2YFp+pBcdlajnmo6Oa3iGEFxEX281qCFhrmCoi6z9cGNSIkbHodMSNC6GCtQ67ZhfVt1KeEwHR6hgBz1zi+0IbRPLq6w4z5LAdxavkhDS05+4DhyMKKyYYLHY0/WqhqSW8V1TzGhR+d15Hxq+NdKbXdP7L3wuHejpuRzEOaVpJ/griHF6ETrOjJOd3syCS28QNgaKshsX+iQ424IjQN/4gql9xzWRKsSKkkMBwvjMgkrwPeM5NmTjr42s4wq0lD5Fqa8WEXzX3HInBbReWPj7jZPJlBW87y9y3/G12iHrKxrTd939GrgsAStUVNywkvubHjvL1i1Gt73xtDlzd1w0IoLYCx1mrVleCn6c99EAc5hSO+xJfT+PS5g508i4W9w2eUZGFDbGDm3QIzJH35Jns5erePllnO7emsG/js/vYOMMKhsnaTsR98lmJKT41uryrsVVRofpdpucOTVij6hrqcH/EEnh74OU4VLmXbM2015fh+WOcRWac5Q9m8ofEuo3HCcgsm6dmZGocORlCNqYdM7q8I/VccZ7elfQHV/dH1fST4r5rnruSvzdbQ0O7teyterRRnNjBXq88XqCzou3e30XoYpf22UHwFUzHIhXRxF/DdWwsBMxSN3/nPWiMRT92bKPMbRoEHTm2kh7XXw1EW/CnznqD54rF5ZGAqY/u8/2MJ/fWQ9bGAcdEFfpWdz3J5hsUT2xJadgnLP64oIHbxvqTlimmw+SxCZz1w3bXiTanSLH9ZJKN9qaFNt52o63QyetRlh09WqENlFc96RcVN17Hja7OlvE3/DbNU0yomR0LSc1v6bN3wD34sjcrUbDZbgtbq5OQ1yykcfo3wI3uLEJ7uMa2FVDE1ZYpbN/KcBsJHVzxzjNN0XbzLNLD3BlNn8sMOFZjjXQLAgSuy+JTIO4zp+cTnqhOcMggtqEG0nWCgsDFPm5kZYc+A7Ryi2P8OTt3YfOGLkho/FOA9F/0uFrdrd3H9Iv83Tcj30cSWCjLoGwFSOZRVZUZoxX3+Nq7LwpME0ZFGc8yqmJblbaQ7N3p9L+jUTT6OoElyt1wu78qPM0kOP9ll46Q88T712t5AfTqASF8opswbTA1t+u5Vjr1pU+7tvosfteEPf5RkTx7sYR4y4Nis8pKP3hUWxCY1e4nMd35z9iHk9sysrfBE1x5/Gp1l2kLgjJovf8ejRvugtVY9CEOa/lGK6r+5OJ8v8CAAD//1na61c=" + return "eJzsXduP27aaf89fQZyXNIupu00X+xAsDjCdnPQETXoG9aTAPulQ5GebMUWqJDUT969f8CZRtmzLuniaYpOnsSXy91343fiR/hZtYfcGrUn5AiHDDIc36OVPUq45oDsuK4ruOTYrqYqXLxBSwAFreIPW+AVCFDRRrDRMijfo7y8QQuinu3tUSFpxeIHQigGn+o374lskcAFxIvvP7Er7t5JV/CR9Pn2H4xy4rj+Or8r8MxCTfNyBJ/7zuAQzUjGxRgUYxYg+HHkfQgqj0qAW/9H66igU+89/mPkntrB7kop2DlyAwRQbPNfgltRZxtY7baCYZWgFWlaKwGSDx4H/VjPE///beb1qjUtllTvt7vg2K3BZMrEOj/6tNfgJ7fwY1NFssEEKTKUEULRSskCtpXh7/x79XoHaLQ7IyhnnTKyPzdca5kf/bFSN5J32+m6zJV2pqHOpRCxEas+PF4eC65J5C+md1MY9qxEThFcUkIJ1xbG6QQZ/uUGYfq60KUCYG4QFRUpWglqmg1JSLTrwMPEoGYGskMJshmCKDFNQSmWQG6drolJJpwuMDpnl3r+N3r9FcoXMBqJQ47w5cCnWGhnZNbmRBvOOeVdcYnN81gf7Wj0TLmQlTNfwelsNpOthAwlNcWFbc0pRvnMfalCPjMCxeZPhhgC4Tf88wGEH6QSD3kmF4AsuSg43CO+9sZIqLKelkQqvATGNlgYLilXz2adlJ01+hkn4GcbytsN+UOkABmstCcMGKHpi3QobgYxksEXUsjAtUF57gTp0bQvkvF6XLuO17gAiQBs4KWgiOQcS5byF3bePmFeASsxUsK+lko+MAsKUMvsg5o0Dbg3dFQs0ELew2/vmFLua9xye3m/Gt2C1smQ9QlYqRroM67llvgFENlitgSI3hFNgrytBk1oSXP78STvzuvz5EzIMlF6gX2FlmasRkcIoTIwbycqRrRAuS84Izu1akWYD6olpuEHMvNRudM60fx4OnRfBKm9p3nHfdeceRe+kNKViwpxyYjmYMW5sMmtuyfckolWNu5dNjxDsX2NA2PdHwBhnr5bBGLx/63SuE4ZTxBktVMSQOoLLwShYD5z/V/dmx5RnZqxBLTSRJXx/+cJf2vfQ9wdU9pjt9YJLgo+wvNe0r7uYaw1GHPjbHFvPW4DZSCq5XO964Sqw2oKZHJUfdhCmH4aC+eFCycjVSoPp8o69Ar0wWRilwxTLoqwM9LPF/tnZ8ogVU/CEOV9QJcsS6CLfGeii3Nqu44S/F0QWVrzudRQGi9FenKTH/FmJyRaMzogLkg89+UVowmAX4GFCGywILEhZLRRY0wg0I1KBPgrmIF3dg/NLVeSgbAjgxkFxWCSFg7Ox6VgIEOL856A5a5YZVsBCAxkA6pMLS6yxxJwHYEwgDUQKqntNvyhJl304N7MNk1Y2tEkyBcydtQKK7u4/+RiSaUQqpUAYvrPIKg2RYX2YRJneLhTgoRp9Z/XPwvMabUfy1QI7cK+JM1mOU+Magh3SI3j/LyRLUM6unxSSQ/GkmIFp6LdDGRDIyH4McFNPzAE3Zn8WFFBItVvkVrekWChcZJr9AQOhWK116X/I3y0qP4NVTquVv31coIcN08FWWwWWgu8QfsSM26DdrbbfPoYcyYeDlqH2ZXiNVrhgvNMPniLJZtUDSfro4TerzGXoz0aNfsJlxsRAfbXyOZCMWzNMBFTrCrTxi5gZjeSTQHZOpEtM4FmolZWZkty4SB2JDcVGPge9AsyTVNsFE2sFWk9lhgmwx1i4tWDCNJcgCVHBwlmm4YhicDEKE0zIHA32r0dQA0FMzJeL4VTliXjm9ORNiBUCmGDD0AZrlAMIpCohmFifVFkPIHNmfhCMf3BcWhtqh0GaCQIRxxPWSBusDNCbJM5aoKWrfVIEj6B26L//s/nmdmVAIW2/Z2J94wp5dqEKadAj0yyu0qq0C/P7182rBxmHfbVUkvRKOd6GhyfPOZIMiFfagFps6EovLDghKXQp3lG2H0T8lNn40ctd1Nrwz7fvlo6gX+wE3phhBVEbrOhdnbJGdB6u9mXnjOASE2Z2Hc73RCR8FHccrkbtN/5qpFJEGFZ9fvrxAqSVYZz94QKmUWCtzylBERAmlDQ90FCHb8cRPfBVYgOYm80uy7kk2znkX0+B/BRR4K5A1Efkn2W+WGHGgc6A7rPMg05u8CMgP4+VdE99tOCiXZsTXbpiLkOnq7ywMckVuOesfD2fNYk9cdbZxHyCbhKWMeJugM4n9H2kw0TfIJ1TATq5OlgNdliJRV2HyHxInTX2bmLT+b+3v/5SJ5C6KYD0AVmWc9hK7Da/PLCw0+SY2wMRkcJgJkDNgcsBSmY4DyfI7ki2P8Q7W0SpxOzQffywg2ODmwILvJ6RPza++Rjm2I9u+qt/Cb7NYyoGuh1RHxY0KfnvNim98dy7qWt9YWq7YHNAq4qvGOdJwwDZAK14LyoemTIV5qFqOz3Dw/hN0dRK4LwvJNJG+nZoX7c9ztjznVcbpo1cK1yc477LRLynlHJruRtgQJOuumd8W4NNOJzt1Ajb12LK7x6pEwO/oakR81XaMOAZ02VZQCt1LgydhXbnGXRp1ZBZJ7Zmj9ZLGGygryN7Pumlbu5iGdYvTyjJhinPIM+EG72kmm402Uyl37bbu/j0jEkwlaQqQJgFBcv0EYbqoWWedEUIaL2qeD0F8lMc8Z01ELddMicMO4H2Svt7BYqBRlIhLuW2Ks+B85sZc6JzM3RUT9bbo0rz8t/rLfw7RifeURRpp+mTdV0Kl3Wv6d09WhpMtlSxR1Cu3TS87XbHD/unV1K5t376+R8vJ9PCZis6hFVuV886tGz0zuJdVVQcu1jy7v5T7H8SyT5j2glYI+hXFTtLA2cFMyM3bC1sj9SNFjcpm4lG40tLMtPtolrcHnHfHdSwC+Ab+AgWQhoEXwgARd8jrIPw2l/4TrOCmemKl6//6wIOhghysk35Rtph5A7lnLlOe47UedSlIfeSTfd9lbHh+loBNq7Qj8We4qRaEyZ8Br2BcgMFKMyzUK3063DgvssHSTBH9Zh1BdSvPSb8fsxQK3GINfJtWrRh1BnwuvrvtGBDx/VIqGEPdozsw6Z5t2OYFOBMq75ICfjK/ERv4xn4WNpQZoUrbgZuajZeonTtUnYofYNyJbcgEJVPwrmKXQk3qMCfpXL93AUT3adkDgCOW9kfWyWVQdo4swCu5cCCUv+lfVhg6QjjGvRlGltqE2Ssxi8sm2noPcXdYNcl6ffLn0F7T3YinNVUVkCSzDSnhtr0tboSLpKBkNR3YYZNA5xzmCwaTsZMIuOwftzM47HOYw32kfc3BldayjUvrppnJ6IbnGLXyF1rzEhd803yoiP/Gqdlh3FoohETh6O+4SrVuKmxM9cSk60UDO1YfacgrXj5AS1O3k3UlLB9C9XATts9/bgOcI94YDOmQ3xMS6bWjBmyKw80LcsNRBtilPELLzFuIbabEeMVHNLIALVPvtUx3XWj/5S7Y5ZTE4yEBVQ3OE9ncMcH0zNqaOzLjZ3FmZfjwL7cBqfYY2zduZzAPejYHYFfgzBzYTcKCx2afCaGXzKa+cLF8KMiBf6C7v0NAv9ajtRVi2fEMY/2xlfsDSmVJKB1PO4xFmRGMRRSPNs2kpM9h0fgsXHWAxoV8kaiJs68Y0vLPs4B6Xgp6YG1GG/PztmJUtKR6yzFba3E9Jg77MM0sB8lr4op4sUGsTvdFlKK+gBO6NewU17LgSfkjdD1X7roStfqaZLO4JolUvODd0RnOVhbmYI/FaGdriLulcbDYcJa3gFDOJx1lWMhXGKaY44F6Xut1QeJKfoxvjLbqfSNMaVe5JhsQdBsXJG87f6Sk1K43vANLSn/fHi4/27puII8W6wgJQo4OlWzG+mwaGfPUUds4bB4vquBuKMXHWD7ANSlFHpoVnaal37owMwa6zdSIYLJBl5ZXsIXA0pg7vB/s3zVl4Br6QDhDITRFuplHJ5Z9JeCuZaYj62ZwMcuiPyHRdSCcccfj8J0yGpNe7i7/+7T2/vo8fewBj1tMEefsOLyyd9P9nB37/7S/qYlJ+Hmrgsbkjbd8kgbBbhwp2n7EZ+NO3bZZkLr7OV0bDhDyLgjvqfFGNCkpqMXJfOKLlA8s+wG096Nnon5V937Dz926NI3qxGyeNWPnJll0U3Y+UVSo7zOIklhzsv1ay+BhLLO6wxJmWnNs1LJL7sF4VK7e3uE8FcVHt+u6pvMJGPF1l4FyIAqmHA317jU0q7P5fID8jDO4hy1EvfrpA3XfvuY6Ki/d6snoHFaehxRI8ffPl6GSMDTFeRIXN42TIiyBDEBxLtwUCo54FgZm2y6c1Ft2EpW640zPWexthMAjg0Iwk5s33Yc2OhxXGP/MlimjWJ5lSbZfuodIpiTijtuO9V42oBIW2v8XQmWE7FeYEmL9ZBo3KwZ48nXh7dw1Nbaza2N00kfXofo+Ci3aj/vbyPE/P/5N5J/vnHga+HeQcHTsWifZWtp0O3dzy0DJ4XnVeSRY9pxRq2UFMbqlbUoyhzfcZiNL78+PKACsK6U5YhUCDDZJNYG5WCeAEQkEAt6ztbUIcLXvGgiEd0Z7mxLCp3n6te9lPrwdYaF9nUzrVU1uwbT0lxXGfOcjHO/PYCMYqU/ABoYedNYrRiztWKkeA1wzbrOjPlckvOMtLcM81ES6+UUCwNt1XCZ3/kI9vndkC6kNBugjuxvmECFftWQn7rhl9oxQhtMtjfeWxVMVAZamSzHO1AhCymxDtXK2mz79ZDuiHRvXLgf7jnc5ejezxhyi++x3+Mpq1xXea/B76t8WeWz7cVogUu9kcZ5cy7Xo/Y8nb9zl2fUp1G0xmu3x+x6o6k/yl1P2gOQ7zDJ8l3mo82rAjw4C+JlEpCcQk+kWLF1VpUUT9IgQ+L9e37gcDIfkQ0Wa9A3XuJ+LSWXAuxK8L/wArrip9ktqiKL3BidhY+RewokEfvMkIZLWnIK2kTIGV4Pu+Pwdg3OMobt3FdRQ/3wEf4Qhh4CTNh6daiXM7rK67kXmGwjIROuqVo3MNkK+cSBrv1Sum3+rrftWmuNAmduD99OfRb9LDa2Ei3UNS3f4MV2gRcoTFp/8SrIIwV2FvjOQEbk4B3aFtP9Wbbmpo8k1mvapFz7BQudJUbGR9DvlTQYpX0i57Bf1xI3SXYKYoR9TkmhgGnGwRhQc66Csso50xvPeDsn8nMiI0tGWrT0BF5ImtmlawfjTMCc6J82UgOKM7mEy8veAf4oKVvtbsn2bXxggnV9jLwsvWBoKkIPKYgGN11SE8hoXM9DP+idzO8J2AYKSQ19huglaEG8aDCd/aVdJnqDQNBSMmHdWmVcl9YOTMuP9KKjEvVc09FxDc8Qgovo660GNSTMFQx1kbUfbkxKxOg4dFqCxsVQgVqnHfPLqlsJj+ngCBXsoGdusR2hbWJ5lRXnWRL4zuJVEkJ6+hPXgYMRhRUTLBZ7ul7VkNQyvmuKGS0qvzvvQ8NvjfRm1/T+C59LB3p6LgdxTmnaCf4KYpxehI4z42SnN7Pg0huEjYGi7MaFPgnOtuAI0De+YGrfcU2kCrGi5FCAMD6zoBJ8z3iODdn4ayPruAItpU9R6qtFBN81t9xJAa0XFv5+42QyZQXv+4vcTwZb7ZD1FY3pu+5+DVyWgBUqKm5Yyf1Nj51l6xaj29542py5Oy4aEUHshQ6z1iwvBT/O+2iAOUypHfakvp/HJcycaWTcLW6bPCMjChtjhzboEZmjb8mz2ctVvPwyTndvzeBfx+d3sHEGlY2TtJ2I+2loJKT41uryrsVVRofpdpucOTVij6hrqcH/EEnh74OU4VLmXbM2015fh+WOcRWac5Q9m8ofEuo3HCcgsm6dmZGocORlCNqYdM7q8I/VccZ7elfQHV/dH1fST4r5rnruSvzdbQ0O7teyterRRnNjBXq88XqCzou3e30XoYpf22UHwFUzHIhXRxF/DdWwsBMxSN3/nPWiMRT92bKPMbRoEHTm2kh7XXw1EW/CnznqD54rF5ZGAqY/u8/2MJ/fWQ9bGAcdEFfpWdz3J5hsUT2xJadgnLP64oIHbxvqTlimmx8kiU3mrhu2vUi0O0WO6yWVbrQ1KbbztB1vh05ajbDo6tUIbaK47km5qLr3PGx2dbaIv+G3a5pkRMnoWk5qfk2bvwHuxZG5W42Gy3Bb3FydhrhkI4/RvwVucGMT3I9rYFcNTVhhlc5+U4DZSOrmjnGab4q2mWeXHuDKbP5YYMKzHGugWRAkdr8kMg3iOn9yOumF5gyDCGoTbiRZKywMUOTnRlpy4DtEK7c8wpO3dx86Y+SGjMY7DUT/SYdfs7u9+5D+Mk/H/djHkQQ26hIIWzGSWWRFZcZ49T2uxs6bAtOUQXHGo5ya6GalPTR7dyrt30g0ja5OcLlSJ+zOH3WeBnK839JLZ+h54r2rlfxgGpWgUF6RLZgW2Pq3aznWunWlj/tt9Nh9L4j7+UZE8e7GEeMuDYrPKSj94VFsQmNXuJzHd+c/Yh5PbMrK3wRNcefxqdZdpC4IyaL3/Ho0b7oLVWPQhDmv5Riuq/uTifL/AgAA//+rygKM" } From 07f18be88671df087aa8d5c3bdd17e203352660d Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 10:20:28 +0200 Subject: [PATCH 08/11] use array instead of map for tags include tags in the event signature --- .../metricbeat/module/gcp/billing/billing.go | 22 +++++++----- .../module/gcp/billing/billing_test.go | 34 +++++++++++-------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 457d50988ee8..157651b32d32 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -307,29 +307,35 @@ func (m *MetricSet) queryBigQuery(ctx context.Context, client *bigquery.Client, return events, nil } -// createTagsMap converts a comma-separated string of key-value pairs into a flattened map. -// It is used to convert the tags string generated by the SQL query to a map. +// tag represents a BigQuery billing tag. +type tag struct { + Key string `json:"key"` + Value string `json:"value"` +} + +// createTagsMap converts a comma-separated string of key-value pairs into an array of tags. +// It is used to convert the tags string generated by the SQL query. // The SQL query generates tags string by concatenating key-value pairs from an array of structs (tags). -func createTagsMap(tagsItem bigquery.Value) mapstr.M { - tagsMap := mapstr.M{} +func createTags(tagsItem bigquery.Value) []tag { + var tagsArray []tag if tags, ok := tagsItem.(string); ok { pairs := strings.Split(tags, ",") for _, pair := range pairs { kv := strings.Split(pair, ":") if len(kv) == 2 { - _, _ = tagsMap.Put(kv[0], kv[1]) + tagsArray = append(tagsArray, tag{Key: kv[0], Value: kv[1]}) } } } - return tagsMap.Flatten() + return tagsArray } func createEvents(row row, tableName, projectID string) mb.Event { event := mb.Event{} - tagsMap := createTagsMap(row.Tags) + tagsMap := createTags(row.Tags) event.MetricSetFields = mapstr.M{ "invoice_month": row.InvoiceMonth, @@ -370,7 +376,7 @@ func getCurrentDate() string { func generateEventID(currentDate string, row row) string { // create eventID using hash of current_date + invoice.month + project.id + project.name // This will prevent more than one billing metric getting collected in the same day. - eventID := currentDate + row.InvoiceMonth + row.ProjectId + row.ProjectName + row.SkuId + row.ServiceId + eventID := currentDate + row.InvoiceMonth + row.ProjectId + row.ProjectName + row.SkuId + row.ServiceId + row.Tags h := sha256.New() h.Write([]byte(eventID)) prefix := hex.EncodeToString(h.Sum(nil)) diff --git a/x-pack/metricbeat/module/gcp/billing/billing_test.go b/x-pack/metricbeat/module/gcp/billing/billing_test.go index f44fbae2d76e..f95009dc5dba 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing_test.go +++ b/x-pack/metricbeat/module/gcp/billing/billing_test.go @@ -44,33 +44,39 @@ func TestCreateTagsMap(t *testing.T) { testCases := []struct { name string tagsItem bigquery.Value - want mapstr.M + want []tag }{ { name: "valid tags", tagsItem: "tag1.a:value1,tag2.b:value2", - want: mapstr.M{"tag1.a": "value1", "tag2.b": "value2"}, + want: []tag{ + {Key: "tag1.a", Value: "value1"}, + {Key: "tag2.b", Value: "value2"}, + }, }, { name: "valid tags no values", tagsItem: "tag1:,tag2:", - want: mapstr.M{"tag1": "", "tag2": ""}, + want: []tag{ + {Key: "tag1", Value: ""}, + {Key: "tag2", Value: ""}, + }, }, { name: "no tags", tagsItem: "", - want: mapstr.M{}, + want: nil, }, { name: "invalid format", tagsItem: "tag1 value1,tag2 value2", - want: mapstr.M{}, + want: nil, }, } for _, tt := range testCases { t.Run(tt.name, func(t *testing.T) { - tags := createTagsMap(tt.tagsItem) + tags := createTags(tt.tagsItem) assert.Equal(tt.want, tags) }) } @@ -141,10 +147,10 @@ func TestCreateEvents(t *testing.T) { "sku_description": "E2 Instance Ram running in Americas", "service_id": "6F81-5844-456A", "service_description": "Compute Engine", - "tags": mapstr.M{ - "tag1": "value1", - "tag2.a.b/c": "value2", - "tag3": "", + "tags": []tag{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2.a.b/c", Value: "value2"}, + {Key: "tag3", Value: ""}, }, }, } @@ -191,10 +197,10 @@ func TestCreateEvents(t *testing.T) { "service_id": "6F81-5844-456A", "service_description": "Compute Engine", "effective_price": 123.45, - "tags": mapstr.M{ - "tag1": "value1", - "tag2.a.b/c": "value2", - "tag3": "", + "tags": []tag{ + {Key: "tag1", Value: "value1"}, + {Key: "tag2.a.b/c", Value: "value2"}, + {Key: "tag3", Value: ""}, }, }, } From f0cdbe84e2cbc7249a52f1347c6a03a60f5543f0 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 12:41:10 +0200 Subject: [PATCH 09/11] update sample doc --- x-pack/metricbeat/module/gcp/billing/_meta/data.json | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/billing/_meta/data.json b/x-pack/metricbeat/module/gcp/billing/_meta/data.json index 67c624ef2ad0..5a2cf619d82f 100644 --- a/x-pack/metricbeat/module/gcp/billing/_meta/data.json +++ b/x-pack/metricbeat/module/gcp/billing/_meta/data.json @@ -22,9 +22,12 @@ "sku_description": "Network Inter Region Ingress from Jakarta to Americas", "service_description": "Compute Engine", "effective_price": 0.00292353, - "tags": { - "size": "medium" - } + "tags": [ + { + "key": "size", + "value": "standard" + } + ] } }, "metricset": { From 0db2b746e5dd049b5153824f8dc600175bad6f8d Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Tue, 31 Oct 2023 12:41:31 +0200 Subject: [PATCH 10/11] use sum instead of max for dashboards --- .../kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json | 2 +- .../kibana/7/lens/73346db0-e88d-11ea-bf8c-d13ebf358a78.json | 2 +- .../kibana/7/lens/e6933020-e88d-11ea-bf8c-d13ebf358a78.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json index 709aae25d964..bb8197009645 100644 --- a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json +++ b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json @@ -14,7 +14,7 @@ "dataType": "number", "isBucketed": false, "label": "Maximum of gcp.billing.total", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, diff --git a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/73346db0-e88d-11ea-bf8c-d13ebf358a78.json b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/73346db0-e88d-11ea-bf8c-d13ebf358a78.json index 032e87dd1c0b..1925cf6e8e49 100644 --- a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/73346db0-e88d-11ea-bf8c-d13ebf358a78.json +++ b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/73346db0-e88d-11ea-bf8c-d13ebf358a78.json @@ -16,7 +16,7 @@ "dataType": "number", "isBucketed": false, "label": "Total Billing", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, diff --git a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/e6933020-e88d-11ea-bf8c-d13ebf358a78.json b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/e6933020-e88d-11ea-bf8c-d13ebf358a78.json index 0a8b8543e0ad..657fc29ffa49 100644 --- a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/e6933020-e88d-11ea-bf8c-d13ebf358a78.json +++ b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/e6933020-e88d-11ea-bf8c-d13ebf358a78.json @@ -16,7 +16,7 @@ "dataType": "number", "isBucketed": false, "label": "Total Billing Cost", - "operationType": "max", + "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" }, From 56a122476ca2301fedcd8c5b141ba3f2c0142d94 Mon Sep 17 00:00:00 2001 From: Gabriel Pop Date: Fri, 3 Nov 2023 19:39:27 +0200 Subject: [PATCH 11/11] fix label minor changes --- .../kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json | 2 +- x-pack/metricbeat/module/gcp/billing/billing.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json index bb8197009645..60ad59b5ca5f 100644 --- a/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json +++ b/x-pack/metricbeat/module/gcp/_meta/kibana/7/lens/057de170-e88d-11ea-bf8c-d13ebf358a78.json @@ -13,7 +13,7 @@ "10b91492-efef-490d-bc7a-c2074b2eae84": { "dataType": "number", "isBucketed": false, - "label": "Maximum of gcp.billing.total", + "label": "Sum of gcp.billing.total", "operationType": "sum", "scale": "ratio", "sourceField": "gcp.billing.total" diff --git a/x-pack/metricbeat/module/gcp/billing/billing.go b/x-pack/metricbeat/module/gcp/billing/billing.go index 157651b32d32..c211a9d11eaf 100644 --- a/x-pack/metricbeat/module/gcp/billing/billing.go +++ b/x-pack/metricbeat/module/gcp/billing/billing.go @@ -313,7 +313,7 @@ type tag struct { Value string `json:"value"` } -// createTagsMap converts a comma-separated string of key-value pairs into an array of tags. +// createTags converts a comma-separated string of key-value pairs into an array of tags. // It is used to convert the tags string generated by the SQL query. // The SQL query generates tags string by concatenating key-value pairs from an array of structs (tags). func createTags(tagsItem bigquery.Value) []tag { @@ -335,7 +335,7 @@ func createTags(tagsItem bigquery.Value) []tag { func createEvents(row row, tableName, projectID string) mb.Event { event := mb.Event{} - tagsMap := createTags(row.Tags) + tags := createTags(row.Tags) event.MetricSetFields = mapstr.M{ "invoice_month": row.InvoiceMonth, @@ -348,7 +348,7 @@ func createEvents(row row, tableName, projectID string) mb.Event { "sku_description": row.SkuDescription, "service_id": row.ServiceId, "service_description": row.ServiceDescription, - "tags": tagsMap, + "tags": tags, } if isDetailedTable(tableName) {