From 134db01037d228c37a56e8c12a75a3c65aa0d15b Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Mon, 15 Apr 2024 12:57:13 +0100 Subject: [PATCH 01/17] Add attribute_name mappings skeleton --- pkg/translator/azure/attribute_name.go | 21 +++++++++++++++++++ .../azure/attribute_name_mappings.yaml | 3 +++ 2 files changed, 24 insertions(+) create mode 100644 pkg/translator/azure/attribute_name.go create mode 100644 pkg/translator/azure/attribute_name_mappings.yaml diff --git a/pkg/translator/azure/attribute_name.go b/pkg/translator/azure/attribute_name.go new file mode 100644 index 000000000000..fcff23f5734a --- /dev/null +++ b/pkg/translator/azure/attribute_name.go @@ -0,0 +1,21 @@ +package azure + +import ( + _ "embed" + "encoding/json" +) + +//go:embed attribute_name_mappings/resource_logs.json +var resource_logs_json []byte +var resource_logs_map = unmarshal(resource_logs_json) + +var mappings = map[string]map[string]string{ + "common": resource_logs_map, +} + +func unmarshal(data []byte) map[string]string { + var m map[string]string + if err := json.Unmarshal(data, &m); err != nil { + } + return m +} diff --git a/pkg/translator/azure/attribute_name_mappings.yaml b/pkg/translator/azure/attribute_name_mappings.yaml new file mode 100644 index 000000000000..6b5110a25b76 --- /dev/null +++ b/pkg/translator/azure/attribute_name_mappings.yaml @@ -0,0 +1,3 @@ +"logs": + "resourceId": "az.resource_id" + "tenantId": "az.tenant_id" From 047800659b4537c340cf4c813071f35829ea3e15 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Wed, 17 Apr 2024 17:39:54 +0200 Subject: [PATCH 02/17] Add property name mappings for Azure Front Door resource logs via Event Hubs --- pkg/translator/azure/attribute_name.go | 21 -------- .../azure/attribute_name_mappings.yaml | 3 -- pkg/translator/azure/property_names.go | 51 +++++++++++++++++++ pkg/translator/azure/resourcelogs_to_logs.go | 20 +++++++- 4 files changed, 70 insertions(+), 25 deletions(-) delete mode 100644 pkg/translator/azure/attribute_name.go delete mode 100644 pkg/translator/azure/attribute_name_mappings.yaml create mode 100644 pkg/translator/azure/property_names.go diff --git a/pkg/translator/azure/attribute_name.go b/pkg/translator/azure/attribute_name.go deleted file mode 100644 index fcff23f5734a..000000000000 --- a/pkg/translator/azure/attribute_name.go +++ /dev/null @@ -1,21 +0,0 @@ -package azure - -import ( - _ "embed" - "encoding/json" -) - -//go:embed attribute_name_mappings/resource_logs.json -var resource_logs_json []byte -var resource_logs_map = unmarshal(resource_logs_json) - -var mappings = map[string]map[string]string{ - "common": resource_logs_map, -} - -func unmarshal(data []byte) map[string]string { - var m map[string]string - if err := json.Unmarshal(data, &m); err != nil { - } - return m -} diff --git a/pkg/translator/azure/attribute_name_mappings.yaml b/pkg/translator/azure/attribute_name_mappings.yaml deleted file mode 100644 index 6b5110a25b76..000000000000 --- a/pkg/translator/azure/attribute_name_mappings.yaml +++ /dev/null @@ -1,3 +0,0 @@ -"logs": - "resourceId": "az.resource_id" - "tenantId": "az.tenant_id" diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go new file mode 100644 index 000000000000..93c05f308cc6 --- /dev/null +++ b/pkg/translator/azure/property_names.go @@ -0,0 +1,51 @@ +package azure + +var mappings = map[string]map[string]string{ + "common": {}, + "FrontDoorAccessLog": { + "trackingReference": "", + "httpMethod": "http.request.method", + "httpVersion": "http.request.version", + "requestUri": "url.full", + "hostName": "server.address", + "requestBytes": "http.request.body.size", + "responseBytes": "http.response.body.size", + "userAgent": "user_agent.original", + "clientIp": "client.address", + "clientPort": "client.port", + "socketIp": "socket.address", + "timeTaken": "http.server.request.duration", + "requestProtocol": "", + "securityProtocol": "", + "securityCipher": "", + "securityCurves": "", + "endpoint": "", + "httpStatusCode": "http.response.status_code", + "pop": "", + "cacheStatus": "", + "matchedRulesSetName": "", + "routeName": "http.route", + "referrer": "http.request.header.referer", + "timeToFirstByte": "", + "errorInfo": "", + "originURL": "", + "originIP": "", + "originName": "", + "result": "", + "sni": "", + }, +} + +func ResourceLogKeyToSemConvKey(azName string, category string) (string, bool) { + if mapping := mappings[category]; mapping != nil { + if mapped := mapping[azName]; mapped != "" { + return mapped, true + } + } + + if name := mappings["common"][azName]; name != "" { + return name, true + } + + return "", false +} diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index b2c96d28f568..088043c91129 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -192,9 +192,11 @@ func extractRawAttributes(log azureLogRecord) map[string]any { } attrs[azureOperationName] = log.OperationName setIf(attrs, azureOperationVersion, log.OperationVersion) + if log.Properties != nil { - attrs[azureProperties] = *log.Properties + copyProperties(log.Category, log.Properties, attrs) } + setIf(attrs, azureResultDescription, log.ResultDescription) setIf(attrs, azureResultSignature, log.ResultSignature) setIf(attrs, azureResultType, log.ResultType) @@ -207,6 +209,22 @@ func extractRawAttributes(log azureLogRecord) map[string]any { return attrs } +func copyProperties(category string, properties *any, attrs map[string]any) { + pmap := (*properties).(map[string]any) + var attrsProps map[string]any = nil + for k, v := range pmap { + if otelKey, found := ResourceLogKeyToSemConvKey(k, category); found { + attrs[otelKey] = v + } else { + if attrsProps == nil { + attrsProps = map[string]any{} + attrs["azure.properties"] = attrsProps + } + attrsProps[k] = v + } + } +} + func setIf(attrs map[string]any, key string, value *string) { if value != nil && *value != "" { attrs[key] = *value From 298619a4590db78c3cdbee63b938163940e01714 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Thu, 18 Apr 2024 18:01:52 +0200 Subject: [PATCH 03/17] Add FrontDoorHealthProbeLog and FrontdoorWebApplicationFirewallLog --- pkg/translator/azure/property_names.go | 33 ++++++++++++++++++-- pkg/translator/azure/resourcelogs_to_logs.go | 11 +++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 93c05f308cc6..4d41a05c7dad 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -34,16 +34,43 @@ var mappings = map[string]map[string]string{ "result": "", "sni": "", }, + "FrontDoorHealthProbeLog": { + "healthProbeId": "", + "POP": "", + "httpVerb": "http.request.method", + "result": "OriginError", + "httpStatusCode": "http.response.status_code", + "probeURL": "url.full", + "originName": "server.address", + "originIP": "", + "totalLatencyMilliseconds": "", + "connectionLatencyMilliseconds": "", + "DNSLatencyMicroseconds": "", + }, + "FrontdoorWebApplicationFirewallLog": { + "clientIP": "client.address", + "clientPort": "client.port", + "socketIP": "socket.address", + "requestUri": "url.full", + "ruleName": "", + "policy": "", + "action": "", + "host": "server.address", + "trackingReference": "", + "policyMode": "", + }, } -func ResourceLogKeyToSemConvKey(azName string, category string) (string, bool) { - if mapping := mappings[category]; mapping != nil { +func resourceLogKeyToSemConvKey(azName string, category string) (string, bool) { + mapping, ok := mappings[category] + if ok { if mapped := mapping[azName]; mapped != "" { return mapped, true } } - if name := mappings["common"][azName]; name != "" { + mapping = mappings["common"] + if name := mapping[azName]; name != "" { return name, true } diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index 088043c91129..1b38a43f20d9 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -211,18 +211,17 @@ func extractRawAttributes(log azureLogRecord) map[string]any { func copyProperties(category string, properties *any, attrs map[string]any) { pmap := (*properties).(map[string]any) - var attrsProps map[string]any = nil + attrsProps := map[string]any{} for k, v := range pmap { - if otelKey, found := ResourceLogKeyToSemConvKey(k, category); found { + if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { attrs[otelKey] = v } else { - if attrsProps == nil { - attrsProps = map[string]any{} - attrs["azure.properties"] = attrsProps - } attrsProps[k] = v } } + if len(attrsProps) > 0 { + attrs["azure.properties"] = attrsProps + } } func setIf(attrs map[string]any, key string, value *string) { From 610340b627c4eb5c5469b4da8728f618b036fd1d Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Tue, 23 Apr 2024 13:29:33 +0100 Subject: [PATCH 04/17] Add complex conversion handling --- pkg/translator/azure/complex_conversions.go | 75 +++++++++++++++++++ .../azure/complex_conversions_test.go | 42 +++++++++++ pkg/translator/azure/go.mod | 2 +- pkg/translator/azure/go.sum | 2 + pkg/translator/azure/property_names.go | 35 ++++++++- pkg/translator/azure/resourcelogs_to_logs.go | 8 +- 6 files changed, 159 insertions(+), 5 deletions(-) create mode 100644 pkg/translator/azure/complex_conversions.go create mode 100644 pkg/translator/azure/complex_conversions_test.go diff --git a/pkg/translator/azure/complex_conversions.go b/pkg/translator/azure/complex_conversions.go new file mode 100644 index 000000000000..4e2a2d8920b5 --- /dev/null +++ b/pkg/translator/azure/complex_conversions.go @@ -0,0 +1,75 @@ +package azure + +import ( + "fmt" + "strconv" + "strings" +) + +type ComplexConversion func(string, any, map[string]any) bool + +var conversions = map[string]ComplexConversion{ + "AppServiceHTTPLogs:Protocol": appServiceHTTPLogsProtocol, + "FrontDoorHealthProbeLog:DNSLatencyMicroseconds": frontDoorHealthProbeLogDNSLatencyMicroseconds, + "FrontDoorHealthProbeLog:totalLatencyMilliseconds": frontDoorHealthProbeLogTotalLatencyMilliseconds, +} + +// Splits the "HTTP/1.1" value into "HTTP" and "1.1" and sets as "network.protocol.name" and "network.protocol.version" +func appServiceHTTPLogsProtocol(key string, value any, attrs map[string]any) bool { + if str, ok := value.(string); ok { + if parts := strings.SplitN(str, "/", 2); len(parts) == 2 { + attrs["network.protocol.name"] = parts[0] + attrs["network.protocol.version"] = parts[1] + return true + } + } + return false +} + +// Converts Microseconds value to Seconds and sets as "dns.lookup.duration" +func frontDoorHealthProbeLogDNSLatencyMicroseconds(key string, value any, attrs map[string]any) bool { + microseconds, ok := tryParseFloat64(value) + if !ok { + return false + } + seconds := microseconds / 1_000_000 + attrs["dns.lookup.duration"] = seconds + return true +} + +// Converts Milliseconds value to Seconds and sets as "http.client.request.duration" +func frontDoorHealthProbeLogTotalLatencyMilliseconds(key string, value any, attrs map[string]any) bool { + milliseconds, ok := tryParseFloat64(value) + if !ok { + return false + } + seconds := milliseconds / 1_000 + attrs["http.client.request.duration"] = seconds + return true +} + +func tryParseFloat64(value any) (float64, bool) { + switch value.(type) { + case float32: + return float64(value.(float32)), true + case float64: + return value.(float64), true + case int: + return float64(value.(int)), true + case int32: + return float64(value.(int32)), true + case int64: + return float64(value.(int64)), true + case string: + f, err := strconv.ParseFloat(value.(string), 64) + return f, err == nil + default: + return 0, false + } +} + +func tryGetComplexConversion(category string, propertyName string) (ComplexConversion, bool) { + key := fmt.Sprintf("%s:%s", category, propertyName) + conversion, ok := conversions[key] + return conversion, ok +} diff --git a/pkg/translator/azure/complex_conversions_test.go b/pkg/translator/azure/complex_conversions_test.go new file mode 100644 index 000000000000..e24f448e1ad3 --- /dev/null +++ b/pkg/translator/azure/complex_conversions_test.go @@ -0,0 +1,42 @@ +package azure + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +func TestAppServiceHTTPLogsProtocol(t *testing.T) { + f, ok := tryGetComplexConversion("AppServiceHTTPLogs", "Protocol") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("Protocol", "HTTP/1.1", attrs) + assert.True(t, ok) + protocolName, ok := attrs["network.protocol.name"] + assert.True(t, ok) + assert.Equal(t, "HTTP", protocolName) + protocolVersion, ok := attrs["network.protocol.version"] + assert.True(t, ok) + assert.Equal(t, "1.1", protocolVersion) +} + +func TestFrontDoorHealthProbeLogDNSLatencyMicroseconds(t *testing.T) { + f, ok := tryGetComplexConversion("FrontDoorHealthProbeLog", "DNSLatencyMicroseconds") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("DNSLatencyMicroseconds", 123456, attrs) + assert.True(t, ok) + duration, ok := attrs["dns.lookup.duration"].(float64) + assert.True(t, ok) + assert.Equal(t, 0.123456, duration) +} + +func TestFrontDoorHealthProbeLogTotalLatencyMilliseconds(t *testing.T) { + f, ok := tryGetComplexConversion("FrontDoorHealthProbeLog", "totalLatencyMilliseconds") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("totalLatencyMilliseconds", 123, attrs) + assert.True(t, ok) + duration, ok := attrs["http.client.request.duration"].(float64) + assert.True(t, ok) + assert.Equal(t, 0.123, duration) +} diff --git a/pkg/translator/azure/go.mod b/pkg/translator/azure/go.mod index f873bcdd07f7..c5145a9cf6a4 100644 --- a/pkg/translator/azure/go.mod +++ b/pkg/translator/azure/go.mod @@ -12,7 +12,7 @@ require ( go.opentelemetry.io/collector/semconv v0.97.1-0.20240409140257-792fac1b62d4 go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f ) require ( diff --git a/pkg/translator/azure/go.sum b/pkg/translator/azure/go.sum index 8055177c94d0..e158f29f5862 100644 --- a/pkg/translator/azure/go.sum +++ b/pkg/translator/azure/go.sum @@ -80,6 +80,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 4d41a05c7dad..9657d53e8bda 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -5,7 +5,7 @@ var mappings = map[string]map[string]string{ "FrontDoorAccessLog": { "trackingReference": "", "httpMethod": "http.request.method", - "httpVersion": "http.request.version", + "httpVersion": "network.protocol.version", "requestUri": "url.full", "hostName": "server.address", "requestBytes": "http.request.body.size", @@ -15,7 +15,7 @@ var mappings = map[string]map[string]string{ "clientPort": "client.port", "socketIp": "socket.address", "timeTaken": "http.server.request.duration", - "requestProtocol": "", + "requestProtocol": "network.protocol.name", "securityProtocol": "", "securityCipher": "", "securityCurves": "", @@ -38,7 +38,7 @@ var mappings = map[string]map[string]string{ "healthProbeId": "", "POP": "", "httpVerb": "http.request.method", - "result": "OriginError", + "result": "", //"OriginError", "httpStatusCode": "http.response.status_code", "probeURL": "url.full", "originName": "server.address", @@ -59,6 +59,35 @@ var mappings = map[string]map[string]string{ "trackingReference": "", "policyMode": "", }, + "AppServiceHTTPLogs": { + "_BilledSize": "", //real The record size in bytes + "CIp": "client.address", //string IP address of the client + "ComputerName": "host.name", //string The name of the server on which the log file entry was generated. + "Cookie": "", //string Cookie on HTTP request + "CsBytes": "http.request.body.size", //int Number of bytes received by server + "CsHost": "url.domain", //string Host name header on HTTP request + "CsMethod": "http.request.method", //string The request HTTP verb + "CsUriQuery": "url.query", //string URI query on HTTP request + "CsUriStem": "url.path", //string The target of the request + "CsUsername": "", //string The name of the authenticated user on HTTP request + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Protocol": "", //string HTTP version + "Referer": "", //string The site that the user last visited. This site provided a link to the current site + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "Result": "", //string Success / Failure of HTTP request + "ScBytes": "http.response.body.size", //int Number of bytes sent by server + "ScStatus": "http.response.status_code", //int HTTP status code + "ScSubStatus": "", //string Substatus error code on HTTP request + "ScWin32Status": "", //string Windows status code on HTTP request + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "SPort": "server.port", //string Server port number + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "TimeTaken": "", //int Time taken by HTTP request in milliseconds + "Type": "", //string The name of the table + "UserAgent": "", //string User agent on HTTP request + }, } func resourceLogKeyToSemConvKey(azName string, category string) (string, bool) { diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index 1b38a43f20d9..9a081290b8d9 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -213,6 +213,12 @@ func copyProperties(category string, properties *any, attrs map[string]any) { pmap := (*properties).(map[string]any) attrsProps := map[string]any{} for k, v := range pmap { + // Check for a complex conversion, e.g. AppServiceHTTPLogs.Protocol + if complexConversion, ok := tryGetComplexConversion(category, k); ok { + if complexConversion(k, v, attrs) { + continue + } + } if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { attrs[otelKey] = v } else { @@ -220,7 +226,7 @@ func copyProperties(category string, properties *any, attrs map[string]any) { } } if len(attrsProps) > 0 { - attrs["azure.properties"] = attrsProps + attrs[azureProperties] = attrsProps } } From 87d1cdd173695fa1fe8d28e2be110ff4ec77682c Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Tue, 23 Apr 2024 16:31:46 +0100 Subject: [PATCH 05/17] Incorporate suggestions from lmolkova --- pkg/translator/azure/property_names.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 9657d53e8bda..3747625352ef 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -3,7 +3,7 @@ package azure var mappings = map[string]map[string]string{ "common": {}, "FrontDoorAccessLog": { - "trackingReference": "", + "trackingReference": "az.service_request_id", "httpMethod": "http.request.method", "httpVersion": "network.protocol.version", "requestUri": "url.full", @@ -13,7 +13,7 @@ var mappings = map[string]map[string]string{ "userAgent": "user_agent.original", "clientIp": "client.address", "clientPort": "client.port", - "socketIp": "socket.address", + "socketIp": "network.peer.address", "timeTaken": "http.server.request.duration", "requestProtocol": "network.protocol.name", "securityProtocol": "", @@ -27,7 +27,7 @@ var mappings = map[string]map[string]string{ "routeName": "http.route", "referrer": "http.request.header.referer", "timeToFirstByte": "", - "errorInfo": "", + "errorInfo": "error.type", "originURL": "", "originIP": "", "originName": "", @@ -38,7 +38,7 @@ var mappings = map[string]map[string]string{ "healthProbeId": "", "POP": "", "httpVerb": "http.request.method", - "result": "", //"OriginError", + "result": "", "httpStatusCode": "http.response.status_code", "probeURL": "url.full", "originName": "server.address", @@ -50,7 +50,7 @@ var mappings = map[string]map[string]string{ "FrontdoorWebApplicationFirewallLog": { "clientIP": "client.address", "clientPort": "client.port", - "socketIP": "socket.address", + "socketIP": "network.peer.address", "requestUri": "url.full", "ruleName": "", "policy": "", From 18afa9f09064699a1455361d89d700610d7ce790 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Tue, 23 Apr 2024 16:53:41 +0100 Subject: [PATCH 06/17] Add empty maps for AppService log types --- pkg/translator/azure/property_names.go | 191 ++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 2 deletions(-) diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 3747625352ef..0870f586f020 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -2,6 +2,31 @@ package azure var mappings = map[string]map[string]string{ "common": {}, + "AzureCdnAccessLog": { + "BackendHostname": "", // If the request is being forwarded to a backend, this field represents the hostname of the backend. This field is blank if the request gets redirected or forwarded to a regional cache (when caching gets enabled for the routing rule). + "CacheStatus": "", // For caching scenarios, this field defines the cache hit/miss at the POP + "ClientIp": "", // The IP address of the client that made the request. If there was an X-Forwarded-For header in the request, then the Client IP is picked from the same. + "ClientPort": "", // The IP port of the client that made the request. + "HttpMethod": "", // HTTP method used by the request. + "HttpStatusCode": "", // The HTTP status code returned from the proxy. If a request to the origin timeouts the value for HttpStatusCode is set to 0. + "HttpStatusDetails": "", // Resulting status on the request. Meaning of this string value can be found at a Status reference table. + "HttpVersion": "", // Type of the request or connection. + "POP": "", // Short name of the edge where the request landed. + "RequestBytes": "", // The size of the HTTP request message in bytes, including the request headers and the request body. + "RequestUri": "", // URI of the received request. + "ResponseBytes": "", // Bytes sent by the backend server as the response. + "RoutingRuleName": "", // The name of the routing rule that the request matched. + "RulesEngineMatchNames": "", // The names of the rules that the request matched. + "SecurityProtocol": "", // The TLS/SSL protocol version used by the request or null if no encryption. + "isReceivedFromClient": "", // If true, it means that the request came from the client. If false, the request is a miss in the edge (child POP) and is responded from origin shield (parent POP). + "TimeTaken": "", // The length of time from first byte of request into Azure Front Door to last byte of response out, in seconds. + "TrackingReference": "", // The unique reference string that identifies a request served by Azure Front Door, also sent as X-Azure-Ref header to the client. Required for searching details in the access logs for a specific request. + "UserAgent": "", // The browser type that the client used. + "ErrorInfo": "", // This field contains the specific type of error to narrow down troubleshooting area. + "TimeToFirstByte": "", // The length of time in milliseconds from when Microsoft CDN receives the request to the time the first byte gets sent to the client. The time is measured only from the Microsoft side. Client-side data isn't measured. + "Result": "", // SSLMismatchedSNI is a status code that signifies a successful request with a mismatch warning between the Server Name Indication (SNI) and the host header. This status code implies domain fronting, a technique that violates Azure Front Door's terms of service. Requests with SSLMismatchedSNI will be rejected after January 22, 2024. + "SNI": "", // This field specifies the Server Name Indication (SNI) that is sent during the TLS/SSL handshake. It can be used to identify the exact SNI value if there was a SSLMismatchedSNI status code. Additionally, it can be compared with the host value in the requestUri field to detect and resolve the mismatch issue. + }, "FrontDoorAccessLog": { "trackingReference": "az.service_request_id", "httpMethod": "http.request.method", @@ -56,9 +81,115 @@ var mappings = map[string]map[string]string{ "policy": "", "action": "", "host": "server.address", - "trackingReference": "", + "trackingReference": "az.service_request_id", "policyMode": "", }, + "AppServiceAppLogs": { + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "ContainerId": "", //string Application container id + "CustomLevel": "", //string Verbosity level of log + "ExceptionClass": "", //string Application class from where log message is emitted + "Host": "", //string Host where the application is running + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", //string Verbosity level of log mapped to standard levels (Informational, Warning, Error, or Critical) + "Logger": "", //string Application logger used to emit log message + "Message": "", //string Log message + "Method": "", //string Application Method from where log message is emitted + "OperationName": "", //string The name of the operation represented by this event. + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "ResultDescription": "", //string Log message description + "Source": "", //string Application source from where log message is emitted + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "Stacktrace": "", //string Complete stack trace of the log message in case of exception + "StackTrace": "", //string Complete stack trace of the log message in case of exception + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "WebSiteInstanceId": "", //string Instance ID of the application running + }, + "AppServiceAuditLogs": { + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", //string Name of the operation + "Protocol": "", //string Authentication protocol + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "User": "", //string Username used for publishing access + "UserAddress": "", //string Client IP address of the publishing user + "UserDisplayName": "", //string Email address of a user in case publishing was authorized via AAD authentication + }, + "AppServiceAuthenticationLogs": { + "_BilledSize": "", //real The record size in bytes + "CorrelationId": "", //string The ID for correlated events. + "Details": "", //string The event details. + "HostName": "", //string The host name of the application. + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", //string The level of log verbosity. + "Message": "", //string The log message. + "ModuleRuntimeVersion": "", //string The version of App Service Authentication running. + "OperationName": "", //string The name of the operation represented by this event. + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SiteName": "", //string The runtime name of the application. + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "StatusCode": "", //int The HTTP status code of the operation. + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "SubStatusCode": "", //int The HTTP sub-status code of the request. + "TaskName": "", //string The name of the task being performed. + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime The timestamp (UTC) of when this event was generated. + "Type": "", //string The name of the table + }, + "AppServiceConsoleLogs": { + "_BilledSize": "", // real The record size in bytes + "Category": "", // string Log category name + "ContainerId": "", // string Application container id + "Host": "", // string Host where the application is running + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", // string Verbosity level of log + "OperationName": "", // string The name of the operation represented by this event. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "ResultDescription": "", // string Log message description + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table + }, + "AppServiceEnvironmentPlatformLogs": { + "_BilledSize": "", // real The record size in bytes + "Category": "", // string + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", // string + "ResourceId": "", // string + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "ResultDescription": "", // string + "ResultType": "", // string + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TimeGenerated": "", // datetime + "Type": "", // string The name of the table + }, + "AppServiceFileAuditLogs": { + "_BilledSize": "", // real The record size in bytes + "Category": "", // string Log category name + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", // string Operation performed on a file + "Path": "", // string Path to the file that was changed + "Process": "", // string Type of the process that change the file + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table + }, "AppServiceHTTPLogs": { "_BilledSize": "", //real The record size in bytes "CIp": "client.address", //string IP address of the client @@ -77,7 +208,7 @@ var mappings = map[string]map[string]string{ "Result": "", //string Success / Failure of HTTP request "ScBytes": "http.response.body.size", //int Number of bytes sent by server "ScStatus": "http.response.status_code", //int HTTP status code - "ScSubStatus": "", //string Substatus error code on HTTP request + "ScSubStatus": "", //string Sub-status error code on HTTP request "ScWin32Status": "", //string Windows status code on HTTP request "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics "SPort": "server.port", //string Server port number @@ -88,6 +219,62 @@ var mappings = map[string]map[string]string{ "Type": "", //string The name of the table "UserAgent": "", //string User agent on HTTP request }, + "AppServiceIPSecAuditLogs": { + "_BilledSize": "", // real The record size in bytes + "CIp": "", // string IP address of the client + "CsHost": "", // string Host header of the HTTP request + "Details": "", // string Additional information + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "Result": "", // string The result whether the access is Allowed or Denied + "ServiceEndpoint": "", // string This indicates whether the access is via Virtual Network Service Endpoint communication + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time of the Http Request + "Type": "", // string The name of the table + "XAzureFDID": "", // string X-Azure-FDID header (Azure Frontdoor ID) of the HTTP request + "XFDHealthProbe": "", // string X-FD-HealthProbe (Azure Frontdoor Health Probe) of the HTTP request + "XForwardedFor": "", // string X-Forwarded-For header of the HTTP request + "XForwardedHost": "", // string X-Forwarded-Host header of the HTTP request + }, + "AppServicePlatformLogs": { + "ActivityId": "", // string Activity ID to correlate events + "_BilledSize": "", // real The record size in bytes + "ContainerId": "", // string Application container id + "DeploymentId": "", // string Deployment ID of the application deployment + "Exception": "", // string Details of the exception + "Host": "", // string Host where the application is running + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", // string Level of log verbosity + "Message": "", // string Log message + "OperationName": "", // string The name of the operation represented by this event. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "StackTrace": "", // string Stack trace for the exception + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table + }, + "AppServiceServerlessSecurityPluginData": { + "_BilledSize": "", // real The record size in bytes + "Index": "", // int Available when multiple payloads exist for the same message. In that case, payloads share the same SlSecRequestId and Index defines the chronological order of payloads. + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "MsgVersion": "", // string The version of the message schema. Used to make code changes backward- and forward- compatible. + "Payload": "", // dynamic An array of messages, where each one is a JSON string. + "PayloadType": "", // string The type of the payload. Mostly used to distinguish between messages meant for different types of security analysis. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "Sender": "", // string The name of the component that published this message. Almost always will be the name of the plugin, but can also be platform. + "SlSecMetadata": "", // dynamic Contains details about the resource like the deployment ID, runtime info, website info, OS, etc. + "SlSecProps": "", // dynamic Contains other details that might be needed for debugging end-to-end requests, e.g., slsec nuget version. + "SlSecRequestId": "", // string The ingestion request ID used for identifying the message and the request for diagnostics and debugging. + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime The date and time (UTC) this message was created on the node. + "Type": "", // string The name of the table + }, } func resourceLogKeyToSemConvKey(azName string, category string) (string, bool) { From cf37abbb95711cb8625ae11f8a7d8fb504e3fb58 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Wed, 24 Apr 2024 16:17:57 +0100 Subject: [PATCH 07/17] Fill out as many name mappings as possible --- pkg/translator/azure/complex_conversions.go | 16 +- pkg/translator/azure/property_names.go | 249 ++++++++++---------- 2 files changed, 140 insertions(+), 125 deletions(-) diff --git a/pkg/translator/azure/complex_conversions.go b/pkg/translator/azure/complex_conversions.go index 4e2a2d8920b5..818b9f1fad82 100644 --- a/pkg/translator/azure/complex_conversions.go +++ b/pkg/translator/azure/complex_conversions.go @@ -9,16 +9,30 @@ import ( type ComplexConversion func(string, any, map[string]any) bool var conversions = map[string]ComplexConversion{ + "AzureCDNAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, + "FrontDoorAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, "AppServiceHTTPLogs:Protocol": appServiceHTTPLogsProtocol, "FrontDoorHealthProbeLog:DNSLatencyMicroseconds": frontDoorHealthProbeLogDNSLatencyMicroseconds, "FrontDoorHealthProbeLog:totalLatencyMilliseconds": frontDoorHealthProbeLogTotalLatencyMilliseconds, } +// Splits the "TLS 1.2" value into "TLS" and "1.2" and sets as "network.protocol.name" and "network.protocol.version" +func azureCDNAccessLogSecurityProtocol(key string, value any, attrs map[string]any) bool { + if str, ok := value.(string); ok { + if parts := strings.SplitN(str, " ", 2); len(parts) == 2 { + attrs["tls.protocol.name"] = strings.ToLower(parts[0]) + attrs["tls.protocol.version"] = parts[1] + return true + } + } + return false +} + // Splits the "HTTP/1.1" value into "HTTP" and "1.1" and sets as "network.protocol.name" and "network.protocol.version" func appServiceHTTPLogsProtocol(key string, value any, attrs map[string]any) bool { if str, ok := value.(string); ok { if parts := strings.SplitN(str, "/", 2); len(parts) == 2 { - attrs["network.protocol.name"] = parts[0] + attrs["network.protocol.name"] = strings.ToLower(parts[0]) attrs["network.protocol.version"] = parts[1] return true } diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 0870f586f020..561b62bd6781 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -3,29 +3,29 @@ package azure var mappings = map[string]map[string]string{ "common": {}, "AzureCdnAccessLog": { - "BackendHostname": "", // If the request is being forwarded to a backend, this field represents the hostname of the backend. This field is blank if the request gets redirected or forwarded to a regional cache (when caching gets enabled for the routing rule). - "CacheStatus": "", // For caching scenarios, this field defines the cache hit/miss at the POP - "ClientIp": "", // The IP address of the client that made the request. If there was an X-Forwarded-For header in the request, then the Client IP is picked from the same. - "ClientPort": "", // The IP port of the client that made the request. - "HttpMethod": "", // HTTP method used by the request. - "HttpStatusCode": "", // The HTTP status code returned from the proxy. If a request to the origin timeouts the value for HttpStatusCode is set to 0. - "HttpStatusDetails": "", // Resulting status on the request. Meaning of this string value can be found at a Status reference table. - "HttpVersion": "", // Type of the request or connection. - "POP": "", // Short name of the edge where the request landed. - "RequestBytes": "", // The size of the HTTP request message in bytes, including the request headers and the request body. - "RequestUri": "", // URI of the received request. - "ResponseBytes": "", // Bytes sent by the backend server as the response. - "RoutingRuleName": "", // The name of the routing rule that the request matched. - "RulesEngineMatchNames": "", // The names of the rules that the request matched. - "SecurityProtocol": "", // The TLS/SSL protocol version used by the request or null if no encryption. - "isReceivedFromClient": "", // If true, it means that the request came from the client. If false, the request is a miss in the edge (child POP) and is responded from origin shield (parent POP). - "TimeTaken": "", // The length of time from first byte of request into Azure Front Door to last byte of response out, in seconds. - "TrackingReference": "", // The unique reference string that identifies a request served by Azure Front Door, also sent as X-Azure-Ref header to the client. Required for searching details in the access logs for a specific request. - "UserAgent": "", // The browser type that the client used. - "ErrorInfo": "", // This field contains the specific type of error to narrow down troubleshooting area. - "TimeToFirstByte": "", // The length of time in milliseconds from when Microsoft CDN receives the request to the time the first byte gets sent to the client. The time is measured only from the Microsoft side. Client-side data isn't measured. - "Result": "", // SSLMismatchedSNI is a status code that signifies a successful request with a mismatch warning between the Server Name Indication (SNI) and the host header. This status code implies domain fronting, a technique that violates Azure Front Door's terms of service. Requests with SSLMismatchedSNI will be rejected after January 22, 2024. - "SNI": "", // This field specifies the Server Name Indication (SNI) that is sent during the TLS/SSL handshake. It can be used to identify the exact SNI value if there was a SSLMismatchedSNI status code. Additionally, it can be compared with the host value in the requestUri field to detect and resolve the mismatch issue. + "BackendHostname": "destination.address", // If the request is being forwarded to a backend, this field represents the hostname of the backend. This field is blank if the request gets redirected or forwarded to a regional cache (when caching gets enabled for the routing rule). + "CacheStatus": "", // For caching scenarios, this field defines the cache hit/miss at the POP + "ClientIp": "client.address", // The IP address of the client that made the request. If there was an X-Forwarded-For header in the request, then the Client IP is picked from the same. + "ClientPort": "client.port", // The IP port of the client that made the request. + "HttpMethod": "http.request.method", // HTTP method used by the request. + "HttpStatusCode": "http.response.status_code", // The HTTP status code returned from the proxy. If a request to the origin timeouts the value for HttpStatusCode is set to 0. + "HttpStatusDetails": "", // Resulting status on the request. Meaning of this string value can be found at a Status reference table. + "HttpVersion": "network.protocol.version", // Type of the request or connection. + "POP": "", // Short name of the edge where the request landed. + "RequestBytes": "http.request.size", // The size of the HTTP request message in bytes, including the request headers and the request body. + "RequestUri": "url.full", // URI of the received request. + "ResponseBytes": "http.response.size", // Bytes sent by the backend server as the response. + "RoutingRuleName": "", // The name of the routing rule that the request matched. + "RulesEngineMatchNames": "", // The names of the rules that the request matched. + "SecurityProtocol": "", // handled by complex_conversions + "isReceivedFromClient": "", // If true, it means that the request came from the client. If false, the request is a miss in the edge (child POP) and is responded from origin shield (parent POP). + "TimeTaken": "", // The length of time from first byte of request into Azure Front Door to last byte of response out, in seconds. + "TrackingReference": "az.service_request_id", // The unique reference string that identifies a request served by Azure Front Door, also sent as X-Azure-Ref header to the client. Required for searching details in the access logs for a specific request. + "UserAgent": "user_agent.original", // The browser type that the client used. + "ErrorInfo": "error.type", // This field contains the specific type of error to narrow down troubleshooting area. + "TimeToFirstByte": "", // The length of time in milliseconds from when Microsoft CDN receives the request to the time the first byte gets sent to the client. The time is measured only from the Microsoft side. Client-side data isn't measured. + "Result": "", // SSLMismatchedSNI is a status code that signifies a successful request with a mismatch warning between the Server Name Indication (SNI) and the host header. This status code implies domain fronting, a technique that violates Azure Front Door's terms of service. Requests with SSLMismatchedSNI will be rejected after January 22, 2024. + "SNI": "", // This field specifies the Server Name Indication (SNI) that is sent during the TLS/SSL handshake. It can be used to identify the exact SNI value if there was a SSLMismatchedSNI status code. Additionally, it can be compared with the host value in the requestUri field to detect and resolve the mismatch issue. }, "FrontDoorAccessLog": { "trackingReference": "az.service_request_id", @@ -33,17 +33,17 @@ var mappings = map[string]map[string]string{ "httpVersion": "network.protocol.version", "requestUri": "url.full", "hostName": "server.address", - "requestBytes": "http.request.body.size", - "responseBytes": "http.response.body.size", + "requestBytes": "http.request.size", + "responseBytes": "http.response.size", "userAgent": "user_agent.original", "clientIp": "client.address", "clientPort": "client.port", "socketIp": "network.peer.address", "timeTaken": "http.server.request.duration", "requestProtocol": "network.protocol.name", - "securityProtocol": "", - "securityCipher": "", - "securityCurves": "", + "securityProtocol": "", // handled by complex_conversions + "securityCipher": "tls.cipher", + "securityCurves": "tls.curve", "endpoint": "", "httpStatusCode": "http.response.status_code", "pop": "", @@ -68,9 +68,9 @@ var mappings = map[string]map[string]string{ "probeURL": "url.full", "originName": "server.address", "originIP": "", - "totalLatencyMilliseconds": "", + "totalLatencyMilliseconds": "", // handled by complex_conversions "connectionLatencyMilliseconds": "", - "DNSLatencyMicroseconds": "", + "DNSLatencyMicroseconds": "", // handled by complex_conversions }, "FrontdoorWebApplicationFirewallLog": { "clientIP": "client.address", @@ -110,41 +110,41 @@ var mappings = map[string]map[string]string{ "WebSiteInstanceId": "", //string Instance ID of the application running }, "AppServiceAuditLogs": { - "_BilledSize": "", //real The record size in bytes - "Category": "", //string Log category name - "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "OperationName": "", //string Name of the operation - "Protocol": "", //string Authentication protocol - "_ResourceId": "", //string A unique identifier for the resource that the record is associated with - "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with - "TenantId": "", //string The Log Analytics workspace ID - "TimeGenerated": "", //datetime Time when event is generated - "Type": "", //string The name of the table - "User": "", //string Username used for publishing access - "UserAddress": "", //string Client IP address of the publishing user - "UserDisplayName": "", //string Email address of a user in case publishing was authorized via AAD authentication + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", //string Name of the operation + "Protocol": "", //string Authentication protocol + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "User": "enduser.id", //string Username used for publishing access + "UserAddress": "client.address", //string Client IP address of the publishing user + "UserDisplayName": "", //string Email address of a user in case publishing was authorized via AAD authentication }, "AppServiceAuthenticationLogs": { - "_BilledSize": "", //real The record size in bytes - "CorrelationId": "", //string The ID for correlated events. - "Details": "", //string The event details. - "HostName": "", //string The host name of the application. - "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "Level": "", //string The level of log verbosity. - "Message": "", //string The log message. - "ModuleRuntimeVersion": "", //string The version of App Service Authentication running. - "OperationName": "", //string The name of the operation represented by this event. - "_ResourceId": "", //string A unique identifier for the resource that the record is associated with - "SiteName": "", //string The runtime name of the application. - "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "StatusCode": "", //int The HTTP status code of the operation. - "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with - "SubStatusCode": "", //int The HTTP sub-status code of the request. - "TaskName": "", //string The name of the task being performed. - "TenantId": "", //string The Log Analytics workspace ID - "TimeGenerated": "", //datetime The timestamp (UTC) of when this event was generated. - "Type": "", //string The name of the table + "_BilledSize": "", //real The record size in bytes + "CorrelationId": "", //string The ID for correlated events. + "Details": "", //string The event details. + "HostName": "", //string The host name of the application. + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", //string The level of log verbosity. + "Message": "", //string The log message. + "ModuleRuntimeVersion": "", //string The version of App Service Authentication running. + "OperationName": "", //string The name of the operation represented by this event. + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SiteName": "", //string The runtime name of the application. + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "StatusCode": "http.response.status_code", //int The HTTP status code of the operation. + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "SubStatusCode": "", //int The HTTP sub-status code of the request. + "TaskName": "", //string The name of the task being performed. + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime The timestamp (UTC) of when this event was generated. + "Type": "", //string The name of the table }, "AppServiceConsoleLogs": { "_BilledSize": "", // real The record size in bytes @@ -191,71 +191,72 @@ var mappings = map[string]map[string]string{ "Type": "", // string The name of the table }, "AppServiceHTTPLogs": { - "_BilledSize": "", //real The record size in bytes - "CIp": "client.address", //string IP address of the client - "ComputerName": "host.name", //string The name of the server on which the log file entry was generated. - "Cookie": "", //string Cookie on HTTP request - "CsBytes": "http.request.body.size", //int Number of bytes received by server - "CsHost": "url.domain", //string Host name header on HTTP request - "CsMethod": "http.request.method", //string The request HTTP verb - "CsUriQuery": "url.query", //string URI query on HTTP request - "CsUriStem": "url.path", //string The target of the request - "CsUsername": "", //string The name of the authenticated user on HTTP request - "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "Protocol": "", //string HTTP version - "Referer": "", //string The site that the user last visited. This site provided a link to the current site - "_ResourceId": "", //string A unique identifier for the resource that the record is associated with - "Result": "", //string Success / Failure of HTTP request - "ScBytes": "http.response.body.size", //int Number of bytes sent by server - "ScStatus": "http.response.status_code", //int HTTP status code - "ScSubStatus": "", //string Sub-status error code on HTTP request - "ScWin32Status": "", //string Windows status code on HTTP request - "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "SPort": "server.port", //string Server port number - "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with - "TenantId": "", //string The Log Analytics workspace ID - "TimeGenerated": "", //datetime Time when event is generated - "TimeTaken": "", //int Time taken by HTTP request in milliseconds - "Type": "", //string The name of the table - "UserAgent": "", //string User agent on HTTP request + "_BilledSize": "", //real The record size in bytes + "CIp": "client.address", //string IP address of the client + "ComputerName": "host.name", //string The name of the server on which the log file entry was generated. + "Cookie": "", //string Cookie on HTTP request + "CsBytes": "http.request.body.size", //int Number of bytes received by server + "CsHost": "url.domain", //string Host name header on HTTP request + "CsMethod": "http.request.method", //string The request HTTP verb + "CsUriQuery": "url.query", //string URI query on HTTP request + "CsUriStem": "url.path", //string The target of the request + "CsUsername": "", //string The name of the authenticated user on HTTP request + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Protocol": "", // handled by complex_conversions + "Referer": "http.request.header.referer", //string The site that the user last visited. This site provided a link to the current site + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "Result": "", //string Success / Failure of HTTP request + "ScBytes": "http.response.body.size", //int Number of bytes sent by server + "ScStatus": "http.response.status_code", //int HTTP status code + "ScSubStatus": "", //string Sub-status error code on HTTP request + "ScWin32Status": "", //string Windows status code on HTTP request + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "SPort": "server.port", //string Server port number + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "TimeTaken": "http.server.request.duration", //int Time taken by HTTP request in milliseconds + "Type": "", //string The name of the table + "UserAgent": "user_agent.original", //string User agent on HTTP request }, "AppServiceIPSecAuditLogs": { - "_BilledSize": "", // real The record size in bytes - "CIp": "", // string IP address of the client - "CsHost": "", // string Host header of the HTTP request - "Details": "", // string Additional information - "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "_ResourceId": "", // string A unique identifier for the resource that the record is associated with - "Result": "", // string The result whether the access is Allowed or Denied - "ServiceEndpoint": "", // string This indicates whether the access is via Virtual Network Service Endpoint communication - "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with - "TenantId": "", // string The Log Analytics workspace ID - "TimeGenerated": "", // datetime Time of the Http Request - "Type": "", // string The name of the table - "XAzureFDID": "", // string X-Azure-FDID header (Azure Frontdoor ID) of the HTTP request - "XFDHealthProbe": "", // string X-FD-HealthProbe (Azure Frontdoor Health Probe) of the HTTP request - "XForwardedFor": "", // string X-Forwarded-For header of the HTTP request - "XForwardedHost": "", // string X-Forwarded-Host header of the HTTP request + "_BilledSize": "", // real The record size in bytes + "CIp": "client.address", // string IP address of the client + "CsHost": "url.domain", // string Host header of the HTTP request + "Details": "", // string Additional information + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "Result": "", // string The result whether the access is Allowed or Denied + "ServiceEndpoint": "", // string This indicates whether the access is via Virtual Network Service Endpoint communication + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time of the Http Request + "Type": "", // string The name of the table + "XAzureFDID": "", // string X-Azure-FDID header (Azure Frontdoor ID) of the HTTP request + "XFDHealthProbe": "", // string X-FD-HealthProbe (Azure Frontdoor Health Probe) of the HTTP request + "XForwardedFor": "", // string X-Forwarded-For header of the HTTP request + "XForwardedHost": "", // string X-Forwarded-Host header of the HTTP request }, "AppServicePlatformLogs": { - "ActivityId": "", // string Activity ID to correlate events - "_BilledSize": "", // real The record size in bytes - "ContainerId": "", // string Application container id - "DeploymentId": "", // string Deployment ID of the application deployment - "Exception": "", // string Details of the exception - "Host": "", // string Host where the application is running - "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "Level": "", // string Level of log verbosity - "Message": "", // string Log message - "OperationName": "", // string The name of the operation represented by this event. - "_ResourceId": "", // string A unique identifier for the resource that the record is associated with - "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "StackTrace": "", // string Stack trace for the exception - "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with - "TenantId": "", // string The Log Analytics workspace ID - "TimeGenerated": "", // datetime Time when event is generated - "Type": "", // string The name of the table + "ActivityId": "", // string Activity ID to correlate events + "_BilledSize": "", // real The record size in bytes + "containerId": "container.id", // string Application container id + "containerName": "container.name", // string Application container id + "DeploymentId": "", // string Deployment ID of the application deployment + "exception": "error.type", // string Details of the exception + "Host": "", // string Host where the application is running + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", // string Level of log verbosity + "Message": "", // string Log message + "OperationName": "", // string The name of the operation represented by this event. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "StackTrace": "", // string Stack trace for the exception + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table }, "AppServiceServerlessSecurityPluginData": { "_BilledSize": "", // real The record size in bytes From f2b6754c3a0606519aeec9729c2ffd32ea8a1782 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Wed, 24 Apr 2024 16:23:10 +0100 Subject: [PATCH 08/17] Add tests for securityProtocol -> tls.protocol.name and tls.protocol.version --- pkg/translator/azure/complex_conversions.go | 2 +- .../azure/complex_conversions_test.go | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/pkg/translator/azure/complex_conversions.go b/pkg/translator/azure/complex_conversions.go index 818b9f1fad82..b1cf1186a9f9 100644 --- a/pkg/translator/azure/complex_conversions.go +++ b/pkg/translator/azure/complex_conversions.go @@ -10,7 +10,7 @@ type ComplexConversion func(string, any, map[string]any) bool var conversions = map[string]ComplexConversion{ "AzureCDNAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, - "FrontDoorAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, + "FrontDoorAccessLog:securityProtocol": azureCDNAccessLogSecurityProtocol, "AppServiceHTTPLogs:Protocol": appServiceHTTPLogsProtocol, "FrontDoorHealthProbeLog:DNSLatencyMicroseconds": frontDoorHealthProbeLogDNSLatencyMicroseconds, "FrontDoorHealthProbeLog:totalLatencyMilliseconds": frontDoorHealthProbeLogTotalLatencyMilliseconds, diff --git a/pkg/translator/azure/complex_conversions_test.go b/pkg/translator/azure/complex_conversions_test.go index e24f448e1ad3..fc2e4790ae6d 100644 --- a/pkg/translator/azure/complex_conversions_test.go +++ b/pkg/translator/azure/complex_conversions_test.go @@ -5,6 +5,36 @@ import ( "testing" ) +func TestFrontDoorAccessLogSecurityProtocol(t *testing.T) { + f, ok := tryGetComplexConversion("FrontDoorAccessLog", "securityProtocol") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("securityProtocol", "TLS 1.2", attrs) + assert.True(t, ok) + protocolName, ok := attrs["tls.protocol.name"] + assert.True(t, ok) + // Protocol name is normalized to lower case + assert.Equal(t, "tls", protocolName) + protocolVersion, ok := attrs["tls.protocol.version"] + assert.True(t, ok) + assert.Equal(t, "1.2", protocolVersion) +} + +func TestAzureCDNAccessLogSecurityProtocol(t *testing.T) { + f, ok := tryGetComplexConversion("AzureCDNAccessLog", "SecurityProtocol") + assert.True(t, ok) + attrs := map[string]any{} + ok = f("SecurityProtocol", "TLS 1.2", attrs) + assert.True(t, ok) + protocolName, ok := attrs["tls.protocol.name"] + assert.True(t, ok) + // Protocol name is normalized to lower case + assert.Equal(t, "tls", protocolName) + protocolVersion, ok := attrs["tls.protocol.version"] + assert.True(t, ok) + assert.Equal(t, "1.2", protocolVersion) +} + func TestAppServiceHTTPLogsProtocol(t *testing.T) { f, ok := tryGetComplexConversion("AppServiceHTTPLogs", "Protocol") assert.True(t, ok) @@ -13,7 +43,7 @@ func TestAppServiceHTTPLogsProtocol(t *testing.T) { assert.True(t, ok) protocolName, ok := attrs["network.protocol.name"] assert.True(t, ok) - assert.Equal(t, "HTTP", protocolName) + assert.Equal(t, "http", protocolName) protocolVersion, ok := attrs["network.protocol.version"] assert.True(t, ok) assert.Equal(t, "1.1", protocolVersion) From bc6b28b42f3186f301527dda31e7cdc5d2cf3ab9 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Thu, 25 Apr 2024 14:31:05 +0100 Subject: [PATCH 09/17] Add value normalizing and full test for FrontDoorAccessLog --- pkg/translator/azure/complex_conversions.go | 1 + pkg/translator/azure/normalize.go | 73 +++++++++++++++++++ pkg/translator/azure/property_names.go | 2 +- pkg/translator/azure/resourcelogs_to_logs.go | 2 +- .../azure/resourcelogs_to_logs_test.go | 37 ++++++++++ .../testdata/log-frontdooraccesslog.json | 46 ++++++++++++ 6 files changed, 159 insertions(+), 2 deletions(-) create mode 100644 pkg/translator/azure/normalize.go create mode 100644 pkg/translator/azure/testdata/log-frontdooraccesslog.json diff --git a/pkg/translator/azure/complex_conversions.go b/pkg/translator/azure/complex_conversions.go index b1cf1186a9f9..2a2e3825aeec 100644 --- a/pkg/translator/azure/complex_conversions.go +++ b/pkg/translator/azure/complex_conversions.go @@ -7,6 +7,7 @@ import ( ) type ComplexConversion func(string, any, map[string]any) bool +type TypeConversion func(string, any, map[string]any, string) bool var conversions = map[string]ComplexConversion{ "AzureCDNAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, diff --git a/pkg/translator/azure/normalize.go b/pkg/translator/azure/normalize.go new file mode 100644 index 000000000000..3d0aca3d41a7 --- /dev/null +++ b/pkg/translator/azure/normalize.go @@ -0,0 +1,73 @@ +package azure + +import ( + "fmt" + "strconv" + "strings" +) + +const maxInt32 = int64(int32(^uint32(0) >> 1)) + +type valueNormalizer func(any) any + +var normalizers = map[string]valueNormalizer{ + "http.request.size": toInt, + "http.response.size": toInt, + "http.server.request.duration": toFloat, + "network.protocol.name": toLower, + "http.response.status_code": toInt, +} + +func normalizeValue(key string, val any) any { + if f, exists := normalizers[key]; exists { + return f(val) + } + return val +} + +func toLower(value any) any { + switch value.(type) { + case string: + return strings.ToLower(value.(string)) + default: + return strings.ToLower(fmt.Sprint(value)) + } +} + +func toFloat(value any) any { + switch value.(type) { + case float32: + return float64(value.(float32)) + case float64: + return value.(float64) + case int: + return float64(value.(int)) + case int32: + return float64(value.(int32)) + case int64: + return float64(value.(int64)) + case string: + f, err := strconv.ParseFloat(value.(string), 64) + if err == nil { + return f + } + } + return value +} + +func toInt(value any) any { + switch value.(type) { + case int: + return int64(value.(int)) + case int32: + return int64(int(value.(int32))) + case int64: + return value.(int64) + case string: + i, err := strconv.ParseInt(value.(string), 10, 64) + if err == nil { + return i + } + } + return value +} diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 561b62bd6781..3f7050b200e6 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -50,7 +50,7 @@ var mappings = map[string]map[string]string{ "cacheStatus": "", "matchedRulesSetName": "", "routeName": "http.route", - "referrer": "http.request.header.referer", + "referer": "http.request.header.referer", "timeToFirstByte": "", "errorInfo": "error.type", "originURL": "", diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index 9a081290b8d9..77f8ae01a5c9 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -220,7 +220,7 @@ func copyProperties(category string, properties *any, attrs map[string]any) { } } if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { - attrs[otelKey] = v + attrs[otelKey] = normalizeValue(otelKey, v) } else { attrsProps[k] = v } diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index 37bb67df9987..9e7096f7c7b2 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -466,3 +466,40 @@ func TestUnmarshalLogs(t *testing.T) { }) } } + +func TestFrontDoorAccessLog(t *testing.T) { + sut := &ResourceLogsUnmarshaler{ + Version: testBuildInfo.Version, + Logger: zap.NewNop(), + } + + data, err := os.ReadFile(filepath.Join("testdata", "log-frontdooraccesslog.json")) + assert.NoError(t, err) + assert.NotNil(t, data) + + logs, err := sut.UnmarshalLogs(data) + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "23.23.23.23", record["network.peer.address"]) + assert.Equal(t, float64(0.23), record["http.server.request.duration"]) + assert.Equal(t, "https", record["network.protocol.name"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, "TLS_AES_256_GCM_SHA384", record["tls.cipher"]) + assert.Equal(t, "secp384r1", record["tls.curve"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "REFERER", record["http.request.header.referer"]) + assert.Equal(t, "NoError", record["error.type"]) +} diff --git a/pkg/translator/azure/testdata/log-frontdooraccesslog.json b/pkg/translator/azure/testdata/log-frontdooraccesslog.json new file mode 100644 index 000000000000..0a775bdb9572 --- /dev/null +++ b/pkg/translator/azure/testdata/log-frontdooraccesslog.json @@ -0,0 +1,46 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "FrontDoorAccessLog", + "operationName": "Microsoft.Cdn/Profiles/AccessLog/Write", + "properties": { + "trackingReference": "TRACKING_REFERENCE", + "httpMethod": "GET", + "httpVersion": "1.1.0.0", + "requestUri": "https://test.net/", + "sni": "originshield|parentcache|https|tier2", + "requestBytes": "1234", + "responseBytes": "12345", + "userAgent": "Mozilla/5.0", + "clientIp": "42.42.42.42", + "clientPort": "0", + "socketIp": "23.23.23.23", + "timeToFirstByte": "0.420", + "timeTaken": "0.230", + "requestProtocol": "HTTPS", + "securityProtocol": "TLS 1.3", + "rulesEngineMatchNames": [], + "httpStatusCode": "200", + "httpStatusDetails": "200", + "pop": "LON", + "cacheStatus": "MISS", + "errorInfo": "NoError", + "ErrorInfo": "NoError", + "result": "N/A", + "endpoint": "dummyapp-eebde0bwehfthfbb.z01.azurefd.net", + "routingRuleName": "default-route", + "hostName": "dummyapp-eebde0bwehfthfbb.z01.azurefd.net", + "originUrl": "https://dummyapp.icysea-e3cd8b77.eastus.azurecontainerapps.io:443/", + "originIp": "4.156.233.180:443", + "originName": "dummyapp.icysea-e3cd8b77.eastus.azurecontainerapps.io:443", + "referer": "REFERER", + "clientCountry": "United Kingdom", + "domain": "dummyapp-eebde0bwehfthfbb.z01.azurefd.net:443", + "securityCipher": "TLS_AES_256_GCM_SHA384", + "securityCurves": "secp384r1" + } + } + ] +} \ No newline at end of file From be58d535dd8ba47bd8703c4ee911e744636e8a8b Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Fri, 26 Apr 2024 13:42:58 +0100 Subject: [PATCH 10/17] Add test for AppServiceHTTPLog --- pkg/translator/azure/complex_conversions.go | 12 ++++ pkg/translator/azure/normalize.go | 5 +- pkg/translator/azure/property_names.go | 28 ++++----- .../azure/resourcelogs_to_logs_test.go | 57 +++++++++++++++++-- .../testdata/log-appserviceauditlogs.json | 16 ++++++ .../testdata/log-appservicehttplogs.json | 34 +++++++++++ 6 files changed, 133 insertions(+), 19 deletions(-) create mode 100644 pkg/translator/azure/testdata/log-appserviceauditlogs.json create mode 100644 pkg/translator/azure/testdata/log-appservicehttplogs.json diff --git a/pkg/translator/azure/complex_conversions.go b/pkg/translator/azure/complex_conversions.go index 2a2e3825aeec..f780f28a1662 100644 --- a/pkg/translator/azure/complex_conversions.go +++ b/pkg/translator/azure/complex_conversions.go @@ -13,6 +13,7 @@ var conversions = map[string]ComplexConversion{ "AzureCDNAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, "FrontDoorAccessLog:securityProtocol": azureCDNAccessLogSecurityProtocol, "AppServiceHTTPLogs:Protocol": appServiceHTTPLogsProtocol, + "AppServiceHTTPLogs:TimeTaken": appServiceHTTPLogTimeTakenMilliseconds, "FrontDoorHealthProbeLog:DNSLatencyMicroseconds": frontDoorHealthProbeLogDNSLatencyMicroseconds, "FrontDoorHealthProbeLog:totalLatencyMilliseconds": frontDoorHealthProbeLogTotalLatencyMilliseconds, } @@ -63,6 +64,17 @@ func frontDoorHealthProbeLogTotalLatencyMilliseconds(key string, value any, attr return true } +// Converts Milliseconds value to Seconds and sets as "http.server.request.duration" +func appServiceHTTPLogTimeTakenMilliseconds(key string, value any, attrs map[string]any) bool { + milliseconds, ok := tryParseFloat64(value) + if !ok { + return false + } + seconds := milliseconds / 1_000 + attrs["http.server.request.duration"] = seconds + return true +} + func tryParseFloat64(value any) (float64, bool) { switch value.(type) { case float32: diff --git a/pkg/translator/azure/normalize.go b/pkg/translator/azure/normalize.go index 3d0aca3d41a7..212fb2b1f44b 100644 --- a/pkg/translator/azure/normalize.go +++ b/pkg/translator/azure/normalize.go @@ -11,11 +11,14 @@ const maxInt32 = int64(int32(^uint32(0) >> 1)) type valueNormalizer func(any) any var normalizers = map[string]valueNormalizer{ + "http.request.body.size": toInt, "http.request.size": toInt, + "http.response.body.size": toInt, "http.response.size": toInt, + "http.response.status_code": toInt, "http.server.request.duration": toFloat, "network.protocol.name": toLower, - "http.response.status_code": toInt, + "server.port": toInt, } func normalizeValue(key string, val any) any { diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/property_names.go index 3f7050b200e6..cb8247cfea8b 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/property_names.go @@ -110,20 +110,20 @@ var mappings = map[string]map[string]string{ "WebSiteInstanceId": "", //string Instance ID of the application running }, "AppServiceAuditLogs": { - "_BilledSize": "", //real The record size in bytes - "Category": "", //string Log category name - "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "OperationName": "", //string Name of the operation - "Protocol": "", //string Authentication protocol - "_ResourceId": "", //string A unique identifier for the resource that the record is associated with - "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with - "TenantId": "", //string The Log Analytics workspace ID - "TimeGenerated": "", //datetime Time when event is generated - "Type": "", //string The name of the table - "User": "enduser.id", //string Username used for publishing access - "UserAddress": "client.address", //string Client IP address of the publishing user - "UserDisplayName": "", //string Email address of a user in case publishing was authorized via AAD authentication + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "OperationName": "", //string Name of the operation + "Protocol": "network.protocol.name", //string Authentication protocol + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "User": "enduser.id", //string Username used for publishing access + "UserAddress": "client.address", //string Client IP address of the publishing user + "UserDisplayName": "", //string Email address of a user in case publishing was authorized via AAD authentication }, "AppServiceAuthenticationLogs": { "_BilledSize": "", //real The record size in bytes diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index 9e7096f7c7b2..bb8216c34057 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -467,18 +467,31 @@ func TestUnmarshalLogs(t *testing.T) { } } -func TestFrontDoorAccessLog(t *testing.T) { +func loadJsonLogs(filename string) (plog.Logs, error) { + l := plog.NewLogs() + sut := &ResourceLogsUnmarshaler{ Version: testBuildInfo.Version, Logger: zap.NewNop(), } - data, err := os.ReadFile(filepath.Join("testdata", "log-frontdooraccesslog.json")) - assert.NoError(t, err) - assert.NotNil(t, data) + data, err := os.ReadFile(filepath.Join("testdata", filename)) + if err != nil { + return l, err + } logs, err := sut.UnmarshalLogs(data) + if err != nil { + return l, err + } + + return logs, nil +} + +func TestFrontDoorAccessLog(t *testing.T) { + logs, err := loadJsonLogs("log-frontdooraccesslog.json") + assert.NoError(t, err) record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() @@ -503,3 +516,39 @@ func TestFrontDoorAccessLog(t *testing.T) { assert.Equal(t, "REFERER", record["http.request.header.referer"]) assert.Equal(t, "NoError", record["error.type"]) } + +func TestAppServiceAuditLog(t *testing.T) { + logs, err := loadJsonLogs("log-appserviceauditlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "USER_ID", record["enduser.id"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "kudu", record["network.protocol.name"]) +} + +func TestAppServiceHTTPLog(t *testing.T) { + logs, err := loadJsonLogs("log-appservicehttplogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "test.com", record["url.domain"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, int64(80), record["server.port"]) + assert.Equal(t, "/api/test/", record["url.path"]) + assert.Equal(t, "foo=42", record["url.query"]) + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, 0.42, record["http.server.request.duration"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, int64(4242), record["http.request.body.size"]) + assert.Equal(t, int64(42), record["http.response.body.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "REFERER", record["http.request.header.referer"]) + assert.Equal(t, "COMPUTER_NAME", record["host.name"]) + assert.Equal(t, "http", record["network.protocol.name"]) + assert.Equal(t, "1.1", record["network.protocol.version"]) +} diff --git a/pkg/translator/azure/testdata/log-appserviceauditlogs.json b/pkg/translator/azure/testdata/log-appserviceauditlogs.json new file mode 100644 index 000000000000..ba64020d5954 --- /dev/null +++ b/pkg/translator/azure/testdata/log-appserviceauditlogs.json @@ -0,0 +1,16 @@ +{ + "records": [ + { + "time": "2024-04-24T12:01:20.8427400Z", + "ResourceId": "/SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP", + "Category": "AppServiceAuditLogs", + "OperationName": "Authorization", + "Properties": { + "User": "USER_ID", + "UserDisplayName": "$fbehtestapp", + "UserAddress": "42.42.42.42", + "Protocol": "Kudu" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure/testdata/log-appservicehttplogs.json b/pkg/translator/azure/testdata/log-appservicehttplogs.json new file mode 100644 index 000000000000..6565dc976b3b --- /dev/null +++ b/pkg/translator/azure/testdata/log-appservicehttplogs.json @@ -0,0 +1,34 @@ +{ + "records": [ + { + "time": "2024-04-24T11:59:40.9893370Z", + "EventTime": "2024-04-24T11:59:40.9893370Z", + "resourceId": "/SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP", + "properties": { + "CsHost": "test.com", + "CIp": "42.42.42.42", + "SPort": "80", + "CsUriStem": "/api/test/", + "CsUriQuery": "foo=42", + "CsMethod": "GET", + "TimeTaken": 420, + "ScStatus": "200", + "Result": "Success", + "CsBytes": "4242", + "ScBytes": "42", + "UserAgent": "Mozilla/5.0", + "Cookie": "", + "CsUsername": "user@test.com", + "Referer": "REFERER", + "ComputerName": "COMPUTER_NAME", + "Protocol": "HTTP/1.1" + }, + "category": "AppServiceHTTPLogs", + "EventStampType": "Stamp", + "EventPrimaryStampName": "waws-prod-blu-479", + "EventStampName": "waws-prod-blu-479", + "Host": "lw0sdlwk0005XR", + "EventIpAddress": "10.50.0.34" + } + ] +} \ No newline at end of file From 09446db349e5cc2da5c6e8d476fc56288f24b722 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Fri, 26 Apr 2024 14:17:17 +0100 Subject: [PATCH 11/17] Add test for AppServicePlatformLog --- .../azure/resourcelogs_to_logs_test.go | 11 ++++++++++ .../testdata/log-appserviceplatformlogs.json | 21 +++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 pkg/translator/azure/testdata/log-appserviceplatformlogs.json diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index bb8216c34057..e281cc1c7dc4 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -552,3 +552,14 @@ func TestAppServiceHTTPLog(t *testing.T) { assert.Equal(t, "http", record["network.protocol.name"]) assert.Equal(t, "1.1", record["network.protocol.version"]) } + +func TestAppServicePlatformLog(t *testing.T) { + logs, err := loadJsonLogs("log-appserviceplatformlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "CONTAINER_NAME", record["container.name"]) +} diff --git a/pkg/translator/azure/testdata/log-appserviceplatformlogs.json b/pkg/translator/azure/testdata/log-appserviceplatformlogs.json new file mode 100644 index 000000000000..903e5dcfcd25 --- /dev/null +++ b/pkg/translator/azure/testdata/log-appserviceplatformlogs.json @@ -0,0 +1,21 @@ +{ + "records": [ + { + "resourceId": "/SUBSCRIPTIONS/DA2DD5CC-E7BC-4DB6-94D9-0AFB3BD30577/RESOURCEGROUPS/FRETBADGER/PROVIDERS/MICROSOFT.WEB/SITES/FBEHTESTAPP", + "category": "AppServicePlatformLogs", + "time": "2024-04-24T12:03:55.630Z", + "level": "Informational", + "operationName": "ContainerLogs", + "properties": { + "message": "Initiating warmup request to container fbehtestapp_1_b8a27b37 for site fbehtestapp", + "containerId": "CONTAINER_ID", + "containerName": "CONTAINER_NAME" + }, + "EventStampType": "Stamp", + "EventPrimaryStampName": "waws-prod-blu-479", + "EventStampName": "waws-prod-blu-479", + "Host": "lw0sdlwk0005XR", + "EventIpAddress": "10.50.0.34" + } + ] +} \ No newline at end of file From b25d353c723c1a76a0905473d2ef75989f9a75b3 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Mon, 29 Apr 2024 16:45:46 +0100 Subject: [PATCH 12/17] Tests for CDN Access Log, Front Door WAF Log and FD Health Probe Log --- ...ersions.go => logs_complex_conversions.go} | 8 +-- ...st.go => logs_complex_conversions_test.go} | 2 +- .../azure/{normalize.go => logs_normalize.go} | 0 ...operty_names.go => logs_property_names.go} | 4 +- .../azure/resourcelogs_to_logs_test.go | 52 +++++++++++++++++++ .../azure/testdata/log-azurecdnaccesslog.json | 34 ++++++++++++ .../testdata/log-frontdoorhealthprobelog.json | 22 ++++++++ .../azure/testdata/log-frontdoorwaflog.json | 18 +++++++ 8 files changed, 133 insertions(+), 7 deletions(-) rename pkg/translator/azure/{complex_conversions.go => logs_complex_conversions.go} (92%) rename pkg/translator/azure/{complex_conversions_test.go => logs_complex_conversions_test.go} (97%) rename pkg/translator/azure/{normalize.go => logs_normalize.go} (100%) rename pkg/translator/azure/{property_names.go => logs_property_names.go} (99%) create mode 100644 pkg/translator/azure/testdata/log-azurecdnaccesslog.json create mode 100644 pkg/translator/azure/testdata/log-frontdoorhealthprobelog.json create mode 100644 pkg/translator/azure/testdata/log-frontdoorwaflog.json diff --git a/pkg/translator/azure/complex_conversions.go b/pkg/translator/azure/logs_complex_conversions.go similarity index 92% rename from pkg/translator/azure/complex_conversions.go rename to pkg/translator/azure/logs_complex_conversions.go index f780f28a1662..c8a86488f8a8 100644 --- a/pkg/translator/azure/complex_conversions.go +++ b/pkg/translator/azure/logs_complex_conversions.go @@ -10,8 +10,8 @@ type ComplexConversion func(string, any, map[string]any) bool type TypeConversion func(string, any, map[string]any, string) bool var conversions = map[string]ComplexConversion{ - "AzureCDNAccessLog:SecurityProtocol": azureCDNAccessLogSecurityProtocol, - "FrontDoorAccessLog:securityProtocol": azureCDNAccessLogSecurityProtocol, + "AzureCdnAccessLog:SecurityProtocol": azureCdnAccessLogSecurityProtocol, + "FrontDoorAccessLog:securityProtocol": azureCdnAccessLogSecurityProtocol, "AppServiceHTTPLogs:Protocol": appServiceHTTPLogsProtocol, "AppServiceHTTPLogs:TimeTaken": appServiceHTTPLogTimeTakenMilliseconds, "FrontDoorHealthProbeLog:DNSLatencyMicroseconds": frontDoorHealthProbeLogDNSLatencyMicroseconds, @@ -19,7 +19,7 @@ var conversions = map[string]ComplexConversion{ } // Splits the "TLS 1.2" value into "TLS" and "1.2" and sets as "network.protocol.name" and "network.protocol.version" -func azureCDNAccessLogSecurityProtocol(key string, value any, attrs map[string]any) bool { +func azureCdnAccessLogSecurityProtocol(key string, value any, attrs map[string]any) bool { if str, ok := value.(string); ok { if parts := strings.SplitN(str, " ", 2); len(parts) == 2 { attrs["tls.protocol.name"] = strings.ToLower(parts[0]) @@ -60,7 +60,7 @@ func frontDoorHealthProbeLogTotalLatencyMilliseconds(key string, value any, attr return false } seconds := milliseconds / 1_000 - attrs["http.client.request.duration"] = seconds + attrs["http.request.duration"] = seconds return true } diff --git a/pkg/translator/azure/complex_conversions_test.go b/pkg/translator/azure/logs_complex_conversions_test.go similarity index 97% rename from pkg/translator/azure/complex_conversions_test.go rename to pkg/translator/azure/logs_complex_conversions_test.go index fc2e4790ae6d..c968478ae21e 100644 --- a/pkg/translator/azure/complex_conversions_test.go +++ b/pkg/translator/azure/logs_complex_conversions_test.go @@ -66,7 +66,7 @@ func TestFrontDoorHealthProbeLogTotalLatencyMilliseconds(t *testing.T) { attrs := map[string]any{} ok = f("totalLatencyMilliseconds", 123, attrs) assert.True(t, ok) - duration, ok := attrs["http.client.request.duration"].(float64) + duration, ok := attrs["http.request.duration"].(float64) assert.True(t, ok) assert.Equal(t, 0.123, duration) } diff --git a/pkg/translator/azure/normalize.go b/pkg/translator/azure/logs_normalize.go similarity index 100% rename from pkg/translator/azure/normalize.go rename to pkg/translator/azure/logs_normalize.go diff --git a/pkg/translator/azure/property_names.go b/pkg/translator/azure/logs_property_names.go similarity index 99% rename from pkg/translator/azure/property_names.go rename to pkg/translator/azure/logs_property_names.go index cb8247cfea8b..d9593e509f3c 100644 --- a/pkg/translator/azure/property_names.go +++ b/pkg/translator/azure/logs_property_names.go @@ -66,8 +66,8 @@ var mappings = map[string]map[string]string{ "result": "", "httpStatusCode": "http.response.status_code", "probeURL": "url.full", - "originName": "server.address", - "originIP": "", + "originName": "", + "originIP": "server.address", "totalLatencyMilliseconds": "", // handled by complex_conversions "connectionLatencyMilliseconds": "", "DNSLatencyMicroseconds": "", // handled by complex_conversions diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index e281cc1c7dc4..8ef3436001d0 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -517,6 +517,21 @@ func TestFrontDoorAccessLog(t *testing.T) { assert.Equal(t, "NoError", record["error.type"]) } +func TestFrontDoorWAFLog(t *testing.T) { + logs, err := loadJsonLogs("log-frontdoorwaflog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, "test.net", record["server.address"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "23.23.23.23", record["network.peer.address"]) +} + func TestAppServiceAuditLog(t *testing.T) { logs, err := loadJsonLogs("log-appserviceauditlogs.json") @@ -563,3 +578,40 @@ func TestAppServicePlatformLog(t *testing.T) { assert.Equal(t, "CONTAINER_ID", record["container.id"]) assert.Equal(t, "CONTAINER_NAME", record["container.name"]) } + +func TestAzureCdnAccessLog(t *testing.T) { + logs, err := loadJsonLogs("log-azurecdnaccesslog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "NoError", record["error.type"]) +} + +func TestFrontDoorHealthProbeLog(t *testing.T) { + logs, err := loadJsonLogs("log-frontdoorhealthprobelog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "https://probe.net/health", record["url.full"]) + assert.Equal(t, "42.42.42.42", record["server.address"]) + assert.Equal(t, 0.042, record["http.request.duration"]) + assert.Equal(t, 0.00023, record["dns.lookup.duration"]) +} diff --git a/pkg/translator/azure/testdata/log-azurecdnaccesslog.json b/pkg/translator/azure/testdata/log-azurecdnaccesslog.json new file mode 100644 index 000000000000..2a905fb85e16 --- /dev/null +++ b/pkg/translator/azure/testdata/log-azurecdnaccesslog.json @@ -0,0 +1,34 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AzureCdnAccessLog", + "operationName": "Microsoft.AzureCdn/Profiles/AccessLog", + "properties": { + "BackendHostName": "backendhost.net", + "ClientIp": "42.42.42.42", + "ClientPort": "0", + "HttpMethod": "GET", + "HttpVersion": "1.1.0.0", + "HttpStatusCode": "200", + "HttpStatusDetails": "200", + "POP": "LON", + "RequestBytes": "1234", + "RequestUri": "https://test.net/", + "ResponseBytes": "12345", + "RoutingRuleName": "default-route", + "RulesEngineMatchNames": [], + "SecurityProtocol": "TLS 1.3", + "isReceivedFromClient": false, + "TimeTaken": "0.230", + "TrackingReference": "TRACKING_REFERENCE", + "UserAgent": "Mozilla/5.0", + "ErrorInfo": "NoError", + "TimeToFirstByte": "0.420", + "Result": "N/A", + "SNI": "originshield|parentcache|https|tier2" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure/testdata/log-frontdoorhealthprobelog.json b/pkg/translator/azure/testdata/log-frontdoorhealthprobelog.json new file mode 100644 index 000000000000..f0cf4bb0560e --- /dev/null +++ b/pkg/translator/azure/testdata/log-frontdoorhealthprobelog.json @@ -0,0 +1,22 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "FrontDoorHealthProbeLog", + "operationName": "WAF/FirewallLog", + "properties": { + "healthProbeId": "AAAA", + "POP": "", + "httpVerb": "GET", + "result": "", + "httpStatusCode": "200", + "probeURL": "https://probe.net/health", + "originName": "https://probe.net/", + "originIP": "42.42.42.42", + "totalLatencyMilliseconds": "42", + "DNSLatencyMicroseconds": "230" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure/testdata/log-frontdoorwaflog.json b/pkg/translator/azure/testdata/log-frontdoorwaflog.json new file mode 100644 index 000000000000..6a894fe955a7 --- /dev/null +++ b/pkg/translator/azure/testdata/log-frontdoorwaflog.json @@ -0,0 +1,18 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "FrontdoorWebApplicationFirewallLog", + "operationName": "WAF/FirewallLog", + "properties": { + "trackingReference": "TRACKING_REFERENCE", + "clientIP": "42.42.42.42", + "clientPort": "0", + "socketIP": "23.23.23.23", + "requestUri": "https://test.net/", + "host": "test.net" + } + } + ] +} \ No newline at end of file From fc758fca71d5493920042471482b9c87ed552789 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Tue, 30 Apr 2024 13:33:49 +0100 Subject: [PATCH 13/17] Finish adding tests --- pkg/translator/azure/logs_property_names.go | 108 +++++++++--------- .../azure/resourcelogs_to_logs_test.go | 99 +++++++++++----- .../azure/testdata/log-appserviceapplogs.json | 18 +++ .../testdata/log-appserviceconsolelogs.json | 14 +++ .../log-appserviceipsecauditlogs.json | 18 +++ 5 files changed, 174 insertions(+), 83 deletions(-) create mode 100644 pkg/translator/azure/testdata/log-appserviceapplogs.json create mode 100644 pkg/translator/azure/testdata/log-appserviceconsolelogs.json create mode 100644 pkg/translator/azure/testdata/log-appserviceipsecauditlogs.json diff --git a/pkg/translator/azure/logs_property_names.go b/pkg/translator/azure/logs_property_names.go index d9593e509f3c..dbd4c079b297 100644 --- a/pkg/translator/azure/logs_property_names.go +++ b/pkg/translator/azure/logs_property_names.go @@ -85,29 +85,29 @@ var mappings = map[string]map[string]string{ "policyMode": "", }, "AppServiceAppLogs": { - "_BilledSize": "", //real The record size in bytes - "Category": "", //string Log category name - "ContainerId": "", //string Application container id - "CustomLevel": "", //string Verbosity level of log - "ExceptionClass": "", //string Application class from where log message is emitted - "Host": "", //string Host where the application is running - "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "Level": "", //string Verbosity level of log mapped to standard levels (Informational, Warning, Error, or Critical) - "Logger": "", //string Application logger used to emit log message - "Message": "", //string Log message - "Method": "", //string Application Method from where log message is emitted - "OperationName": "", //string The name of the operation represented by this event. - "_ResourceId": "", //string A unique identifier for the resource that the record is associated with - "ResultDescription": "", //string Log message description - "Source": "", //string Application source from where log message is emitted - "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "Stacktrace": "", //string Complete stack trace of the log message in case of exception - "StackTrace": "", //string Complete stack trace of the log message in case of exception - "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with - "TenantId": "", //string The Log Analytics workspace ID - "TimeGenerated": "", //datetime Time when event is generated - "Type": "", //string The name of the table - "WebSiteInstanceId": "", //string Instance ID of the application running + "_BilledSize": "", //real The record size in bytes + "Category": "", //string Log category name + "ContainerId": "container.id", //string Application container id + "CustomLevel": "", //string Verbosity level of log + "ExceptionClass": "exception.type", //string Application class from where log message is emitted + "Host": "host.id", //string Host where the application is running + "_IsBillable": "", //string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", //string Verbosity level of log mapped to standard levels (Informational, Warning, Error, or Critical) + "Logger": "", //string Application logger used to emit log message + "Message": "", //string Log message + "Method": "code.function", //string Application Method from where log message is emitted + "OperationName": "", //string The name of the operation represented by this event. + "_ResourceId": "", //string A unique identifier for the resource that the record is associated with + "ResultDescription": "", //string Log message description + "Source": "code.filepath", //string Application source from where log message is emitted + "SourceSystem": "", //string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "Stacktrace": "exception.stacktrace", //string Complete stack trace of the log message in case of exception + "StackTrace": "exception.stacktrace", //string Complete stack trace of the log message in case of exception + "_SubscriptionId": "", //string A unique identifier for the subscription that the record is associated with + "TenantId": "", //string The Log Analytics workspace ID + "TimeGenerated": "", //datetime Time when event is generated + "Type": "", //string The name of the table + "WebSiteInstanceId": "", //string Instance ID of the application running }, "AppServiceAuditLogs": { "_BilledSize": "", //real The record size in bytes @@ -147,20 +147,20 @@ var mappings = map[string]map[string]string{ "Type": "", //string The name of the table }, "AppServiceConsoleLogs": { - "_BilledSize": "", // real The record size in bytes - "Category": "", // string Log category name - "ContainerId": "", // string Application container id - "Host": "", // string Host where the application is running - "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "Level": "", // string Verbosity level of log - "OperationName": "", // string The name of the operation represented by this event. - "_ResourceId": "", // string A unique identifier for the resource that the record is associated with - "ResultDescription": "", // string Log message description - "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with - "TenantId": "", // string The Log Analytics workspace ID - "TimeGenerated": "", // datetime Time when event is generated - "Type": "", // string The name of the table + "_BilledSize": "", // real The record size in bytes + "Category": "", // string Log category name + "ContainerId": "container.id", // string Application container id + "Host": "host.id", // string Host where the application is running + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "Level": "", // string Verbosity level of log + "OperationName": "", // string The name of the operation represented by this event. + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "ResultDescription": "", // string Log message description + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time when event is generated + "Type": "", // string The name of the table }, "AppServiceEnvironmentPlatformLogs": { "_BilledSize": "", // real The record size in bytes @@ -220,23 +220,23 @@ var mappings = map[string]map[string]string{ "UserAgent": "user_agent.original", //string User agent on HTTP request }, "AppServiceIPSecAuditLogs": { - "_BilledSize": "", // real The record size in bytes - "CIp": "client.address", // string IP address of the client - "CsHost": "url.domain", // string Host header of the HTTP request - "Details": "", // string Additional information - "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account - "_ResourceId": "", // string A unique identifier for the resource that the record is associated with - "Result": "", // string The result whether the access is Allowed or Denied - "ServiceEndpoint": "", // string This indicates whether the access is via Virtual Network Service Endpoint communication - "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics - "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with - "TenantId": "", // string The Log Analytics workspace ID - "TimeGenerated": "", // datetime Time of the Http Request - "Type": "", // string The name of the table - "XAzureFDID": "", // string X-Azure-FDID header (Azure Frontdoor ID) of the HTTP request - "XFDHealthProbe": "", // string X-FD-HealthProbe (Azure Frontdoor Health Probe) of the HTTP request - "XForwardedFor": "", // string X-Forwarded-For header of the HTTP request - "XForwardedHost": "", // string X-Forwarded-Host header of the HTTP request + "_BilledSize": "", // real The record size in bytes + "CIp": "client.address", // string IP address of the client + "CsHost": "url.domain", // string Host header of the HTTP request + "Details": "", // string Additional information + "_IsBillable": "", // string Specifies whether ingesting the data is billable. When _IsBillable is false ingestion isn't billed to your Azure account + "_ResourceId": "", // string A unique identifier for the resource that the record is associated with + "Result": "", // string The result whether the access is Allowed or Denied + "ServiceEndpoint": "", // string This indicates whether the access is via Virtual Network Service Endpoint communication + "SourceSystem": "", // string The type of agent the event was collected by. For example, OpsManager for Windows agent, either direct connect or Operations Manager, Linux for all Linux agents, or Azure for Azure Diagnostics + "_SubscriptionId": "", // string A unique identifier for the subscription that the record is associated with + "TenantId": "", // string The Log Analytics workspace ID + "TimeGenerated": "", // datetime Time of the Http Request + "Type": "", // string The name of the table + "XAzureFDID": "http.request.header.x-azure-fdid", // string X-Azure-FDID header (Azure Frontdoor ID) of the HTTP request + "XFDHealthProbe": "http.request.header.x-fd-healthprobe", // string X-FD-HealthProbe (Azure Frontdoor Health Probe) of the HTTP request + "XForwardedFor": "http.request.header.x-forwarded-for", // string X-Forwarded-For header of the HTTP request + "XForwardedHost": "http.request.header.x-forwarded-host", // string X-Forwarded-Host header of the HTTP request }, "AppServicePlatformLogs": { "ActivityId": "", // string Activity ID to correlate events diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index 8ef3436001d0..5d3e72112dc2 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -489,6 +489,28 @@ func loadJsonLogs(filename string) (plog.Logs, error) { return logs, nil } +func TestAzureCdnAccessLog(t *testing.T) { + logs, err := loadJsonLogs("log-azurecdnaccesslog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "NoError", record["error.type"]) +} + func TestFrontDoorAccessLog(t *testing.T) { logs, err := loadJsonLogs("log-frontdooraccesslog.json") @@ -517,6 +539,21 @@ func TestFrontDoorAccessLog(t *testing.T) { assert.Equal(t, "NoError", record["error.type"]) } +func TestFrontDoorHealthProbeLog(t *testing.T) { + logs, err := loadJsonLogs("log-frontdoorhealthprobelog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "https://probe.net/health", record["url.full"]) + assert.Equal(t, "42.42.42.42", record["server.address"]) + assert.Equal(t, 0.042, record["http.request.duration"]) + assert.Equal(t, 0.00023, record["dns.lookup.duration"]) +} + func TestFrontDoorWAFLog(t *testing.T) { logs, err := loadJsonLogs("log-frontdoorwaflog.json") @@ -532,6 +569,32 @@ func TestFrontDoorWAFLog(t *testing.T) { assert.Equal(t, "23.23.23.23", record["network.peer.address"]) } +func TestAppServiceAppLog(t *testing.T) { + logs, err := loadJsonLogs("log-appserviceapplogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "EXCEPTION_CLASS", record["exception.type"]) + assert.Equal(t, "HOST", record["host.id"]) + assert.Equal(t, "METHOD", record["code.function"]) + assert.Equal(t, "FILEPATH", record["code.filepath"]) + assert.Equal(t, "STACKTRACE", record["exception.stacktrace"]) +} + +func TestAppServiceConsoleLog(t *testing.T) { + logs, err := loadJsonLogs("log-appserviceconsolelogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "HOST", record["host.id"]) +} + func TestAppServiceAuditLog(t *testing.T) { logs, err := loadJsonLogs("log-appserviceauditlogs.json") @@ -579,39 +642,17 @@ func TestAppServicePlatformLog(t *testing.T) { assert.Equal(t, "CONTAINER_NAME", record["container.name"]) } -func TestAzureCdnAccessLog(t *testing.T) { - logs, err := loadJsonLogs("log-azurecdnaccesslog.json") +func TestAppServiceIPSecAuditLog(t *testing.T) { + logs, err := loadJsonLogs("log-appserviceipsecauditlogs.json") assert.NoError(t, err) record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - assert.Equal(t, "GET", record["http.request.method"]) - assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) - assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) - assert.Equal(t, "https://test.net/", record["url.full"]) - assert.Equal(t, int64(1234), record["http.request.size"]) - assert.Equal(t, int64(12345), record["http.response.size"]) - assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, "0", record["client.port"]) - assert.Equal(t, "tls", record["tls.protocol.name"]) - assert.Equal(t, "1.3", record["tls.protocol.version"]) - assert.Equal(t, int64(200), record["http.response.status_code"]) - assert.Equal(t, "NoError", record["error.type"]) -} - -func TestFrontDoorHealthProbeLog(t *testing.T) { - logs, err := loadJsonLogs("log-frontdoorhealthprobelog.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "GET", record["http.request.method"]) - assert.Equal(t, int64(200), record["http.response.status_code"]) - assert.Equal(t, "https://probe.net/health", record["url.full"]) - assert.Equal(t, "42.42.42.42", record["server.address"]) - assert.Equal(t, 0.042, record["http.request.duration"]) - assert.Equal(t, 0.00023, record["dns.lookup.duration"]) + assert.Equal(t, "HOST", record["url.domain"]) + assert.Equal(t, "FDID", record["http.request.header.x-azure-fdid"]) + assert.Equal(t, "HEALTH_PROBE", record["http.request.header.x-fd-healthprobe"]) + assert.Equal(t, "FORWARDED_FOR", record["http.request.header.x-forwarded-for"]) + assert.Equal(t, "FORWARDED_HOST", record["http.request.header.x-forwarded-host"]) } diff --git a/pkg/translator/azure/testdata/log-appserviceapplogs.json b/pkg/translator/azure/testdata/log-appserviceapplogs.json new file mode 100644 index 000000000000..202a755a2286 --- /dev/null +++ b/pkg/translator/azure/testdata/log-appserviceapplogs.json @@ -0,0 +1,18 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AppServiceAppLogs", + "operationName": "AppLog", + "properties": { + "ContainerId": "CONTAINER_ID", + "ExceptionClass": "EXCEPTION_CLASS", + "Host": "HOST", + "Method": "METHOD", + "Source": "FILEPATH", + "Stacktrace": "STACKTRACE" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure/testdata/log-appserviceconsolelogs.json b/pkg/translator/azure/testdata/log-appserviceconsolelogs.json new file mode 100644 index 000000000000..8158b619f8a8 --- /dev/null +++ b/pkg/translator/azure/testdata/log-appserviceconsolelogs.json @@ -0,0 +1,14 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AppServiceConsoleLogs", + "operationName": "ConsoleLog", + "properties": { + "ContainerId": "CONTAINER_ID", + "Host": "HOST" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure/testdata/log-appserviceipsecauditlogs.json b/pkg/translator/azure/testdata/log-appserviceipsecauditlogs.json new file mode 100644 index 000000000000..207d81ff629d --- /dev/null +++ b/pkg/translator/azure/testdata/log-appserviceipsecauditlogs.json @@ -0,0 +1,18 @@ +{ + "records": [ + { + "time": "2024-04-24T12:06:12.0000000Z", + "resourceId": "/RESOURCE_ID", + "category": "AppServiceIPSecAuditLogs", + "operationName": "IPSecAuditLog", + "properties": { + "CIp": "42.42.42.42", + "CsHost": "HOST", + "XAzureFDID": "FDID", + "XFDHealthProbe": "HEALTH_PROBE", + "XForwardedFor": "FORWARDED_FOR", + "XForwardedHost": "FORWARDED_HOST" + } + } + ] +} \ No newline at end of file From 77b4024850821cecd9baf3970914cb862daac223 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Tue, 30 Apr 2024 13:48:12 +0100 Subject: [PATCH 14/17] Add chloggen entry --- .../32486-apply-semantic-conventions.yaml | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/32486-apply-semantic-conventions.yaml diff --git a/.chloggen/32486-apply-semantic-conventions.yaml b/.chloggen/32486-apply-semantic-conventions.yaml new file mode 100644 index 000000000000..3cdf0e1fce40 --- /dev/null +++ b/.chloggen/32486-apply-semantic-conventions.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: azureeventhubreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "Map Azure Resource Log property names to the Semantic Conventions equivalent" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [32764] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: '[user]' \ No newline at end of file From a02352f17a393a2eaa82fc27cb9b3f58de15abdb Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Mon, 6 May 2024 12:56:41 +0100 Subject: [PATCH 15/17] Add feature flag for applying semantic conventions --- pkg/translator/azure/resourcelogs_to_logs.go | 21 ++++++++++---- .../azure/resourcelogs_to_logs_test.go | 29 ++++++++++--------- .../azureresourcelogs_unmarshaler.go | 7 +++-- receiver/azureeventhubreceiver/config.go | 13 +++++---- receiver/azureeventhubreceiver/config_test.go | 3 ++ receiver/azureeventhubreceiver/factory.go | 2 +- receiver/azureeventhubreceiver/go.mod | 4 +-- receiver/azureeventhubreceiver/go.sum | 2 ++ .../testdata/config.yaml | 1 + .../azureresourcelogs_unmarshaler.go | 5 ++-- receiver/kafkareceiver/go.mod | 2 +- receiver/kafkareceiver/go.sum | 1 + 12 files changed, 55 insertions(+), 35 deletions(-) diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index 77f8ae01a5c9..4d5358233945 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -73,8 +73,9 @@ type azureLogRecord struct { var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil) type ResourceLogsUnmarshaler struct { - Version string - Logger *zap.Logger + Version string + Logger *zap.Logger + ApplySemanticConventions bool } func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { @@ -122,7 +123,7 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { lr.SetSeverityText(log.Level.String()) } - if err := lr.Attributes().FromRaw(extractRawAttributes(log)); err != nil { + if err := lr.Attributes().FromRaw(extractRawAttributes(log, r.ApplySemanticConventions)); err != nil { return l, err } } @@ -176,7 +177,7 @@ func asSeverity(number json.Number) plog.SeverityNumber { } } -func extractRawAttributes(log azureLogRecord) map[string]any { +func extractRawAttributes(log azureLogRecord, applySemanticConventions bool) map[string]any { var attrs = map[string]any{} attrs[azureCategory] = log.Category @@ -194,7 +195,11 @@ func extractRawAttributes(log azureLogRecord) map[string]any { setIf(attrs, azureOperationVersion, log.OperationVersion) if log.Properties != nil { - copyProperties(log.Category, log.Properties, attrs) + if applySemanticConventions { + copyPropertiesAndApplySemanticConventions(log.Category, log.Properties, attrs) + } else { + attrs[azureProperties] = *log.Properties + } } setIf(attrs, azureResultDescription, log.ResultDescription) @@ -209,9 +214,11 @@ func extractRawAttributes(log azureLogRecord) map[string]any { return attrs } -func copyProperties(category string, properties *any, attrs map[string]any) { +func copyPropertiesAndApplySemanticConventions(category string, properties *any, attrs map[string]any) { + pmap := (*properties).(map[string]any) attrsProps := map[string]any{} + for k, v := range pmap { // Check for a complex conversion, e.g. AppServiceHTTPLogs.Protocol if complexConversion, ok := tryGetComplexConversion(category, k); ok { @@ -219,12 +226,14 @@ func copyProperties(category string, properties *any, attrs map[string]any) { continue } } + // Check for an equivalent Semantic Convention key if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { attrs[otelKey] = normalizeValue(otelKey, v) } else { attrsProps[k] = v } } + if len(attrsProps) > 0 { attrs[azureProperties] = attrsProps } diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index 5d3e72112dc2..40788d9da296 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -358,7 +358,7 @@ func TestExtractRawAttributes(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, extractRawAttributes(tt.log)) + assert.Equal(t, tt.expected, extractRawAttributes(tt.log, false)) }) } @@ -467,12 +467,13 @@ func TestUnmarshalLogs(t *testing.T) { } } -func loadJsonLogs(filename string) (plog.Logs, error) { +func loadJsonLogsAndApplySemanticConventions(filename string) (plog.Logs, error) { l := plog.NewLogs() sut := &ResourceLogsUnmarshaler{ - Version: testBuildInfo.Version, - Logger: zap.NewNop(), + Version: testBuildInfo.Version, + Logger: zap.NewNop(), + ApplySemanticConventions: true, } data, err := os.ReadFile(filepath.Join("testdata", filename)) @@ -490,7 +491,7 @@ func loadJsonLogs(filename string) (plog.Logs, error) { } func TestAzureCdnAccessLog(t *testing.T) { - logs, err := loadJsonLogs("log-azurecdnaccesslog.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-azurecdnaccesslog.json") assert.NoError(t, err) @@ -512,7 +513,7 @@ func TestAzureCdnAccessLog(t *testing.T) { } func TestFrontDoorAccessLog(t *testing.T) { - logs, err := loadJsonLogs("log-frontdooraccesslog.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdooraccesslog.json") assert.NoError(t, err) @@ -540,7 +541,7 @@ func TestFrontDoorAccessLog(t *testing.T) { } func TestFrontDoorHealthProbeLog(t *testing.T) { - logs, err := loadJsonLogs("log-frontdoorhealthprobelog.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorhealthprobelog.json") assert.NoError(t, err) @@ -555,7 +556,7 @@ func TestFrontDoorHealthProbeLog(t *testing.T) { } func TestFrontDoorWAFLog(t *testing.T) { - logs, err := loadJsonLogs("log-frontdoorwaflog.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorwaflog.json") assert.NoError(t, err) @@ -570,7 +571,7 @@ func TestFrontDoorWAFLog(t *testing.T) { } func TestAppServiceAppLog(t *testing.T) { - logs, err := loadJsonLogs("log-appserviceapplogs.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceapplogs.json") assert.NoError(t, err) @@ -585,7 +586,7 @@ func TestAppServiceAppLog(t *testing.T) { } func TestAppServiceConsoleLog(t *testing.T) { - logs, err := loadJsonLogs("log-appserviceconsolelogs.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceconsolelogs.json") assert.NoError(t, err) @@ -596,7 +597,7 @@ func TestAppServiceConsoleLog(t *testing.T) { } func TestAppServiceAuditLog(t *testing.T) { - logs, err := loadJsonLogs("log-appserviceauditlogs.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceauditlogs.json") assert.NoError(t, err) @@ -608,7 +609,7 @@ func TestAppServiceAuditLog(t *testing.T) { } func TestAppServiceHTTPLog(t *testing.T) { - logs, err := loadJsonLogs("log-appservicehttplogs.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-appservicehttplogs.json") assert.NoError(t, err) @@ -632,7 +633,7 @@ func TestAppServiceHTTPLog(t *testing.T) { } func TestAppServicePlatformLog(t *testing.T) { - logs, err := loadJsonLogs("log-appserviceplatformlogs.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceplatformlogs.json") assert.NoError(t, err) @@ -643,7 +644,7 @@ func TestAppServicePlatformLog(t *testing.T) { } func TestAppServiceIPSecAuditLog(t *testing.T) { - logs, err := loadJsonLogs("log-appserviceipsecauditlogs.json") + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceipsecauditlogs.json") assert.NoError(t, err) diff --git a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go index 1665c9f2d833..0ad2b398ff7e 100644 --- a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go +++ b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go @@ -16,12 +16,13 @@ type AzureResourceLogsEventUnmarshaler struct { unmarshaler *azure.ResourceLogsUnmarshaler } -func newAzureResourceLogsUnmarshaler(buildInfo component.BuildInfo, logger *zap.Logger) eventLogsUnmarshaler { +func newAzureResourceLogsUnmarshaler(buildInfo component.BuildInfo, logger *zap.Logger, applySemanticConventions bool) eventLogsUnmarshaler { return AzureResourceLogsEventUnmarshaler{ unmarshaler: &azure.ResourceLogsUnmarshaler{ - Version: buildInfo.Version, - Logger: logger, + Version: buildInfo.Version, + Logger: logger, + ApplySemanticConventions: applySemanticConventions, }, } } diff --git a/receiver/azureeventhubreceiver/config.go b/receiver/azureeventhubreceiver/config.go index 9ccb99205fc8..e5e01c98cf87 100644 --- a/receiver/azureeventhubreceiver/config.go +++ b/receiver/azureeventhubreceiver/config.go @@ -25,12 +25,13 @@ var ( ) type Config struct { - Connection string `mapstructure:"connection"` - Partition string `mapstructure:"partition"` - Offset string `mapstructure:"offset"` - StorageID *component.ID `mapstructure:"storage"` - Format string `mapstructure:"format"` - ConsumerGroup string `mapstructure:"group"` + Connection string `mapstructure:"connection"` + Partition string `mapstructure:"partition"` + Offset string `mapstructure:"offset"` + StorageID *component.ID `mapstructure:"storage"` + Format string `mapstructure:"format"` + ConsumerGroup string `mapstructure:"group"` + ApplySemanticConventions bool `mapstructure:"apply_semantic_conventions"` } func isValidFormat(format string) bool { diff --git a/receiver/azureeventhubreceiver/config_test.go b/receiver/azureeventhubreceiver/config_test.go index 7d4dcf4c546f..36497f0d8e13 100644 --- a/receiver/azureeventhubreceiver/config_test.go +++ b/receiver/azureeventhubreceiver/config_test.go @@ -33,12 +33,15 @@ func TestLoadConfig(t *testing.T) { assert.Equal(t, "", r0.(*Config).Offset) assert.Equal(t, "", r0.(*Config).Partition) assert.Equal(t, defaultLogFormat, logFormat(r0.(*Config).Format)) + assert.False(t, r0.(*Config).ApplySemanticConventions) r1 := cfg.Receivers[component.NewIDWithName(metadata.Type, "all")] assert.Equal(t, "Endpoint=sb://namespace.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=superSecret1234=;EntityPath=hubName", r1.(*Config).Connection) assert.Equal(t, "1234-5566", r1.(*Config).Offset) assert.Equal(t, "foo", r1.(*Config).Partition) assert.Equal(t, rawLogFormat, logFormat(r1.(*Config).Format)) + assert.True(t, r1.(*Config).ApplySemanticConventions) + } func TestMissingConnection(t *testing.T) { diff --git a/receiver/azureeventhubreceiver/factory.go b/receiver/azureeventhubreceiver/factory.go index a9d0bfea544d..aa09c9ed1f53 100644 --- a/receiver/azureeventhubreceiver/factory.go +++ b/receiver/azureeventhubreceiver/factory.go @@ -100,7 +100,7 @@ func (f *eventhubReceiverFactory) getReceiver( if logFormat(receiverConfig.Format) == rawLogFormat { logsUnmarshaler = newRawLogsUnmarshaler(settings.Logger) } else { - logsUnmarshaler = newAzureResourceLogsUnmarshaler(settings.BuildInfo, settings.Logger) + logsUnmarshaler = newAzureResourceLogsUnmarshaler(settings.BuildInfo, settings.Logger, receiverConfig.ApplySemanticConventions) } case component.DataTypeMetrics: if logFormat(receiverConfig.Format) == rawLogFormat { diff --git a/receiver/azureeventhubreceiver/go.mod b/receiver/azureeventhubreceiver/go.mod index f6f4ea357adb..fceb1fc47167 100644 --- a/receiver/azureeventhubreceiver/go.mod +++ b/receiver/azureeventhubreceiver/go.mod @@ -8,7 +8,7 @@ require ( github.com/json-iterator/go v1.1.12 github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.97.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.97.0 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.97.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.99.0 github.com/relvacode/iso8601 v1.4.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.97.1-0.20240409140257-792fac1b62d4 @@ -112,7 +112,7 @@ require ( go.opentelemetry.io/proto/otlp v1.1.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/receiver/azureeventhubreceiver/go.sum b/receiver/azureeventhubreceiver/go.sum index 45aa127396e6..bd71a8048c38 100644 --- a/receiver/azureeventhubreceiver/go.sum +++ b/receiver/azureeventhubreceiver/go.sum @@ -305,6 +305,8 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOM golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= diff --git a/receiver/azureeventhubreceiver/testdata/config.yaml b/receiver/azureeventhubreceiver/testdata/config.yaml index 4afddc4c7175..091b2cbe0a48 100644 --- a/receiver/azureeventhubreceiver/testdata/config.yaml +++ b/receiver/azureeventhubreceiver/testdata/config.yaml @@ -7,6 +7,7 @@ receivers: partition: foo offset: "1234-5566" format: "raw" + apply_semantic_conventions: true processors: nop: diff --git a/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go b/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go index b042bc052390..029424e8c92a 100644 --- a/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go +++ b/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go @@ -17,8 +17,9 @@ type azureResourceLogsUnmarshaler struct { func newAzureResourceLogsUnmarshaler(version string, logger *zap.Logger) LogsUnmarshaler { return azureResourceLogsUnmarshaler{ unmarshaler: &azure.ResourceLogsUnmarshaler{ - Version: version, - Logger: logger, + Version: version, + Logger: logger, + ApplySemanticConventions: false, }, } } diff --git a/receiver/kafkareceiver/go.mod b/receiver/kafkareceiver/go.mod index 6ad31dfb951c..c02fbf56e570 100644 --- a/receiver/kafkareceiver/go.mod +++ b/receiver/kafkareceiver/go.mod @@ -88,7 +88,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.23.0 // indirect golang.org/x/sys v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect diff --git a/receiver/kafkareceiver/go.sum b/receiver/kafkareceiver/go.sum index 4e9a9f32cd6d..f13637e00e34 100644 --- a/receiver/kafkareceiver/go.sum +++ b/receiver/kafkareceiver/go.sum @@ -226,6 +226,7 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOM golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= From 623574bca180021e1d27cb579756456e442cf285 Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Mon, 6 May 2024 14:42:43 +0100 Subject: [PATCH 16/17] Refactor translator into azure_logs with SemConv, azure without --- pkg/translator/azure/go.mod | 2 +- pkg/translator/azure/go.sum | 2 - pkg/translator/azure/resourcelogs_to_logs.go | 42 +- .../azure/resourcelogs_to_logs_test.go | 193 +---- pkg/translator/azure_logs/Makefile | 1 + .../complex_conversions.go} | 2 +- .../complex_conversions_test.go} | 2 +- pkg/translator/azure_logs/go.mod | 53 ++ pkg/translator/azure_logs/go.sum | 85 +++ pkg/translator/azure_logs/metadata.yaml | 3 + .../normalize.go} | 2 +- pkg/translator/azure_logs/package_test.go | 14 + .../property_names.go} | 2 +- .../azure_logs/resourcelogs_to_logs.go | 241 +++++++ .../azure_logs/resourcelogs_to_logs_test.go | 658 ++++++++++++++++++ .../testdata/log-appserviceapplogs.json | 0 .../testdata/log-appserviceauditlogs.json | 0 .../testdata/log-appserviceconsolelogs.json | 0 .../testdata/log-appservicehttplogs.json | 0 .../log-appserviceipsecauditlogs.json | 0 .../testdata/log-appserviceplatformlogs.json | 0 .../testdata/log-azurecdnaccesslog.json | 0 .../azure_logs/testdata/log-bad-level.json | 39 ++ .../azure_logs/testdata/log-bad-time.json | 45 ++ .../testdata/log-frontdooraccesslog.json | 0 .../testdata/log-frontdoorhealthprobelog.json | 0 .../testdata/log-frontdoorwaflog.json | 0 .../azure_logs/testdata/log-maximum.json | 85 +++ .../azure_logs/testdata/log-minimum-2.json | 16 + .../azure_logs/testdata/log-minimum.json | 10 + .../azureresourcelogs_unmarshaler.go | 27 +- receiver/azureeventhubreceiver/go.mod | 5 +- receiver/azureeventhubreceiver/go.sum | 2 - versions.yaml | 1 + 34 files changed, 1286 insertions(+), 246 deletions(-) create mode 100644 pkg/translator/azure_logs/Makefile rename pkg/translator/{azure/logs_complex_conversions.go => azure_logs/complex_conversions.go} (99%) rename pkg/translator/{azure/logs_complex_conversions_test.go => azure_logs/complex_conversions_test.go} (99%) create mode 100644 pkg/translator/azure_logs/go.mod create mode 100644 pkg/translator/azure_logs/go.sum create mode 100644 pkg/translator/azure_logs/metadata.yaml rename pkg/translator/{azure/logs_normalize.go => azure_logs/normalize.go} (98%) create mode 100644 pkg/translator/azure_logs/package_test.go rename pkg/translator/{azure/logs_property_names.go => azure_logs/property_names.go} (99%) create mode 100644 pkg/translator/azure_logs/resourcelogs_to_logs.go create mode 100644 pkg/translator/azure_logs/resourcelogs_to_logs_test.go rename pkg/translator/{azure => azure_logs}/testdata/log-appserviceapplogs.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-appserviceauditlogs.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-appserviceconsolelogs.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-appservicehttplogs.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-appserviceipsecauditlogs.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-appserviceplatformlogs.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-azurecdnaccesslog.json (100%) create mode 100644 pkg/translator/azure_logs/testdata/log-bad-level.json create mode 100644 pkg/translator/azure_logs/testdata/log-bad-time.json rename pkg/translator/{azure => azure_logs}/testdata/log-frontdooraccesslog.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-frontdoorhealthprobelog.json (100%) rename pkg/translator/{azure => azure_logs}/testdata/log-frontdoorwaflog.json (100%) create mode 100644 pkg/translator/azure_logs/testdata/log-maximum.json create mode 100644 pkg/translator/azure_logs/testdata/log-minimum-2.json create mode 100644 pkg/translator/azure_logs/testdata/log-minimum.json diff --git a/pkg/translator/azure/go.mod b/pkg/translator/azure/go.mod index dc1823fb6ca3..0767961eb53c 100644 --- a/pkg/translator/azure/go.mod +++ b/pkg/translator/azure/go.mod @@ -12,7 +12,7 @@ require ( go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac go.uber.org/goleak v1.3.0 go.uber.org/zap v1.27.0 - golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f + golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc ) require ( diff --git a/pkg/translator/azure/go.sum b/pkg/translator/azure/go.sum index fdbba94e7af3..7583052f14ab 100644 --- a/pkg/translator/azure/go.sum +++ b/pkg/translator/azure/go.sum @@ -76,8 +76,6 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= -golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= diff --git a/pkg/translator/azure/resourcelogs_to_logs.go b/pkg/translator/azure/resourcelogs_to_logs.go index 4d5358233945..b2c96d28f568 100644 --- a/pkg/translator/azure/resourcelogs_to_logs.go +++ b/pkg/translator/azure/resourcelogs_to_logs.go @@ -73,9 +73,8 @@ type azureLogRecord struct { var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil) type ResourceLogsUnmarshaler struct { - Version string - Logger *zap.Logger - ApplySemanticConventions bool + Version string + Logger *zap.Logger } func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { @@ -123,7 +122,7 @@ func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { lr.SetSeverityText(log.Level.String()) } - if err := lr.Attributes().FromRaw(extractRawAttributes(log, r.ApplySemanticConventions)); err != nil { + if err := lr.Attributes().FromRaw(extractRawAttributes(log)); err != nil { return l, err } } @@ -177,7 +176,7 @@ func asSeverity(number json.Number) plog.SeverityNumber { } } -func extractRawAttributes(log azureLogRecord, applySemanticConventions bool) map[string]any { +func extractRawAttributes(log azureLogRecord) map[string]any { var attrs = map[string]any{} attrs[azureCategory] = log.Category @@ -193,15 +192,9 @@ func extractRawAttributes(log azureLogRecord, applySemanticConventions bool) map } attrs[azureOperationName] = log.OperationName setIf(attrs, azureOperationVersion, log.OperationVersion) - if log.Properties != nil { - if applySemanticConventions { - copyPropertiesAndApplySemanticConventions(log.Category, log.Properties, attrs) - } else { - attrs[azureProperties] = *log.Properties - } + attrs[azureProperties] = *log.Properties } - setIf(attrs, azureResultDescription, log.ResultDescription) setIf(attrs, azureResultSignature, log.ResultSignature) setIf(attrs, azureResultType, log.ResultType) @@ -214,31 +207,6 @@ func extractRawAttributes(log azureLogRecord, applySemanticConventions bool) map return attrs } -func copyPropertiesAndApplySemanticConventions(category string, properties *any, attrs map[string]any) { - - pmap := (*properties).(map[string]any) - attrsProps := map[string]any{} - - for k, v := range pmap { - // Check for a complex conversion, e.g. AppServiceHTTPLogs.Protocol - if complexConversion, ok := tryGetComplexConversion(category, k); ok { - if complexConversion(k, v, attrs) { - continue - } - } - // Check for an equivalent Semantic Convention key - if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { - attrs[otelKey] = normalizeValue(otelKey, v) - } else { - attrsProps[k] = v - } - } - - if len(attrsProps) > 0 { - attrs[azureProperties] = attrsProps - } -} - func setIf(attrs map[string]any, key string, value *string) { if value != nil && *value != "" { attrs[key] = *value diff --git a/pkg/translator/azure/resourcelogs_to_logs_test.go b/pkg/translator/azure/resourcelogs_to_logs_test.go index 40788d9da296..37bb67df9987 100644 --- a/pkg/translator/azure/resourcelogs_to_logs_test.go +++ b/pkg/translator/azure/resourcelogs_to_logs_test.go @@ -358,7 +358,7 @@ func TestExtractRawAttributes(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.expected, extractRawAttributes(tt.log, false)) + assert.Equal(t, tt.expected, extractRawAttributes(tt.log)) }) } @@ -466,194 +466,3 @@ func TestUnmarshalLogs(t *testing.T) { }) } } - -func loadJsonLogsAndApplySemanticConventions(filename string) (plog.Logs, error) { - l := plog.NewLogs() - - sut := &ResourceLogsUnmarshaler{ - Version: testBuildInfo.Version, - Logger: zap.NewNop(), - ApplySemanticConventions: true, - } - - data, err := os.ReadFile(filepath.Join("testdata", filename)) - if err != nil { - return l, err - } - - logs, err := sut.UnmarshalLogs(data) - - if err != nil { - return l, err - } - - return logs, nil -} - -func TestAzureCdnAccessLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-azurecdnaccesslog.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "GET", record["http.request.method"]) - assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) - assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) - assert.Equal(t, "https://test.net/", record["url.full"]) - assert.Equal(t, int64(1234), record["http.request.size"]) - assert.Equal(t, int64(12345), record["http.response.size"]) - assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) - assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, "0", record["client.port"]) - assert.Equal(t, "tls", record["tls.protocol.name"]) - assert.Equal(t, "1.3", record["tls.protocol.version"]) - assert.Equal(t, int64(200), record["http.response.status_code"]) - assert.Equal(t, "NoError", record["error.type"]) -} - -func TestFrontDoorAccessLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdooraccesslog.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "GET", record["http.request.method"]) - assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) - assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) - assert.Equal(t, "https://test.net/", record["url.full"]) - assert.Equal(t, int64(1234), record["http.request.size"]) - assert.Equal(t, int64(12345), record["http.response.size"]) - assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) - assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, "0", record["client.port"]) - assert.Equal(t, "23.23.23.23", record["network.peer.address"]) - assert.Equal(t, float64(0.23), record["http.server.request.duration"]) - assert.Equal(t, "https", record["network.protocol.name"]) - assert.Equal(t, "tls", record["tls.protocol.name"]) - assert.Equal(t, "1.3", record["tls.protocol.version"]) - assert.Equal(t, "TLS_AES_256_GCM_SHA384", record["tls.cipher"]) - assert.Equal(t, "secp384r1", record["tls.curve"]) - assert.Equal(t, int64(200), record["http.response.status_code"]) - assert.Equal(t, "REFERER", record["http.request.header.referer"]) - assert.Equal(t, "NoError", record["error.type"]) -} - -func TestFrontDoorHealthProbeLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorhealthprobelog.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "GET", record["http.request.method"]) - assert.Equal(t, int64(200), record["http.response.status_code"]) - assert.Equal(t, "https://probe.net/health", record["url.full"]) - assert.Equal(t, "42.42.42.42", record["server.address"]) - assert.Equal(t, 0.042, record["http.request.duration"]) - assert.Equal(t, 0.00023, record["dns.lookup.duration"]) -} - -func TestFrontDoorWAFLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorwaflog.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) - assert.Equal(t, "https://test.net/", record["url.full"]) - assert.Equal(t, "test.net", record["server.address"]) - assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, "0", record["client.port"]) - assert.Equal(t, "23.23.23.23", record["network.peer.address"]) -} - -func TestAppServiceAppLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceapplogs.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "CONTAINER_ID", record["container.id"]) - assert.Equal(t, "EXCEPTION_CLASS", record["exception.type"]) - assert.Equal(t, "HOST", record["host.id"]) - assert.Equal(t, "METHOD", record["code.function"]) - assert.Equal(t, "FILEPATH", record["code.filepath"]) - assert.Equal(t, "STACKTRACE", record["exception.stacktrace"]) -} - -func TestAppServiceConsoleLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceconsolelogs.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "CONTAINER_ID", record["container.id"]) - assert.Equal(t, "HOST", record["host.id"]) -} - -func TestAppServiceAuditLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceauditlogs.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "USER_ID", record["enduser.id"]) - assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, "kudu", record["network.protocol.name"]) -} - -func TestAppServiceHTTPLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-appservicehttplogs.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "test.com", record["url.domain"]) - assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, int64(80), record["server.port"]) - assert.Equal(t, "/api/test/", record["url.path"]) - assert.Equal(t, "foo=42", record["url.query"]) - assert.Equal(t, "GET", record["http.request.method"]) - assert.Equal(t, 0.42, record["http.server.request.duration"]) - assert.Equal(t, int64(200), record["http.response.status_code"]) - assert.Equal(t, int64(4242), record["http.request.body.size"]) - assert.Equal(t, int64(42), record["http.response.body.size"]) - assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) - assert.Equal(t, "REFERER", record["http.request.header.referer"]) - assert.Equal(t, "COMPUTER_NAME", record["host.name"]) - assert.Equal(t, "http", record["network.protocol.name"]) - assert.Equal(t, "1.1", record["network.protocol.version"]) -} - -func TestAppServicePlatformLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceplatformlogs.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "CONTAINER_ID", record["container.id"]) - assert.Equal(t, "CONTAINER_NAME", record["container.name"]) -} - -func TestAppServiceIPSecAuditLog(t *testing.T) { - logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceipsecauditlogs.json") - - assert.NoError(t, err) - - record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() - - assert.Equal(t, "42.42.42.42", record["client.address"]) - assert.Equal(t, "HOST", record["url.domain"]) - assert.Equal(t, "FDID", record["http.request.header.x-azure-fdid"]) - assert.Equal(t, "HEALTH_PROBE", record["http.request.header.x-fd-healthprobe"]) - assert.Equal(t, "FORWARDED_FOR", record["http.request.header.x-forwarded-for"]) - assert.Equal(t, "FORWARDED_HOST", record["http.request.header.x-forwarded-host"]) -} diff --git a/pkg/translator/azure_logs/Makefile b/pkg/translator/azure_logs/Makefile new file mode 100644 index 000000000000..bdd863a203be --- /dev/null +++ b/pkg/translator/azure_logs/Makefile @@ -0,0 +1 @@ +include ../../../Makefile.Common diff --git a/pkg/translator/azure/logs_complex_conversions.go b/pkg/translator/azure_logs/complex_conversions.go similarity index 99% rename from pkg/translator/azure/logs_complex_conversions.go rename to pkg/translator/azure_logs/complex_conversions.go index c8a86488f8a8..48875ff269fa 100644 --- a/pkg/translator/azure/logs_complex_conversions.go +++ b/pkg/translator/azure_logs/complex_conversions.go @@ -1,4 +1,4 @@ -package azure +package azure_logs import ( "fmt" diff --git a/pkg/translator/azure/logs_complex_conversions_test.go b/pkg/translator/azure_logs/complex_conversions_test.go similarity index 99% rename from pkg/translator/azure/logs_complex_conversions_test.go rename to pkg/translator/azure_logs/complex_conversions_test.go index c968478ae21e..c8963fd5df4b 100644 --- a/pkg/translator/azure/logs_complex_conversions_test.go +++ b/pkg/translator/azure_logs/complex_conversions_test.go @@ -1,4 +1,4 @@ -package azure +package azure_logs import ( "github.com/stretchr/testify/assert" diff --git a/pkg/translator/azure_logs/go.mod b/pkg/translator/azure_logs/go.mod new file mode 100644 index 000000000000..3709bc5bda0b --- /dev/null +++ b/pkg/translator/azure_logs/go.mod @@ -0,0 +1,53 @@ +//module github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs + +go 1.21.0 + +require ( + github.com/json-iterator/go v1.1.12 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest v0.99.0 + github.com/relvacode/iso8601 v1.4.0 + github.com/stretchr/testify v1.9.0 + go.opentelemetry.io/collector/component v0.99.1-0.20240503221155-67d37183e6ac + go.opentelemetry.io/collector/pdata v1.6.1-0.20240503221155-67d37183e6ac + go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac + go.uber.org/goleak v1.3.0 + go.uber.org/zap v1.27.0 + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f +) + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/knadh/koanf/maps v0.1.1 // indirect + github.com/knadh/koanf/providers/confmap v0.1.0 // indirect + github.com/knadh/koanf/v2 v2.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil v0.99.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + go.opentelemetry.io/collector/config/configtelemetry v0.99.1-0.20240503221155-67d37183e6ac // indirect + go.opentelemetry.io/collector/confmap v0.99.1-0.20240503221155-67d37183e6ac // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/metric v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/sys v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect + google.golang.org/grpc v1.63.2 // indirect + google.golang.org/protobuf v1.34.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatautil => ../../pdatautil + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest => ../../pdatatest + +module github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../golden diff --git a/pkg/translator/azure_logs/go.sum b/pkg/translator/azure_logs/go.sum new file mode 100644 index 000000000000..a84ac60f8d4e --- /dev/null +++ b/pkg/translator/azure_logs/go.sum @@ -0,0 +1,85 @@ +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-viper/mapstructure/v2 v2.0.0-alpha.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= +github.com/knadh/koanf/providers/confmap v0.1.0/go.mod h1:2uLhxQzJnyHKfxG927awZC7+fyHFdQkd697K4MdLnIU= +github.com/knadh/koanf/v2 v2.1.1/go.mod h1:4mnTRbZCK+ALuBXHZMjDfG9y714L7TykVnZkXbMU3Es= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/relvacode/iso8601 v1.4.0 h1:GsInVSEJfkYuirYFxa80nMLbH2aydgZpIf52gYZXUJs= +github.com/relvacode/iso8601 v1.4.0/go.mod h1:FlNp+jz+TXpyRqgmM7tnzHHzBnz776kmAH2h3sZCn0I= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opentelemetry.io/collector/component v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:+b56nMIvo3CO5TShFn38RwX4FsXv0lVt2HoGmsaXObo= +go.opentelemetry.io/collector/config/configtelemetry v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:YV5PaOdtnU1xRomPcYqoHmyCr48tnaAREeGO96EZw8o= +go.opentelemetry.io/collector/confmap v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:BWKPIpYeUzSG6ZgCJMjF7xsLvyrvJCfYURl57E5vhiQ= +go.opentelemetry.io/collector/pdata v1.6.1-0.20240503221155-67d37183e6ac h1:+FnNEftMuQPg86UOZnLUXzdIjxmHKNsnmSiRTYTCVok= +go.opentelemetry.io/collector/pdata v1.6.1-0.20240503221155-67d37183e6ac/go.mod h1:ehCBBA5GoFrMZkwyZAKGY/lAVSgZf6rzUt3p9mddmPU= +go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac h1:FGz+i1cQrlahOmW3XAcM372XeNJPk57FYDphGVPaFwU= +go.opentelemetry.io/collector/semconv v0.99.1-0.20240503221155-67d37183e6ac/go.mod h1:8ElcRZ8Cdw5JnvhTOQOdYizkJaQ10Z2fS+R6djOnj6A= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/metric v1.26.0/go.mod h1:SY+rHOI4cEawI9a7N1A4nIg/nTQXe1ccCNWYOJUrpX4= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= +golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= +google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM= +google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA= +google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= +google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/translator/azure_logs/metadata.yaml b/pkg/translator/azure_logs/metadata.yaml new file mode 100644 index 000000000000..1fb3dbfbcb4d --- /dev/null +++ b/pkg/translator/azure_logs/metadata.yaml @@ -0,0 +1,3 @@ +status: + codeowners: + active: [open-telemetry/collector-approvers, atoulme, cparkins] diff --git a/pkg/translator/azure/logs_normalize.go b/pkg/translator/azure_logs/normalize.go similarity index 98% rename from pkg/translator/azure/logs_normalize.go rename to pkg/translator/azure_logs/normalize.go index 212fb2b1f44b..06c47bb76363 100644 --- a/pkg/translator/azure/logs_normalize.go +++ b/pkg/translator/azure_logs/normalize.go @@ -1,4 +1,4 @@ -package azure +package azure_logs import ( "fmt" diff --git a/pkg/translator/azure_logs/package_test.go b/pkg/translator/azure_logs/package_test.go new file mode 100644 index 000000000000..c4b240f8c62f --- /dev/null +++ b/pkg/translator/azure_logs/package_test.go @@ -0,0 +1,14 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azure_logs + +import ( + "testing" + + "go.uber.org/goleak" +) + +func TestMain(m *testing.M) { + goleak.VerifyTestMain(m) +} diff --git a/pkg/translator/azure/logs_property_names.go b/pkg/translator/azure_logs/property_names.go similarity index 99% rename from pkg/translator/azure/logs_property_names.go rename to pkg/translator/azure_logs/property_names.go index dbd4c079b297..f3067b6716f0 100644 --- a/pkg/translator/azure/logs_property_names.go +++ b/pkg/translator/azure_logs/property_names.go @@ -1,4 +1,4 @@ -package azure +package azure_logs var mappings = map[string]map[string]string{ "common": {}, diff --git a/pkg/translator/azure_logs/resourcelogs_to_logs.go b/pkg/translator/azure_logs/resourcelogs_to_logs.go new file mode 100644 index 000000000000..c24a145cf86f --- /dev/null +++ b/pkg/translator/azure_logs/resourcelogs_to_logs.go @@ -0,0 +1,241 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azure_logs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs" + +import ( + "bytes" + "encoding/json" + "errors" + "strconv" + + jsoniter "github.com/json-iterator/go" + "github.com/relvacode/iso8601" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + conventions "go.opentelemetry.io/collector/semconv/v1.13.0" + "go.uber.org/zap" + "golang.org/x/exp/slices" +) + +const ( + // Constants for OpenTelemetry Specs + scopeName = "otelcol/azureresourcelogs" + + // Constants for Azure Log Records + azureCategory = "azure.category" + azureCorrelationID = "azure.correlation.id" + azureDuration = "azure.duration" + azureIdentity = "azure.identity" + azureOperationName = "azure.operation.name" + azureOperationVersion = "azure.operation.version" + azureProperties = "azure.properties" + azureResourceID = "azure.resource.id" + azureResultType = "azure.result.type" + azureResultSignature = "azure.result.signature" + azureResultDescription = "azure.result.description" + azureTenantID = "azure.tenant.id" +) + +var ( + errMissingTimestamp = errors.New("missing timestamp") +) + +// azureRecords represents an array of Azure log records +// as exported via an Azure Event Hub +type azureRecords struct { + Records []azureLogRecord `json:"records"` +} + +// azureLogRecord represents a single Azure log following +// the common schema: +// https://learn.microsoft.com/en-us/azure/azure-monitor/essentials/resource-logs-schema +type azureLogRecord struct { + Time string `json:"time"` + Timestamp string `json:"timeStamp"` + ResourceID string `json:"resourceId"` + TenantID *string `json:"tenantId"` + OperationName string `json:"operationName"` + OperationVersion *string `json:"operationVersion"` + Category string `json:"category"` + ResultType *string `json:"resultType"` + ResultSignature *string `json:"resultSignature"` + ResultDescription *string `json:"resultDescription"` + DurationMs *json.Number `json:"durationMs"` + CallerIPAddress *string `json:"callerIpAddress"` + CorrelationID *string `json:"correlationId"` + Identity *any `json:"identity"` + Level *json.Number `json:"Level"` + Location *string `json:"location"` + Properties *any `json:"properties"` +} + +var _ plog.Unmarshaler = (*ResourceLogsUnmarshaler)(nil) + +type ResourceLogsUnmarshaler struct { + Version string + Logger *zap.Logger +} + +func (r ResourceLogsUnmarshaler) UnmarshalLogs(buf []byte) (plog.Logs, error) { + l := plog.NewLogs() + + var azureLogs azureRecords + decoder := jsoniter.NewDecoder(bytes.NewReader(buf)) + if err := decoder.Decode(&azureLogs); err != nil { + return l, err + } + + var resourceIDs []string + azureResourceLogs := make(map[string][]azureLogRecord) + for _, azureLog := range azureLogs.Records { + azureResourceLogs[azureLog.ResourceID] = append(azureResourceLogs[azureLog.ResourceID], azureLog) + keyExists := slices.Contains(resourceIDs, azureLog.ResourceID) + if !keyExists { + resourceIDs = append(resourceIDs, azureLog.ResourceID) + } + } + + for _, resourceID := range resourceIDs { + logs := azureResourceLogs[resourceID] + resourceLogs := l.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, resourceID) + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName(scopeName) + scopeLogs.Scope().SetVersion(r.Version) + logRecords := scopeLogs.LogRecords() + + for i := 0; i < len(logs); i++ { + log := logs[i] + nanos, err := getTimestamp(log) + if err != nil { + r.Logger.Warn("Unable to convert timestamp from log", zap.String("timestamp", log.Time)) + continue + } + + lr := logRecords.AppendEmpty() + lr.SetTimestamp(nanos) + + if log.Level != nil { + severity := asSeverity(*log.Level) + lr.SetSeverityNumber(severity) + lr.SetSeverityText(log.Level.String()) + } + + if err := lr.Attributes().FromRaw(extractRawAttributes(log)); err != nil { + return l, err + } + } + } + + return l, nil +} + +func getTimestamp(record azureLogRecord) (pcommon.Timestamp, error) { + if record.Time != "" { + return asTimestamp(record.Time) + } else if record.Timestamp != "" { + return asTimestamp(record.Timestamp) + } + + return 0, errMissingTimestamp +} + +// asTimestamp will parse an ISO8601 string into an OpenTelemetry +// nanosecond timestamp. If the string cannot be parsed, it will +// return zero and the error. +func asTimestamp(s string) (pcommon.Timestamp, error) { + t, err := iso8601.ParseString(s) + if err != nil { + return 0, err + } + + return pcommon.Timestamp(t.UnixNano()), nil +} + +// asSeverity converts the Azure log level to equivalent +// OpenTelemetry severity numbers. If the log level is not +// valid, then the 'Unspecified' value is returned. +func asSeverity(number json.Number) plog.SeverityNumber { + switch number.String() { + case "Informational": + return plog.SeverityNumberInfo + case "Warning": + return plog.SeverityNumberWarn + case "Error": + return plog.SeverityNumberError + case "Critical": + return plog.SeverityNumberFatal + default: + var levelNumber, _ = number.Int64() + if levelNumber > 0 { + return plog.SeverityNumber(levelNumber) + } + + return plog.SeverityNumberUnspecified + } +} + +func extractRawAttributes(log azureLogRecord) map[string]any { + var attrs = map[string]any{} + + attrs[azureCategory] = log.Category + setIf(attrs, azureCorrelationID, log.CorrelationID) + if log.DurationMs != nil { + duration, err := strconv.ParseInt(log.DurationMs.String(), 10, 64) + if err == nil { + attrs[azureDuration] = duration + } + } + if log.Identity != nil { + attrs[azureIdentity] = *log.Identity + } + attrs[azureOperationName] = log.OperationName + setIf(attrs, azureOperationVersion, log.OperationVersion) + + if log.Properties != nil { + copyPropertiesAndApplySemanticConventions(log.Category, log.Properties, attrs) + } + + setIf(attrs, azureResultDescription, log.ResultDescription) + setIf(attrs, azureResultSignature, log.ResultSignature) + setIf(attrs, azureResultType, log.ResultType) + setIf(attrs, azureTenantID, log.TenantID) + + setIf(attrs, conventions.AttributeCloudRegion, log.Location) + attrs[conventions.AttributeCloudProvider] = conventions.AttributeCloudProviderAzure + + setIf(attrs, conventions.AttributeNetSockPeerAddr, log.CallerIPAddress) + return attrs +} + +func copyPropertiesAndApplySemanticConventions(category string, properties *any, attrs map[string]any) { + + pmap := (*properties).(map[string]any) + attrsProps := map[string]any{} + + for k, v := range pmap { + // Check for a complex conversion, e.g. AppServiceHTTPLogs.Protocol + if complexConversion, ok := tryGetComplexConversion(category, k); ok { + if complexConversion(k, v, attrs) { + continue + } + } + // Check for an equivalent Semantic Convention key + if otelKey, ok := resourceLogKeyToSemConvKey(k, category); ok { + attrs[otelKey] = normalizeValue(otelKey, v) + } else { + attrsProps[k] = v + } + } + + if len(attrsProps) > 0 { + attrs[azureProperties] = attrsProps + } +} + +func setIf(attrs map[string]any, key string, value *string) { + if value != nil && *value != "" { + attrs[key] = *value + } +} diff --git a/pkg/translator/azure_logs/resourcelogs_to_logs_test.go b/pkg/translator/azure_logs/resourcelogs_to_logs_test.go new file mode 100644 index 000000000000..124c3506d06d --- /dev/null +++ b/pkg/translator/azure_logs/resourcelogs_to_logs_test.go @@ -0,0 +1,658 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package azure_logs // import "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs" + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + conventions "go.opentelemetry.io/collector/semconv/v1.13.0" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/plogtest" +) + +var testBuildInfo = component.BuildInfo{ + Version: "1.2.3", +} + +var minimumLogRecord = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") + lr.SetTimestamp(ts) + lr.Attributes().PutStr(azureOperationName, "SecretGet") + lr.Attributes().PutStr(azureCategory, "AuditEvent") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + return lr +}() + +var maximumLogRecord1 = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2022-11-11T04:48:27.6767145Z") + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberWarn) + lr.SetSeverityText("Warning") + guid := "607964b6-41a5-4e24-a5db-db7aab3b9b34" + + lr.Attributes().PutStr(azureTenantID, "/TENANT_ID") + lr.Attributes().PutStr(azureOperationName, "SecretGet") + lr.Attributes().PutStr(azureOperationVersion, "7.0") + lr.Attributes().PutStr(azureCategory, "AuditEvent") + lr.Attributes().PutStr(azureCorrelationID, guid) + lr.Attributes().PutStr(azureResultType, "Success") + lr.Attributes().PutStr(azureResultSignature, "Signature") + lr.Attributes().PutStr(azureResultDescription, "Description") + lr.Attributes().PutInt(azureDuration, 1234) + lr.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "127.0.0.1") + lr.Attributes().PutStr(conventions.AttributeCloudRegion, "ukso") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + lr.Attributes().PutEmptyMap(azureIdentity).PutEmptyMap("claim").PutStr("oid", guid) + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("string", "string") + m.PutDouble("int", 429) + m.PutDouble("float", 3.14) + m.PutBool("bool", false) + + return lr +}() + +var maximumLogRecord2 = func() []plog.LogRecord { + sl := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty() + lr := sl.LogRecords().AppendEmpty() + lr2 := sl.LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2022-11-11T04:48:29.6767145Z") + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberWarn) + lr.SetSeverityText("Warning") + guid := "96317703-2132-4a8d-a5d7-e18d2f486783" + + lr.Attributes().PutStr(azureTenantID, "/TENANT_ID") + lr.Attributes().PutStr(azureOperationName, "SecretSet") + lr.Attributes().PutStr(azureOperationVersion, "7.0") + lr.Attributes().PutStr(azureCategory, "AuditEvent") + lr.Attributes().PutStr(azureCorrelationID, guid) + lr.Attributes().PutStr(azureResultType, "Success") + lr.Attributes().PutStr(azureResultSignature, "Signature") + lr.Attributes().PutStr(azureResultDescription, "Description") + lr.Attributes().PutInt(azureDuration, 4321) + lr.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "127.0.0.1") + lr.Attributes().PutStr(conventions.AttributeCloudRegion, "ukso") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + lr.Attributes().PutEmptyMap(azureIdentity).PutEmptyMap("claim").PutStr("oid", guid) + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("string", "string") + m.PutDouble("int", 924) + m.PutDouble("float", 41.3) + m.PutBool("bool", true) + + ts, _ = asTimestamp("2022-11-11T04:48:31.6767145Z") + lr2.SetTimestamp(ts) + lr2.SetSeverityNumber(plog.SeverityNumberWarn) + lr2.SetSeverityText("Warning") + guid = "4ae807da-39d9-4327-b5b4-0ab685a57f9a" + + lr2.Attributes().PutStr(azureTenantID, "/TENANT_ID") + lr2.Attributes().PutStr(azureOperationName, "SecretGet") + lr2.Attributes().PutStr(azureOperationVersion, "7.0") + lr2.Attributes().PutStr(azureCategory, "AuditEvent") + lr2.Attributes().PutStr(azureCorrelationID, guid) + lr2.Attributes().PutStr(azureResultType, "Success") + lr2.Attributes().PutStr(azureResultSignature, "Signature") + lr2.Attributes().PutStr(azureResultDescription, "Description") + lr2.Attributes().PutInt(azureDuration, 321) + lr2.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "127.0.0.1") + lr2.Attributes().PutStr(conventions.AttributeCloudRegion, "ukso") + lr2.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + lr2.Attributes().PutEmptyMap(azureIdentity).PutEmptyMap("claim").PutStr("oid", guid) + m = lr2.Attributes().PutEmptyMap(azureProperties) + m.PutStr("string", "string") + m.PutDouble("int", 925) + m.PutDouble("float", 41.4) + m.PutBool("bool", false) + + var records []plog.LogRecord + return append(records, lr, lr2) +}() + +var badLevelLogRecord = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2023-10-26T14:22:43.3416357Z") + lr.SetTimestamp(ts) + lr.SetSeverityNumber(plog.SeverityNumberTrace4) + lr.SetSeverityText("4") + guid := "128bc026-5ead-40c7-8853-ebb32bc077a3" + + lr.Attributes().PutStr(azureOperationName, "Microsoft.ApiManagement/GatewayLogs") + lr.Attributes().PutStr(azureCategory, "GatewayLogs") + lr.Attributes().PutStr(azureCorrelationID, guid) + lr.Attributes().PutStr(azureResultType, "Succeeded") + lr.Attributes().PutInt(azureDuration, 243) + lr.Attributes().PutStr(conventions.AttributeNetSockPeerAddr, "13.14.15.16") + lr.Attributes().PutStr(conventions.AttributeCloudRegion, "West US") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("method", "GET") + m.PutStr("url", "https://api.azure-api.net/sessions") + m.PutDouble("backendResponseCode", 200) + m.PutDouble("responseCode", 200) + m.PutDouble("responseSize", 102945) + m.PutStr("cache", "none") + m.PutDouble("backendTime", 54) + m.PutDouble("requestSize", 632) + m.PutStr("apiId", "demo-api") + m.PutStr("operationId", "GetSessions") + m.PutStr("apimSubscriptionId", "master") + m.PutDouble("clientTime", 190) + m.PutStr("clientProtocol", "HTTP/1.1") + m.PutStr("backendProtocol", "HTTP/1.1") + m.PutStr("apiRevision", "1") + m.PutStr("clientTlsVersion", "1.2") + m.PutStr("backendMethod", "GET") + m.PutStr("backendUrl", "https://api.azurewebsites.net/sessions") + return lr +}() + +var badTimeLogRecord = func() plog.LogRecord { + lr := plog.NewLogs().ResourceLogs().AppendEmpty().ScopeLogs().AppendEmpty().LogRecords().AppendEmpty() + + ts, _ := asTimestamp("2021-10-14T22:17:11+00:00") + lr.SetTimestamp(ts) + + lr.Attributes().PutStr(azureOperationName, "ApplicationGatewayAccess") + lr.Attributes().PutStr(azureCategory, "ApplicationGatewayAccessLog") + lr.Attributes().PutStr(conventions.AttributeCloudProvider, conventions.AttributeCloudProviderAzure) + + m := lr.Attributes().PutEmptyMap(azureProperties) + m.PutStr("instanceId", "appgw_2") + m.PutStr("clientIP", "185.42.129.24") + m.PutDouble("clientPort", 45057) + m.PutStr("httpMethod", "GET") + m.PutStr("originalRequestUriWithArgs", "/") + m.PutStr("requestUri", "/") + m.PutStr("requestQuery", "") + m.PutStr("userAgent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36") + m.PutDouble("httpStatus", 200) + m.PutStr("httpVersion", "HTTP/1.1") + m.PutDouble("receivedBytes", 184) + m.PutDouble("sentBytes", 466) + m.PutDouble("clientResponseTime", 0) + m.PutDouble("timeTaken", 0.034) + m.PutStr("WAFEvaluationTime", "0.000") + m.PutStr("WAFMode", "Detection") + m.PutStr("transactionId", "592d1649f75a8d480a3c4dc6a975309d") + m.PutStr("sslEnabled", "on") + m.PutStr("sslCipher", "ECDHE-RSA-AES256-GCM-SHA384") + m.PutStr("sslProtocol", "TLSv1.2") + m.PutStr("sslClientVerify", "NONE") + m.PutStr("sslClientCertificateFingerprint", "") + m.PutStr("sslClientCertificateIssuerName", "") + m.PutStr("serverRouted", "52.239.221.65:443") + m.PutStr("serverStatus", "200") + m.PutStr("serverResponseLatency", "0.028") + m.PutStr("upstreamSourcePort", "21564") + m.PutStr("originalHost", "20.110.30.194") + m.PutStr("host", "20.110.30.194") + return lr +}() + +func TestAsTimestamp(t *testing.T) { + timestamp := "2022-11-11T04:48:27.6767145Z" + nanos, err := asTimestamp(timestamp) + assert.NoError(t, err) + assert.Less(t, pcommon.Timestamp(0), nanos) + + timestamp = "invalid-time" + nanos, err = asTimestamp(timestamp) + assert.Error(t, err) + assert.Equal(t, pcommon.Timestamp(0), nanos) +} + +func TestAsSeverity(t *testing.T) { + tests := map[string]plog.SeverityNumber{ + "Informational": plog.SeverityNumberInfo, + "Warning": plog.SeverityNumberWarn, + "Error": plog.SeverityNumberError, + "Critical": plog.SeverityNumberFatal, + "unknown": plog.SeverityNumberUnspecified, + } + + for input, expected := range tests { + t.Run(input, func(t *testing.T) { + assert.Equal(t, expected, asSeverity(json.Number(input))) + }) + } +} + +func TestSetIf(t *testing.T) { + m := map[string]any{} + + setIf(m, "key", nil) + actual, found := m["key"] + assert.False(t, found) + assert.Nil(t, actual) + + v := "" + setIf(m, "key", &v) + actual, found = m["key"] + assert.False(t, found) + assert.Nil(t, actual) + + v = "ok" + setIf(m, "key", &v) + actual, found = m["key"] + assert.True(t, found) + assert.Equal(t, "ok", actual) +} + +func TestExtractRawAttributes(t *testing.T) { + badDuration := json.Number("invalid") + goodDuration := json.Number("1234") + + tenantID := "tenant.id" + operationVersion := "operation.version" + resultType := "result.type" + resultSignature := "result.signature" + resultDescription := "result.description" + callerIPAddress := "127.0.0.1" + correlationID := "edb70d1a-eec2-4b4c-b2f4-60e3510160ee" + level := json.Number("Informational") + location := "location" + + identity := any("someone") + + properties := any(map[string]any{ + "a": uint64(1), + "b": true, + "c": 1.23, + "d": "ok", + }) + + tests := []struct { + name string + log azureLogRecord + expected map[string]any + }{ + { + name: "minimal", + log: azureLogRecord{ + Time: "", + ResourceID: "resource.id", + OperationName: "operation.name", + Category: "category", + DurationMs: &badDuration, + }, + expected: map[string]any{ + azureOperationName: "operation.name", + azureCategory: "category", + conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure, + }, + }, + { + name: "bad-duration", + log: azureLogRecord{ + Time: "", + ResourceID: "resource.id", + OperationName: "operation.name", + Category: "category", + DurationMs: &badDuration, + }, + expected: map[string]any{ + azureOperationName: "operation.name", + azureCategory: "category", + conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure, + }, + }, + { + name: "everything", + log: azureLogRecord{ + Time: "", + ResourceID: "resource.id", + TenantID: &tenantID, + OperationName: "operation.name", + OperationVersion: &operationVersion, + Category: "category", + ResultType: &resultType, + ResultSignature: &resultSignature, + ResultDescription: &resultDescription, + DurationMs: &goodDuration, + CallerIPAddress: &callerIPAddress, + CorrelationID: &correlationID, + Identity: &identity, + Level: &level, + Location: &location, + Properties: &properties, + }, + expected: map[string]any{ + azureTenantID: "tenant.id", + azureOperationName: "operation.name", + azureOperationVersion: "operation.version", + azureCategory: "category", + azureCorrelationID: correlationID, + azureResultType: "result.type", + azureResultSignature: "result.signature", + azureResultDescription: "result.description", + azureDuration: int64(1234), + conventions.AttributeNetSockPeerAddr: "127.0.0.1", + azureIdentity: "someone", + conventions.AttributeCloudRegion: "location", + conventions.AttributeCloudProvider: conventions.AttributeCloudProviderAzure, + azureProperties: properties, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.expected, extractRawAttributes(tt.log)) + }) + } + +} + +func TestUnmarshalLogs(t *testing.T) { + expectedMinimum := plog.NewLogs() + resourceLogs := expectedMinimum.ResourceLogs().AppendEmpty() + scopeLogs := resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr := scopeLogs.LogRecords().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + minimumLogRecord.CopyTo(lr) + + expectedMinimum2 := plog.NewLogs() + resourceLogs = expectedMinimum2.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + logRecords := scopeLogs.LogRecords() + lr = logRecords.AppendEmpty() + minimumLogRecord.CopyTo(lr) + lr = logRecords.AppendEmpty() + minimumLogRecord.CopyTo(lr) + + expectedMaximum := plog.NewLogs() + resourceLogs = expectedMaximum.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID-1") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + maximumLogRecord1.CopyTo(lr) + + resourceLogs = expectedMaximum.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID-2") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + lr2 := scopeLogs.LogRecords().AppendEmpty() + maximumLogRecord2[0].CopyTo(lr) + maximumLogRecord2[1].CopyTo(lr2) + + expectedBadLevel := plog.NewLogs() + resourceLogs = expectedBadLevel.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + badLevelLogRecord.CopyTo(lr) + + expectedBadTime := plog.NewLogs() + resourceLogs = expectedBadTime.ResourceLogs().AppendEmpty() + resourceLogs.Resource().Attributes().PutStr(azureResourceID, "/RESOURCE_ID") + scopeLogs = resourceLogs.ScopeLogs().AppendEmpty() + scopeLogs.Scope().SetName("otelcol/azureresourcelogs") + scopeLogs.Scope().SetVersion(testBuildInfo.Version) + lr = scopeLogs.LogRecords().AppendEmpty() + badTimeLogRecord.CopyTo(lr) + + tests := []struct { + file string + expected plog.Logs + }{ + { + file: "log-minimum.json", + expected: expectedMinimum, + }, + { + file: "log-minimum-2.json", + expected: expectedMinimum2, + }, + { + file: "log-maximum.json", + expected: expectedMaximum, + }, + { + file: "log-bad-level.json", + expected: expectedBadLevel, + }, + { + file: "log-bad-time.json", + expected: expectedBadTime, + }, + } + + sut := &ResourceLogsUnmarshaler{ + Version: testBuildInfo.Version, + Logger: zap.NewNop(), + } + for _, tt := range tests { + t.Run(tt.file, func(t *testing.T) { + data, err := os.ReadFile(filepath.Join("testdata", tt.file)) + assert.NoError(t, err) + assert.NotNil(t, data) + + logs, err := sut.UnmarshalLogs(data) + assert.NoError(t, err) + + assert.NoError(t, plogtest.CompareLogs(tt.expected, logs)) + }) + } +} + +func loadJsonLogsAndApplySemanticConventions(filename string) (plog.Logs, error) { + l := plog.NewLogs() + + sut := &ResourceLogsUnmarshaler{ + Version: testBuildInfo.Version, + Logger: zap.NewNop(), + } + + data, err := os.ReadFile(filepath.Join("testdata", filename)) + if err != nil { + return l, err + } + + logs, err := sut.UnmarshalLogs(data) + + if err != nil { + return l, err + } + + return logs, nil +} + +func TestAzureCdnAccessLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-azurecdnaccesslog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "NoError", record["error.type"]) +} + +func TestFrontDoorAccessLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdooraccesslog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, "1.1.0.0", record["network.protocol.version"]) + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, int64(1234), record["http.request.size"]) + assert.Equal(t, int64(12345), record["http.response.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "23.23.23.23", record["network.peer.address"]) + assert.Equal(t, float64(0.23), record["http.server.request.duration"]) + assert.Equal(t, "https", record["network.protocol.name"]) + assert.Equal(t, "tls", record["tls.protocol.name"]) + assert.Equal(t, "1.3", record["tls.protocol.version"]) + assert.Equal(t, "TLS_AES_256_GCM_SHA384", record["tls.cipher"]) + assert.Equal(t, "secp384r1", record["tls.curve"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "REFERER", record["http.request.header.referer"]) + assert.Equal(t, "NoError", record["error.type"]) +} + +func TestFrontDoorHealthProbeLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorhealthprobelog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, "https://probe.net/health", record["url.full"]) + assert.Equal(t, "42.42.42.42", record["server.address"]) + assert.Equal(t, 0.042, record["http.request.duration"]) + assert.Equal(t, 0.00023, record["dns.lookup.duration"]) +} + +func TestFrontDoorWAFLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-frontdoorwaflog.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "TRACKING_REFERENCE", record["az.service_request_id"]) + assert.Equal(t, "https://test.net/", record["url.full"]) + assert.Equal(t, "test.net", record["server.address"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "0", record["client.port"]) + assert.Equal(t, "23.23.23.23", record["network.peer.address"]) +} + +func TestAppServiceAppLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceapplogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "EXCEPTION_CLASS", record["exception.type"]) + assert.Equal(t, "HOST", record["host.id"]) + assert.Equal(t, "METHOD", record["code.function"]) + assert.Equal(t, "FILEPATH", record["code.filepath"]) + assert.Equal(t, "STACKTRACE", record["exception.stacktrace"]) +} + +func TestAppServiceConsoleLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceconsolelogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "HOST", record["host.id"]) +} + +func TestAppServiceAuditLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceauditlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "USER_ID", record["enduser.id"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "kudu", record["network.protocol.name"]) +} + +func TestAppServiceHTTPLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appservicehttplogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "test.com", record["url.domain"]) + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, int64(80), record["server.port"]) + assert.Equal(t, "/api/test/", record["url.path"]) + assert.Equal(t, "foo=42", record["url.query"]) + assert.Equal(t, "GET", record["http.request.method"]) + assert.Equal(t, 0.42, record["http.server.request.duration"]) + assert.Equal(t, int64(200), record["http.response.status_code"]) + assert.Equal(t, int64(4242), record["http.request.body.size"]) + assert.Equal(t, int64(42), record["http.response.body.size"]) + assert.Equal(t, "Mozilla/5.0", record["user_agent.original"]) + assert.Equal(t, "REFERER", record["http.request.header.referer"]) + assert.Equal(t, "COMPUTER_NAME", record["host.name"]) + assert.Equal(t, "http", record["network.protocol.name"]) + assert.Equal(t, "1.1", record["network.protocol.version"]) +} + +func TestAppServicePlatformLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceplatformlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "CONTAINER_ID", record["container.id"]) + assert.Equal(t, "CONTAINER_NAME", record["container.name"]) +} + +func TestAppServiceIPSecAuditLog(t *testing.T) { + logs, err := loadJsonLogsAndApplySemanticConventions("log-appserviceipsecauditlogs.json") + + assert.NoError(t, err) + + record := logs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().AsRaw() + + assert.Equal(t, "42.42.42.42", record["client.address"]) + assert.Equal(t, "HOST", record["url.domain"]) + assert.Equal(t, "FDID", record["http.request.header.x-azure-fdid"]) + assert.Equal(t, "HEALTH_PROBE", record["http.request.header.x-fd-healthprobe"]) + assert.Equal(t, "FORWARDED_FOR", record["http.request.header.x-forwarded-for"]) + assert.Equal(t, "FORWARDED_HOST", record["http.request.header.x-forwarded-host"]) +} diff --git a/pkg/translator/azure/testdata/log-appserviceapplogs.json b/pkg/translator/azure_logs/testdata/log-appserviceapplogs.json similarity index 100% rename from pkg/translator/azure/testdata/log-appserviceapplogs.json rename to pkg/translator/azure_logs/testdata/log-appserviceapplogs.json diff --git a/pkg/translator/azure/testdata/log-appserviceauditlogs.json b/pkg/translator/azure_logs/testdata/log-appserviceauditlogs.json similarity index 100% rename from pkg/translator/azure/testdata/log-appserviceauditlogs.json rename to pkg/translator/azure_logs/testdata/log-appserviceauditlogs.json diff --git a/pkg/translator/azure/testdata/log-appserviceconsolelogs.json b/pkg/translator/azure_logs/testdata/log-appserviceconsolelogs.json similarity index 100% rename from pkg/translator/azure/testdata/log-appserviceconsolelogs.json rename to pkg/translator/azure_logs/testdata/log-appserviceconsolelogs.json diff --git a/pkg/translator/azure/testdata/log-appservicehttplogs.json b/pkg/translator/azure_logs/testdata/log-appservicehttplogs.json similarity index 100% rename from pkg/translator/azure/testdata/log-appservicehttplogs.json rename to pkg/translator/azure_logs/testdata/log-appservicehttplogs.json diff --git a/pkg/translator/azure/testdata/log-appserviceipsecauditlogs.json b/pkg/translator/azure_logs/testdata/log-appserviceipsecauditlogs.json similarity index 100% rename from pkg/translator/azure/testdata/log-appserviceipsecauditlogs.json rename to pkg/translator/azure_logs/testdata/log-appserviceipsecauditlogs.json diff --git a/pkg/translator/azure/testdata/log-appserviceplatformlogs.json b/pkg/translator/azure_logs/testdata/log-appserviceplatformlogs.json similarity index 100% rename from pkg/translator/azure/testdata/log-appserviceplatformlogs.json rename to pkg/translator/azure_logs/testdata/log-appserviceplatformlogs.json diff --git a/pkg/translator/azure/testdata/log-azurecdnaccesslog.json b/pkg/translator/azure_logs/testdata/log-azurecdnaccesslog.json similarity index 100% rename from pkg/translator/azure/testdata/log-azurecdnaccesslog.json rename to pkg/translator/azure_logs/testdata/log-azurecdnaccesslog.json diff --git a/pkg/translator/azure_logs/testdata/log-bad-level.json b/pkg/translator/azure_logs/testdata/log-bad-level.json new file mode 100644 index 000000000000..662d34821f28 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-bad-level.json @@ -0,0 +1,39 @@ +{ + "records": [ + { + "DeploymentVersion": "0.40.16708.0", + "Level": 4, + "isRequestSuccess": true, + "time": "2023-10-26T14:22:43.3416357Z", + "operationName": "Microsoft.ApiManagement/GatewayLogs", + "category": "GatewayLogs", + "durationMs": 243, + "callerIpAddress": "13.14.15.16", + "correlationId": "128bc026-5ead-40c7-8853-ebb32bc077a3", + "location": "West US", + "properties": { + "method": "GET", + "url": "https://api.azure-api.net/sessions", + "backendResponseCode": 200, + "responseCode": 200, + "responseSize": 102945, + "cache": "none", + "backendTime": 54, + "requestSize": 632, + "apiId": "demo-api", + "operationId": "GetSessions", + "apimSubscriptionId": "master", + "clientTime": 190, + "clientProtocol": "HTTP/1.1", + "backendProtocol": "HTTP/1.1", + "apiRevision": "1", + "clientTlsVersion": "1.2", + "backendMethod": "GET", + "backendUrl": "https://api.azurewebsites.net/sessions" + }, + "resourceId": "/RESOURCE_ID", + "resultType": "Succeeded", + "truncated": 0 + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure_logs/testdata/log-bad-time.json b/pkg/translator/azure_logs/testdata/log-bad-time.json new file mode 100644 index 000000000000..614d170378ec --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-bad-time.json @@ -0,0 +1,45 @@ +{ + "records": [ + { + "timeStamp": "2021-10-14T22:17:11+00:00", + "resourceId": "/RESOURCE_ID", + "listenerName": "HTTP-Listener", + "ruleName": "Storage-Static-Rule", + "backendPoolName": "StaticStorageAccount", + "backendSettingName": "StorageStatic-HTTPS-Setting", + "operationName": "ApplicationGatewayAccess", + "category": "ApplicationGatewayAccessLog", + "properties": { + "instanceId": "appgw_2", + "clientIP": "185.42.129.24", + "clientPort": 45057, + "httpMethod": "GET", + "originalRequestUriWithArgs": "\/", + "requestUri": "\/", + "requestQuery": "", + "userAgent": "Mozilla\/5.0 (Windows NT 6.1; WOW64) AppleWebKit\/537.36 (KHTML, like Gecko) Chrome\/52.0.2743.116 Safari\/537.36", + "httpStatus": 200, + "httpVersion": "HTTP\/1.1", + "receivedBytes": 184, + "sentBytes": 466, + "clientResponseTime": 0, + "timeTaken": 0.034, + "WAFEvaluationTime": "0.000", + "WAFMode": "Detection", + "transactionId": "592d1649f75a8d480a3c4dc6a975309d", + "sslEnabled": "on", + "sslCipher": "ECDHE-RSA-AES256-GCM-SHA384", + "sslProtocol": "TLSv1.2", + "sslClientVerify": "NONE", + "sslClientCertificateFingerprint": "", + "sslClientCertificateIssuerName": "", + "serverRouted": "52.239.221.65:443", + "serverStatus": "200", + "serverResponseLatency": "0.028", + "upstreamSourcePort": "21564", + "originalHost": "20.110.30.194", + "host": "20.110.30.194" + } + } + ] +} \ No newline at end of file diff --git a/pkg/translator/azure/testdata/log-frontdooraccesslog.json b/pkg/translator/azure_logs/testdata/log-frontdooraccesslog.json similarity index 100% rename from pkg/translator/azure/testdata/log-frontdooraccesslog.json rename to pkg/translator/azure_logs/testdata/log-frontdooraccesslog.json diff --git a/pkg/translator/azure/testdata/log-frontdoorhealthprobelog.json b/pkg/translator/azure_logs/testdata/log-frontdoorhealthprobelog.json similarity index 100% rename from pkg/translator/azure/testdata/log-frontdoorhealthprobelog.json rename to pkg/translator/azure_logs/testdata/log-frontdoorhealthprobelog.json diff --git a/pkg/translator/azure/testdata/log-frontdoorwaflog.json b/pkg/translator/azure_logs/testdata/log-frontdoorwaflog.json similarity index 100% rename from pkg/translator/azure/testdata/log-frontdoorwaflog.json rename to pkg/translator/azure_logs/testdata/log-frontdoorwaflog.json diff --git a/pkg/translator/azure_logs/testdata/log-maximum.json b/pkg/translator/azure_logs/testdata/log-maximum.json new file mode 100644 index 000000000000..b105c6b168bc --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-maximum.json @@ -0,0 +1,85 @@ +{ + "records": [ + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID-1", + "tenantId": "/TENANT_ID", + "operationName": "SecretGet", + "operationVersion": "7.0", + "category": "AuditEvent", + "resultType": "Success", + "resultSignature": "Signature", + "resultDescription": "Description", + "durationMs": "1234", + "callerIpAddress": "127.0.0.1", + "correlationId": "607964b6-41a5-4e24-a5db-db7aab3b9b34", + "Level": "Warning", + "location": "ukso", + "identity": { + "claim": { + "oid": "607964b6-41a5-4e24-a5db-db7aab3b9b34" + } + }, + "properties": { + "string": "string", + "int": 429, + "float": 3.14, + "bool": false + } + }, + { + "time": "2022-11-11T04:48:29.6767145Z", + "resourceId": "/RESOURCE_ID-2", + "tenantId": "/TENANT_ID", + "operationName": "SecretSet", + "operationVersion": "7.0", + "category": "AuditEvent", + "resultType": "Success", + "resultSignature": "Signature", + "resultDescription": "Description", + "durationMs": "4321", + "callerIpAddress": "127.0.0.1", + "correlationId": "96317703-2132-4a8d-a5d7-e18d2f486783", + "Level": "Warning", + "location": "ukso", + "identity": { + "claim": { + "oid": "96317703-2132-4a8d-a5d7-e18d2f486783" + } + }, + "properties": { + "string": "string", + "int": 924, + "float": 41.3, + "bool": true + } + }, + { + "time": "2022-11-11T04:48:31.6767145Z", + "resourceId": "/RESOURCE_ID-2", + "tenantId": "/TENANT_ID", + "operationName": "SecretGet", + "operationVersion": "7.0", + "category": "AuditEvent", + "resultType": "Success", + "resultSignature": "Signature", + "resultDescription": "Description", + "durationMs": "321", + "callerIpAddress": "127.0.0.1", + "correlationId": "4ae807da-39d9-4327-b5b4-0ab685a57f9a", + "Level": "Warning", + "location": "ukso", + "identity": { + "claim": { + "oid": "4ae807da-39d9-4327-b5b4-0ab685a57f9a" + } + }, + "properties": { + "string": "string", + "int": 925, + "float": 41.4, + "bool": false + } + } + ] +} diff --git a/pkg/translator/azure_logs/testdata/log-minimum-2.json b/pkg/translator/azure_logs/testdata/log-minimum-2.json new file mode 100644 index 000000000000..6eac63fa0389 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-minimum-2.json @@ -0,0 +1,16 @@ +{ + "records": [ + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID", + "operationName": "SecretGet", + "category": "AuditEvent" + }, + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID", + "operationName": "SecretGet", + "category": "AuditEvent" + } + ] +} diff --git a/pkg/translator/azure_logs/testdata/log-minimum.json b/pkg/translator/azure_logs/testdata/log-minimum.json new file mode 100644 index 000000000000..16d4f2e71177 --- /dev/null +++ b/pkg/translator/azure_logs/testdata/log-minimum.json @@ -0,0 +1,10 @@ +{ + "records": [ + { + "time": "2022-11-11T04:48:27.6767145Z", + "resourceId": "/RESOURCE_ID", + "operationName": "SecretGet", + "category": "AuditEvent" + } + ] +} diff --git a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go index 0ad2b398ff7e..e6971f1dbc8d 100644 --- a/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go +++ b/receiver/azureeventhubreceiver/azureresourcelogs_unmarshaler.go @@ -10,20 +10,33 @@ import ( "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure" + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs" ) +type logsUnmarshaler interface { + UnmarshalLogs([]byte) (plog.Logs, error) +} + type AzureResourceLogsEventUnmarshaler struct { - unmarshaler *azure.ResourceLogsUnmarshaler + unmarshaler logsUnmarshaler } func newAzureResourceLogsUnmarshaler(buildInfo component.BuildInfo, logger *zap.Logger, applySemanticConventions bool) eventLogsUnmarshaler { - return AzureResourceLogsEventUnmarshaler{ - unmarshaler: &azure.ResourceLogsUnmarshaler{ - Version: buildInfo.Version, - Logger: logger, - ApplySemanticConventions: applySemanticConventions, - }, + if applySemanticConventions { + return AzureResourceLogsEventUnmarshaler{ + unmarshaler: &azure_logs.ResourceLogsUnmarshaler{ + Version: buildInfo.Version, + Logger: logger, + }, + } + } else { + return AzureResourceLogsEventUnmarshaler{ + unmarshaler: &azure.ResourceLogsUnmarshaler{ + Version: buildInfo.Version, + Logger: logger, + }, + } } } diff --git a/receiver/azureeventhubreceiver/go.mod b/receiver/azureeventhubreceiver/go.mod index 8333a20674a1..0cba394a1336 100644 --- a/receiver/azureeventhubreceiver/go.mod +++ b/receiver/azureeventhubreceiver/go.mod @@ -9,6 +9,7 @@ require ( github.com/open-telemetry/opentelemetry-collector-contrib/internal/sharedcomponent v0.99.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza v0.99.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure v0.99.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs v0.99.0 github.com/relvacode/iso8601 v1.4.0 github.com/stretchr/testify v1.9.0 go.opentelemetry.io/collector/component v0.99.1-0.20240503221155-67d37183e6ac @@ -112,7 +113,7 @@ require ( go.opentelemetry.io/proto/otlp v1.2.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/crypto v0.22.0 // indirect - golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc // indirect + golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f // indirect golang.org/x/net v0.24.0 // indirect golang.org/x/sys v0.19.0 // indirect golang.org/x/text v0.14.0 // indirect @@ -144,6 +145,8 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/share replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure => ../../pkg/translator/azure +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs => ../../pkg/translator/azure_logs + replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/golden => ../../pkg/golden replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/common => ../../internal/common diff --git a/receiver/azureeventhubreceiver/go.sum b/receiver/azureeventhubreceiver/go.sum index e6ed8d4a9222..dbaf6601bea2 100644 --- a/receiver/azureeventhubreceiver/go.sum +++ b/receiver/azureeventhubreceiver/go.sum @@ -299,8 +299,6 @@ golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc h1:ao2WRsKSzW6KuUY9IWPwWahcHCgR0s52IfwutMfEbdM= -golang.org/x/exp v0.0.0-20240103183307-be819d1f06fc/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY= golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= diff --git a/versions.yaml b/versions.yaml index 553569f6852b..1ffb85162f93 100644 --- a/versions.yaml +++ b/versions.yaml @@ -142,6 +142,7 @@ module-sets: - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/ottl - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure + - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/azure_logs - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/jaeger - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/loki - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus From ce3e549a6a49cc2aab356b01cdbf28a06cc50eec Mon Sep 17 00:00:00 2001 From: Mark Rendle Date: Mon, 6 May 2024 14:43:47 +0100 Subject: [PATCH 17/17] Revert previous change --- receiver/kafkareceiver/azureresourcelogs_unmarshaler.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go b/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go index 029424e8c92a..b042bc052390 100644 --- a/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go +++ b/receiver/kafkareceiver/azureresourcelogs_unmarshaler.go @@ -17,9 +17,8 @@ type azureResourceLogsUnmarshaler struct { func newAzureResourceLogsUnmarshaler(version string, logger *zap.Logger) LogsUnmarshaler { return azureResourceLogsUnmarshaler{ unmarshaler: &azure.ResourceLogsUnmarshaler{ - Version: version, - Logger: logger, - ApplySemanticConventions: false, + Version: version, + Logger: logger, }, } }