From 184e954bf6599d1ef725307a2fe2a5af7c780540 Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Fri, 30 Aug 2024 11:03:10 +0200 Subject: [PATCH] [processor/redaction] add support for redacting metrics and logs attributes (#34609) **Description:** This PR extends the redaction processor to also support the redaction of attributes within logs and metrics. **Link to tracking Issue:** #34479 **Testing:** Extended the existing unit tests to also cover the redaction of logs and metrics **Documentation:** Adapted the readme to reflect the changes --------- Signed-off-by: Florian Bacher --- .chloggen/redaction-add-metrics-and-logs.yaml | 27 + processor/redactionprocessor/README.md | 22 +- processor/redactionprocessor/factory.go | 46 ++ processor/redactionprocessor/factory_test.go | 18 + .../generated_component_test.go | 14 + .../internal/metadata/generated_status.go | 4 +- processor/redactionprocessor/metadata.yaml | 1 + processor/redactionprocessor/processor.go | 75 +++ .../redactionprocessor/processor_test.go | 520 +++++++++++++----- 9 files changed, 582 insertions(+), 145 deletions(-) create mode 100644 .chloggen/redaction-add-metrics-and-logs.yaml diff --git a/.chloggen/redaction-add-metrics-and-logs.yaml b/.chloggen/redaction-add-metrics-and-logs.yaml new file mode 100644 index 000000000000..d4acba3f6889 --- /dev/null +++ b/.chloggen/redaction-add-metrics-and-logs.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: redactionprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add support for logs and metrics + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34479] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/processor/redactionprocessor/README.md b/processor/redactionprocessor/README.md index f5c54d306206..aa2d5956b1db 100644 --- a/processor/redactionprocessor/README.md +++ b/processor/redactionprocessor/README.md @@ -3,19 +3,21 @@ | Status | | | ------------- |-----------| -| Stability | [beta]: traces | +| Stability | [alpha]: logs, metrics | +| | [beta]: traces | | Distributions | [contrib] | | Issues | [![Open issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aopen%20label%3Aprocessor%2Fredaction%20&label=open&color=orange&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aopen+is%3Aissue+label%3Aprocessor%2Fredaction) [![Closed issues](https://img.shields.io/github/issues-search/open-telemetry/opentelemetry-collector-contrib?query=is%3Aissue%20is%3Aclosed%20label%3Aprocessor%2Fredaction%20&label=closed&color=blue&logo=opentelemetry)](https://github.com/open-telemetry/opentelemetry-collector-contrib/issues?q=is%3Aclosed+is%3Aissue+label%3Aprocessor%2Fredaction) | | [Code Owners](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/CONTRIBUTING.md#becoming-a-code-owner) | [@dmitryax](https://www.github.com/dmitryax), [@mx-psi](https://www.github.com/mx-psi), [@TylerHelmuth](https://www.github.com/TylerHelmuth) | | Emeritus | [@leonsp-ai](https://www.github.com/leonsp-ai) | +[alpha]: https://github.com/open-telemetry/opentelemetry-collector#alpha [beta]: https://github.com/open-telemetry/opentelemetry-collector#beta [contrib]: https://github.com/open-telemetry/opentelemetry-collector-releases/tree/main/distributions/otelcol-contrib -This processor deletes span attributes that don't match a list of allowed span -attributes. It also masks span attribute values that match a blocked value -list. Span attributes that aren't on the allowed list are removed before any +This processor deletes span, log, and metric datapoint attributes that don't match a list of allowed +attributes. It also masks attribute values that match a blocked value +list. Attributes that aren't on the allowed list are removed before any value checks are done. ## Use Cases @@ -57,9 +59,9 @@ processors: # allowed_keys list. The list of blocked_values is applied regardless. If # you just want to block values, set this to true. allow_all_keys: false - # allowed_keys is a list of span attribute keys that are kept on the span and + # allowed_keys is a list of span/log/datapoint attribute keys that are kept on the span/log/datapoint and # processed. The list is designed to fail closed. If allowed_keys is empty, - # no span attributes are allowed and all span attributes are removed. To + # no attributes are allowed and all span attributes are removed. To # allow all keys, set allow_all_keys to true. allowed_keys: - description @@ -76,7 +78,7 @@ processors: - "4[0-9]{12}(?:[0-9]{3})?" ## Visa credit card number - "(5[1-5][0-9]{14})" ## MasterCard number # summary controls the verbosity level of the diagnostic attributes that - # the processor adds to the spans when it redacts or masks other + # the processor adds to the spans/logs/datapoints when it redacts or masks other # attributes. In some contexts a list of redacted attributes leaks # information, while it is valuable when integrating and testing a new # configuration. Possible values: @@ -93,8 +95,8 @@ Ignored attributes are processed first so they're always allowed and never blocked. This field should only be used where you know the data is always safe to send to the telemetry system. -Only span attributes included on the list of allowed keys list are retained. -If `allowed_keys` is empty, then no span attributes are allowed. All span +Only span/log/datapoint attributes included on the list of allowed keys list are retained. +If `allowed_keys` is empty, then no attributes are allowed. All attributes are removed in that case. To keep all span attributes, you should explicitly set `allow_all_keys` to true. @@ -102,7 +104,7 @@ explicitly set `allow_all_keys` to true. allowed key matches the regular expression for a blocked value, the matching part of the value is then masked with a fixed length of asterisks. -For example, if `notes` is on the list of allowed keys, then the `notes` span +For example, if `notes` is on the list of allowed keys, then the `notes` attribute is retained. However, if there is a value such as a credit card number in the `notes` field that matched a regular expression on the list of blocked values, then that value is masked. diff --git a/processor/redactionprocessor/factory.go b/processor/redactionprocessor/factory.go index 7ef213e475ef..b900a03e512f 100644 --- a/processor/redactionprocessor/factory.go +++ b/processor/redactionprocessor/factory.go @@ -23,6 +23,8 @@ func NewFactory() processor.Factory { metadata.Type, createDefaultConfig, processor.WithTraces(createTracesProcessor, metadata.TracesStability), + processor.WithLogs(createLogsProcessor, metadata.LogsStability), + processor.WithMetrics(createMetricsProcessor, metadata.MetricsStability), ) } @@ -53,3 +55,47 @@ func createTracesProcessor( redaction.processTraces, processorhelper.WithCapabilities(consumer.Capabilities{MutatesData: true})) } + +// createLogsProcessor creates an instance of redaction for processing logs +func createLogsProcessor( + ctx context.Context, + set processor.Settings, + cfg component.Config, + next consumer.Logs) (processor.Logs, error) { + oCfg := cfg.(*Config) + + red, err := newRedaction(ctx, oCfg, set.Logger) + if err != nil { + return nil, fmt.Errorf("error creating a redaction processor: %w", err) + } + + return processorhelper.NewLogsProcessor( + ctx, + set, + cfg, + next, + red.processLogs, + processorhelper.WithCapabilities(consumer.Capabilities{MutatesData: true})) +} + +// createMetricsProcessor creates an instance of redaction for processing metrics +func createMetricsProcessor( + ctx context.Context, + set processor.Settings, + cfg component.Config, + next consumer.Metrics) (processor.Metrics, error) { + oCfg := cfg.(*Config) + + red, err := newRedaction(ctx, oCfg, set.Logger) + if err != nil { + return nil, fmt.Errorf("error creating a redaction processor: %w", err) + } + + return processorhelper.NewMetricsProcessor( + ctx, + set, + cfg, + next, + red.processMetrics, + processorhelper.WithCapabilities(consumer.Capabilities{MutatesData: true})) +} diff --git a/processor/redactionprocessor/factory_test.go b/processor/redactionprocessor/factory_test.go index 7afe5741c794..490b59a38ab9 100644 --- a/processor/redactionprocessor/factory_test.go +++ b/processor/redactionprocessor/factory_test.go @@ -26,3 +26,21 @@ func TestCreateTestProcessor(t *testing.T) { assert.NotNil(t, tp) assert.True(t, tp.Capabilities().MutatesData) } + +func TestCreateTestLogsProcessor(t *testing.T) { + cfg := &Config{} + + tp, err := createLogsProcessor(context.Background(), processortest.NewNopSettings(), cfg, consumertest.NewNop()) + assert.NoError(t, err) + assert.NotNil(t, tp) + assert.Equal(t, true, tp.Capabilities().MutatesData) +} + +func TestCreateTestMetricsProcessor(t *testing.T) { + cfg := &Config{} + + tp, err := createMetricsProcessor(context.Background(), processortest.NewNopSettings(), cfg, consumertest.NewNop()) + assert.NoError(t, err) + assert.NotNil(t, tp) + assert.Equal(t, true, tp.Capabilities().MutatesData) +} diff --git a/processor/redactionprocessor/generated_component_test.go b/processor/redactionprocessor/generated_component_test.go index a59a3b22d7d3..baa14ee81730 100644 --- a/processor/redactionprocessor/generated_component_test.go +++ b/processor/redactionprocessor/generated_component_test.go @@ -36,6 +36,20 @@ func TestComponentLifecycle(t *testing.T) { createFn func(ctx context.Context, set processor.Settings, cfg component.Config) (component.Component, error) }{ + { + name: "logs", + createFn: func(ctx context.Context, set processor.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateLogsProcessor(ctx, set, cfg, consumertest.NewNop()) + }, + }, + + { + name: "metrics", + createFn: func(ctx context.Context, set processor.Settings, cfg component.Config) (component.Component, error) { + return factory.CreateMetricsProcessor(ctx, set, cfg, consumertest.NewNop()) + }, + }, + { name: "traces", createFn: func(ctx context.Context, set processor.Settings, cfg component.Config) (component.Component, error) { diff --git a/processor/redactionprocessor/internal/metadata/generated_status.go b/processor/redactionprocessor/internal/metadata/generated_status.go index 4dd78cb1284d..4d9e987c1961 100644 --- a/processor/redactionprocessor/internal/metadata/generated_status.go +++ b/processor/redactionprocessor/internal/metadata/generated_status.go @@ -12,5 +12,7 @@ var ( ) const ( - TracesStability = component.StabilityLevelBeta + LogsStability = component.StabilityLevelAlpha + MetricsStability = component.StabilityLevelAlpha + TracesStability = component.StabilityLevelBeta ) diff --git a/processor/redactionprocessor/metadata.yaml b/processor/redactionprocessor/metadata.yaml index 8b4d5d286edc..466150b8ce13 100644 --- a/processor/redactionprocessor/metadata.yaml +++ b/processor/redactionprocessor/metadata.yaml @@ -4,6 +4,7 @@ status: class: processor stability: beta: [traces] + alpha: [logs,metrics] distributions: [contrib] codeowners: active: [dmitryax, mx-psi, TylerHelmuth] diff --git a/processor/redactionprocessor/processor.go b/processor/redactionprocessor/processor.go index 57368a1878a1..ae07d3819a3f 100644 --- a/processor/redactionprocessor/processor.go +++ b/processor/redactionprocessor/processor.go @@ -11,6 +11,8 @@ import ( "strings" "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap" ) @@ -59,6 +61,22 @@ func (s *redaction) processTraces(ctx context.Context, batch ptrace.Traces) (ptr return batch, nil } +func (s *redaction) processLogs(ctx context.Context, logs plog.Logs) (plog.Logs, error) { + for i := 0; i < logs.ResourceLogs().Len(); i++ { + rl := logs.ResourceLogs().At(i) + s.processResourceLog(ctx, rl) + } + return logs, nil +} + +func (s *redaction) processMetrics(ctx context.Context, metrics pmetric.Metrics) (pmetric.Metrics, error) { + for i := 0; i < metrics.ResourceMetrics().Len(); i++ { + rm := metrics.ResourceMetrics().At(i) + s.processResourceMetric(ctx, rm) + } + return metrics, nil +} + // processResourceSpan processes the RS and all of its spans and then returns the last // view metric context. The context can be used for tests func (s *redaction) processResourceSpan(ctx context.Context, rs ptrace.ResourceSpans) { @@ -79,6 +97,63 @@ func (s *redaction) processResourceSpan(ctx context.Context, rs ptrace.ResourceS } } +// processResourceLog processes the log resource and all of its logs and then returns the last +// view metric context. The context can be used for tests +func (s *redaction) processResourceLog(ctx context.Context, rl plog.ResourceLogs) { + rsAttrs := rl.Resource().Attributes() + + s.processAttrs(ctx, rsAttrs) + + for j := 0; j < rl.ScopeLogs().Len(); j++ { + ils := rl.ScopeLogs().At(j) + for k := 0; k < rl.ScopeLogs().Len(); k++ { + log := ils.LogRecords().At(k) + s.processAttrs(ctx, log.Attributes()) + } + } +} + +func (s *redaction) processResourceMetric(ctx context.Context, rm pmetric.ResourceMetrics) { + rsAttrs := rm.Resource().Attributes() + + s.processAttrs(ctx, rsAttrs) + + for j := 0; j < rm.ScopeMetrics().Len(); j++ { + ils := rm.ScopeMetrics().At(j) + for k := 0; k < ils.Metrics().Len(); k++ { + metric := ils.Metrics().At(k) + switch metric.Type() { + case pmetric.MetricTypeGauge: + dps := metric.Gauge().DataPoints() + for i := 0; i < dps.Len(); i++ { + s.processAttrs(ctx, dps.At(i).Attributes()) + } + case pmetric.MetricTypeSum: + dps := metric.Sum().DataPoints() + for i := 0; i < dps.Len(); i++ { + s.processAttrs(ctx, dps.At(i).Attributes()) + } + case pmetric.MetricTypeHistogram: + dps := metric.Histogram().DataPoints() + for i := 0; i < dps.Len(); i++ { + s.processAttrs(ctx, dps.At(i).Attributes()) + } + case pmetric.MetricTypeExponentialHistogram: + dps := metric.ExponentialHistogram().DataPoints() + for i := 0; i < dps.Len(); i++ { + s.processAttrs(ctx, dps.At(i).Attributes()) + } + case pmetric.MetricTypeSummary: + dps := metric.Summary().DataPoints() + for i := 0; i < dps.Len(); i++ { + s.processAttrs(ctx, dps.At(i).Attributes()) + } + case pmetric.MetricTypeEmpty: + } + } + } +} + // processAttrs redacts the attributes of a resource span or a span func (s *redaction) processAttrs(_ context.Context, attributes pcommon.Map) { // TODO: Use the context for recording metrics diff --git a/processor/redactionprocessor/processor_test.go b/processor/redactionprocessor/processor_test.go index b44ab20c950e..acc11d817c81 100644 --- a/processor/redactionprocessor/processor_test.go +++ b/processor/redactionprocessor/processor_test.go @@ -12,6 +12,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/pdata/pcommon" + "go.opentelemetry.io/collector/pdata/plog" + "go.opentelemetry.io/collector/pdata/pmetric" "go.opentelemetry.io/collector/pdata/ptrace" "go.uber.org/zap/zaptest" ) @@ -35,16 +37,33 @@ func TestRedactUnknownAttributes(t *testing.T) { } outTraces := runTest(t, allowed, redacted, nil, ignored, config) - - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - for k, v := range allowed { - val, ok := attr.Get(k) - assert.True(t, ok) - assert.Equal(t, v.AsRaw(), val.AsRaw()) - } - for k := range redacted { - _, ok := attr.Get(k) - assert.False(t, ok) + outLogs := runLogsTest(t, allowed, redacted, nil, ignored, config) + outMetricsGauge := runMetricsTest(t, allowed, redacted, nil, ignored, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, redacted, nil, ignored, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, redacted, nil, ignored, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, redacted, nil, ignored, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, redacted, nil, ignored, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + for k, v := range allowed { + val, ok := attr.Get(k) + assert.True(t, ok) + assert.Equal(t, v.AsRaw(), val.AsRaw()) + } + for k := range redacted { + _, ok := attr.Get(k) + assert.False(t, ok) + } } } @@ -63,15 +82,32 @@ func TestAllowAllKeys(t *testing.T) { } outTraces := runTest(t, allowed, nil, nil, nil, config) - - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - for k, v := range allowed { - val, ok := attr.Get(k) - assert.True(t, ok) - assert.Equal(t, v.AsRaw(), val.AsRaw()) + outLogs := runLogsTest(t, allowed, nil, nil, nil, config) + outMetricsGauge := runMetricsTest(t, allowed, nil, nil, nil, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, nil, nil, nil, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, nil, nil, nil, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, nil, nil, nil, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, nil, nil, nil, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + for k, v := range allowed { + val, ok := attr.Get(k) + assert.True(t, ok) + assert.Equal(t, v.AsRaw(), val.AsRaw()) + } + value, _ := attr.Get("name") + assert.Equal(t, "placeholder", value.Str()) } - value, _ := attr.Get("name") - assert.Equal(t, "placeholder", value.Str()) } // TestAllowAllKeysMaskValues validates that the processor still redacts @@ -92,15 +128,32 @@ func TestAllowAllKeysMaskValues(t *testing.T) { } outTraces := runTest(t, allowed, nil, masked, nil, config) - - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - for k, v := range allowed { - val, ok := attr.Get(k) - assert.True(t, ok) - assert.Equal(t, v.AsRaw(), val.AsRaw()) + outLogs := runLogsTest(t, allowed, nil, masked, nil, config) + outMetricsGauge := runMetricsTest(t, allowed, nil, masked, nil, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, nil, masked, nil, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, nil, masked, nil, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, nil, masked, nil, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, nil, masked, nil, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + for k, v := range allowed { + val, ok := attr.Get(k) + assert.True(t, ok) + assert.Equal(t, v.AsRaw(), val.AsRaw()) + } + value, _ := attr.Get("credit_card") + assert.Equal(t, "placeholder ****", value.Str()) } - value, _ := attr.Get("credit_card") - assert.Equal(t, "placeholder ****", value.Str()) } // TODO: Test redaction with metric tags in a metrics PR @@ -131,35 +184,52 @@ func TestRedactSummaryDebug(t *testing.T) { } outTraces := runTest(t, allowed, redacted, masked, ignored, config) + outLogs := runLogsTest(t, allowed, redacted, masked, ignored, config) + outMetricsGauge := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + deleted := make([]string, 0, len(redacted)) + for k := range redacted { + _, ok := attr.Get(k) + assert.False(t, ok) + deleted = append(deleted, k) + } + maskedKeys, ok := attr.Get(redactedKeys) + assert.True(t, ok) + sort.Strings(deleted) + assert.Equal(t, strings.Join(deleted, ","), maskedKeys.Str()) + maskedKeyCount, ok := attr.Get(redactedKeyCount) + assert.True(t, ok) + assert.Equal(t, int64(len(deleted)), maskedKeyCount.Int()) - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - deleted := make([]string, 0, len(redacted)) - for k := range redacted { - _, ok := attr.Get(k) - assert.False(t, ok) - deleted = append(deleted, k) - } - maskedKeys, ok := attr.Get(redactedKeys) - assert.True(t, ok) - sort.Strings(deleted) - assert.Equal(t, strings.Join(deleted, ","), maskedKeys.Str()) - maskedKeyCount, ok := attr.Get(redactedKeyCount) - assert.True(t, ok) - assert.Equal(t, int64(len(deleted)), maskedKeyCount.Int()) - - ignoredKeyCount, ok := attr.Get(ignoredKeyCount) - assert.True(t, ok) - assert.Equal(t, int64(len(ignored)), ignoredKeyCount.Int()) - - blockedKeys := []string{"name"} - maskedValues, ok := attr.Get(maskedValues) - assert.True(t, ok) - assert.Equal(t, strings.Join(blockedKeys, ","), maskedValues.Str()) - maskedValueCount, ok := attr.Get(maskedValueCount) - assert.True(t, ok) - assert.Equal(t, int64(1), maskedValueCount.Int()) - value, _ := attr.Get("name") - assert.Equal(t, "placeholder ****", value.Str()) + ignoredKeyCount, ok := attr.Get(ignoredKeyCount) + assert.True(t, ok) + assert.Equal(t, int64(len(ignored)), ignoredKeyCount.Int()) + + blockedKeys := []string{"name"} + maskedValues, ok := attr.Get(maskedValues) + assert.True(t, ok) + assert.Equal(t, strings.Join(blockedKeys, ","), maskedValues.Str()) + maskedValueCount, ok := attr.Get(maskedValueCount) + assert.True(t, ok) + assert.Equal(t, int64(1), maskedValueCount.Int()) + value, _ := attr.Get("name") + assert.Equal(t, "placeholder ****", value.Str()) + } } // TestRedactSummaryInfo validates that the processor writes a verbose summary @@ -186,33 +256,50 @@ func TestRedactSummaryInfo(t *testing.T) { } outTraces := runTest(t, allowed, redacted, masked, ignored, config) - - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - deleted := make([]string, 0, len(redacted)) - for k := range redacted { - _, ok := attr.Get(k) + outLogs := runLogsTest(t, allowed, redacted, masked, ignored, config) + outMetricsGauge := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, redacted, masked, ignored, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + deleted := make([]string, 0, len(redacted)) + for k := range redacted { + _, ok := attr.Get(k) + assert.False(t, ok) + deleted = append(deleted, k) + } + _, ok := attr.Get(redactedKeys) assert.False(t, ok) - deleted = append(deleted, k) - } - _, ok := attr.Get(redactedKeys) - assert.False(t, ok) - maskedKeyCount, ok := attr.Get(redactedKeyCount) - assert.True(t, ok) - assert.Equal(t, int64(len(deleted)), maskedKeyCount.Int()) - _, ok = attr.Get(maskedValues) - assert.False(t, ok) - - maskedValueCount, ok := attr.Get(maskedValueCount) - assert.True(t, ok) - assert.Equal(t, int64(1), maskedValueCount.Int()) - value, _ := attr.Get("name") - assert.Equal(t, "placeholder ****", value.Str()) - - ignoredKeyCount, ok := attr.Get(ignoredKeyCount) - assert.True(t, ok) - assert.Equal(t, int64(1), ignoredKeyCount.Int()) - value, _ = attr.Get("safe_attribute") - assert.Equal(t, "harmless but suspicious 4111111111111141", value.Str()) + maskedKeyCount, ok := attr.Get(redactedKeyCount) + assert.True(t, ok) + assert.Equal(t, int64(len(deleted)), maskedKeyCount.Int()) + _, ok = attr.Get(maskedValues) + assert.False(t, ok) + + maskedValueCount, ok := attr.Get(maskedValueCount) + assert.True(t, ok) + assert.Equal(t, int64(1), maskedValueCount.Int()) + value, _ := attr.Get("name") + assert.Equal(t, "placeholder ****", value.Str()) + + ignoredKeyCount, ok := attr.Get(ignoredKeyCount) + assert.True(t, ok) + assert.Equal(t, int64(1), ignoredKeyCount.Int()) + value, _ = attr.Get("safe_attribute") + assert.Equal(t, "harmless but suspicious 4111111111111141", value.Str()) + } } // TestRedactSummarySilent validates that the processor does not create the @@ -232,22 +319,39 @@ func TestRedactSummarySilent(t *testing.T) { } outTraces := runTest(t, allowed, redacted, masked, nil, config) - - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - for k := range redacted { - _, ok := attr.Get(k) + outLogs := runLogsTest(t, allowed, redacted, masked, nil, config) + outMetricsGauge := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + for k := range redacted { + _, ok := attr.Get(k) + assert.False(t, ok) + } + _, ok := attr.Get(redactedKeys) + assert.False(t, ok) + _, ok = attr.Get(redactedKeyCount) + assert.False(t, ok) + _, ok = attr.Get(maskedValues) + assert.False(t, ok) + _, ok = attr.Get(maskedValueCount) assert.False(t, ok) + value, _ := attr.Get("name") + assert.Equal(t, "placeholder ****", value.Str()) } - _, ok := attr.Get(redactedKeys) - assert.False(t, ok) - _, ok = attr.Get(redactedKeyCount) - assert.False(t, ok) - _, ok = attr.Get(maskedValues) - assert.False(t, ok) - _, ok = attr.Get(maskedValueCount) - assert.False(t, ok) - value, _ := attr.Get("name") - assert.Equal(t, "placeholder ****", value.Str()) } // TestRedactSummaryDefault validates that the processor does not create the @@ -265,18 +369,35 @@ func TestRedactSummaryDefault(t *testing.T) { } outTraces := runTest(t, allowed, nil, masked, ignored, config) - - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - _, ok := attr.Get(redactedKeys) - assert.False(t, ok) - _, ok = attr.Get(redactedKeyCount) - assert.False(t, ok) - _, ok = attr.Get(maskedValues) - assert.False(t, ok) - _, ok = attr.Get(maskedValueCount) - assert.False(t, ok) - _, ok = attr.Get(ignoredKeyCount) - assert.False(t, ok) + outLogs := runLogsTest(t, allowed, nil, masked, ignored, config) + outMetricsGauge := runMetricsTest(t, allowed, nil, masked, ignored, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, nil, masked, ignored, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, nil, masked, ignored, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, nil, masked, ignored, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, nil, masked, ignored, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + _, ok := attr.Get(redactedKeys) + assert.False(t, ok) + _, ok = attr.Get(redactedKeyCount) + assert.False(t, ok) + _, ok = attr.Get(maskedValues) + assert.False(t, ok) + _, ok = attr.Get(maskedValueCount) + assert.False(t, ok) + _, ok = attr.Get(ignoredKeyCount) + assert.False(t, ok) + } } // TestMultipleBlockValues validates that the processor can block multiple @@ -297,35 +418,52 @@ func TestMultipleBlockValues(t *testing.T) { } outTraces := runTest(t, allowed, redacted, masked, nil, config) + outLogs := runLogsTest(t, allowed, redacted, masked, nil, config) + outMetricsGauge := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeGauge) + outMetricsSum := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeSum) + outMetricsHistogram := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeHistogram) + outMetricsExponentialHistogram := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeExponentialHistogram) + outMetricsSummary := runMetricsTest(t, allowed, redacted, masked, nil, config, pmetric.MetricTypeSummary) + + attrs := []pcommon.Map{ + outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes(), + outLogs.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes(), + outMetricsGauge.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Gauge().DataPoints().At(0).Attributes(), + outMetricsSum.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Sum().DataPoints().At(0).Attributes(), + outMetricsHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Histogram().DataPoints().At(0).Attributes(), + outMetricsExponentialHistogram.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).ExponentialHistogram().DataPoints().At(0).Attributes(), + outMetricsSummary.ResourceMetrics().At(0).ScopeMetrics().At(0).Metrics().At(0).Summary().DataPoints().At(0).Attributes(), + } + + for _, attr := range attrs { + deleted := make([]string, 0, len(redacted)) + for k := range redacted { + _, ok := attr.Get(k) + assert.False(t, ok) + deleted = append(deleted, k) + } + maskedKeys, ok := attr.Get(redactedKeys) + assert.True(t, ok) + assert.Equal(t, strings.Join(deleted, ","), maskedKeys.Str()) + maskedKeyCount, ok := attr.Get(redactedKeyCount) + assert.True(t, ok) + assert.Equal(t, int64(len(deleted)), maskedKeyCount.Int()) - attr := outTraces.ResourceSpans().At(0).ScopeSpans().At(0).Spans().At(0).Attributes() - deleted := make([]string, 0, len(redacted)) - for k := range redacted { - _, ok := attr.Get(k) - assert.False(t, ok) - deleted = append(deleted, k) - } - maskedKeys, ok := attr.Get(redactedKeys) - assert.True(t, ok) - assert.Equal(t, strings.Join(deleted, ","), maskedKeys.Str()) - maskedKeyCount, ok := attr.Get(redactedKeyCount) - assert.True(t, ok) - assert.Equal(t, int64(len(deleted)), maskedKeyCount.Int()) - - blockedKeys := []string{"name", "mystery"} - maskedValues, ok := attr.Get(maskedValues) - assert.True(t, ok) - sort.Strings(blockedKeys) - assert.Equal(t, strings.Join(blockedKeys, ","), maskedValues.Str()) - assert.Equal(t, pcommon.ValueTypeStr, maskedValues.Type()) - assert.Equal(t, strings.Join(blockedKeys, ","), maskedValues.Str()) - maskedValueCount, ok := attr.Get(maskedValueCount) - assert.True(t, ok) - assert.Equal(t, int64(len(blockedKeys)), maskedValueCount.Int()) - nameValue, _ := attr.Get("name") - mysteryValue, _ := attr.Get("mystery") - assert.Equal(t, "placeholder **** ****", nameValue.Str()) - assert.Equal(t, "mystery ****", mysteryValue.Str()) + blockedKeys := []string{"name", "mystery"} + maskedValues, ok := attr.Get(maskedValues) + assert.True(t, ok) + sort.Strings(blockedKeys) + assert.Equal(t, strings.Join(blockedKeys, ","), maskedValues.Str()) + assert.Equal(t, pcommon.ValueTypeStr, maskedValues.Type()) + assert.Equal(t, strings.Join(blockedKeys, ","), maskedValues.Str()) + maskedValueCount, ok := attr.Get(maskedValueCount) + assert.True(t, ok) + assert.Equal(t, int64(len(blockedKeys)), maskedValueCount.Int()) + nameValue, _ := attr.Get("name") + mysteryValue, _ := attr.Get("mystery") + assert.Equal(t, "placeholder **** ****", nameValue.Str()) + assert.Equal(t, "mystery ****", mysteryValue.Str()) + } } // TestProcessAttrsAppliedTwice validates a use case when data is coming through redaction processor more than once. @@ -415,6 +553,120 @@ func runTest( return outBatch } +// runLogsTest transforms the test input log data and passes it through the processor +func runLogsTest( + t *testing.T, + allowed map[string]pcommon.Value, + redacted map[string]pcommon.Value, + masked map[string]pcommon.Value, + ignored map[string]pcommon.Value, + config *Config, +) plog.Logs { + inBatch := plog.NewLogs() + rl := inBatch.ResourceLogs().AppendEmpty() + ils := rl.ScopeLogs().AppendEmpty() + + library := ils.Scope() + library.SetName("first-library") + logEntry := ils.LogRecords().AppendEmpty() + logEntry.Body().SetStr("first-batch-first-logEntry") + logEntry.SetTraceID([16]byte{1, 2, 3, 4}) + + length := len(allowed) + len(masked) + len(redacted) + len(ignored) + for k, v := range allowed { + v.CopyTo(logEntry.Attributes().PutEmpty(k)) + } + for k, v := range masked { + v.CopyTo(logEntry.Attributes().PutEmpty(k)) + } + for k, v := range redacted { + v.CopyTo(logEntry.Attributes().PutEmpty(k)) + } + for k, v := range ignored { + v.CopyTo(logEntry.Attributes().PutEmpty(k)) + } + + assert.Equal(t, logEntry.Attributes().Len(), length) + assert.Equal(t, ils.LogRecords().At(0).Attributes().Len(), length) + assert.Equal(t, inBatch.ResourceLogs().At(0).ScopeLogs().At(0).LogRecords().At(0).Attributes().Len(), length) + + // test + ctx := context.Background() + processor, err := newRedaction(ctx, config, zaptest.NewLogger(t)) + assert.NoError(t, err) + outBatch, err := processor.processLogs(ctx, inBatch) + + // verify + assert.NoError(t, err) + return outBatch +} + +// runMetricsTest transforms the test input metric data and passes it through the processor +func runMetricsTest( + t *testing.T, + allowed map[string]pcommon.Value, + redacted map[string]pcommon.Value, + masked map[string]pcommon.Value, + ignored map[string]pcommon.Value, + config *Config, + metricType pmetric.MetricType, +) pmetric.Metrics { + inBatch := pmetric.NewMetrics() + rl := inBatch.ResourceMetrics().AppendEmpty() + ils := rl.ScopeMetrics().AppendEmpty() + + library := ils.Scope() + library.SetName("first-library") + metric := ils.Metrics().AppendEmpty() + metric.SetDescription("first-batch-first-metric") + + length := len(allowed) + len(masked) + len(redacted) + len(ignored) + + var dataPointAttrs pcommon.Map + switch metricType { + case pmetric.MetricTypeGauge: + dataPointAttrs = metric.SetEmptyGauge().DataPoints().AppendEmpty().Attributes() + case pmetric.MetricTypeSum: + dataPointAttrs = metric.SetEmptySum().DataPoints().AppendEmpty().Attributes() + case pmetric.MetricTypeHistogram: + dataPointAttrs = metric.SetEmptyHistogram().DataPoints().AppendEmpty().Attributes() + case pmetric.MetricTypeExponentialHistogram: + dataPointAttrs = metric.SetEmptyExponentialHistogram().DataPoints().AppendEmpty().Attributes() + case pmetric.MetricTypeSummary: + dataPointAttrs = metric.SetEmptySummary().DataPoints().AppendEmpty().Attributes() + case pmetric.MetricTypeEmpty: + } + for k, v := range allowed { + v.CopyTo(dataPointAttrs.PutEmpty(k)) + v.CopyTo(rl.Resource().Attributes().PutEmpty(k)) + } + for k, v := range masked { + v.CopyTo(dataPointAttrs.PutEmpty(k)) + v.CopyTo(rl.Resource().Attributes().PutEmpty(k)) + } + for k, v := range redacted { + v.CopyTo(dataPointAttrs.PutEmpty(k)) + v.CopyTo(rl.Resource().Attributes().PutEmpty(k)) + } + for k, v := range ignored { + v.CopyTo(dataPointAttrs.PutEmpty(k)) + v.CopyTo(rl.Resource().Attributes().PutEmpty(k)) + } + + assert.Equal(t, length, dataPointAttrs.Len()) + assert.Equal(t, length, rl.Resource().Attributes().Len()) + + // test + ctx := context.Background() + processor, err := newRedaction(ctx, config, zaptest.NewLogger(t)) + assert.NoError(t, err) + outBatch, err := processor.processMetrics(ctx, inBatch) + + // verify + assert.NoError(t, err) + return outBatch +} + // BenchmarkRedactSummaryDebug measures the performance impact of running the processor // with full debug level of output for redacting span attributes not on the allowed // keys list