diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9a1dbb55644..b852d9d5b47 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -17,7 +17,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - `queue.mem.flush.timeout` is changing from `1s` to `10s`. - `output.elasticsearch.bulk_max_size` is changing from `50` to `1600`. - `output.elasticsearch.idle_connection_timeout` is changing from `60s` to `3s`. - +- Avoid logging fields values when handling Elasticsearch output errors except at the debug log level. The debug log level must now be used to see detailed errors, for example mapping errors and their cause. {pull}37229[37229] *Auditbeat* diff --git a/libbeat/outputs/elasticsearch/client.go b/libbeat/outputs/elasticsearch/client.go index b485807776e..d8538d39615 100644 --- a/libbeat/outputs/elasticsearch/client.go +++ b/libbeat/outputs/elasticsearch/client.go @@ -427,10 +427,12 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat result, _ := data[i].Content.Meta.HasKey(dead_letter_marker_field) if result { stats.nonIndexable++ - client.log.Errorf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg) + client.log.Errorf("Can't deliver to dead letter index event (status=%v). Enable debug logs to view the event and cause.", status) + client.log.Debugf("Can't deliver to dead letter index event %#v (status=%v): %s", data[i], status, msg) // poison pill - this will clog the pipeline if the underlying failure is non transient. } else if client.NonIndexableAction == dead_letter_index { - client.log.Warnf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg) + client.log.Warnf("Cannot index event (status=%v), trying dead letter index. Enable debug logs to view the event and cause.", status) + client.log.Debugf("Cannot index event %#v (status=%v): %s, trying dead letter index", data[i], status, msg) if data[i].Content.Meta == nil { data[i].Content.Meta = mapstr.M{ dead_letter_marker_field: true, @@ -445,7 +447,8 @@ func (client *Client) bulkCollectPublishFails(result eslegclient.BulkResult, dat } } else { // drop stats.nonIndexable++ - client.log.Warnf("Cannot index event %#v (status=%v): %s, dropping event!", data[i], status, msg) + client.log.Warnf("Cannot index event (status=%v): dropping event!", status) + client.log.Debugf("Cannot index event %#v (status=%v): %s, dropping event! Enable debug logs to view the event and cause.", data[i], status, msg) continue } }