From 79d41dd2623220cb13ce503faca6d3f9105c6ede Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Tue, 25 Jul 2023 09:09:03 -0700 Subject: [PATCH 01/13] Add chlog --- .chloggen/prw5xx.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 .chloggen/prw5xx.yaml diff --git a/.chloggen/prw5xx.yaml b/.chloggen/prw5xx.yaml new file mode 100755 index 000000000000..94ccbdfeaa2a --- /dev/null +++ b/.chloggen/prw5xx.yaml @@ -0,0 +1,20 @@ +# Use this changelog template to create an entry for release notes. +# If your change doesn't affect end users, such as a test fix or a tooling change, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: prometheusremotewriteexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Retry on 5xx status codes using go-retryablehttp client + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [20304] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: From 2630d0620e4ed894abb0b42367727ceb67b4637b Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 27 Jul 2023 14:57:00 -0700 Subject: [PATCH 02/13] Use retryablehttp client --- exporter/prometheusremotewriteexporter/exporter.go | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 296e13d3dac8..8cc520bbc715 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -17,6 +17,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" + "github.com/hashicorp/go-retryablehttp" "github.com/prometheus/prometheus/prompb" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/confighttp" @@ -34,7 +35,7 @@ const maxBatchByteSize = 3000000 // prwExporter converts OTLP metrics to Prometheus remote write TimeSeries and sends them to a remote endpoint. type prwExporter struct { endpointURL *url.URL - client *http.Client + client *retryablehttp.Client wg *sync.WaitGroup closeChan chan struct{} concurrency int @@ -68,6 +69,7 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err concurrency: cfg.RemoteWriteQueue.NumConsumers, clientSettings: &cfg.HTTPClientSettings, settings: set.TelemetrySettings, + client: retryablehttp.NewClient(), exporterSettings: prometheusremotewrite.Settings{ Namespace: cfg.Namespace, ExternalLabels: sanitizedLabels, @@ -89,10 +91,14 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err // Start creates the prometheus client func (prwe *prwExporter) Start(ctx context.Context, host component.Host) (err error) { - prwe.client, err = prwe.clientSettings.ToClient(host, prwe.settings) + httpClient, err := prwe.clientSettings.ToClient(host, prwe.settings) if err != nil { return err } + + // Set the HTTP client as the underlying client for the retryable client + prwe.client.HTTPClient = httpClient + return prwe.turnOnWALIfEnabled(contextWithLogger(ctx, prwe.settings.Logger.Named("prw.wal"))) } @@ -237,7 +243,8 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") req.Header.Set("User-Agent", prwe.userAgentHeader) - resp, err := prwe.client.Do(req) + retryReq, _ := retryablehttp.FromRequest(req) + resp, err := prwe.client.Do(retryReq) if err != nil { return consumererror.NewPermanent(err) } From 31ff4205f9cf64e8797c0eaf5e96fb0a5480eb80 Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 27 Jul 2023 15:01:24 -0700 Subject: [PATCH 03/13] Modify Test case to count retry attempts --- .../exporter_test.go | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index 35ff6028e427..919ec7383386 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -5,6 +5,7 @@ package prometheusremotewriteexporter import ( "context" + "github.com/hashicorp/go-retryablehttp" "io" "net/http" "net/http/httptest" @@ -978,3 +979,33 @@ func TestWALOnExporterRoundTrip(t *testing.T) { assert.Equal(t, want, gotFromUpload) assert.Equal(t, gotFromWAL, gotFromUpload) } + +func TestRetryOn5xx(t *testing.T) { + // Create a mock HTTP server with a counter to simulate a 5xx error on the first attempt and a 2xx success on the second attempt + attempts := 0 + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if attempts < 4 { + attempts++ + http.Error(w, "Internal Server Error", http.StatusInternalServerError) + } else { + w.WriteHeader(http.StatusOK) + } + })) + defer mockServer.Close() + + endpointURL, err := url.Parse(mockServer.URL) + assert.NoError(t, err) + + // Create the prwExporter + exporter := &prwExporter{ + endpointURL: endpointURL, + client: retryablehttp.NewClient(), + } + + ctx := context.Background() + + // Execute the write request and verify that the exporter returns a non-permanent error on the first attempt. + err = exporter.execute(ctx, &prompb.WriteRequest{}) + assert.NoError(t, err) + assert.Equal(t, 4, attempts) +} From 8df157cf68435eb7157978d7b41b5c189e26b5c8 Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Fri, 28 Jul 2023 14:06:20 -0700 Subject: [PATCH 04/13] Use require.NoError --- exporter/prometheusremotewriteexporter/exporter_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index 919ec7383386..3242044f8de3 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -994,7 +994,7 @@ func TestRetryOn5xx(t *testing.T) { defer mockServer.Close() endpointURL, err := url.Parse(mockServer.URL) - assert.NoError(t, err) + require.NoError(t, err) // Create the prwExporter exporter := &prwExporter{ From 4242c3bf2657e071fee63dde03167e90467abb5a Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Mon, 31 Jul 2023 10:53:55 -0700 Subject: [PATCH 05/13] Rebased --- exporter/prometheusremotewriteexporter/exporter.go | 2 +- exporter/prometheusremotewriteexporter/exporter_test.go | 2 +- exporter/prometheusremotewriteexporter/go.mod | 2 ++ exporter/prometheusremotewriteexporter/go.sum | 9 +++++++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 8cc520bbc715..e68ec9760cf5 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -69,7 +69,7 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err concurrency: cfg.RemoteWriteQueue.NumConsumers, clientSettings: &cfg.HTTPClientSettings, settings: set.TelemetrySettings, - client: retryablehttp.NewClient(), + client: retryablehttp.NewClient(), exporterSettings: prometheusremotewrite.Settings{ Namespace: cfg.Namespace, ExternalLabels: sanitizedLabels, diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index 3242044f8de3..be49a7cb88cf 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -5,7 +5,6 @@ package prometheusremotewriteexporter import ( "context" - "github.com/hashicorp/go-retryablehttp" "io" "net/http" "net/http/httptest" @@ -15,6 +14,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" + "github.com/hashicorp/go-retryablehttp" "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/assert" diff --git a/exporter/prometheusremotewriteexporter/go.mod b/exporter/prometheusremotewriteexporter/go.mod index c7e3bc54cc37..2e22a129f947 100644 --- a/exporter/prometheusremotewriteexporter/go.mod +++ b/exporter/prometheusremotewriteexporter/go.mod @@ -7,6 +7,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 + github.com/hashicorp/go-retryablehttp v0.7.2 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.82.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.82.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.82.0 @@ -32,6 +33,7 @@ require ( github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/knadh/koanf v1.5.0 // indirect diff --git a/exporter/prometheusremotewriteexporter/go.sum b/exporter/prometheusremotewriteexporter/go.sum index b23b71f6346e..5c0d4b8e127c 100644 --- a/exporter/prometheusremotewriteexporter/go.sum +++ b/exporter/prometheusremotewriteexporter/go.sum @@ -49,6 +49,7 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -120,15 +121,21 @@ github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOj github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= +github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -180,11 +187,13 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= From fc6099e1280c17fac391f6bbb2819762ae57dd2b Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Wed, 2 Aug 2023 23:31:10 -0700 Subject: [PATCH 06/13] Use retry settings form cfg --- exporter/prometheusremotewriteexporter/exporter.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index e68ec9760cf5..30d881539d8a 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -14,6 +14,7 @@ import ( "net/url" "strings" "sync" + "time" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" @@ -42,6 +43,8 @@ type prwExporter struct { userAgentHeader string clientSettings *confighttp.HTTPClientSettings settings component.TelemetrySettings + retryWaitMin time.Duration + retryWaitMax time.Duration wal *prweWAL exporterSettings prometheusremotewrite.Settings @@ -70,6 +73,8 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err clientSettings: &cfg.HTTPClientSettings, settings: set.TelemetrySettings, client: retryablehttp.NewClient(), + retryWaitMax: cfg.MaxInterval, + retryWaitMin: cfg.InitialInterval, exporterSettings: prometheusremotewrite.Settings{ Namespace: cfg.Namespace, ExternalLabels: sanitizedLabels, @@ -98,6 +103,8 @@ func (prwe *prwExporter) Start(ctx context.Context, host component.Host) (err er // Set the HTTP client as the underlying client for the retryable client prwe.client.HTTPClient = httpClient + prwe.client.RetryWaitMin = prwe.retryWaitMin + prwe.client.RetryWaitMax = prwe.retryWaitMax return prwe.turnOnWALIfEnabled(contextWithLogger(ctx, prwe.settings.Logger.Named("prw.wal"))) } @@ -243,7 +250,10 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") req.Header.Set("User-Agent", prwe.userAgentHeader) - retryReq, _ := retryablehttp.FromRequest(req) + retryReq, err := retryablehttp.FromRequest(req) + if err != nil { + return consumererror.NewPermanent(err) + } resp, err := prwe.client.Do(retryReq) if err != nil { return consumererror.NewPermanent(err) From f03e816c6e20376348f8bd66af579785f62a211f Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 10 Aug 2023 08:35:27 -0700 Subject: [PATCH 07/13] Use cenkalti/backoff for retry --- .../prometheusremotewriteexporter/exporter.go | 103 +++++++++--------- .../exporter_test.go | 3 +- exporter/prometheusremotewriteexporter/go.mod | 2 - exporter/prometheusremotewriteexporter/go.sum | 9 -- 4 files changed, 51 insertions(+), 66 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 30d881539d8a..bba1211ed735 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -16,9 +16,9 @@ import ( "sync" "time" + "github.com/cenkalti/backoff/v4" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" - "github.com/hashicorp/go-retryablehttp" "github.com/prometheus/prometheus/prompb" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config/confighttp" @@ -36,15 +36,13 @@ const maxBatchByteSize = 3000000 // prwExporter converts OTLP metrics to Prometheus remote write TimeSeries and sends them to a remote endpoint. type prwExporter struct { endpointURL *url.URL - client *retryablehttp.Client + client *http.Client wg *sync.WaitGroup closeChan chan struct{} concurrency int userAgentHeader string clientSettings *confighttp.HTTPClientSettings settings component.TelemetrySettings - retryWaitMin time.Duration - retryWaitMax time.Duration wal *prweWAL exporterSettings prometheusremotewrite.Settings @@ -72,9 +70,6 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err concurrency: cfg.RemoteWriteQueue.NumConsumers, clientSettings: &cfg.HTTPClientSettings, settings: set.TelemetrySettings, - client: retryablehttp.NewClient(), - retryWaitMax: cfg.MaxInterval, - retryWaitMin: cfg.InitialInterval, exporterSettings: prometheusremotewrite.Settings{ Namespace: cfg.Namespace, ExternalLabels: sanitizedLabels, @@ -96,16 +91,10 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err // Start creates the prometheus client func (prwe *prwExporter) Start(ctx context.Context, host component.Host) (err error) { - httpClient, err := prwe.clientSettings.ToClient(host, prwe.settings) + prwe.client, err = prwe.clientSettings.ToClient(host, prwe.settings) if err != nil { return err } - - // Set the HTTP client as the underlying client for the retryable client - prwe.client.HTTPClient = httpClient - prwe.client.RetryWaitMin = prwe.retryWaitMin - prwe.client.RetryWaitMax = prwe.retryWaitMax - return prwe.turnOnWALIfEnabled(contextWithLogger(ctx, prwe.settings.Logger.Named("prw.wal"))) } @@ -229,50 +218,58 @@ func (prwe *prwExporter) export(ctx context.Context, requests []*prompb.WriteReq } func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequest) error { - // Uses proto.Marshal to convert the WriteRequest into bytes array - data, err := proto.Marshal(writeReq) - if err != nil { - return consumererror.NewPermanent(err) - } - buf := make([]byte, len(data), cap(data)) - compressedData := snappy.Encode(buf, data) + // Attempt the HTTP request with retry logic + backOff := backoff.NewExponentialBackOff() + backOff.InitialInterval = 1 * time.Second + backOff.MaxInterval = 10 * time.Second + backOff.MaxElapsedTime = 1 * time.Minute + + retryFunc := func() error { + // Uses proto.Marshal to convert the WriteRequest into bytes array + data, err := proto.Marshal(writeReq) + if err != nil { + return consumererror.NewPermanent(err) + } + buf := make([]byte, len(data), cap(data)) + compressedData := snappy.Encode(buf, data) - // Create the HTTP POST request to send to the endpoint - req, err := http.NewRequestWithContext(ctx, "POST", prwe.endpointURL.String(), bytes.NewReader(compressedData)) - if err != nil { - return consumererror.NewPermanent(err) - } + // Create the HTTP POST request to send to the endpoint + req, err := http.NewRequestWithContext(ctx, "POST", prwe.endpointURL.String(), bytes.NewReader(compressedData)) + if err != nil { + return consumererror.NewPermanent(err) + } - // Add necessary headers specified by: - // https://cortexmetrics.io/docs/apis/#remote-api - req.Header.Add("Content-Encoding", "snappy") - req.Header.Set("Content-Type", "application/x-protobuf") - req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") - req.Header.Set("User-Agent", prwe.userAgentHeader) + // Add necessary headers specified by: + // https://cortexmetrics.io/docs/apis/#remote-api + req.Header.Add("Content-Encoding", "snappy") + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("X-Prometheus-Remote-Write-Version", "0.1.0") + req.Header.Set("User-Agent", prwe.userAgentHeader) - retryReq, err := retryablehttp.FromRequest(req) - if err != nil { - return consumererror.NewPermanent(err) - } - resp, err := prwe.client.Do(retryReq) - if err != nil { - return consumererror.NewPermanent(err) - } - defer resp.Body.Close() + resp, err := prwe.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + // 2xx status code is considered a success + // 5xx errors are recoverable and the exporter should retry + // Reference for different behavior according to status code: + // https://github.com/prometheus/prometheus/pull/2552/files#diff-ae8db9d16d8057358e49d694522e7186 + if resp.StatusCode >= 200 && resp.StatusCode < 300 { + return nil + } - // 2xx status code is considered a success - // 5xx errors are recoverable and the exporter should retry - // Reference for different behavior according to status code: - // https://github.com/prometheus/prometheus/pull/2552/files#diff-ae8db9d16d8057358e49d694522e7186 - if resp.StatusCode >= 200 && resp.StatusCode < 300 { - return nil - } - body, err := io.ReadAll(io.LimitReader(resp.Body, 256)) - rerr := fmt.Errorf("remote write returned HTTP status %v; err = %w: %s", resp.Status, err, body) - if resp.StatusCode >= 500 && resp.StatusCode < 600 { - return rerr + body, err := io.ReadAll(io.LimitReader(resp.Body, 256)) + rerr := fmt.Errorf("remote write returned HTTP status %v; err = %w: %s", resp.Status, err, body) + if resp.StatusCode >= 500 && resp.StatusCode < 600 { + return rerr + } + return consumererror.NewPermanent(rerr) } - return consumererror.NewPermanent(rerr) + + // Use the BackOff instance to retry the operation with exponential backoff. + return backoff.Retry(retryFunc, backOff) } func (prwe *prwExporter) walEnabled() bool { return prwe.wal != nil } diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index be49a7cb88cf..7fe8e92aeb8c 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -14,7 +14,6 @@ import ( "github.com/gogo/protobuf/proto" "github.com/golang/snappy" - "github.com/hashicorp/go-retryablehttp" "github.com/prometheus/prometheus/model/value" "github.com/prometheus/prometheus/prompb" "github.com/stretchr/testify/assert" @@ -999,7 +998,7 @@ func TestRetryOn5xx(t *testing.T) { // Create the prwExporter exporter := &prwExporter{ endpointURL: endpointURL, - client: retryablehttp.NewClient(), + client: http.DefaultClient, } ctx := context.Background() diff --git a/exporter/prometheusremotewriteexporter/go.mod b/exporter/prometheusremotewriteexporter/go.mod index 2e22a129f947..c7e3bc54cc37 100644 --- a/exporter/prometheusremotewriteexporter/go.mod +++ b/exporter/prometheusremotewriteexporter/go.mod @@ -7,7 +7,6 @@ require ( github.com/fsnotify/fsnotify v1.6.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 - github.com/hashicorp/go-retryablehttp v0.7.2 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.82.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry v0.82.0 github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheus v0.82.0 @@ -33,7 +32,6 @@ require ( github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/knadh/koanf v1.5.0 // indirect diff --git a/exporter/prometheusremotewriteexporter/go.sum b/exporter/prometheusremotewriteexporter/go.sum index 5c0d4b8e127c..b23b71f6346e 100644 --- a/exporter/prometheusremotewriteexporter/go.sum +++ b/exporter/prometheusremotewriteexporter/go.sum @@ -49,7 +49,6 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= @@ -121,21 +120,15 @@ github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOj github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= -github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.4.0 h1:ctuWFGrhFha8BnnzxqeRGidlEcQkDyL5u8J8t5eA11I= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= @@ -187,13 +180,11 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= From 243f3acc9febcf7d15c073aee299cfd8caae8c66 Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 10 Aug 2023 12:18:24 -0700 Subject: [PATCH 08/13] Add 4xx test case --- .chloggen/prw5xx.yaml | 2 +- .../prometheusremotewriteexporter/exporter.go | 6 ++-- .../exporter_test.go | 32 +++++++++++++++++++ 3 files changed, 36 insertions(+), 4 deletions(-) diff --git a/.chloggen/prw5xx.yaml b/.chloggen/prw5xx.yaml index 94ccbdfeaa2a..9febfe6a2265 100755 --- a/.chloggen/prw5xx.yaml +++ b/.chloggen/prw5xx.yaml @@ -9,7 +9,7 @@ change_type: bug_fix component: prometheusremotewriteexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: Retry on 5xx status codes using go-retryablehttp client +note: Retry on 5xx status codes using `cenkalti/backoff` client # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [20304] diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index bba1211ed735..8897459b267e 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -228,7 +228,7 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ // Uses proto.Marshal to convert the WriteRequest into bytes array data, err := proto.Marshal(writeReq) if err != nil { - return consumererror.NewPermanent(err) + return backoff.Permanent(consumererror.NewPermanent(err)) } buf := make([]byte, len(data), cap(data)) compressedData := snappy.Encode(buf, data) @@ -236,7 +236,7 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ // Create the HTTP POST request to send to the endpoint req, err := http.NewRequestWithContext(ctx, "POST", prwe.endpointURL.String(), bytes.NewReader(compressedData)) if err != nil { - return consumererror.NewPermanent(err) + return backoff.Permanent(consumererror.NewPermanent(err)) } // Add necessary headers specified by: @@ -265,7 +265,7 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ if resp.StatusCode >= 500 && resp.StatusCode < 600 { return rerr } - return consumererror.NewPermanent(rerr) + return backoff.Permanent(consumererror.NewPermanent(rerr)) } // Use the BackOff instance to retry the operation with exponential backoff. diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index 7fe8e92aeb8c..c3da552fc303 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -5,6 +5,7 @@ package prometheusremotewriteexporter import ( "context" + "go.opentelemetry.io/collector/consumer/consumererror" "io" "net/http" "net/http/httptest" @@ -1008,3 +1009,34 @@ func TestRetryOn5xx(t *testing.T) { assert.NoError(t, err) assert.Equal(t, 4, attempts) } + +func TestNoRetryOn4xx(t *testing.T) { + // Create a mock HTTP server with a counter to simulate a 4xx error + attempts := 0 + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if attempts < 1 { + attempts++ + http.Error(w, "Bad Request", http.StatusBadRequest) + } else { + w.WriteHeader(http.StatusOK) + } + })) + defer mockServer.Close() + + endpointURL, err := url.Parse(mockServer.URL) + require.NoError(t, err) + + // Create the prwExporter + exporter := &prwExporter{ + endpointURL: endpointURL, + client: http.DefaultClient, + } + + ctx := context.Background() + + // Execute the write request and verify that the exporter returns an error due to the 4xx response. + err = exporter.execute(ctx, &prompb.WriteRequest{}) + assert.Error(t, err) + assert.True(t, consumererror.IsPermanent(err)) + assert.Equal(t, 1, attempts) +} From 188c426a0cedbeff8741d42c1bf50bde2dcd5e8f Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 10 Aug 2023 15:06:41 -0700 Subject: [PATCH 09/13] Add Retry in testcase to run faster --- exporter/prometheusremotewriteexporter/exporter.go | 12 ++++++++---- .../prometheusremotewriteexporter/exporter_test.go | 11 ++++++++++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 8897459b267e..bba57b904605 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -14,7 +14,6 @@ import ( "net/url" "strings" "sync" - "time" "github.com/cenkalti/backoff/v4" "github.com/gogo/protobuf/proto" @@ -24,6 +23,7 @@ import ( "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/consumer/consumererror" "go.opentelemetry.io/collector/exporter" + "go.opentelemetry.io/collector/exporter/exporterhelper" "go.opentelemetry.io/collector/pdata/pmetric" "go.uber.org/multierr" @@ -43,6 +43,7 @@ type prwExporter struct { userAgentHeader string clientSettings *confighttp.HTTPClientSettings settings component.TelemetrySettings + retrySettings exporterhelper.RetrySettings wal *prweWAL exporterSettings prometheusremotewrite.Settings @@ -70,6 +71,7 @@ func newPRWExporter(cfg *Config, set exporter.CreateSettings) (*prwExporter, err concurrency: cfg.RemoteWriteQueue.NumConsumers, clientSettings: &cfg.HTTPClientSettings, settings: set.TelemetrySettings, + retrySettings: cfg.RetrySettings, exporterSettings: prometheusremotewrite.Settings{ Namespace: cfg.Namespace, ExternalLabels: sanitizedLabels, @@ -220,9 +222,11 @@ func (prwe *prwExporter) export(ctx context.Context, requests []*prompb.WriteReq func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequest) error { // Attempt the HTTP request with retry logic backOff := backoff.NewExponentialBackOff() - backOff.InitialInterval = 1 * time.Second - backOff.MaxInterval = 10 * time.Second - backOff.MaxElapsedTime = 1 * time.Minute + backOff.InitialInterval = prwe.retrySettings.InitialInterval + backOff.RandomizationFactor = prwe.retrySettings.RandomizationFactor + backOff.Multiplier = prwe.retrySettings.Multiplier + backOff.MaxInterval = prwe.retrySettings.MaxInterval + backOff.MaxElapsedTime = prwe.retrySettings.MaxElapsedTime retryFunc := func() error { // Uses proto.Marshal to convert the WriteRequest into bytes array diff --git a/exporter/prometheusremotewriteexporter/exporter_test.go b/exporter/prometheusremotewriteexporter/exporter_test.go index c3da552fc303..476f2e2c44d0 100644 --- a/exporter/prometheusremotewriteexporter/exporter_test.go +++ b/exporter/prometheusremotewriteexporter/exporter_test.go @@ -5,13 +5,13 @@ package prometheusremotewriteexporter import ( "context" - "go.opentelemetry.io/collector/consumer/consumererror" "io" "net/http" "net/http/httptest" "net/url" "sync" "testing" + "time" "github.com/gogo/protobuf/proto" "github.com/golang/snappy" @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config/confighttp" "go.opentelemetry.io/collector/config/configtls" + "go.opentelemetry.io/collector/consumer/consumererror" "go.opentelemetry.io/collector/exporter" "go.opentelemetry.io/collector/exporter/exporterhelper" "go.opentelemetry.io/collector/exporter/exportertest" @@ -668,6 +669,13 @@ func Test_PushMetrics(t *testing.T) { defer server.Close() + // Adjusted retry settings for faster testing + retrySettings := exporterhelper.RetrySettings{ + Enabled: true, + InitialInterval: 100 * time.Millisecond, // Shorter initial interval + MaxInterval: 1 * time.Second, // Shorter max interval + MaxElapsedTime: 2 * time.Second, // Shorter max elapsed time + } cfg := &Config{ Namespace: "", HTTPClientSettings: confighttp.HTTPClientSettings{ @@ -683,6 +691,7 @@ func Test_PushMetrics(t *testing.T) { CreatedMetric: &CreatedMetric{ Enabled: true, }, + RetrySettings: retrySettings, } if useWAL { From aa9d7bb288bc66fcdc6ee2e2520cf8e5fbb538f2 Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 10 Aug 2023 17:44:53 -0700 Subject: [PATCH 10/13] Pass retry settings inline --- .../prometheusremotewriteexporter/exporter.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index bba57b904605..991469c99e39 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -220,13 +220,6 @@ func (prwe *prwExporter) export(ctx context.Context, requests []*prompb.WriteReq } func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequest) error { - // Attempt the HTTP request with retry logic - backOff := backoff.NewExponentialBackOff() - backOff.InitialInterval = prwe.retrySettings.InitialInterval - backOff.RandomizationFactor = prwe.retrySettings.RandomizationFactor - backOff.Multiplier = prwe.retrySettings.Multiplier - backOff.MaxInterval = prwe.retrySettings.MaxInterval - backOff.MaxElapsedTime = prwe.retrySettings.MaxElapsedTime retryFunc := func() error { // Uses proto.Marshal to convert the WriteRequest into bytes array @@ -272,8 +265,16 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ return backoff.Permanent(consumererror.NewPermanent(rerr)) } - // Use the BackOff instance to retry the operation with exponential backoff. - return backoff.Retry(retryFunc, backOff) + // Use the BackOff instance to retry the func with exponential backoff. + return backoff.Retry(retryFunc, &backoff.ExponentialBackOff{ + InitialInterval: prwe.retrySettings.InitialInterval, + RandomizationFactor: prwe.retrySettings.RandomizationFactor, + Multiplier: prwe.retrySettings.Multiplier, + MaxInterval: prwe.retrySettings.MaxInterval, + MaxElapsedTime: prwe.retrySettings.MaxElapsedTime, + Stop: backoff.Stop, + Clock: backoff.SystemClock, + }) } func (prwe *prwExporter) walEnabled() bool { return prwe.wal != nil } From e5fa7e907ae21865e424603f30ecdaeb42bbe334 Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Thu, 10 Aug 2023 18:09:28 -0700 Subject: [PATCH 11/13] Wrap to permanent error --- exporter/prometheusremotewriteexporter/exporter.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 991469c99e39..75bed9ef53f4 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -220,7 +220,7 @@ func (prwe *prwExporter) export(ctx context.Context, requests []*prompb.WriteReq } func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequest) error { - + // Retry function for backoff retryFunc := func() error { // Uses proto.Marshal to convert the WriteRequest into bytes array data, err := proto.Marshal(writeReq) @@ -266,7 +266,7 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ } // Use the BackOff instance to retry the func with exponential backoff. - return backoff.Retry(retryFunc, &backoff.ExponentialBackOff{ + err := backoff.Retry(retryFunc, &backoff.ExponentialBackOff{ InitialInterval: prwe.retrySettings.InitialInterval, RandomizationFactor: prwe.retrySettings.RandomizationFactor, Multiplier: prwe.retrySettings.Multiplier, @@ -275,6 +275,12 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ Stop: backoff.Stop, Clock: backoff.SystemClock, }) + + if err != nil { + return consumererror.NewPermanent(err) + } + + return nil } func (prwe *prwExporter) walEnabled() bool { return prwe.wal != nil } From fe660ebaf2fc10623a7d0db1554092c08b811cc4 Mon Sep 17 00:00:00 2001 From: "Pavan Sai Vasireddy (Vasi)" Date: Tue, 15 Aug 2023 11:41:37 -0700 Subject: [PATCH 12/13] Return Error --- exporter/prometheusremotewriteexporter/exporter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/exporter/prometheusremotewriteexporter/exporter.go b/exporter/prometheusremotewriteexporter/exporter.go index 75bed9ef53f4..dd2899ade7f1 100644 --- a/exporter/prometheusremotewriteexporter/exporter.go +++ b/exporter/prometheusremotewriteexporter/exporter.go @@ -280,7 +280,7 @@ func (prwe *prwExporter) execute(ctx context.Context, writeReq *prompb.WriteRequ return consumererror.NewPermanent(err) } - return nil + return err } func (prwe *prwExporter) walEnabled() bool { return prwe.wal != nil } From 6751b22694d036a983688dfe1420bea53cfebefe Mon Sep 17 00:00:00 2001 From: Vasi Vasireddy <41936996+vasireddy99@users.noreply.github.com> Date: Mon, 21 Aug 2023 20:50:50 -0700 Subject: [PATCH 13/13] Remove withRetry from exporterhelper --- exporter/prometheusremotewriteexporter/factory.go | 1 - 1 file changed, 1 deletion(-) diff --git a/exporter/prometheusremotewriteexporter/factory.go b/exporter/prometheusremotewriteexporter/factory.go index 859612a8a36a..0e3aa6fc898a 100644 --- a/exporter/prometheusremotewriteexporter/factory.go +++ b/exporter/prometheusremotewriteexporter/factory.go @@ -57,7 +57,6 @@ func createMetricsExporter(ctx context.Context, set exporter.CreateSettings, NumConsumers: 1, QueueSize: prwCfg.RemoteWriteQueue.QueueSize, }), - exporterhelper.WithRetry(prwCfg.RetrySettings), exporterhelper.WithStart(prwe.Start), exporterhelper.WithShutdown(prwe.Shutdown), )