Merge pull request openshift#168 from rhobs/automated-updates-master

[bot] Bump openshift/prometheus to v2.47.0

.github/CODEOWNERS

@@ -1,6 +1,7 @@
 /web/ui @juliusv
 /web/ui/module @juliusv @nexucis
 /storage/remote @csmarchbanks @cstyan @bwplotka @tomwilkie
+/storage/remote/otlptranslator @gouthamve @jesusvazquez
 /discovery/kubernetes @brancz
 /tsdb @jesusvazquez
 /promql @roidelapluie

.github/workflows/ci.yml

@@ -11,7 +11,7 @@ jobs:
     # Whenever the Go version is updated here, .promu.yml
     # should also be updated.
     container:
-      image: quay.io/prometheus/golang-builder:1.20-base
+      image: quay.io/prometheus/golang-builder:1.21-base
     steps:
       - uses: actions/checkout@v3
       - uses: prometheus/promci@v0.1.0
@@ -32,7 +32,7 @@ jobs:
     # Whenever the Go version is updated here, .promu.yml
     # should also be updated.
     container:
-      image: quay.io/prometheus/golang-builder:1.20-base
+      image: quay.io/prometheus/golang-builder:1.21-base
 
     steps:
       - uses: actions/checkout@v3
@@ -55,7 +55,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '>=1.20 <1.21'
+          go-version: '>=1.21 <1.22'
       - run: |
           $TestTargets = go list ./... | Where-Object { $_ -NotMatch "(github.com/prometheus/prometheus/discovery.*|github.com/prometheus/prometheus/config|github.com/prometheus/prometheus/web)"}
           go test $TestTargets -vet=off -v
@@ -66,7 +66,7 @@ jobs:
     runs-on: ubuntu-latest
     # The go verson in this image should be N-1 wrt test_go.
     container:
-      image: quay.io/prometheus/golang-builder:1.19-base
+      image: quay.io/prometheus/golang-builder:1.20-base
     steps:
       - uses: actions/checkout@v3
       - run: make build
@@ -79,7 +79,7 @@ jobs:
     # Whenever the Go version is updated here, .promu.yml
     # should also be updated.
     container:
-      image: quay.io/prometheus/golang-builder:1.19-base
+      image: quay.io/prometheus/golang-builder:1.20-base
     steps:
       - uses: actions/checkout@v3
       - run: go install ./cmd/promtool/.
@@ -172,7 +172,7 @@ jobs:
           quay_io_login: ${{ secrets.quay_io_login }}
           quay_io_password: ${{ secrets.quay_io_password }}
   publish_release:
-    name: Publish release arfefacts
+    name: Publish release artefacts
     runs-on: ubuntu-latest
     needs: [test_ui, test_go, test_windows, golangci, codeql, build_all]
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v2.')

.github/workflows/codeql-analysis.yml

@@ -27,7 +27,7 @@ jobs:
         uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '>=1.20 <1.21'
+          go-version: '>=1.21 <1.22'
 
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v2

.golangci.yml

@@ -3,6 +3,9 @@ run:
   skip-files:
     # Skip autogenerated files.
     - ^.*\.(pb|y)\.go$
+  skip-dirs:
+    # Copied it from a different source
+    - storage/remote/otlptranslator/prometheusremotewrite
 
 output:
   sort-results: true

.promu.yml

@@ -1,7 +1,7 @@
 go:
     # Whenever the Go version is updated here,
     # .circle/config.yml should also be updated.
-    version: 1.20
+    version: 1.21
 repository:
     path: github.com/prometheus/prometheus
 build:

CHANGELOG.md

@@ -1,5 +1,27 @@
 # Changelog
 
+## 2.47.0 / 2023-09-06
+
+This release adds an experimental OpenTelemetry (OTLP) Ingestion feature,
+and also new setting `keep_dropped_targets` to limit the amount of dropped
+targets held in memory. This defaults to 0 meaning 'no limit', so we encourage
+users with large Prometheus to try setting a limit such as 100.
+
+* [FEATURE] Web: Add OpenTelemetry (OTLP) Ingestion endpoint. #12571 #12643
+* [FEATURE] Scraping: Optionally limit detail on dropped targets, to save memory. #12647
+* [ENHANCEMENT] TSDB: Write head chunks to disk in the background to reduce blocking. #11818
+* [ENHANCEMENT] PromQL: Speed up aggregate and function queries. #12682
+* [ENHANCEMENT] PromQL: More efficient evaluation of query with `timestamp()`. #12579
+* [ENHANCEMENT] API: Faster streaming of Labels to JSON. #12598
+* [ENHANCEMENT] Agent: Memory pooling optimisation. #12651
+* [ENHANCEMENT] TSDB: Prevent storage space leaks due to terminated snapshots on shutdown. #12664
+* [ENHANCEMENT] Histograms: Refactoring and optimisations. #12352 #12584 #12596 #12711 #12054
+* [ENHANCEMENT] Histograms: Add `histogram_stdvar` and `histogram_stddev` functions. #12614
+* [ENHANCEMENT] Remote-write: add http.resend_count tracing attribute. #12676
+* [ENHANCEMENT] TSDB: Support native histograms in snapshot on shutdown. #12722
+* [BUGFIX] TSDB/Agent: ensure that new series get written to WAL on rollback. #12592
+* [BUGFIX] Scraping: fix infinite loop on exemplar in protobuf format. #12737
+
 ## 2.46.0 / 2023-07-25
 
 * [FEATURE] Promtool: Add PromQL format and label matcher set/delete commands to promtool. #11411

RELEASE.md

@@ -51,7 +51,7 @@ Release cadence of first pre-releases being cut is 6 weeks.
 | v2.44          | 2023-04-19                                 | Bryan Boreham (GitHub: @bboreham)           |
 | v2.45 LTS      | 2023-05-31                                 | Jesus Vazquez (Github: @jesusvazquez)       |
 | v2.46          | 2023-07-12                                 | Julien Pivotto (GitHub: @roidelapluie)      |
-| v2.47          | 2023-08-23                                 | **searching for volunteer**                 |
+| v2.47          | 2023-08-23                                 | Bryan Boreham (GitHub: @bboreham)           |
 | v2.48          | 2023-10-04                                 | **searching for volunteer**                 |
 
 If you are interested in volunteering please create a pull request against the [prometheus/prometheus](https://github.com/prometheus/prometheus) repository and propose yourself for the release series of your choice.

VERSION

@@ -1 +1 @@
-2.46.0
+2.47.0

cmd/prometheus/main.go

@@ -170,6 +170,9 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 			case "remote-write-receiver":
 				c.web.EnableRemoteWriteReceiver = true
 				level.Warn(logger).Log("msg", "Remote write receiver enabled via feature flag remote-write-receiver. This is DEPRECATED. Use --web.enable-remote-write-receiver.")
+			case "otlp-write-receiver":
+				c.web.EnableOTLPWriteReceiver = true
+				level.Info(logger).Log("msg", "Experimental OTLP write receiver enabled")
 			case "expand-external-labels":
 				c.enableExpandExternalLabels = true
 				level.Info(logger).Log("msg", "Experimental expand-external-labels enabled")
@@ -211,11 +214,6 @@ func (c *flagConfig) setFeatureListOptions(logger log.Logger) error {
 		}
 	}
 
-	if c.tsdb.EnableNativeHistograms && c.tsdb.EnableMemorySnapshotOnShutdown {
-		c.tsdb.EnableMemorySnapshotOnShutdown = false
-		level.Warn(logger).Log("msg", "memory-snapshot-on-shutdown has been disabled automatically because memory-snapshot-on-shutdown and native-histograms cannot be enabled at the same time.")
-	}
-
 	return nil
 }
 
@@ -420,7 +418,7 @@ func main() {
 	a.Flag("scrape.discovery-reload-interval", "Interval used by scrape manager to throttle target groups updates.").
 		Hidden().Default("5s").SetValue(&cfg.scrape.DiscoveryReloadInterval)
 
-	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
+	a.Flag("enable-feature", "Comma separated feature names to enable. Valid options: agent, exemplar-storage, expand-external-labels, memory-snapshot-on-shutdown, promql-at-modifier, promql-negative-offset, promql-per-step-stats, remote-write-receiver (DEPRECATED), extra-scrape-metrics, new-service-discovery-manager, auto-gomaxprocs, no-default-scrape-port, native-histograms, otlp-write-receiver. See https://prometheus.io/docs/prometheus/latest/feature_flags/ for more details.").
 		Default("").StringsVar(&cfg.featureList)
 
 	promlogflag.AddFlags(a, &cfg.promlogConfig)

cmd/promtool/testdata/unittest.yml

@@ -10,6 +10,21 @@ tests:
       - series: test_full
         values: "0 0"
 
+      - series: test_repeat
+        values: "1x2"
+
+      - series: test_increase
+        values: "1+1x2"
+
+      - series: test_histogram
+        values: "{{schema:1 sum:-0.3 count:32.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}"
+
+      - series: test_histogram_repeat
+        values: "{{sum:3 count:2 buckets:[2]}}x2"
+
+      - series: test_histogram_increase
+        values: "{{sum:3 count:2 buckets:[2]}}+{{sum:1.3 count:1 buckets:[1]}}x2"
+
       - series: test_stale
         values: "0 stale"
 
@@ -31,6 +46,37 @@ tests:
         exp_samples:
           - value: 60
 
+      # Repeat & increase
+      - expr: test_repeat
+        eval_time: 2m
+        exp_samples:
+          - value: 1
+            labels: "test_repeat"
+      - expr: test_increase
+        eval_time: 2m
+        exp_samples:
+          - value: 3
+            labels: "test_increase"
+
+      # Histograms
+      - expr: test_histogram
+        eval_time: 1m
+        exp_samples:
+          - labels: "test_histogram"
+            histogram: "{{schema:1 sum:-0.3 count:32.1 z_bucket:7.1 z_bucket_w:0.05 buckets:[5.1 10 7] offset:-3 n_buckets:[4.1 5] n_offset:-5}}"
+
+      - expr: test_histogram_repeat
+        eval_time: 2m
+        exp_samples:
+          - labels: "test_histogram_repeat"
+            histogram: "{{count:2 sum:3 buckets:[2]}}"
+
+      - expr: test_histogram_increase
+        eval_time: 2m
+        exp_samples:
+          - labels: "test_histogram_increase"
+            histogram: "{{count:4 sum:5.6 buckets:[4]}}"
+
       # Ensure a value is stale as soon as it is marked as such.
       - expr: test_stale
         eval_time: 59s

cmd/promtool/unittest.go

@@ -29,6 +29,7 @@ import (
 	"github.com/prometheus/common/model"
 	"gopkg.in/yaml.v2"
 
+	"github.com/prometheus/prometheus/model/histogram"
 	"github.com/prometheus/prometheus/model/labels"
 	"github.com/prometheus/prometheus/promql"
 	"github.com/prometheus/prometheus/promql/parser"
@@ -346,23 +347,39 @@ Outer:
 		var gotSamples []parsedSample
 		for _, s := range got {
 			gotSamples = append(gotSamples, parsedSample{
-				Labels: s.Metric.Copy(),
-				Value:  s.F,
+				Labels:    s.Metric.Copy(),
+				Value:     s.F,
+				Histogram: promql.HistogramTestExpression(s.H),
 			})
 		}
 
 		var expSamples []parsedSample
 		for _, s := range testCase.ExpSamples {
 			lb, err := parser.ParseMetric(s.Labels)
+			var hist *histogram.FloatHistogram
+			if err == nil && s.Histogram != "" {
+				_, values, parseErr := parser.ParseSeriesDesc("{} " + s.Histogram)
+				switch {
+				case parseErr != nil:
+					err = parseErr
+				case len(values) != 1:
+					err = fmt.Errorf("expected 1 value, got %d", len(values))
+				case values[0].Histogram == nil:
+					err = fmt.Errorf("expected histogram, got %v", values[0])
+				default:
+					hist = values[0].Histogram
+				}
+			}
 			if err != nil {
 				err = fmt.Errorf("labels %q: %w", s.Labels, err)
 				errs = append(errs, fmt.Errorf("    expr: %q, time: %s, err: %w", testCase.Expr,
 					testCase.EvalTime.String(), err))
 				continue Outer
 			}
 			expSamples = append(expSamples, parsedSample{
-				Labels: lb,
-				Value:  s.Value,
+				Labels:    lb,
+				Value:     s.Value,
+				Histogram: promql.HistogramTestExpression(hist),
 			})
 		}
 
@@ -530,14 +547,16 @@ type promqlTestCase struct {
 }
 
 type sample struct {
-	Labels string  `yaml:"labels"`
-	Value  float64 `yaml:"value"`
+	Labels    string  `yaml:"labels"`
+	Value     float64 `yaml:"value"`
+	Histogram string  `yaml:"histogram"` // A non-empty string means Value is ignored.
 }
 
 // parsedSample is a sample with parsed Labels.
 type parsedSample struct {
-	Labels labels.Labels
-	Value  float64
+	Labels    labels.Labels
+	Value     float64
+	Histogram string // TestExpression() of histogram.FloatHistogram
 }
 
 func parsedSamplesString(pss []parsedSample) string {
@@ -552,5 +571,8 @@ func parsedSamplesString(pss []parsedSample) string {
 }
 
 func (ps *parsedSample) String() string {
+	if ps.Histogram != "" {
+		return ps.Labels.String() + " " + ps.Histogram
+	}
 	return ps.Labels.String() + " " + strconv.FormatFloat(ps.Value, 'E', -1, 64)
 }

config/config.go

@@ -409,6 +409,9 @@ type GlobalConfig struct {
 	// More than this label value length post metric-relabeling will cause the
 	// scrape to fail. 0 means no limit.
 	LabelValueLengthLimit uint `yaml:"label_value_length_limit,omitempty"`
+	// Keep no more than this many dropped targets per job.
+	// 0 means no limit.
+	KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.
@@ -514,6 +517,9 @@ type ScrapeConfig struct {
 	// More than this many buckets in a native histogram will cause the scrape to
 	// fail.
 	NativeHistogramBucketLimit uint `yaml:"native_histogram_bucket_limit,omitempty"`
+	// Keep no more than this many dropped targets per job.
+	// 0 means no limit.
+	KeepDroppedTargets uint `yaml:"keep_dropped_targets,omitempty"`
 
 	// We cannot do proper Go type embedding below as the parser will then parse
 	// values arbitrarily into the overflow maps of further-down types.
@@ -608,6 +614,9 @@ func (c *ScrapeConfig) Validate(globalConfig GlobalConfig) error {
 	if c.LabelValueLengthLimit == 0 {
 		c.LabelValueLengthLimit = globalConfig.LabelValueLengthLimit
 	}
+	if c.KeepDroppedTargets == 0 {
+		c.KeepDroppedTargets = globalConfig.KeepDroppedTargets
+	}
 
 	return nil
 }

discovery/hetzner/hcloud.go

@@ -91,7 +91,7 @@ func (d *hcloudDiscovery) refresh(ctx context.Context) ([]*targetgroup.Group, er
 	targets := make([]model.LabelSet, len(servers))
 	for i, server := range servers {
 		labels := model.LabelSet{
-			hetznerLabelRole:              model.LabelValue(hetznerRoleHcloud),
+			hetznerLabelRole:              model.LabelValue(HetznerRoleHcloud),
 			hetznerLabelServerID:          model.LabelValue(fmt.Sprintf("%d", server.ID)),
 			hetznerLabelServerName:        model.LabelValue(server.Name),
 			hetznerLabelDatacenter:        model.LabelValue(server.Datacenter.Name),

discovery/hetzner/hetzner.go

@@ -57,7 +57,7 @@ type SDConfig struct {
 
 	RefreshInterval model.Duration `yaml:"refresh_interval"`
 	Port            int            `yaml:"port"`
-	Role            role           `yaml:"role"`
+	Role            Role           `yaml:"role"`
 	hcloudEndpoint  string         // For tests only.
 	robotEndpoint   string         // For tests only.
 }
@@ -74,26 +74,26 @@ type refresher interface {
 	refresh(context.Context) ([]*targetgroup.Group, error)
 }
 
-// role is the role of the target within the Hetzner Ecosystem.
-type role string
+// Role is the Role of the target within the Hetzner Ecosystem.
+type Role string
 
 // The valid options for role.
 const (
 	// Hetzner Robot Role (Dedicated Server)
 	// https://robot.hetzner.com
-	hetznerRoleRobot role = "robot"
+	HetznerRoleRobot Role = "robot"
 	// Hetzner Cloud Role
 	// https://console.hetzner.cloud
-	hetznerRoleHcloud role = "hcloud"
+	HetznerRoleHcloud Role = "hcloud"
 )
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface.
-func (c *role) UnmarshalYAML(unmarshal func(interface{}) error) error {
+func (c *Role) UnmarshalYAML(unmarshal func(interface{}) error) error {
 	if err := unmarshal((*string)(c)); err != nil {
 		return err
 	}
 	switch *c {
-	case hetznerRoleRobot, hetznerRoleHcloud:
+	case HetznerRoleRobot, HetznerRoleHcloud:
 		return nil
 	default:
 		return fmt.Errorf("unknown role %q", *c)
@@ -143,12 +143,12 @@ func NewDiscovery(conf *SDConfig, logger log.Logger) (*refresh.Discovery, error)
 
 func newRefresher(conf *SDConfig, l log.Logger) (refresher, error) {
 	switch conf.Role {
-	case hetznerRoleHcloud:
+	case HetznerRoleHcloud:
 		if conf.hcloudEndpoint == "" {
 			conf.hcloudEndpoint = hcloud.Endpoint
 		}
 		return newHcloudDiscovery(conf, l)
-	case hetznerRoleRobot:
+	case HetznerRoleRobot:
 		if conf.robotEndpoint == "" {
 			conf.robotEndpoint = "https://robot-ws.your-server.de"
 		}