From 73cda17d556b9191994f95ce9a9b8a354799c63b Mon Sep 17 00:00:00 2001 From: prombot Date: Tue, 14 Jun 2022 19:51:35 +0000 Subject: [PATCH 01/23] Update common Prometheus files Signed-off-by: prombot --- Makefile.common | 75 ++++++++----------------------------------------- 1 file changed, 11 insertions(+), 64 deletions(-) diff --git a/Makefile.common b/Makefile.common index c263b733f..6c8e3e219 100644 --- a/Makefile.common +++ b/Makefile.common @@ -36,29 +36,6 @@ GO_VERSION ?= $(shell $(GO) version) GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION)) PRE_GO_111 ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.') -GOVENDOR := -GO111MODULE := -ifeq (, $(PRE_GO_111)) - ifneq (,$(wildcard go.mod)) - # Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI). - GO111MODULE := on - - ifneq (,$(wildcard vendor)) - # Always use the local vendor/ directory to satisfy the dependencies. - GOOPTS := $(GOOPTS) -mod=vendor - endif - endif -else - ifneq (,$(wildcard go.mod)) - ifneq (,$(wildcard vendor)) -$(warning This repository requires Go >= 1.11 because of Go modules) -$(warning Some recipes may not work as expected as the current Go runtime is '$(GO_VERSION_NUMBER)') - endif - else - # This repository isn't using Go modules (yet). - GOVENDOR := $(FIRST_GOPATH)/bin/govendor - endif -endif PROMU := $(FIRST_GOPATH)/bin/promu pkgs = ./... @@ -150,11 +127,7 @@ common-check_license: .PHONY: common-deps common-deps: @echo ">> getting dependencies" -ifdef GO111MODULE - GO111MODULE=$(GO111MODULE) $(GO) mod download -else - $(GO) get $(GOOPTS) -t ./... -endif + $(GO) mod download .PHONY: update-go-deps update-go-deps: @@ -162,20 +135,17 @@ update-go-deps: @for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \ $(GO) get -d $$m; \ done - GO111MODULE=$(GO111MODULE) $(GO) mod tidy -ifneq (,$(wildcard vendor)) - GO111MODULE=$(GO111MODULE) $(GO) mod vendor -endif + $(GO) mod tidy .PHONY: common-test-short common-test-short: $(GOTEST_DIR) @echo ">> running short tests" - GO111MODULE=$(GO111MODULE) $(GOTEST) -short $(GOOPTS) $(pkgs) + $(GOTEST) -short $(GOOPTS) $(pkgs) .PHONY: common-test common-test: $(GOTEST_DIR) @echo ">> running all tests" - GO111MODULE=$(GO111MODULE) $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs) + $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs) $(GOTEST_DIR): @mkdir -p $@ @@ -183,25 +153,21 @@ $(GOTEST_DIR): .PHONY: common-format common-format: @echo ">> formatting code" - GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs) + $(GO) fmt $(pkgs) .PHONY: common-vet common-vet: @echo ">> vetting code" - GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs) + $(GO) vet $(GOOPTS) $(pkgs) .PHONY: common-lint common-lint: $(GOLANGCI_LINT) ifdef GOLANGCI_LINT @echo ">> running golangci-lint" -ifdef GO111MODULE # 'go list' needs to be executed before staticcheck to prepopulate the modules cache. # Otherwise staticcheck might fail randomly for some reason not yet explained. - GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null - GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs) -else - $(GOLANGCI_LINT) run $(pkgs) -endif + $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null + $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs) endif .PHONY: common-yamllint @@ -218,28 +184,15 @@ endif common-staticcheck: lint .PHONY: common-unused -common-unused: $(GOVENDOR) -ifdef GOVENDOR - @echo ">> running check for unused packages" - @$(GOVENDOR) list +unused | grep . && exit 1 || echo 'No unused packages' -else -ifdef GO111MODULE +common-unused: @echo ">> running check for unused/missing packages in go.mod" - GO111MODULE=$(GO111MODULE) $(GO) mod tidy -ifeq (,$(wildcard vendor)) + $(GO) mod tidy @git diff --exit-code -- go.sum go.mod -else - @echo ">> running check for unused packages in vendor/" - GO111MODULE=$(GO111MODULE) $(GO) mod vendor - @git diff --exit-code -- go.sum go.mod vendor/ -endif -endif -endif .PHONY: common-build common-build: promu @echo ">> building binaries" - GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) + $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES) .PHONY: common-tarball common-tarball: promu @@ -295,12 +248,6 @@ $(GOLANGCI_LINT): | sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION) endif -ifdef GOVENDOR -.PHONY: $(GOVENDOR) -$(GOVENDOR): - GOOS= GOARCH= $(GO) get -u github.com/kardianos/govendor -endif - .PHONY: precheck precheck:: From a167b9165406f9411b845b731c9a142d4267e648 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Jul 2022 17:06:17 +0000 Subject: [PATCH 02/23] Bump github.com/prometheus/common from 0.34.0 to 0.35.0 Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.34.0 to 0.35.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.34.0...v0.35.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c2505f7fe..f27c896ba 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/lib/pq v1.10.6 github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.34.0 + github.com/prometheus/common v0.35.0 github.com/prometheus/exporter-toolkit v0.7.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 8bc754b16..3f9003a23 100644 --- a/go.sum +++ b/go.sum @@ -189,8 +189,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.34.0 h1:RBmGO9d/FVjqHT0yUGQwBJhkwKV+wPCn7KGpvfab0uE= -github.com/prometheus/common v0.34.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/exporter-toolkit v0.7.1 h1:c6RXaK8xBVercEeUQ4tRNL8UGWzDHfvj9dseo1FcK1Y= github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= From da63601f5718aa61c55c890601dd42513ee8eadd Mon Sep 17 00:00:00 2001 From: SuperQ Date: Thu, 28 Jul 2022 13:42:32 +0200 Subject: [PATCH 03/23] Release v0.11.0 NOTE: pg_stat_bgwriter counter metrics had the `_total` suffix added #556 * [CHANGE] refactor pg_stat_bgwriter metrics into standalone collector #556 * [FEATURE] Add pg_database collector #613 * [ENHANCEMENT] Add pg_database_size_bytes metric #613 * [BUGFIX] Avoid parsing error from bogus Azure Flexible Server custom GUC #587 * [BUGFIX] Fix pg_stat_archiver error in 9.4 and earlier. #599 * [BUGFIX] Sanitize setting values because of Aurora irregularity #620 Signed-off-by: SuperQ --- .circleci/config.yml | 18 +++++++++++++++--- CHANGELOG.md | 10 +++++++++- VERSION | 2 +- 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index db3cad4e5..ae168f04e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,16 +63,28 @@ workflows: - cimg/postgres:14.1 - prometheus/build: name: build + parallelism: 3 + promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386" filters: tags: - only: /.*/ + ignore: /^v.*/ + branches: + ignore: /^(main|master|release-.*|.*build-all.*)$/ + - prometheus/build: + name: build_all + parallelism: 12 + filters: + branches: + only: /^(main|master|release-.*|.*build-all.*)$/ + tags: + only: /^v.*/ - prometheus/publish_master: context: org-context docker_hub_organization: prometheuscommunity quay_io_organization: prometheuscommunity requires: - test - - build + - build_all filters: branches: only: master @@ -82,7 +94,7 @@ workflows: quay_io_organization: prometheuscommunity requires: - test - - build + - build_all filters: tags: only: /^v.*/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 0572bf5dc..a8f7b92f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,15 @@ ## master / unreleased -* [CHANGE] pg_stat_bgwriter counter metrics had the `_total` suffix added #556 +## 0.11.0 / 2022-07-28 + +NOTE: pg_stat_bgwriter counter metrics had the `_total` suffix added #556 + +* [CHANGE] refactor pg_stat_bgwriter metrics into standalone collector #556 +* [FEATURE] Add pg_database collector #613 * [ENHANCEMENT] Add pg_database_size_bytes metric #613 +* [BUGFIX] Avoid parsing error from bogus Azure Flexible Server custom GUC #587 +* [BUGFIX] Fix pg_stat_archiver error in 9.4 and earlier. #599 +* [BUGFIX] Sanitize setting values because of Aurora irregularity #620 ## 0.10.1 / 2022-01-14 diff --git a/VERSION b/VERSION index 571215736..d9df1bbc0 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.10.1 +0.11.0 From 713461df9847bd50d47b0df2b5a4f1f01206bdb8 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Fri, 25 Feb 2022 11:45:15 -0500 Subject: [PATCH 04/23] WIP: Add prelim multi-target support - Remove multi server support from new collector package - Add http handler for multi-target support Signed-off-by: Joe Adams --- cmd/postgres_exporter/datasource.go | 6 ++ cmd/postgres_exporter/main.go | 25 ++++--- cmd/postgres_exporter/probe.go | 91 +++++++++++++++++++++++++ collector/collector.go | 42 +++++------- collector/pg_database.go | 11 ++- collector/pg_stat_bgwriter.go | 43 ++++-------- collector/probe.go | 90 +++++++++++++++++++++++++ collector/server.go | 100 ---------------------------- 8 files changed, 238 insertions(+), 170 deletions(-) create mode 100644 cmd/postgres_exporter/probe.go create mode 100644 collector/probe.go delete mode 100644 collector/server.go diff --git a/cmd/postgres_exporter/datasource.go b/cmd/postgres_exporter/datasource.go index 90d797b63..716138f32 100644 --- a/cmd/postgres_exporter/datasource.go +++ b/cmd/postgres_exporter/datasource.go @@ -162,6 +162,12 @@ func getDataSources() ([]string, error) { uri = os.Getenv("DATA_SOURCE_URI") } + // No datasources found. This allows us to support the multi-target pattern + // withouth an explicit datasource. + if uri == "" { + return []string{}, nil + } + dsn = "postgresql://" + ui + "@" + uri return []string{dsn}, nil diff --git a/cmd/postgres_exporter/main.go b/cmd/postgres_exporter/main.go index 35db723a2..2ec1bf269 100644 --- a/cmd/postgres_exporter/main.go +++ b/cmd/postgres_exporter/main.go @@ -85,16 +85,17 @@ func main() { return } - dsn, err := getDataSources() + dsns, err := getDataSources() if err != nil { level.Error(logger).Log("msg", "Failed reading data sources", "err", err.Error()) os.Exit(1) } - if len(dsn) == 0 { - level.Error(logger).Log("msg", "Couldn't find environment variables describing the datasource to use") - os.Exit(1) - } + // TODO(@sysadmind): Remove this with multi-target support + // if len(dsn) == 0 { + // level.Error(logger).Log("msg", "Couldn't find environment variables describing the datasource to use") + // os.Exit(1) + // } opts := []ExporterOpt{ DisableDefaultMetrics(*disableDefaultMetrics), @@ -106,7 +107,7 @@ func main() { IncludeDatabases(*includeDatabases), } - exporter := NewExporter(dsn, opts...) + exporter := NewExporter(dsns, opts...) defer func() { exporter.servers.Close() }() @@ -115,6 +116,12 @@ func main() { prometheus.MustRegister(exporter) + // TODO(@sysadmind): Remove this with multi-target support. We are removing multiple DSN support + dsn := "" + if len(dsns) > 0 { + dsn = dsns[0] + } + pe, err := collector.NewPostgresCollector( logger, dsn, @@ -122,9 +129,9 @@ func main() { ) if err != nil { level.Error(logger).Log("msg", "Failed to create PostgresCollector", "err", err.Error()) - os.Exit(1) + } else { + prometheus.MustRegister(pe) } - prometheus.MustRegister(pe) http.Handle(*metricPath, promhttp.Handler()) http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { @@ -132,6 +139,8 @@ func main() { w.Write(landingPage) // nolint: errcheck }) + http.HandleFunc("/probe", handleProbe(logger)) + level.Info(logger).Log("msg", "Listening on address", "address", *listenAddress) srv := &http.Server{Addr: *listenAddress} if err := web.ListenAndServe(srv, *webConfig, logger); err != nil { diff --git a/cmd/postgres_exporter/probe.go b/cmd/postgres_exporter/probe.go new file mode 100644 index 000000000..c23777b45 --- /dev/null +++ b/cmd/postgres_exporter/probe.go @@ -0,0 +1,91 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + "time" + + "github.com/go-kit/log" + "github.com/prometheus-community/postgres_exporter/collector" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func handleProbe(logger log.Logger) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + params := r.URL.Query() + target := params.Get("target") + if target == "" { + http.Error(w, "target is required", http.StatusBadRequest) + return + } + + // TODO: Timeout + // TODO: Auth Module + + probeSuccessGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_success", + Help: "Displays whether or not the probe was a success", + }) + probeDurationGauge := prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "probe_duration_seconds", + Help: "Returns how long the probe took to complete in seconds", + }) + + tl := log.With(logger, "target", target) + _ = tl + + start := time.Now() + registry := prometheus.NewRegistry() + registry.MustRegister(probeSuccessGauge) + registry.MustRegister(probeDurationGauge) + + // TODO(@sysadmind): this is a temp hack until we have a proper auth module + target = "postgres://postgres:test@localhost:5432/circle_test?sslmode=disable" + + // Run the probe + pc, err := collector.NewProbeCollector(tl, registry, target) + if err != nil { + probeSuccessGauge.Set(0) + probeDurationGauge.Set(time.Since(start).Seconds()) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + _ = ctx + + // TODO: Which way should this be? Register or handle the collection manually? + // Also, what about the context? + + // Option 1: Register the collector + registry.MustRegister(pc) + + // Option 2: Handle the collection manually. This allows us to collect duration metrics. + // The collectors themselves already support their own duration metrics. + // err = pc.Update(ctx) + // if err != nil { + // probeSuccessGauge.Set(0) + // } else { + // probeSuccessGauge.Set(1) + // } + + duration := time.Since(start).Seconds() + probeDurationGauge.Set(duration) + + // TODO check success, etc + h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{}) + h.ServeHTTP(w, r) + } +} diff --git a/collector/collector.go b/collector/collector.go index 58765b659..6d1a4dd11 100644 --- a/collector/collector.go +++ b/collector/collector.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "errors" "fmt" "sync" @@ -58,7 +59,7 @@ var ( ) type Collector interface { - Update(ctx context.Context, server *server, ch chan<- prometheus.Metric) error + Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error } func registerCollector(name string, isDefaultEnabled bool, createFunc func(logger log.Logger) (Collector, error)) { @@ -86,13 +87,13 @@ type PostgresCollector struct { Collectors map[string]Collector logger log.Logger - servers map[string]*server + db *sql.DB } type Option func(*PostgresCollector) error // NewPostgresCollector creates a new PostgresCollector. -func NewPostgresCollector(logger log.Logger, dsns []string, filters []string, options ...Option) (*PostgresCollector, error) { +func NewPostgresCollector(logger log.Logger, dsn string, filters []string, options ...Option) (*PostgresCollector, error) { p := &PostgresCollector{ logger: logger, } @@ -136,17 +137,18 @@ func NewPostgresCollector(logger log.Logger, dsns []string, filters []string, op p.Collectors = collectors - servers := make(map[string]*server) - for _, dsn := range dsns { - s, err := makeServer(dsn) - if err != nil { - return nil, err - } + if dsn == "" { + return nil, errors.New("empty dsn") + } - servers[dsn] = s + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, err } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) - p.servers = servers + p.db = db return p, nil } @@ -160,32 +162,20 @@ func (p PostgresCollector) Describe(ch chan<- *prometheus.Desc) { // Collect implements the prometheus.Collector interface. func (p PostgresCollector) Collect(ch chan<- prometheus.Metric) { ctx := context.TODO() - wg := sync.WaitGroup{} - wg.Add(len(p.servers)) - for _, s := range p.servers { - go func(s *server) { - p.subCollect(ctx, s, ch) - wg.Done() - }(s) - } - wg.Wait() -} - -func (p PostgresCollector) subCollect(ctx context.Context, server *server, ch chan<- prometheus.Metric) { wg := sync.WaitGroup{} wg.Add(len(p.Collectors)) for name, c := range p.Collectors { go func(name string, c Collector) { - execute(ctx, name, c, server, ch, p.logger) + execute(ctx, name, c, p.db, ch, p.logger) wg.Done() }(name, c) } wg.Wait() } -func execute(ctx context.Context, name string, c Collector, s *server, ch chan<- prometheus.Metric, logger log.Logger) { +func execute(ctx context.Context, name string, c Collector, db *sql.DB, ch chan<- prometheus.Metric, logger log.Logger) { begin := time.Now() - err := c.Update(ctx, s, ch) + err := c.Update(ctx, db, ch) duration := time.Since(begin) var success float64 diff --git a/collector/pg_database.go b/collector/pg_database.go index 5868f66d8..8fa4dab8e 100644 --- a/collector/pg_database.go +++ b/collector/pg_database.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" @@ -36,15 +37,11 @@ var pgDatabase = map[string]*prometheus.Desc{ "size_bytes": prometheus.NewDesc( "pg_database_size_bytes", "Disk space used by the database", - []string{"datname", "server"}, nil, + []string{"datname"}, nil, ), } -func (PGDatabaseCollector) Update(ctx context.Context, server *server, ch chan<- prometheus.Metric) error { - db, err := server.GetDB() - if err != nil { - return err - } +func (PGDatabaseCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { rows, err := db.QueryContext(ctx, `SELECT pg_database.datname ,pg_database_size(pg_database.datname) @@ -63,7 +60,7 @@ func (PGDatabaseCollector) Update(ctx context.Context, server *server, ch chan<- ch <- prometheus.MustNewConstMetric( pgDatabase["size_bytes"], - prometheus.GaugeValue, float64(size), datname, server.GetName(), + prometheus.GaugeValue, float64(size), datname, ) } if err := rows.Err(); err != nil { diff --git a/collector/pg_stat_bgwriter.go b/collector/pg_stat_bgwriter.go index 7e7d09c72..f897dbe2c 100644 --- a/collector/pg_stat_bgwriter.go +++ b/collector/pg_stat_bgwriter.go @@ -15,6 +15,7 @@ package collector import ( "context" + "database/sql" "time" "github.com/go-kit/log" @@ -38,77 +39,72 @@ var statBGWriter = map[string]*prometheus.Desc{ "checkpoints_timed": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoints_timed_total"), "Number of scheduled checkpoints that have been performed", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "checkpoints_req": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoints_req_total"), "Number of requested checkpoints that have been performed", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "checkpoint_write_time": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoint_write_time_total"), "Total amount of time that has been spent in the portion of checkpoint processing where files are written to disk, in milliseconds", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "checkpoint_sync_time": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "checkpoint_sync_time_total"), "Total amount of time that has been spent in the portion of checkpoint processing where files are synchronized to disk, in milliseconds", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "buffers_checkpoint": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "buffers_checkpoint_total"), "Number of buffers written during checkpoints", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "buffers_clean": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "buffers_clean_total"), "Number of buffers written by the background writer", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "maxwritten_clean": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "maxwritten_clean_total"), "Number of times the background writer stopped a cleaning scan because it had written too many buffers", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "buffers_backend": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "buffers_backend_total"), "Number of buffers written directly by a backend", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "buffers_backend_fsync": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "buffers_backend_fsync_total"), "Number of times a backend had to execute its own fsync call (normally the background writer handles those even when the backend does its own write)", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "buffers_alloc": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "buffers_alloc_total"), "Number of buffers allocated", - []string{"server"}, + []string{}, prometheus.Labels{}, ), "stats_reset": prometheus.NewDesc( prometheus.BuildFQName(namespace, bgWriterSubsystem, "stats_reset_total"), "Time at which these statistics were last reset", - []string{"server"}, + []string{}, prometheus.Labels{}, ), } -func (PGStatBGWriterCollector) Update(ctx context.Context, server *server, ch chan<- prometheus.Metric) error { - db, err := server.GetDB() - if err != nil { - return err - } - +func (PGStatBGWriterCollector) Update(ctx context.Context, db *sql.DB, ch chan<- prometheus.Metric) error { row := db.QueryRowContext(ctx, `SELECT checkpoints_timed @@ -136,7 +132,7 @@ func (PGStatBGWriterCollector) Update(ctx context.Context, server *server, ch ch var ba int var sr time.Time - err = row.Scan(&cpt, &cpr, &cpwt, &cpst, &bcp, &bc, &mwc, &bb, &bbf, &ba, &sr) + err := row.Scan(&cpt, &cpr, &cpwt, &cpst, &bcp, &bc, &mwc, &bb, &bbf, &ba, &sr) if err != nil { return err } @@ -145,67 +141,56 @@ func (PGStatBGWriterCollector) Update(ctx context.Context, server *server, ch ch statBGWriter["checkpoints_timed"], prometheus.CounterValue, float64(cpt), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["checkpoints_req"], prometheus.CounterValue, float64(cpr), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["checkpoint_write_time"], prometheus.CounterValue, float64(cpwt), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["checkpoint_sync_time"], prometheus.CounterValue, float64(cpst), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["buffers_checkpoint"], prometheus.CounterValue, float64(bcp), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["buffers_clean"], prometheus.CounterValue, float64(bc), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["maxwritten_clean"], prometheus.CounterValue, float64(mwc), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["buffers_backend"], prometheus.CounterValue, float64(bb), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["buffers_backend_fsync"], prometheus.CounterValue, float64(bbf), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["buffers_alloc"], prometheus.CounterValue, float64(ba), - server.GetName(), ) ch <- prometheus.MustNewConstMetric( statBGWriter["stats_reset"], prometheus.CounterValue, float64(sr.Unix()), - server.GetName(), ) return nil diff --git a/collector/probe.go b/collector/probe.go new file mode 100644 index 000000000..8aa18b92c --- /dev/null +++ b/collector/probe.go @@ -0,0 +1,90 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package collector + +import ( + "context" + "database/sql" + "fmt" + "strings" + "sync" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" +) + +type ProbeCollector struct { + registry *prometheus.Registry + collectors map[string]Collector + logger log.Logger + db *sql.DB +} + +func NewProbeCollector(logger log.Logger, registry *prometheus.Registry, dsn string) (*ProbeCollector, error) { + collectors := make(map[string]Collector) + initiatedCollectorsMtx.Lock() + defer initiatedCollectorsMtx.Unlock() + for key, enabled := range collectorState { + // TODO: Handle filters + // if !*enabled || (len(f) > 0 && !f[key]) { + // continue + // } + if !*enabled { + continue + } + if collector, ok := initiatedCollectors[key]; ok { + collectors[key] = collector + } else { + collector, err := factories[key](log.With(logger, "collector", key)) + if err != nil { + return nil, err + } + collectors[key] = collector + initiatedCollectors[key] = collector + } + } + + if !strings.HasPrefix(dsn, "postgres://") { + dsn = fmt.Sprintf("postgres://%s", dsn) + } + + db, err := sql.Open("postgres", dsn) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + return &ProbeCollector{ + registry: registry, + collectors: collectors, + logger: logger, + db: db, + }, nil +} + +func (pc *ProbeCollector) Describe(ch chan<- *prometheus.Desc) { +} + +func (pc *ProbeCollector) Collect(ch chan<- prometheus.Metric) { + wg := sync.WaitGroup{} + wg.Add(len(pc.collectors)) + for name, c := range pc.collectors { + go func(name string, c Collector) { + execute(context.TODO(), name, c, pc.db, ch, pc.logger) + wg.Done() + }(name, c) + } + wg.Wait() +} diff --git a/collector/server.go b/collector/server.go deleted file mode 100644 index fa490a2c6..000000000 --- a/collector/server.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2022 The Prometheus Authors -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package collector - -import ( - "database/sql" - "fmt" - "strings" - - "github.com/lib/pq" -) - -type server struct { - dsn string - name string - db *sql.DB -} - -func makeServer(dsn string) (*server, error) { - name, err := parseServerName(dsn) - if err != nil { - return nil, err - } - return &server{ - dsn: dsn, - name: name, - }, nil -} - -func (s *server) GetDB() (*sql.DB, error) { - if s.db != nil { - return s.db, nil - } - - db, err := sql.Open("postgres", s.dsn) - if err != nil { - return nil, err - } - db.SetMaxOpenConns(1) - db.SetMaxIdleConns(1) - - s.db = db - - return s.db, nil -} - -func (s *server) GetName() string { - return s.name -} - -func (s *server) String() string { - return s.name -} - -func parseServerName(url string) (string, error) { - dsn, err := pq.ParseURL(url) - if err != nil { - dsn = url - } - - pairs := strings.Split(dsn, " ") - kv := make(map[string]string, len(pairs)) - for _, pair := range pairs { - splitted := strings.SplitN(pair, "=", 2) - if len(splitted) != 2 { - return "", fmt.Errorf("malformed dsn %q", dsn) - } - // Newer versions of pq.ParseURL quote values so trim them off if they exist - key := strings.Trim(splitted[0], "'\"") - value := strings.Trim(splitted[1], "'\"") - kv[key] = value - } - - var fingerprint string - - if host, ok := kv["host"]; ok { - fingerprint += host - } else { - fingerprint += "localhost" - } - - if port, ok := kv["port"]; ok { - fingerprint += ":" + port - } else { - fingerprint += ":5432" - } - - return fingerprint, nil -} From cc751b7966f66a60512609c7497379d32bc2847b Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Fri, 4 Mar 2022 16:39:48 -0500 Subject: [PATCH 05/23] Add config module The config module supports adding configuration to the exporter via a config file. This supports adding authentication details in a config file so that /probe requests can specify authentication for endpoints Signed-off-by: Joe Adams --- cmd/postgres_exporter/main.go | 11 ++ cmd/postgres_exporter/probe.go | 34 +++++- config/config.go | 126 ++++++++++++++++++++ config/config_test.go | 58 +++++++++ config/testdata/config-bad-auth-module.yaml | 7 ++ config/testdata/config-bad-extra-field.yaml | 8 ++ config/testdata/config-good.yaml | 8 ++ go.mod | 1 + go.sum | 2 + 9 files changed, 249 insertions(+), 6 deletions(-) create mode 100644 config/config.go create mode 100644 config/config_test.go create mode 100644 config/testdata/config-bad-auth-module.yaml create mode 100644 config/testdata/config-bad-extra-field.yaml create mode 100644 config/testdata/config-good.yaml diff --git a/cmd/postgres_exporter/main.go b/cmd/postgres_exporter/main.go index 2ec1bf269..aee325039 100644 --- a/cmd/postgres_exporter/main.go +++ b/cmd/postgres_exporter/main.go @@ -20,6 +20,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" "github.com/prometheus-community/postgres_exporter/collector" + "github.com/prometheus-community/postgres_exporter/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/common/promlog" @@ -31,6 +32,11 @@ import ( ) var ( + c = config.ConfigHandler{ + Config: &config.Config{}, + } + + configFile = kingpin.Flag("config.file", "Promehteus exporter configuration file.").Default("postres_exporter.yml").String() listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9187").Envar("PG_EXPORTER_WEB_LISTEN_ADDRESS").String() webConfig = webflag.AddFlags(kingpin.CommandLine) metricPath = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").Envar("PG_EXPORTER_WEB_TELEMETRY_PATH").String() @@ -85,6 +91,11 @@ func main() { return } + if err := c.ReloadConfig(*configFile, logger); err != nil { + // This is not fatal, but it means that auth must be provided for every dsn. + level.Error(logger).Log("msg", "Error loading config", "err", err) + } + dsns, err := getDataSources() if err != nil { level.Error(logger).Log("msg", "Failed reading data sources", "err", err.Error()) diff --git a/cmd/postgres_exporter/probe.go b/cmd/postgres_exporter/probe.go index c23777b45..813f4ea81 100644 --- a/cmd/postgres_exporter/probe.go +++ b/cmd/postgres_exporter/probe.go @@ -14,11 +14,14 @@ package main import ( + "fmt" "net/http" "time" "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/prometheus-community/postgres_exporter/collector" + "github.com/prometheus-community/postgres_exporter/config" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -26,15 +29,38 @@ import ( func handleProbe(logger log.Logger) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + conf := c.GetConfig() params := r.URL.Query() target := params.Get("target") if target == "" { http.Error(w, "target is required", http.StatusBadRequest) return } + var authModule config.AuthModule + authModuleName := params.Get("auth_module") + if authModuleName == "" { + level.Info(logger).Log("msg", "no auth_module specified, using default") + } else { + var ok bool + authModule, ok = conf.AuthModules[authModuleName] + if !ok { + http.Error(w, fmt.Sprintf("auth_module %s not found", authModuleName), http.StatusBadRequest) + return + } + if authModule.UserPass.Username == "" || authModule.UserPass.Password == "" { + http.Error(w, fmt.Sprintf("auth_module %s has no username or password", authModuleName), http.StatusBadRequest) + return + } + } + + dsn, err := authModule.ConfigureTarget(target) + if err != nil { + level.Error(logger).Log("msg", "failed to configure target", "err", err) + http.Error(w, fmt.Sprintf("could not configure dsn for target: %v", err), http.StatusBadRequest) + return + } // TODO: Timeout - // TODO: Auth Module probeSuccessGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "probe_success", @@ -46,18 +72,14 @@ func handleProbe(logger log.Logger) http.HandlerFunc { }) tl := log.With(logger, "target", target) - _ = tl start := time.Now() registry := prometheus.NewRegistry() registry.MustRegister(probeSuccessGauge) registry.MustRegister(probeDurationGauge) - // TODO(@sysadmind): this is a temp hack until we have a proper auth module - target = "postgres://postgres:test@localhost:5432/circle_test?sslmode=disable" - // Run the probe - pc, err := collector.NewProbeCollector(tl, registry, target) + pc, err := collector.NewProbeCollector(tl, registry, dsn) if err != nil { probeSuccessGauge.Set(0) probeDurationGauge.Set(time.Since(start).Seconds()) diff --git a/config/config.go b/config/config.go new file mode 100644 index 000000000..49a2dbd63 --- /dev/null +++ b/config/config.go @@ -0,0 +1,126 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "fmt" + "net/url" + "os" + "strings" + "sync" + + "github.com/go-kit/log" + "github.com/prometheus/client_golang/prometheus" + "gopkg.in/yaml.v3" +) + +var ( + configReloadSuccess = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "postgres_exporter", + Name: "config_last_reload_successful", + Help: "Postgres exporter config loaded successfully.", + }) + + configReloadSeconds = prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: "postgres_exporter", + Name: "config_last_reload_success_timestamp_seconds", + Help: "Timestamp of the last successful configuration reload.", + }) +) + +func init() { + prometheus.MustRegister(configReloadSuccess) + prometheus.MustRegister(configReloadSeconds) +} + +type Config struct { + AuthModules map[string]AuthModule `yaml:"auth_modules"` +} + +type AuthModule struct { + Type string `yaml:"type"` + UserPass UserPass `yaml:"userpass,omitempty"` + // Add alternative auth modules here + Options map[string]string `yaml:"options"` +} + +type UserPass struct { + Username string `yaml:"username"` + Password string `yaml:"password"` +} + +type ConfigHandler struct { + sync.RWMutex + Config *Config +} + +func (ch *ConfigHandler) GetConfig() *Config { + ch.RLock() + defer ch.RUnlock() + return ch.Config +} + +func (ch *ConfigHandler) ReloadConfig(f string, logger log.Logger) error { + config := &Config{} + var err error + defer func() { + if err != nil { + configReloadSuccess.Set(0) + } else { + configReloadSuccess.Set(1) + configReloadSeconds.SetToCurrentTime() + } + }() + + yamlReader, err := os.Open(f) + if err != nil { + return fmt.Errorf("Error opening config file %q: %s", f, err) + } + defer yamlReader.Close() + decoder := yaml.NewDecoder(yamlReader) + decoder.KnownFields(true) + + if err = decoder.Decode(config); err != nil { + return fmt.Errorf("Error parsing config file %q: %s", f, err) + } + + ch.Lock() + ch.Config = config + ch.Unlock() + return nil +} + +func (m AuthModule) ConfigureTarget(target string) (string, error) { + // ip:port urls do not parse properly and that is the typical way users interact with postgres + t := fmt.Sprintf("exporter://%s", target) + u, err := url.Parse(t) + if err != nil { + return "", err + } + + if m.Type == "userpass" { + u.User = url.UserPassword(m.UserPass.Username, m.UserPass.Password) + } + + query := u.Query() + for k, v := range m.Options { + query.Set(k, v) + } + u.RawQuery = query.Encode() + + parsed := u.String() + trim := strings.TrimPrefix(parsed, "exporter://") + + return trim, nil +} diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 000000000..63b932adb --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "testing" +) + +func TestLoadConfig(t *testing.T) { + ch := &ConfigHandler{ + Config: &Config{}, + } + + err := ch.ReloadConfig("testdata/config-good.yaml", nil) + if err != nil { + t.Errorf("Error loading config: %s", err) + } +} + +func TestLoadBadConfigs(t *testing.T) { + ch := &ConfigHandler{ + Config: &Config{}, + } + + tests := []struct { + input string + want string + }{ + { + input: "testdata/config-bad-auth-module.yaml", + want: "Error parsing config file \"testdata/config-bad-auth-module.yaml\": yaml: unmarshal errors:\n line 3: field pretendauth not found in type config.AuthModule", + }, + { + input: "testdata/config-bad-extra-field.yaml", + want: "Error parsing config file \"testdata/config-bad-extra-field.yaml\": yaml: unmarshal errors:\n line 8: field doesNotExist not found in type config.AuthModule", + }, + } + + for _, test := range tests { + t.Run(test.input, func(t *testing.T) { + got := ch.ReloadConfig(test.input, nil) + if got == nil || got.Error() != test.want { + t.Fatalf("ReloadConfig(%q) = %v, want %s", test.input, got, test.want) + } + }) + } +} diff --git a/config/testdata/config-bad-auth-module.yaml b/config/testdata/config-bad-auth-module.yaml new file mode 100644 index 000000000..8f718dd5a --- /dev/null +++ b/config/testdata/config-bad-auth-module.yaml @@ -0,0 +1,7 @@ +auth_modules: + foo: + pretendauth: + username: test + password: pass + options: + extra: "1" diff --git a/config/testdata/config-bad-extra-field.yaml b/config/testdata/config-bad-extra-field.yaml new file mode 100644 index 000000000..f6ff6d6cf --- /dev/null +++ b/config/testdata/config-bad-extra-field.yaml @@ -0,0 +1,8 @@ +auth_modules: + foo: + userpass: + username: test + password: pass + options: + extra: "1" + doesNotExist: test diff --git a/config/testdata/config-good.yaml b/config/testdata/config-good.yaml new file mode 100644 index 000000000..13453e26f --- /dev/null +++ b/config/testdata/config-good.yaml @@ -0,0 +1,8 @@ +auth_modules: + first: + type: userpass + userpass: + username: first + password: firstpass + options: + sslmode: disable diff --git a/go.mod b/go.mod index f27c896ba..f2ccd64e3 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) require ( diff --git a/go.sum b/go.sum index 3f9003a23..1ad1ca5b1 100644 --- a/go.sum +++ b/go.sum @@ -494,6 +494,8 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 8f8d2208f593056b0f159466625b42d2c8a956c9 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Tue, 28 Jun 2022 22:22:14 -0400 Subject: [PATCH 06/23] cleanup and README Signed-off-by: Joe Adams Co-authored-by: Ben Kochie --- README.md | 30 ++++++++++++++++++++++++++++++ cmd/postgres_exporter/main.go | 6 ------ cmd/postgres_exporter/probe.go | 20 ++++++-------------- config/config.go | 10 +++------- 4 files changed, 39 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 4d23603a6..2f50fba07 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,36 @@ docker run \ quay.io/prometheuscommunity/postgres-exporter ``` +## Multi-Target Support (BETA) +**This Feature is in beta and may require changes in future releases. Feedback is welcome.** + +This exporter supports the [multi-target pattern](https://prometheus.io/docs/guides/multi-target-exporter/). This allows running a single instance of this exporter for multiple postgres targets. Using the milti-target funcationality of this exporter is **optional** and meant for users where it is impossible to install the exporter as a sidecar. For example SaaS-managed services. + +To use the multi-target functionality, send an http request to the endpoint `/probe?target=foo:5432` where target is set to the DSN of the postgres instance to scrape metrics from. + +To avoid putting sensitive information like username and password in the URL, preconfigured auth modules are supported via the [auth_modules](#auth_modules) section of the config file. auth_modules for DSNs can be used with the `/probe` endpoint by specifying the `?auth_module=foo` http parameter. + +## Configuration File + +The configuration file controls the behavior of the exporter. It can be set using the `--config.file` command line flag and defaults to `postres_exporter.yml`. + +### auth_modules +This section defines preset authentication and connection parameters for use in the [multi-target endpoint](#multi-target-support-beta). `auth_modules` is a map of modules with the key being the identifier which can be used in the `/probe` endpoint. +Currently only the `userpass` type is supported. + +Example: +```yaml +auth_modules: + foo1: # Set this to any name you want + type: userpass + userpass: + username: first + password: firstpass + options: + # options become key=value parameters of the DSN + sslmode: disable +``` + ## Building and running git clone https://github.com/prometheus-community/postgres_exporter.git diff --git a/cmd/postgres_exporter/main.go b/cmd/postgres_exporter/main.go index aee325039..d8f612956 100644 --- a/cmd/postgres_exporter/main.go +++ b/cmd/postgres_exporter/main.go @@ -102,12 +102,6 @@ func main() { os.Exit(1) } - // TODO(@sysadmind): Remove this with multi-target support - // if len(dsn) == 0 { - // level.Error(logger).Log("msg", "Couldn't find environment variables describing the datasource to use") - // os.Exit(1) - // } - opts := []ExporterOpt{ DisableDefaultMetrics(*disableDefaultMetrics), DisableSettingsMetrics(*disableSettingsMetrics), diff --git a/cmd/postgres_exporter/probe.go b/cmd/postgres_exporter/probe.go index 813f4ea81..7b2154319 100644 --- a/cmd/postgres_exporter/probe.go +++ b/cmd/postgres_exporter/probe.go @@ -60,7 +60,7 @@ func handleProbe(logger log.Logger) http.HandlerFunc { return } - // TODO: Timeout + // TODO(@sysadmind): Timeout probeSuccessGauge := prometheus.NewGauge(prometheus.GaugeOpts{ Name: "probe_success", @@ -86,23 +86,15 @@ func handleProbe(logger log.Logger) http.HandlerFunc { http.Error(w, err.Error(), http.StatusInternalServerError) return } - _ = ctx - // TODO: Which way should this be? Register or handle the collection manually? - // Also, what about the context? + // TODO(@sysadmind): Remove the registry.MustRegister() call below and instead handle the collection here. That will allow + // for the passing of context, handling of timeouts, and more control over the collection. + // The current NewProbeCollector() implementation relies on the MustNewConstMetric() call to create the metrics which is not + // ideal to use without the registry.MustRegister() call. + _ = ctx - // Option 1: Register the collector registry.MustRegister(pc) - // Option 2: Handle the collection manually. This allows us to collect duration metrics. - // The collectors themselves already support their own duration metrics. - // err = pc.Update(ctx) - // if err != nil { - // probeSuccessGauge.Set(0) - // } else { - // probeSuccessGauge.Set(1) - // } - duration := time.Since(start).Seconds() probeDurationGauge.Set(duration) diff --git a/config/config.go b/config/config.go index 49a2dbd63..10e7b7337 100644 --- a/config/config.go +++ b/config/config.go @@ -22,28 +22,24 @@ import ( "github.com/go-kit/log" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" "gopkg.in/yaml.v3" ) var ( - configReloadSuccess = prometheus.NewGauge(prometheus.GaugeOpts{ + configReloadSuccess = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "postgres_exporter", Name: "config_last_reload_successful", Help: "Postgres exporter config loaded successfully.", }) - configReloadSeconds = prometheus.NewGauge(prometheus.GaugeOpts{ + configReloadSeconds = promauto.NewGauge(prometheus.GaugeOpts{ Namespace: "postgres_exporter", Name: "config_last_reload_success_timestamp_seconds", Help: "Timestamp of the last successful configuration reload.", }) ) -func init() { - prometheus.MustRegister(configReloadSuccess) - prometheus.MustRegister(configReloadSeconds) -} - type Config struct { AuthModules map[string]AuthModule `yaml:"auth_modules"` } From 72430f8d2a7e541526a45a275b6615adabbdef48 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Thu, 28 Jul 2022 10:13:47 -0400 Subject: [PATCH 07/23] Update cmd/postgres_exporter/main.go Signed-off-by: Joe Adams --- README.md | 2 +- cmd/postgres_exporter/main.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2f50fba07..c7cb46212 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ To avoid putting sensitive information like username and password in the URL, pr ## Configuration File -The configuration file controls the behavior of the exporter. It can be set using the `--config.file` command line flag and defaults to `postres_exporter.yml`. +The configuration file controls the behavior of the exporter. It can be set using the `--config.file` command line flag and defaults to `postgres_exporter.yml`. ### auth_modules This section defines preset authentication and connection parameters for use in the [multi-target endpoint](#multi-target-support-beta). `auth_modules` is a map of modules with the key being the identifier which can be used in the `/probe` endpoint. diff --git a/cmd/postgres_exporter/main.go b/cmd/postgres_exporter/main.go index d8f612956..8759f4372 100644 --- a/cmd/postgres_exporter/main.go +++ b/cmd/postgres_exporter/main.go @@ -36,7 +36,7 @@ var ( Config: &config.Config{}, } - configFile = kingpin.Flag("config.file", "Promehteus exporter configuration file.").Default("postres_exporter.yml").String() + configFile = kingpin.Flag("config.file", "Postgres exporter configuration file.").Default("postgres_exporter.yml").String() listenAddress = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9187").Envar("PG_EXPORTER_WEB_LISTEN_ADDRESS").String() webConfig = webflag.AddFlags(kingpin.CommandLine) metricPath = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").Envar("PG_EXPORTER_WEB_TELEMETRY_PATH").String() From a627e7fce7027dbfe13b587ede66ccaad6e1f6ba Mon Sep 17 00:00:00 2001 From: bravosierrasierra Date: Fri, 29 Jul 2022 13:33:23 +0300 Subject: [PATCH 08/23] fix for exporter issue 633 fix for exporter issue 633: https://github.com/prometheus-community/postgres_exporter/issues/633 "Scan error on column index 2, name \"checkpoint_write_time\": converting driver.Value type float64 (\"6.594096e+06\") to a int: invalid syntax #633" Signed-off-by: bravosierrasierra --- collector/pg_stat_bgwriter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_stat_bgwriter.go b/collector/pg_stat_bgwriter.go index f897dbe2c..482b74e98 100644 --- a/collector/pg_stat_bgwriter.go +++ b/collector/pg_stat_bgwriter.go @@ -122,7 +122,7 @@ func (PGStatBGWriterCollector) Update(ctx context.Context, db *sql.DB, ch chan<- var cpt int var cpr int - var cpwt int + var cpwt float64 var cpst int var bcp int var bc int From 8855b7a604256e019b47fab2a170bb94a0784eb6 Mon Sep 17 00:00:00 2001 From: Nicolas Rodriguez Date: Sat, 30 Jul 2022 01:54:49 +0200 Subject: [PATCH 09/23] Fix checkpoint_sync_time value type Error: sql: Scan error on column index 3, name \"checkpoint_sync_time\": converting driver.Value type float64 (\"1.876469e+06\") to a int: invalid syntax See also: https://github.com/prometheus-community/postgres_exporter/issues/633 https://github.com/prometheus-community/postgres_exporter/pull/666 Signed-off-by: Nicolas Rodriguez --- collector/pg_stat_bgwriter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/collector/pg_stat_bgwriter.go b/collector/pg_stat_bgwriter.go index 482b74e98..69c75653f 100644 --- a/collector/pg_stat_bgwriter.go +++ b/collector/pg_stat_bgwriter.go @@ -123,7 +123,7 @@ func (PGStatBGWriterCollector) Update(ctx context.Context, db *sql.DB, ch chan<- var cpt int var cpr int var cpwt float64 - var cpst int + var cpst float64 var bcp int var bc int var mwc int From 926b8659a0c2b9a540c998965cdedcfc785a1eb1 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Aug 2022 17:07:11 +0000 Subject: [PATCH 10/23] Bump github.com/prometheus/common from 0.35.0 to 0.37.0 Bumps [github.com/prometheus/common](https://github.com/prometheus/common) from 0.35.0 to 0.37.0. - [Release notes](https://github.com/prometheus/common/releases) - [Commits](https://github.com/prometheus/common/compare/v0.35.0...v0.37.0) --- updated-dependencies: - dependency-name: github.com/prometheus/common dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f2ccd64e3..08fad017a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/lib/pq v1.10.6 github.com/prometheus/client_golang v1.12.2 github.com/prometheus/client_model v0.2.0 - github.com/prometheus/common v0.35.0 + github.com/prometheus/common v0.37.0 github.com/prometheus/exporter-toolkit v0.7.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c diff --git a/go.sum b/go.sum index 1ad1ca5b1..8532a8c2b 100644 --- a/go.sum +++ b/go.sum @@ -189,8 +189,8 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.35.0 h1:Eyr+Pw2VymWejHqCugNaQXkAi6KayVNxaHeu6khmFBE= -github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/exporter-toolkit v0.7.1 h1:c6RXaK8xBVercEeUQ4tRNL8UGWzDHfvj9dseo1FcK1Y= github.com/prometheus/exporter-toolkit v0.7.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= From adf8fc1d958e7b9808eb80a309247551baf21704 Mon Sep 17 00:00:00 2001 From: Luckz <224748+Luckz@users.noreply.github.com> Date: Wed, 10 Aug 2022 18:20:20 +0200 Subject: [PATCH 11/23] Correct minor typos in README.md Signed-off-by: Luckz <224748+Luckz@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c7cb46212..6af134175 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ docker run \ ## Multi-Target Support (BETA) **This Feature is in beta and may require changes in future releases. Feedback is welcome.** -This exporter supports the [multi-target pattern](https://prometheus.io/docs/guides/multi-target-exporter/). This allows running a single instance of this exporter for multiple postgres targets. Using the milti-target funcationality of this exporter is **optional** and meant for users where it is impossible to install the exporter as a sidecar. For example SaaS-managed services. +This exporter supports the [multi-target pattern](https://prometheus.io/docs/guides/multi-target-exporter/). This allows running a single instance of this exporter for multiple postgres targets. Using the multi-target funcationality of this exporter is **optional** and meant for cases where it is impossible to install the exporter as a sidecar, for example SaaS-managed services. To use the multi-target functionality, send an http request to the endpoint `/probe?target=foo:5432` where target is set to the DSN of the postgres instance to scrape metrics from. From e4b3cb580580766b9918f1a95ed75e6552cee4d3 Mon Sep 17 00:00:00 2001 From: SuperQ Date: Thu, 18 Aug 2022 22:52:25 +0200 Subject: [PATCH 12/23] Release 0.11.1 * [BUGFIX] Fix checkpoint_write_time value type #666 * [BUGFIX] Fix checkpoint_sync_time value type #667 Signed-off-by: SuperQ --- CHANGELOG.md | 5 +++++ VERSION | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8f7b92f7..7d792d41d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## master / unreleased +## 0.11.1 / 2022-08-01 + +* [BUGFIX] Fix checkpoint_write_time value type #666 +* [BUGFIX] Fix checkpoint_sync_time value type #667 + ## 0.11.0 / 2022-07-28 NOTE: pg_stat_bgwriter counter metrics had the `_total` suffix added #556 diff --git a/VERSION b/VERSION index d9df1bbc0..af88ba824 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.11.0 +0.11.1 From de9d21df6aae924a64a798f8b2c5e734f41dfcca Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Wed, 24 Aug 2022 22:07:37 -0400 Subject: [PATCH 13/23] Add dsn type for handling datasources dsn is designed to replace the other uses of dsn as a string in the long term. dsn is designed to be safe to log, properly redacting passwords. The goal is eventually always parse datasource information into a dsn type object which can safely be passed around and logged without worrying about wrapping calls in a redaction function (today this function is loggableDSN(). This should solve the root issue in #648, #677, and #643, although the full fix will require more changes to update all code references over to use the dsn type. Signed-off-by: Joe Adams --- cmd/postgres_exporter/datasource.go | 194 +++++++++++++++++++++ cmd/postgres_exporter/datasource_test.go | 206 +++++++++++++++++++++++ 2 files changed, 400 insertions(+) create mode 100644 cmd/postgres_exporter/datasource_test.go diff --git a/cmd/postgres_exporter/datasource.go b/cmd/postgres_exporter/datasource.go index 716138f32..fdfcbd6af 100644 --- a/cmd/postgres_exporter/datasource.go +++ b/cmd/postgres_exporter/datasource.go @@ -20,6 +20,7 @@ import ( "os" "regexp" "strings" + "unicode" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -172,3 +173,196 @@ func getDataSources() ([]string, error) { return []string{dsn}, nil } + +// dsn represents a parsed datasource. It contains fields for the individual connection components. +type dsn struct { + scheme string + username string + password string + host string + path string + query string +} + +// String makes a dsn safe to print by excluding any passwords. This allows dsn to be used in +// strings and log messages without needing to call a redaction function first. +func (d dsn) String() string { + if d.password != "" { + return fmt.Sprintf("%s://%s:******@%s%s?%s", d.scheme, d.username, d.host, d.path, d.query) + } + + if d.username != "" { + return fmt.Sprintf("%s://%s@%s%s?%s", d.scheme, d.username, d.host, d.path, d.query) + } + + return fmt.Sprintf("%s://%s%s?%s", d.scheme, d.host, d.path, d.query) +} + +// dsnFromString parses a connection string into a dsn. It will attempt to parse the string as +// a URL and as a set of key=value pairs. If both attempts fail, dsnFromString will return an error. +func dsnFromString(in string) (dsn, error) { + if strings.HasPrefix(in, "postgresql://") { + return dsnFromURL(in) + } + + // Try to parse as key=value pairs + d, err := dsnFromKeyValue(in) + if err == nil { + return d, nil + } + + return dsn{}, fmt.Errorf("could not understand DSN") +} + +// dsnFromURL parses the input as a URL and returns the dsn representation. +func dsnFromURL(in string) (dsn, error) { + u, err := url.Parse(in) + if err != nil { + return dsn{}, err + } + pass, _ := u.User.Password() + user := u.User.Username() + + query := u.Query() + + if queryPass := query.Get("password"); queryPass != "" { + if pass == "" { + pass = queryPass + } + } + query.Del("password") + + if queryUser := query.Get("user"); queryUser != "" { + if user == "" { + user = queryUser + } + } + query.Del("user") + + d := dsn{ + scheme: u.Scheme, + username: user, + password: pass, + host: u.Host, + path: u.Path, + query: query.Encode(), + } + + return d, nil +} + +// dsnFromKeyValue parses the input as a set of key=value pairs and returns the dsn representation. +func dsnFromKeyValue(in string) (dsn, error) { + // Attempt to confirm at least one key=value pair before starting the rune parser + connstringRe := regexp.MustCompile(`^ *[a-zA-Z0-9]+ *= *[^= ]+`) + if !connstringRe.MatchString(in) { + return dsn{}, fmt.Errorf("input is not a key-value DSN") + } + + // Anything other than known fields should be part of the querystring + query := url.Values{} + + pairs, err := parseKeyValue(in) + if err != nil { + return dsn{}, fmt.Errorf("failed to parse key-value DSN: %v", err) + } + + // Build the dsn from the key=value pairs + d := dsn{ + scheme: "postgresql", + } + + hostname := "" + port := "" + + for k, v := range pairs { + switch k { + case "host": + hostname = v + case "port": + port = v + case "user": + d.username = v + case "password": + d.password = v + default: + query.Set(k, v) + } + } + + if hostname == "" { + hostname = "localhost" + } + + if port == "" { + d.host = hostname + } else { + d.host = fmt.Sprintf("%s:%s", hostname, port) + } + + d.query = query.Encode() + + return d, nil +} + +// parseKeyValue is a key=value parser. It loops over each rune to split out keys and values +// and attempting to honor quoted values. parseKeyValue will return an error if it is unable +// to properly parse the input. +func parseKeyValue(in string) (map[string]string, error) { + out := map[string]string{} + + inPart := false + inQuote := false + part := []rune{} + key := "" + for _, c := range in { + switch { + case unicode.In(c, unicode.Quotation_Mark): + if inQuote { + inQuote = false + } else { + inQuote = true + } + case unicode.In(c, unicode.White_Space): + if inPart { + if inQuote { + part = append(part, c) + } else { + // Are we finishing a key=value? + if key == "" { + return out, fmt.Errorf("invalid input") + } + out[key] = string(part) + inPart = false + part = []rune{} + } + } else { + // Are we finishing a key=value? + if key == "" { + return out, fmt.Errorf("invalid input") + } + out[key] = string(part) + inPart = false + part = []rune{} + // Do something with the value + } + case c == '=': + if inPart { + inPart = false + key = string(part) + part = []rune{} + } else { + return out, fmt.Errorf("invalid input") + } + default: + inPart = true + part = append(part, c) + } + } + + if key != "" && len(part) > 0 { + out[key] = string(part) + } + + return out, nil +} diff --git a/cmd/postgres_exporter/datasource_test.go b/cmd/postgres_exporter/datasource_test.go new file mode 100644 index 000000000..02fb8ddef --- /dev/null +++ b/cmd/postgres_exporter/datasource_test.go @@ -0,0 +1,206 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + "testing" +) + +// Test_dsn_String is designed to test different dsn combinations for their string representation. +// dsn.String() is designed to be safe to print, redacting any password information and these test +// cases are intended to cover known cases. +func Test_dsn_String(t *testing.T) { + type fields struct { + scheme string + username string + password string + host string + path string + query string + } + tests := []struct { + name string + fields fields + want string + }{ + { + name: "Without Password", + fields: fields{ + scheme: "postgresql", + username: "test", + host: "localhost:5432", + query: "", + }, + want: "postgresql://test@localhost:5432?", + }, + { + name: "With Password", + fields: fields{ + scheme: "postgresql", + username: "test", + password: "supersecret", + host: "localhost:5432", + query: "", + }, + want: "postgresql://test:******@localhost:5432?", + }, + { + name: "With Password and Query String", + fields: fields{ + scheme: "postgresql", + username: "test", + password: "supersecret", + host: "localhost:5432", + query: "ssldisable=true", + }, + want: "postgresql://test:******@localhost:5432?ssldisable=true", + }, + { + name: "With Password, Path, and Query String", + fields: fields{ + scheme: "postgresql", + username: "test", + password: "supersecret", + host: "localhost:5432", + path: "/somevalue", + query: "ssldisable=true", + }, + want: "postgresql://test:******@localhost:5432/somevalue?ssldisable=true", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := dsn{ + scheme: tt.fields.scheme, + username: tt.fields.username, + password: tt.fields.password, + host: tt.fields.host, + path: tt.fields.path, + query: tt.fields.query, + } + if got := d.String(); got != tt.want { + t.Errorf("dsn.String() = %v, want %v", got, tt.want) + } + }) + } +} + +// Test_dsnFromString tests the dsnFromString function with known variations +// of connection string inputs to ensure that it properly parses the input into +// a dsn. +func Test_dsnFromString(t *testing.T) { + + tests := []struct { + name string + input string + want dsn + wantErr bool + }{ + { + name: "Key value with password", + input: "host=host.example.com user=postgres port=5432 password=s3cr3t", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + username: "postgres", + password: "s3cr3t", + }, + wantErr: false, + }, + { + name: "Key value with quoted password and space", + input: "host=host.example.com user=postgres port=5432 password=\"s3cr 3t\"", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + username: "postgres", + password: "s3cr 3t", + }, + wantErr: false, + }, + { + name: "Key value with different order", + input: "password=abcde host=host.example.com user=postgres port=5432", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + username: "postgres", + password: "abcde", + }, + wantErr: false, + }, + { + name: "Key value with different order, quoted password, duplicate password", + input: "password=abcde host=host.example.com user=postgres port=5432 password=\"s3cr 3t\"", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + username: "postgres", + password: "s3cr 3t", + }, + wantErr: false, + }, + { + name: "URL with user in query string", + input: "postgresql://host.example.com:5432/tsdb?user=postgres", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + path: "/tsdb", + query: "", + username: "postgres", + }, + wantErr: false, + }, + { + name: "URL with user and password", + input: "postgresql://user:s3cret@host.example.com:5432/tsdb?user=postgres", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + path: "/tsdb", + query: "", + username: "user", + password: "s3cret", + }, + wantErr: false, + }, + { + name: "URL with user and password in query string", + input: "postgresql://host.example.com:5432/tsdb?user=postgres&password=s3cr3t", + want: dsn{ + scheme: "postgresql", + host: "host.example.com:5432", + path: "/tsdb", + query: "", + username: "postgres", + password: "s3cr3t", + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := dsnFromString(tt.input) + if (err != nil) != tt.wantErr { + t.Errorf("dsnFromString() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("dsnFromString() = %+v, want %+v", got, tt.want) + } + }) + } +} From d86dd06ffe9af7b5c1faa37ea3098c3552df5904 Mon Sep 17 00:00:00 2001 From: SuperQ Date: Fri, 26 Aug 2022 16:49:54 +0200 Subject: [PATCH 14/23] Release 0.12.0-rc.0 BREAKING CHANGES: This release changes support for multiple postgres servers to use the multi-target exporter pattern. This makes it much easier to monitor multiple PostgreSQL servers from a single exporter by passing the target via URL params. See the Multi-Target Support section of the README. * [CHANGE] Add multi-target support #618 * [BUGFIX] Add dsn type for handling datasources #678 Signed-off-by: SuperQ --- CHANGELOG.md | 12 +++++++++ VERSION | 2 +- cmd/postgres_exporter/namespace.go | 2 +- cmd/postgres_exporter/postgres_exporter.go | 2 +- .../postgres_exporter_test.go | 2 +- cmd/postgres_exporter/queries.go | 2 +- cmd/postgres_exporter/server.go | 2 +- go.mod | 12 ++++----- go.sum | 26 +++++++++++-------- 9 files changed, 39 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7d792d41d..38d7e4d90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ ## master / unreleased +## 0.12.0-rc.0 / 2022-08-26 + +BREAKING CHANGES: + +This release changes support for multiple postgres servers to use the +multi-target exporter pattern. This makes it much easier to monitor multiple +PostgreSQL servers from a single exporter by passing the target via URL +params. See the Multi-Target Support section of the README. + +* [CHANGE] Add multi-target support #618 +* [BUGFIX] Add dsn type for handling datasources #678 + ## 0.11.1 / 2022-08-01 * [BUGFIX] Fix checkpoint_write_time value type #666 diff --git a/VERSION b/VERSION index af88ba824..68f8b7694 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.11.1 +0.12.0-rc.0 diff --git a/cmd/postgres_exporter/namespace.go b/cmd/postgres_exporter/namespace.go index ab1e74ea0..41674007d 100644 --- a/cmd/postgres_exporter/namespace.go +++ b/cmd/postgres_exporter/namespace.go @@ -19,7 +19,7 @@ import ( "fmt" "time" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/go-kit/log/level" "github.com/lib/pq" "github.com/prometheus/client_golang/prometheus" diff --git a/cmd/postgres_exporter/postgres_exporter.go b/cmd/postgres_exporter/postgres_exporter.go index 2574e0252..af973d5d5 100644 --- a/cmd/postgres_exporter/postgres_exporter.go +++ b/cmd/postgres_exporter/postgres_exporter.go @@ -24,7 +24,7 @@ import ( "strings" "time" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" ) diff --git a/cmd/postgres_exporter/postgres_exporter_test.go b/cmd/postgres_exporter/postgres_exporter_test.go index 6017756a8..468aa39b9 100644 --- a/cmd/postgres_exporter/postgres_exporter_test.go +++ b/cmd/postgres_exporter/postgres_exporter_test.go @@ -24,7 +24,7 @@ import ( "testing" "time" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/prometheus/client_golang/prometheus" . "gopkg.in/check.v1" ) diff --git a/cmd/postgres_exporter/queries.go b/cmd/postgres_exporter/queries.go index e27480f4f..b17420d28 100644 --- a/cmd/postgres_exporter/queries.go +++ b/cmd/postgres_exporter/queries.go @@ -17,7 +17,7 @@ import ( "errors" "fmt" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/go-kit/log/level" "gopkg.in/yaml.v2" ) diff --git a/cmd/postgres_exporter/server.go b/cmd/postgres_exporter/server.go index 8747dffa0..bcfee6812 100644 --- a/cmd/postgres_exporter/server.go +++ b/cmd/postgres_exporter/server.go @@ -19,7 +19,7 @@ import ( "sync" "time" - "github.com/blang/semver" + "github.com/blang/semver/v4" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" ) diff --git a/go.mod b/go.mod index 08fad017a..5f5d6195f 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,17 @@ module github.com/prometheus-community/postgres_exporter go 1.17 require ( - github.com/blang/semver v3.5.1+incompatible + github.com/blang/semver/v4 v4.0.0 github.com/go-kit/log v0.2.1 github.com/lib/pq v1.10.6 - github.com/prometheus/client_golang v1.12.2 + github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.37.0 github.com/prometheus/exporter-toolkit v0.7.1 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c gopkg.in/yaml.v2 v2.4.0 - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -29,12 +29,12 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/procfs v0.7.3 // indirect + github.com/prometheus/procfs v0.8.0 // indirect golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b // indirect - golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/appengine v1.6.6 // indirect - google.golang.org/protobuf v1.26.0 // indirect + google.golang.org/protobuf v1.28.1 // indirect ) diff --git a/go.sum b/go.sum index 8532a8c2b..fcd14c6b4 100644 --- a/go.sum +++ b/go.sum @@ -44,8 +44,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= @@ -116,8 +116,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -177,8 +178,8 @@ github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5Fsn github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34= -github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0 h1:b71QUfeo5M8gq2+evJdTPfZhYMAU0uKPkyPJ7TPsloU= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -197,8 +198,9 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= @@ -305,6 +307,7 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -342,8 +345,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -400,7 +404,6 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -477,8 +480,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -494,8 +498,8 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 9ae347519b63942d85565b33f815f52df7394df9 Mon Sep 17 00:00:00 2001 From: Yoan Blanc Date: Tue, 30 Aug 2022 06:48:06 +0200 Subject: [PATCH 15/23] fix: typo Signed-off-by: Yoan Blanc --- cmd/postgres_exporter/datasource.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/datasource.go b/cmd/postgres_exporter/datasource.go index fdfcbd6af..0a2b943a7 100644 --- a/cmd/postgres_exporter/datasource.go +++ b/cmd/postgres_exporter/datasource.go @@ -164,7 +164,7 @@ func getDataSources() ([]string, error) { } // No datasources found. This allows us to support the multi-target pattern - // withouth an explicit datasource. + // without an explicit datasource. if uri == "" { return []string{}, nil } From 7ffba684de577fe07673b10ddc1dace286d099ec Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Fri, 2 Sep 2022 10:32:44 -0400 Subject: [PATCH 16/23] Update multi-target handler to use new DSN type - Moves new dsn type to config.DSN. This will prevent circular dependencies. - Change DSN.query to be url.Values. This allows the multi-target functionality to merge values without re-parsing the query string - Change NewProbeCollector to use the new config.DSN type - Add DSN.GetConnectionString to return a string formatted for the sql driver to use during connection Signed-off-by: Joe Adams --- cmd/postgres_exporter/datasource.go | 194 --------------- collector/probe.go | 11 +- config/config.go | 28 +-- config/dsn.go | 225 ++++++++++++++++++ .../datasource_test.go => config/dsn_test.go | 45 ++-- 5 files changed, 268 insertions(+), 235 deletions(-) create mode 100644 config/dsn.go rename cmd/postgres_exporter/datasource_test.go => config/dsn_test.go (90%) diff --git a/cmd/postgres_exporter/datasource.go b/cmd/postgres_exporter/datasource.go index 0a2b943a7..97f7ecd8b 100644 --- a/cmd/postgres_exporter/datasource.go +++ b/cmd/postgres_exporter/datasource.go @@ -20,7 +20,6 @@ import ( "os" "regexp" "strings" - "unicode" "github.com/go-kit/log/level" "github.com/prometheus/client_golang/prometheus" @@ -173,196 +172,3 @@ func getDataSources() ([]string, error) { return []string{dsn}, nil } - -// dsn represents a parsed datasource. It contains fields for the individual connection components. -type dsn struct { - scheme string - username string - password string - host string - path string - query string -} - -// String makes a dsn safe to print by excluding any passwords. This allows dsn to be used in -// strings and log messages without needing to call a redaction function first. -func (d dsn) String() string { - if d.password != "" { - return fmt.Sprintf("%s://%s:******@%s%s?%s", d.scheme, d.username, d.host, d.path, d.query) - } - - if d.username != "" { - return fmt.Sprintf("%s://%s@%s%s?%s", d.scheme, d.username, d.host, d.path, d.query) - } - - return fmt.Sprintf("%s://%s%s?%s", d.scheme, d.host, d.path, d.query) -} - -// dsnFromString parses a connection string into a dsn. It will attempt to parse the string as -// a URL and as a set of key=value pairs. If both attempts fail, dsnFromString will return an error. -func dsnFromString(in string) (dsn, error) { - if strings.HasPrefix(in, "postgresql://") { - return dsnFromURL(in) - } - - // Try to parse as key=value pairs - d, err := dsnFromKeyValue(in) - if err == nil { - return d, nil - } - - return dsn{}, fmt.Errorf("could not understand DSN") -} - -// dsnFromURL parses the input as a URL and returns the dsn representation. -func dsnFromURL(in string) (dsn, error) { - u, err := url.Parse(in) - if err != nil { - return dsn{}, err - } - pass, _ := u.User.Password() - user := u.User.Username() - - query := u.Query() - - if queryPass := query.Get("password"); queryPass != "" { - if pass == "" { - pass = queryPass - } - } - query.Del("password") - - if queryUser := query.Get("user"); queryUser != "" { - if user == "" { - user = queryUser - } - } - query.Del("user") - - d := dsn{ - scheme: u.Scheme, - username: user, - password: pass, - host: u.Host, - path: u.Path, - query: query.Encode(), - } - - return d, nil -} - -// dsnFromKeyValue parses the input as a set of key=value pairs and returns the dsn representation. -func dsnFromKeyValue(in string) (dsn, error) { - // Attempt to confirm at least one key=value pair before starting the rune parser - connstringRe := regexp.MustCompile(`^ *[a-zA-Z0-9]+ *= *[^= ]+`) - if !connstringRe.MatchString(in) { - return dsn{}, fmt.Errorf("input is not a key-value DSN") - } - - // Anything other than known fields should be part of the querystring - query := url.Values{} - - pairs, err := parseKeyValue(in) - if err != nil { - return dsn{}, fmt.Errorf("failed to parse key-value DSN: %v", err) - } - - // Build the dsn from the key=value pairs - d := dsn{ - scheme: "postgresql", - } - - hostname := "" - port := "" - - for k, v := range pairs { - switch k { - case "host": - hostname = v - case "port": - port = v - case "user": - d.username = v - case "password": - d.password = v - default: - query.Set(k, v) - } - } - - if hostname == "" { - hostname = "localhost" - } - - if port == "" { - d.host = hostname - } else { - d.host = fmt.Sprintf("%s:%s", hostname, port) - } - - d.query = query.Encode() - - return d, nil -} - -// parseKeyValue is a key=value parser. It loops over each rune to split out keys and values -// and attempting to honor quoted values. parseKeyValue will return an error if it is unable -// to properly parse the input. -func parseKeyValue(in string) (map[string]string, error) { - out := map[string]string{} - - inPart := false - inQuote := false - part := []rune{} - key := "" - for _, c := range in { - switch { - case unicode.In(c, unicode.Quotation_Mark): - if inQuote { - inQuote = false - } else { - inQuote = true - } - case unicode.In(c, unicode.White_Space): - if inPart { - if inQuote { - part = append(part, c) - } else { - // Are we finishing a key=value? - if key == "" { - return out, fmt.Errorf("invalid input") - } - out[key] = string(part) - inPart = false - part = []rune{} - } - } else { - // Are we finishing a key=value? - if key == "" { - return out, fmt.Errorf("invalid input") - } - out[key] = string(part) - inPart = false - part = []rune{} - // Do something with the value - } - case c == '=': - if inPart { - inPart = false - key = string(part) - part = []rune{} - } else { - return out, fmt.Errorf("invalid input") - } - default: - inPart = true - part = append(part, c) - } - } - - if key != "" && len(part) > 0 { - out[key] = string(part) - } - - return out, nil -} diff --git a/collector/probe.go b/collector/probe.go index 8aa18b92c..37cb151b9 100644 --- a/collector/probe.go +++ b/collector/probe.go @@ -16,11 +16,10 @@ package collector import ( "context" "database/sql" - "fmt" - "strings" "sync" "github.com/go-kit/log" + "github.com/prometheus-community/postgres_exporter/config" "github.com/prometheus/client_golang/prometheus" ) @@ -31,7 +30,7 @@ type ProbeCollector struct { db *sql.DB } -func NewProbeCollector(logger log.Logger, registry *prometheus.Registry, dsn string) (*ProbeCollector, error) { +func NewProbeCollector(logger log.Logger, registry *prometheus.Registry, dsn config.DSN) (*ProbeCollector, error) { collectors := make(map[string]Collector) initiatedCollectorsMtx.Lock() defer initiatedCollectorsMtx.Unlock() @@ -55,11 +54,7 @@ func NewProbeCollector(logger log.Logger, registry *prometheus.Registry, dsn str } } - if !strings.HasPrefix(dsn, "postgres://") { - dsn = fmt.Sprintf("postgres://%s", dsn) - } - - db, err := sql.Open("postgres", dsn) + db, err := sql.Open("postgres", dsn.GetConnectionString()) if err != nil { return nil, err } diff --git a/config/config.go b/config/config.go index 10e7b7337..9e514f413 100644 --- a/config/config.go +++ b/config/config.go @@ -15,9 +15,7 @@ package config import ( "fmt" - "net/url" "os" - "strings" "sync" "github.com/go-kit/log" @@ -97,26 +95,26 @@ func (ch *ConfigHandler) ReloadConfig(f string, logger log.Logger) error { return nil } -func (m AuthModule) ConfigureTarget(target string) (string, error) { - // ip:port urls do not parse properly and that is the typical way users interact with postgres - t := fmt.Sprintf("exporter://%s", target) - u, err := url.Parse(t) +func (m AuthModule) ConfigureTarget(target string) (DSN, error) { + dsn, err := dsnFromString(target) if err != nil { - return "", err + return DSN{}, err } + // Set the credentials from the authentication module + // TODO(@sysadmind): What should the order of precedence be? if m.Type == "userpass" { - u.User = url.UserPassword(m.UserPass.Username, m.UserPass.Password) + if m.UserPass.Username != "" { + dsn.username = m.UserPass.Username + } + if m.UserPass.Password != "" { + dsn.password = m.UserPass.Password + } } - query := u.Query() for k, v := range m.Options { - query.Set(k, v) + dsn.query.Set(k, v) } - u.RawQuery = query.Encode() - - parsed := u.String() - trim := strings.TrimPrefix(parsed, "exporter://") - return trim, nil + return dsn, nil } diff --git a/config/dsn.go b/config/dsn.go new file mode 100644 index 000000000..182a0aae7 --- /dev/null +++ b/config/dsn.go @@ -0,0 +1,225 @@ +package config + +import ( + "fmt" + "net/url" + "regexp" + "strings" + "unicode" +) + +// DSN represents a parsed datasource. It contains fields for the individual connection components. +type DSN struct { + scheme string + username string + password string + host string + path string + query url.Values +} + +// String makes a dsn safe to print by excluding any passwords. This allows dsn to be used in +// strings and log messages without needing to call a redaction function first. +func (d DSN) String() string { + if d.password != "" { + return fmt.Sprintf("%s://%s:******@%s%s?%s", d.scheme, d.username, d.host, d.path, d.query.Encode()) + } + + if d.username != "" { + return fmt.Sprintf("%s://%s@%s%s?%s", d.scheme, d.username, d.host, d.path, d.query.Encode()) + } + + return fmt.Sprintf("%s://%s%s?%s", d.scheme, d.host, d.path, d.query.Encode()) +} + +// GetConnectionString returns the URL to pass to the driver for database connections. This value should not be logged. +func (d DSN) GetConnectionString() string { + u := url.URL{ + Scheme: d.scheme, + Host: d.host, + Path: d.path, + RawQuery: d.query.Encode(), + } + + // Username and Password + if d.username != "" { + u.User = url.UserPassword(d.username, d.password) + } + + return u.String() +} + +// dsnFromString parses a connection string into a dsn. It will attempt to parse the string as +// a URL and as a set of key=value pairs. If both attempts fail, dsnFromString will return an error. +func dsnFromString(in string) (DSN, error) { + if strings.HasPrefix(in, "postgresql://") { + return dsnFromURL(in) + } + + // Try to parse as key=value pairs + d, err := dsnFromKeyValue(in) + if err == nil { + return d, nil + } + + // Parse the string as a URL, with the scheme prefixed + d, err = dsnFromURL(fmt.Sprintf("postgresql://%s", in)) + if err == nil { + return d, nil + } + + return DSN{}, fmt.Errorf("could not understand DSN") +} + +// dsnFromURL parses the input as a URL and returns the dsn representation. +func dsnFromURL(in string) (DSN, error) { + u, err := url.Parse(in) + if err != nil { + return DSN{}, err + } + pass, _ := u.User.Password() + user := u.User.Username() + + query := u.Query() + + if queryPass := query.Get("password"); queryPass != "" { + if pass == "" { + pass = queryPass + } + } + query.Del("password") + + if queryUser := query.Get("user"); queryUser != "" { + if user == "" { + user = queryUser + } + } + query.Del("user") + + d := DSN{ + scheme: u.Scheme, + username: user, + password: pass, + host: u.Host, + path: u.Path, + query: query, + } + + return d, nil +} + +// dsnFromKeyValue parses the input as a set of key=value pairs and returns the dsn representation. +func dsnFromKeyValue(in string) (DSN, error) { + // Attempt to confirm at least one key=value pair before starting the rune parser + connstringRe := regexp.MustCompile(`^ *[a-zA-Z0-9]+ *= *[^= ]+`) + if !connstringRe.MatchString(in) { + return DSN{}, fmt.Errorf("input is not a key-value DSN") + } + + // Anything other than known fields should be part of the querystring + query := url.Values{} + + pairs, err := parseKeyValue(in) + if err != nil { + return DSN{}, fmt.Errorf("failed to parse key-value DSN: %v", err) + } + + // Build the dsn from the key=value pairs + d := DSN{ + scheme: "postgresql", + } + + hostname := "" + port := "" + + for k, v := range pairs { + switch k { + case "host": + hostname = v + case "port": + port = v + case "user": + d.username = v + case "password": + d.password = v + default: + query.Set(k, v) + } + } + + if hostname == "" { + hostname = "localhost" + } + + if port == "" { + d.host = hostname + } else { + d.host = fmt.Sprintf("%s:%s", hostname, port) + } + + d.query = query + + return d, nil +} + +// parseKeyValue is a key=value parser. It loops over each rune to split out keys and values +// and attempting to honor quoted values. parseKeyValue will return an error if it is unable +// to properly parse the input. +func parseKeyValue(in string) (map[string]string, error) { + out := map[string]string{} + + inPart := false + inQuote := false + part := []rune{} + key := "" + for _, c := range in { + switch { + case unicode.In(c, unicode.Quotation_Mark): + if inQuote { + inQuote = false + } else { + inQuote = true + } + case unicode.In(c, unicode.White_Space): + if inPart { + if inQuote { + part = append(part, c) + } else { + // Are we finishing a key=value? + if key == "" { + return out, fmt.Errorf("invalid input") + } + out[key] = string(part) + inPart = false + part = []rune{} + } + } else { + // Are we finishing a key=value? + if key == "" { + return out, fmt.Errorf("invalid input") + } + out[key] = string(part) + inPart = false + part = []rune{} + // Do something with the value + } + case c == '=': + if inPart { + inPart = false + key = string(part) + part = []rune{} + } else { + return out, fmt.Errorf("invalid input") + } + default: + inPart = true + part = append(part, c) + } + } + + if key != "" && len(part) > 0 { + out[key] = string(part) + } + + return out, nil +} diff --git a/cmd/postgres_exporter/datasource_test.go b/config/dsn_test.go similarity index 90% rename from cmd/postgres_exporter/datasource_test.go rename to config/dsn_test.go index 02fb8ddef..637a3568e 100644 --- a/cmd/postgres_exporter/datasource_test.go +++ b/config/dsn_test.go @@ -11,9 +11,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +package config import ( + "net/url" "reflect" "testing" ) @@ -28,7 +29,7 @@ func Test_dsn_String(t *testing.T) { password string host string path string - query string + query url.Values } tests := []struct { name string @@ -41,7 +42,7 @@ func Test_dsn_String(t *testing.T) { scheme: "postgresql", username: "test", host: "localhost:5432", - query: "", + query: url.Values{}, }, want: "postgresql://test@localhost:5432?", }, @@ -52,7 +53,7 @@ func Test_dsn_String(t *testing.T) { username: "test", password: "supersecret", host: "localhost:5432", - query: "", + query: url.Values{}, }, want: "postgresql://test:******@localhost:5432?", }, @@ -63,7 +64,9 @@ func Test_dsn_String(t *testing.T) { username: "test", password: "supersecret", host: "localhost:5432", - query: "ssldisable=true", + query: url.Values{ + "ssldisable": []string{"true"}, + }, }, want: "postgresql://test:******@localhost:5432?ssldisable=true", }, @@ -75,14 +78,16 @@ func Test_dsn_String(t *testing.T) { password: "supersecret", host: "localhost:5432", path: "/somevalue", - query: "ssldisable=true", + query: url.Values{ + "ssldisable": []string{"true"}, + }, }, want: "postgresql://test:******@localhost:5432/somevalue?ssldisable=true", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := dsn{ + d := DSN{ scheme: tt.fields.scheme, username: tt.fields.username, password: tt.fields.password, @@ -105,61 +110,65 @@ func Test_dsnFromString(t *testing.T) { tests := []struct { name string input string - want dsn + want DSN wantErr bool }{ { name: "Key value with password", input: "host=host.example.com user=postgres port=5432 password=s3cr3t", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", username: "postgres", password: "s3cr3t", + query: url.Values{}, }, wantErr: false, }, { name: "Key value with quoted password and space", input: "host=host.example.com user=postgres port=5432 password=\"s3cr 3t\"", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", username: "postgres", password: "s3cr 3t", + query: url.Values{}, }, wantErr: false, }, { name: "Key value with different order", input: "password=abcde host=host.example.com user=postgres port=5432", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", username: "postgres", password: "abcde", + query: url.Values{}, }, wantErr: false, }, { name: "Key value with different order, quoted password, duplicate password", input: "password=abcde host=host.example.com user=postgres port=5432 password=\"s3cr 3t\"", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", username: "postgres", password: "s3cr 3t", + query: url.Values{}, }, wantErr: false, }, { name: "URL with user in query string", input: "postgresql://host.example.com:5432/tsdb?user=postgres", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", path: "/tsdb", - query: "", + query: url.Values{}, username: "postgres", }, wantErr: false, @@ -167,11 +176,11 @@ func Test_dsnFromString(t *testing.T) { { name: "URL with user and password", input: "postgresql://user:s3cret@host.example.com:5432/tsdb?user=postgres", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", path: "/tsdb", - query: "", + query: url.Values{}, username: "user", password: "s3cret", }, @@ -180,11 +189,11 @@ func Test_dsnFromString(t *testing.T) { { name: "URL with user and password in query string", input: "postgresql://host.example.com:5432/tsdb?user=postgres&password=s3cr3t", - want: dsn{ + want: DSN{ scheme: "postgresql", host: "host.example.com:5432", path: "/tsdb", - query: "", + query: url.Values{}, username: "postgres", password: "s3cr3t", }, From 69a802493b821aa67d9813d95841521819ad86d5 Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Fri, 2 Sep 2022 11:09:01 -0400 Subject: [PATCH 17/23] Add missing license header Signed-off-by: Joe Adams --- config/dsn.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/config/dsn.go b/config/dsn.go index 182a0aae7..78d798d5f 100644 --- a/config/dsn.go +++ b/config/dsn.go @@ -1,3 +1,16 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( From ad8b5608f8f48d11740099c021116338fcedae83 Mon Sep 17 00:00:00 2001 From: Ildar Valiullin Date: Thu, 1 Sep 2022 12:30:06 +0300 Subject: [PATCH 18/23] extended /probe path metrics Signed-off-by: Ildar Valiullin --- cmd/postgres_exporter/probe.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/cmd/postgres_exporter/probe.go b/cmd/postgres_exporter/probe.go index 7b2154319..6c9a46bfc 100644 --- a/cmd/postgres_exporter/probe.go +++ b/cmd/postgres_exporter/probe.go @@ -78,6 +78,23 @@ func handleProbe(logger log.Logger) http.HandlerFunc { registry.MustRegister(probeSuccessGauge) registry.MustRegister(probeDurationGauge) + opts := []ExporterOpt{ + DisableDefaultMetrics(*disableDefaultMetrics), + DisableSettingsMetrics(*disableSettingsMetrics), + AutoDiscoverDatabases(*autoDiscoverDatabases), + WithUserQueriesPath(*queriesPath), + WithConstantLabels(*constantLabelsList), + ExcludeDatabases(*excludeDatabases), + IncludeDatabases(*includeDatabases), + } + + dsns := []string{dsn.GetConnectionString()} + exporter := NewExporter(dsns, opts...) + defer func() { + exporter.servers.Close() + }() + registry.MustRegister(exporter) + // Run the probe pc, err := collector.NewProbeCollector(tl, registry, dsn) if err != nil { From 8268b7e6ffe698b6baa7588098636d10bb12ca6b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 1 Oct 2022 17:05:04 +0000 Subject: [PATCH 19/23] Bump github.com/lib/pq from 1.10.6 to 1.10.7 Bumps [github.com/lib/pq](https://github.com/lib/pq) from 1.10.6 to 1.10.7. - [Release notes](https://github.com/lib/pq/releases) - [Commits](https://github.com/lib/pq/compare/v1.10.6...v1.10.7) --- updated-dependencies: - dependency-name: github.com/lib/pq dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5f5d6195f..f8a0c26f0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.17 require ( github.com/blang/semver/v4 v4.0.0 github.com/go-kit/log v0.2.1 - github.com/lib/pq v1.10.6 + github.com/lib/pq v1.10.7 github.com/prometheus/client_golang v1.13.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.37.0 diff --git a/go.sum b/go.sum index fcd14c6b4..631253203 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,8 @@ github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= From fc2e8f083714c11c6e9468e0b95b76726d61f8cc Mon Sep 17 00:00:00 2001 From: prombot Date: Tue, 4 Oct 2022 17:59:51 +0000 Subject: [PATCH 20/23] Update common Prometheus files Signed-off-by: prombot --- Makefile.common | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile.common b/Makefile.common index 6c8e3e219..7642c4485 100644 --- a/Makefile.common +++ b/Makefile.common @@ -58,16 +58,19 @@ endif PROMU_VERSION ?= 0.13.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz +SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.45.2 +GOLANGCI_LINT_VERSION ?= v1.49.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) # If we're in CI and there is an Actions file, that means the linter # is being run in Actions, so we don't need to run it here. - ifeq (,$(CIRCLE_JOB)) + ifneq (,$(SKIP_GOLANGCI_LINT)) + GOLANGCI_LINT := + else ifeq (,$(CIRCLE_JOB)) GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint else ifeq (,$(wildcard .github/workflows/golangci-lint.yml)) GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint From 14c9d0370f52fdc2d69cba567b1562307dc066b5 Mon Sep 17 00:00:00 2001 From: Sergey Morozov <38383507+ken3122@users.noreply.github.com> Date: Sun, 9 Oct 2022 01:50:45 +0600 Subject: [PATCH 21/23] 4kB size added for postgres with 4kB block_size Signed-off-by: Sergey Morozov <38383507+ken3122@users.noreply.github.com> --- cmd/postgres_exporter/pg_setting.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/postgres_exporter/pg_setting.go b/cmd/postgres_exporter/pg_setting.go index bfe4c6c52..28a08a70b 100644 --- a/cmd/postgres_exporter/pg_setting.go +++ b/cmd/postgres_exporter/pg_setting.go @@ -129,7 +129,7 @@ func (s *pgSetting) normaliseUnit() (val float64, unit string, err error) { return case "ms", "s", "min", "h", "d": unit = "seconds" - case "B", "kB", "MB", "GB", "TB", "8kB", "16kB", "32kB", "16MB", "32MB", "64MB": + case "B", "kB", "MB", "GB", "TB", "4kB", "8kB", "16kB", "32kB", "16MB", "32MB", "64MB": unit = "bytes" default: err = fmt.Errorf("Unknown unit for runtime variable: %q", s.unit) @@ -158,6 +158,8 @@ func (s *pgSetting) normaliseUnit() (val float64, unit string, err error) { val *= math.Pow(2, 30) case "TB": val *= math.Pow(2, 40) + case "4kB": + val *= math.Pow(2, 12) case "8kB": val *= math.Pow(2, 13) case "16kB": From b10a942f40e6e41192afd5493d0f09a1a1032922 Mon Sep 17 00:00:00 2001 From: Luckz <224748+Luckz@users.noreply.github.com> Date: Sun, 6 Nov 2022 16:24:43 +0100 Subject: [PATCH 22/23] Correct additional typo in README.md Signed-off-by: Luckz <224748+Luckz@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6af134175..2fc0ad8bc 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ docker run \ ## Multi-Target Support (BETA) **This Feature is in beta and may require changes in future releases. Feedback is welcome.** -This exporter supports the [multi-target pattern](https://prometheus.io/docs/guides/multi-target-exporter/). This allows running a single instance of this exporter for multiple postgres targets. Using the multi-target funcationality of this exporter is **optional** and meant for cases where it is impossible to install the exporter as a sidecar, for example SaaS-managed services. +This exporter supports the [multi-target pattern](https://prometheus.io/docs/guides/multi-target-exporter/). This allows running a single instance of this exporter for multiple postgres targets. Using the multi-target functionality of this exporter is **optional** and meant for cases where it is impossible to install the exporter as a sidecar, for example SaaS-managed services. To use the multi-target functionality, send an http request to the endpoint `/probe?target=foo:5432` where target is set to the DSN of the postgres instance to scrape metrics from. From 5fcceb4105830b9642670729af55b709465bde5b Mon Sep 17 00:00:00 2001 From: Julien Pivotto Date: Thu, 24 Nov 2022 14:02:37 +0100 Subject: [PATCH 23/23] Set gauge to 1 when collector is successful Signed-off-by: Julien Pivotto --- cmd/postgres_exporter/probe.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/postgres_exporter/probe.go b/cmd/postgres_exporter/probe.go index 6c9a46bfc..28d1395ee 100644 --- a/cmd/postgres_exporter/probe.go +++ b/cmd/postgres_exporter/probe.go @@ -114,6 +114,7 @@ func handleProbe(logger log.Logger) http.HandlerFunc { duration := time.Since(start).Seconds() probeDurationGauge.Set(duration) + probeSuccessGauge.Set(1) // TODO check success, etc h := promhttp.HandlerFor(registry, promhttp.HandlerOpts{})