Merge pull request #65 from sysdiglabs/master

Master

.circleci/config.yml

@@ -63,16 +63,28 @@ workflows:
             - cimg/postgres:14.1
     - prometheus/build:
         name: build
+        parallelism: 3
+        promu_opts: "-p linux/amd64 -p windows/amd64 -p linux/arm64 -p darwin/amd64 -p darwin/arm64 -p linux/386"
         filters:
           tags:
-            only: /.*/
+            ignore: /^v.*/
+          branches:
+            ignore: /^(main|master|release-.*|.*build-all.*)$/
+    - prometheus/build:
+        name: build_all
+        parallelism: 12
+        filters:
+          branches:
+            only: /^(main|master|release-.*|.*build-all.*)$/
+          tags:
+            only: /^v.*/
     - prometheus/publish_master:
         context: org-context
         docker_hub_organization: prometheuscommunity
         quay_io_organization: prometheuscommunity
         requires:
         - test
-        - build
+        - build_all
         filters:
           branches:
             only: master
@@ -82,7 +94,7 @@ workflows:
         quay_io_organization: prometheuscommunity
         requires:
         - test
-        - build
+        - build_all
         filters:
           tags:
             only: /^v.*/

CHANGELOG.md

@@ -1,7 +1,32 @@
 ## master / unreleased
 
-* [CHANGE] pg_stat_bgwriter counter metrics had the `_total` suffix added #556
+## 0.12.0-rc.0 / 2022-08-26
+
+BREAKING CHANGES:
+
+This release changes support for multiple postgres servers to use the
+multi-target exporter pattern. This makes it much easier to monitor multiple
+PostgreSQL servers from a single exporter by passing the target via URL
+params. See the Multi-Target Support section of the README.
+
+* [CHANGE] Add multi-target support #618
+* [BUGFIX] Add dsn type for handling datasources #678
+
+## 0.11.1 / 2022-08-01
+
+* [BUGFIX] Fix checkpoint_write_time value type #666
+* [BUGFIX] Fix checkpoint_sync_time value type #667
+
+## 0.11.0 / 2022-07-28
+
+NOTE: pg_stat_bgwriter counter metrics had the `_total` suffix added #556
+
+* [CHANGE] refactor pg_stat_bgwriter metrics into standalone collector #556
+* [FEATURE] Add pg_database collector #613
 * [ENHANCEMENT] Add pg_database_size_bytes metric #613
+* [BUGFIX] Avoid parsing error from bogus Azure Flexible Server custom GUC #587
+* [BUGFIX] Fix pg_stat_archiver error in 9.4 and earlier. #599
+* [BUGFIX] Sanitize setting values because of Aurora irregularity #620
 
 ## 0.10.1 / 2022-01-14
 

Makefile.common

@@ -36,29 +36,6 @@ GO_VERSION        ?= $(shell $(GO) version)
 GO_VERSION_NUMBER ?= $(word 3, $(GO_VERSION))
 PRE_GO_111        ?= $(shell echo $(GO_VERSION_NUMBER) | grep -E 'go1\.(10|[0-9])\.')
 
-GOVENDOR :=
-GO111MODULE :=
-ifeq (, $(PRE_GO_111))
-	ifneq (,$(wildcard go.mod))
-		# Enforce Go modules support just in case the directory is inside GOPATH (and for Travis CI).
-		GO111MODULE := on
-
-		ifneq (,$(wildcard vendor))
-			# Always use the local vendor/ directory to satisfy the dependencies.
-			GOOPTS := $(GOOPTS) -mod=vendor
-		endif
-	endif
-else
-	ifneq (,$(wildcard go.mod))
-		ifneq (,$(wildcard vendor))
-$(warning This repository requires Go >= 1.11 because of Go modules)
-$(warning Some recipes may not work as expected as the current Go runtime is '$(GO_VERSION_NUMBER)')
-		endif
-	else
-		# This repository isn't using Go modules (yet).
-		GOVENDOR := $(FIRST_GOPATH)/bin/govendor
-	endif
-endif
 PROMU        := $(FIRST_GOPATH)/bin/promu
 pkgs          = ./...
 
@@ -81,16 +58,19 @@ endif
 PROMU_VERSION ?= 0.13.0
 PROMU_URL     := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz
 
+SKIP_GOLANGCI_LINT :=
 GOLANGCI_LINT :=
 GOLANGCI_LINT_OPTS ?=
-GOLANGCI_LINT_VERSION ?= v1.45.2
+GOLANGCI_LINT_VERSION ?= v1.49.0
 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64.
 # windows isn't included here because of the path separator being different.
 ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin))
 	ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386))
 		# If we're in CI and there is an Actions file, that means the linter
 		# is being run in Actions, so we don't need to run it here.
-		ifeq (,$(CIRCLE_JOB))
+		ifneq (,$(SKIP_GOLANGCI_LINT))
+			GOLANGCI_LINT :=
+		else ifeq (,$(CIRCLE_JOB))
 			GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
 		else ifeq (,$(wildcard .github/workflows/golangci-lint.yml))
 			GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint
@@ -150,58 +130,47 @@ common-check_license:
 .PHONY: common-deps
 common-deps:
 	@echo ">> getting dependencies"
-ifdef GO111MODULE
-	GO111MODULE=$(GO111MODULE) $(GO) mod download
-else
-	$(GO) get $(GOOPTS) -t ./...
-endif
+	$(GO) mod download
 
 .PHONY: update-go-deps
 update-go-deps:
 	@echo ">> updating Go dependencies"
 	@for m in $$($(GO) list -mod=readonly -m -f '{{ if and (not .Indirect) (not .Main)}}{{.Path}}{{end}}' all); do \
 		$(GO) get -d $$m; \
 	done
-	GO111MODULE=$(GO111MODULE) $(GO) mod tidy
-ifneq (,$(wildcard vendor))
-	GO111MODULE=$(GO111MODULE) $(GO) mod vendor
-endif
+	$(GO) mod tidy
 
 .PHONY: common-test-short
 common-test-short: $(GOTEST_DIR)
 	@echo ">> running short tests"
-	GO111MODULE=$(GO111MODULE) $(GOTEST) -short $(GOOPTS) $(pkgs)
+	$(GOTEST) -short $(GOOPTS) $(pkgs)
 
 .PHONY: common-test
 common-test: $(GOTEST_DIR)
 	@echo ">> running all tests"
-	GO111MODULE=$(GO111MODULE) $(GOTEST) $(test-flags) $(GOOPTS) $(pkgs)
+	$(GOTEST) $(test-flags) $(GOOPTS) $(pkgs)
 
 $(GOTEST_DIR):
 	@mkdir -p $@
 
 .PHONY: common-format
 common-format:
 	@echo ">> formatting code"
-	GO111MODULE=$(GO111MODULE) $(GO) fmt $(pkgs)
+	$(GO) fmt $(pkgs)
 
 .PHONY: common-vet
 common-vet:
 	@echo ">> vetting code"
-	GO111MODULE=$(GO111MODULE) $(GO) vet $(GOOPTS) $(pkgs)
+	$(GO) vet $(GOOPTS) $(pkgs)
 
 .PHONY: common-lint
 common-lint: $(GOLANGCI_LINT)
 ifdef GOLANGCI_LINT
 	@echo ">> running golangci-lint"
-ifdef GO111MODULE
 # 'go list' needs to be executed before staticcheck to prepopulate the modules cache.
 # Otherwise staticcheck might fail randomly for some reason not yet explained.
-	GO111MODULE=$(GO111MODULE) $(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
-	GO111MODULE=$(GO111MODULE) $(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs)
-else
-	$(GOLANGCI_LINT) run $(pkgs)
-endif
+	$(GO) list -e -compiled -test=true -export=false -deps=true -find=false -tags= -- ./... > /dev/null
+	$(GOLANGCI_LINT) run $(GOLANGCI_LINT_OPTS) $(pkgs)
 endif
 
 .PHONY: common-yamllint
@@ -218,28 +187,15 @@ endif
 common-staticcheck: lint
 
 .PHONY: common-unused
-common-unused: $(GOVENDOR)
-ifdef GOVENDOR
-	@echo ">> running check for unused packages"
-	@$(GOVENDOR) list +unused | grep . && exit 1 || echo 'No unused packages'
-else
-ifdef GO111MODULE
+common-unused:
 	@echo ">> running check for unused/missing packages in go.mod"
-	GO111MODULE=$(GO111MODULE) $(GO) mod tidy
-ifeq (,$(wildcard vendor))
+	$(GO) mod tidy
 	@git diff --exit-code -- go.sum go.mod
-else
-	@echo ">> running check for unused packages in vendor/"
-	GO111MODULE=$(GO111MODULE) $(GO) mod vendor
-	@git diff --exit-code -- go.sum go.mod vendor/
-endif
-endif
-endif
 
 .PHONY: common-build
 common-build: promu
 	@echo ">> building binaries"
-	GO111MODULE=$(GO111MODULE) $(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES)
+	$(PROMU) build --prefix $(PREFIX) $(PROMU_BINARIES)
 
 .PHONY: common-tarball
 common-tarball: promu
@@ -295,12 +251,6 @@ $(GOLANGCI_LINT):
 		| sh -s -- -b $(FIRST_GOPATH)/bin $(GOLANGCI_LINT_VERSION)
 endif
 
-ifdef GOVENDOR
-.PHONY: $(GOVENDOR)
-$(GOVENDOR):
-	GOOS= GOARCH= $(GO) get -u github.com/kardianos/govendor
-endif
-
 .PHONY: precheck
 precheck::
 

README.md

@@ -21,6 +21,36 @@ docker run \
   quay.io/prometheuscommunity/postgres-exporter
 ```
 
+## Multi-Target Support (BETA)
+**This Feature is in beta and may require changes in future releases. Feedback is welcome.**
+
+This exporter supports the [multi-target pattern](https://prometheus.io/docs/guides/multi-target-exporter/). This allows running a single instance of this exporter for multiple postgres targets. Using the multi-target functionality of this exporter is **optional** and meant for cases where it is impossible to install the exporter as a sidecar, for example SaaS-managed services.
+
+To use the multi-target functionality, send an http request to the endpoint `/probe?target=foo:5432` where target is set to the DSN of the postgres instance to scrape metrics from.
+
+To avoid putting sensitive information like username and password in the URL, preconfigured auth modules are supported via the [auth_modules](#auth_modules) section of the config file. auth_modules for DSNs can be used with the `/probe` endpoint by specifying the `?auth_module=foo` http parameter.
+
+## Configuration File
+
+The configuration file controls the behavior of the exporter. It can be set using the `--config.file` command line flag and defaults to `postgres_exporter.yml`.
+
+### auth_modules
+This section defines preset authentication and connection parameters for use in the [multi-target endpoint](#multi-target-support-beta). `auth_modules` is a map of modules with the key being the identifier which can be used in the `/probe` endpoint.
+Currently only the `userpass` type is supported.
+
+Example:
+```yaml
+auth_modules:
+  foo1: # Set this to any name you want
+    type: userpass
+    userpass:
+      username: first
+      password: firstpass
+    options:
+      # options become key=value parameters of the DSN
+      sslmode: disable
+```
+
 ## Building and running
 
     git clone https://github.com/prometheus-community/postgres_exporter.git

VERSION

@@ -1 +1 @@
-0.10.1
+0.12.0-rc.0

cmd/postgres_exporter/datasource.go

@@ -162,6 +162,12 @@ func getDataSources() ([]string, error) {
 		uri = os.Getenv("DATA_SOURCE_URI")
 	}
 
+	// No datasources found. This allows us to support the multi-target pattern
+	// without an explicit datasource.
+	if uri == "" {
+		return []string{}, nil
+	}
+
 	dsn = "postgresql://" + ui + "@" + uri
 
 	return []string{dsn}, nil

cmd/postgres_exporter/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/prometheus-community/postgres_exporter/collector"
+	"github.com/prometheus-community/postgres_exporter/config"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/prometheus/common/promlog"
@@ -31,6 +32,11 @@ import (
 )
 
 var (
+	c = config.ConfigHandler{
+		Config: &config.Config{},
+	}
+
+	configFile             = kingpin.Flag("config.file", "Postgres exporter configuration file.").Default("postgres_exporter.yml").String()
 	listenAddress          = kingpin.Flag("web.listen-address", "Address to listen on for web interface and telemetry.").Default(":9187").Envar("PG_EXPORTER_WEB_LISTEN_ADDRESS").String()
 	webConfig              = webflag.AddFlags(kingpin.CommandLine)
 	metricPath             = kingpin.Flag("web.telemetry-path", "Path under which to expose metrics.").Default("/metrics").Envar("PG_EXPORTER_WEB_TELEMETRY_PATH").String()
@@ -85,14 +91,14 @@ func main() {
 		return
 	}
 
-	dsn, err := getDataSources()
-	if err != nil {
-		level.Error(logger).Log("msg", "Failed reading data sources", "err", err.Error())
-		os.Exit(1)
+	if err := c.ReloadConfig(*configFile, logger); err != nil {
+		// This is not fatal, but it means that auth must be provided for every dsn.
+		level.Error(logger).Log("msg", "Error loading config", "err", err)
 	}
 
-	if len(dsn) == 0 {
-		level.Error(logger).Log("msg", "Couldn't find environment variables describing the datasource to use")
+	dsns, err := getDataSources()
+	if err != nil {
+		level.Error(logger).Log("msg", "Failed reading data sources", "err", err.Error())
 		os.Exit(1)
 	}
 
@@ -106,7 +112,7 @@ func main() {
 		IncludeDatabases(*includeDatabases),
 	}
 
-	exporter := NewExporter(dsn, opts...)
+	exporter := NewExporter(dsns, opts...)
 	defer func() {
 		exporter.servers.Close()
 	}()
@@ -115,23 +121,31 @@ func main() {
 
 	prometheus.MustRegister(exporter)
 
+	// TODO(@sysadmind): Remove this with multi-target support. We are removing multiple DSN support
+	dsn := ""
+	if len(dsns) > 0 {
+		dsn = dsns[0]
+	}
+
 	pe, err := collector.NewPostgresCollector(
 		logger,
 		dsn,
 		[]string{},
 	)
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to create PostgresCollector", "err", err.Error())
-		os.Exit(1)
+	} else {
+		prometheus.MustRegister(pe)
 	}
-	prometheus.MustRegister(pe)
 
 	http.Handle(*metricPath, promhttp.Handler())
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set("Content-Type", "text/html; charset=UTF-8") // nolint: errcheck
 		w.Write(landingPage)                                       // nolint: errcheck
 	})
 
+	http.HandleFunc("/probe", handleProbe(logger))
+
 	level.Info(logger).Log("msg", "Listening on address", "address", *listenAddress)
 	srv := &http.Server{Addr: *listenAddress}
 	if err := web.ListenAndServe(srv, *webConfig, logger); err != nil {

cmd/postgres_exporter/namespace.go

@@ -19,7 +19,7 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/blang/semver"
+	"github.com/blang/semver/v4"
 	"github.com/go-kit/log/level"
 	"github.com/lib/pq"
 	"github.com/prometheus/client_golang/prometheus"