Add core etcd metrics

* Refuse to run standalone in passthrough mode. It didn't actually do anything anyway. * Simplify metrics. Switch to standard DefaultRegisterer/DefaultGatherer instead of passing around a Registry. * Add etcd metrics from upstream. Copied from mvcc and server; they are not exported so we cannot reuse them. Signed-off-by: Brad Davidson <brad.davidson@rancher.com>
k3s-io · Aug 22, 2024 · eb9ac0e · eb9ac0e
1 parent 227f4d3
commit eb9ac0e
Show file tree

Hide file tree

Showing 11 changed files with 321 additions and 68 deletions.
diff --git a/main.go b/main.go
@@ -129,13 +129,17 @@ func run(c *cli.Context) error {
 	}
 	ctx := signals.SetupSignalContext()
 
-	metricsConfig.ServerTLSConfig = config.ServerTLSConfig
-	go metrics.Serve(ctx, metricsConfig)
-	config.MetricsRegisterer = metrics.Registry
-	_, err := endpoint.Listen(ctx, config)
+	ep, err := endpoint.Listen(ctx, config)
 	if err != nil {
 		return err
 	}
+	if ep.Driver == endpoint.ETCDBackend {
+		return fmt.Errorf("Endpoint driver %s cannot be used in standalone mode", ep.Driver)
+	}
+
+	metricsConfig.ServerTLSConfig = config.ServerTLSConfig
+	go metrics.Serve(ctx, metricsConfig)
+
 	<-ctx.Done()
 	return ctx.Err()
 }
diff --git a/pkg/drivers/dqlite/dqlite.go b/pkg/drivers/dqlite/dqlite.go
@@ -69,7 +69,7 @@ outer:
 	return nil
 }
 
-func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
+func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
 	opts, err := parseOpts(datasourceName)
 	if err != nil {
 		return nil, err

diff --git a/pkg/drivers/dqlite/no_dqlite.go b/pkg/drivers/dqlite/no_dqlite.go
@@ -9,9 +9,8 @@ import (
 
 	"github.com/k3s-io/kine/pkg/drivers/generic"
 	"github.com/k3s-io/kine/pkg/server"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
-func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
+func New(ctx context.Context, datasourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
 	return nil, errors.New(`this binary is built without dqlite support, compile with "-tags dqlite"`)
 }
diff --git a/pkg/drivers/generic/generic.go b/pkg/drivers/generic/generic.go
@@ -16,7 +16,6 @@ import (
 	"github.com/k3s-io/kine/pkg/metrics"
 	"github.com/k3s-io/kine/pkg/server"
 	"github.com/k3s-io/kine/pkg/util"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/collectors"
 	"github.com/sirupsen/logrus"
 )
@@ -171,7 +170,7 @@ func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
 	return db, nil
 }
 
-func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool, metricsRegisterer prometheus.Registerer) (*Generic, error) {
+func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig ConnectionPoolConfig, paramCharacter string, numbered bool) (*Generic, error) {
 	var (
 		db  *sql.DB
 		err error
@@ -193,9 +192,7 @@ func Open(ctx context.Context, driverName, dataSourceName string, connPoolConfig
 
 	configureConnectionPooling(connPoolConfig, db, driverName)
 
-	if metricsRegisterer != nil {
-		metricsRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))
-	}
+	metrics.DefaultRegisterer.MustRegister(collectors.NewDBStatsCollector(db, "kine"))
 
 	return &Generic{
 		DB: db,

diff --git a/pkg/drivers/mysql/mysql.go b/pkg/drivers/mysql/mysql.go
@@ -9,7 +9,6 @@ import (
 	"strconv"
 
 	"github.com/go-sql-driver/mysql"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
 	"github.com/k3s-io/kine/pkg/drivers/generic"
@@ -52,7 +51,7 @@ var (
 	createDB = "CREATE DATABASE IF NOT EXISTS "
 )
 
-func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
+func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
 	tlsConfig, err := tlsInfo.ClientConfig()
 	if err != nil {
 		return nil, err
@@ -71,7 +70,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
 		return nil, err
 	}
 
-	dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false, metricsRegisterer)
+	dialect, err := generic.Open(ctx, "mysql", parsedDSN, connPoolConfig, "?", false)
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/drivers/pgsql/pgsql.go b/pkg/drivers/pgsql/pgsql.go
@@ -19,7 +19,6 @@ import (
 	"github.com/k3s-io/kine/pkg/server"
 	"github.com/k3s-io/kine/pkg/tls"
 	"github.com/k3s-io/kine/pkg/util"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 )
 
@@ -53,7 +52,7 @@ var (
 	createDB = "CREATE DATABASE "
 )
 
-func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
+func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
 	parsedDSN, err := prepareDSN(dataSourceName, tlsInfo)
 	if err != nil {
 		return nil, err
@@ -63,7 +62,7 @@ func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config, connPoo
 		return nil, err
 	}
 
-	dialect, err := generic.Open(ctx, "pgx", parsedDSN, connPoolConfig, "$", true, metricsRegisterer)
+	dialect, err := generic.Open(ctx, "pgx", parsedDSN, connPoolConfig, "$", true)
 	if err != nil {
 		return nil, err
 	}

diff --git a/pkg/drivers/sqlite/sqlite.go b/pkg/drivers/sqlite/sqlite.go
@@ -17,7 +17,6 @@ import (
 	"github.com/k3s-io/kine/pkg/util"
 	"github.com/mattn/go-sqlite3"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
 	// sqlite db driver
@@ -47,20 +46,20 @@ var (
 	}
 )
 
-func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, error) {
-	backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig, metricsRegisterer)
+func New(ctx context.Context, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, error) {
+	backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName, connPoolConfig)
 	return backend, err
 }
 
-func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig, metricsRegisterer prometheus.Registerer) (server.Backend, *generic.Generic, error) {
+func NewVariant(ctx context.Context, driverName, dataSourceName string, connPoolConfig generic.ConnectionPoolConfig) (server.Backend, *generic.Generic, error) {
 	if dataSourceName == "" {
 		if err := os.MkdirAll("./db", 0700); err != nil {
 			return nil, nil, err
 		}
 		dataSourceName = "./db/state.db?_journal=WAL&cache=shared&_busy_timeout=30000"
 	}
 
-	dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false, metricsRegisterer)
+	dialect, err := generic.Open(ctx, driverName, dataSourceName, connPoolConfig, "?", false)
 	if err != nil {
 		return nil, nil, err
 	}

diff --git a/pkg/endpoint/endpoint.go b/pkg/endpoint/endpoint.go
@@ -14,11 +14,9 @@ import (
 	"github.com/k3s-io/kine/pkg/drivers/nats"
 	"github.com/k3s-io/kine/pkg/drivers/pgsql"
 	"github.com/k3s-io/kine/pkg/drivers/sqlite"
-	"github.com/k3s-io/kine/pkg/metrics"
 	"github.com/k3s-io/kine/pkg/server"
 	"github.com/k3s-io/kine/pkg/tls"
 	"github.com/pkg/errors"
-	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"go.etcd.io/etcd/server/v3/embed"
 	"google.golang.org/grpc"
@@ -44,21 +42,22 @@ type Config struct {
 	ConnectionPoolConfig generic.ConnectionPoolConfig
 	ServerTLSConfig      tls.Config
 	BackendTLSConfig     tls.Config
-	MetricsRegisterer    prometheus.Registerer
 	NotifyInterval       time.Duration
 	EmulatedETCDVersion  string
 }
 
 type ETCDConfig struct {
+	Driver      string
 	Endpoints   []string
 	TLSConfig   tls.Config
 	LeaderElect bool
 }
 
-func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
+func Listen(ctx context.Context, config Config) (*ETCDConfig, error) {
 	driver, dsn := ParseStorageEndpoint(config.Endpoint)
 	if driver == ETCDBackend {
-		return ETCDConfig{
+		return &ETCDConfig{
+			Driver:      driver,
 			Endpoints:   strings.Split(config.Endpoint, ","),
 			TLSConfig:   config.BackendTLSConfig,
 			LeaderElect: true,
@@ -67,33 +66,25 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
 
 	leaderelect, backend, err := getKineStorageBackend(ctx, driver, dsn, config)
 	if err != nil {
-		return ETCDConfig{}, errors.Wrap(err, "building kine")
-	}
-
-	if config.MetricsRegisterer != nil {
-		config.MetricsRegisterer.MustRegister(
-			metrics.SQLTotal,
-			metrics.SQLTime,
-			metrics.CompactTotal,
-		)
+		return nil, errors.Wrap(err, "building kine")
 	}
 
 	if err := backend.Start(ctx); err != nil {
-		return ETCDConfig{}, errors.Wrap(err, "starting kine backend")
+		return nil, errors.Wrap(err, "starting kine backend")
 	}
 
 	// set up GRPC server and register services
 	b := server.New(backend, endpointScheme(config), config.NotifyInterval, config.EmulatedETCDVersion)
 	grpcServer, err := grpcServer(config)
 	if err != nil {
-		return ETCDConfig{}, errors.Wrap(err, "creating GRPC server")
+		return nil, errors.Wrap(err, "creating GRPC server")
 	}
 	b.Register(grpcServer)
 
 	// Create raw listener and wrap in cmux for protocol switching
 	listener, err := createListener(config)
 	if err != nil {
-		return ETCDConfig{}, errors.Wrap(err, "creating listener")
+		return nil, errors.Wrap(err, "creating listener")
 	}
 
 	go func() {
@@ -105,12 +96,11 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
 	endpoint := endpointURL(config, listener)
 	logrus.Infof("Kine available at %s", endpoint)
 
-	return ETCDConfig{
-		LeaderElect: leaderelect,
+	return &ETCDConfig{
+		Driver:      driver,
 		Endpoints:   []string{endpoint},
-		TLSConfig: tls.Config{
-			CAFile: config.ServerTLSConfig.CAFile,
-		},
+		TLSConfig:   tls.Config{CAFile: config.ServerTLSConfig.CAFile},
+		LeaderElect: leaderelect,
 	}, nil
 }
 
@@ -214,13 +204,13 @@ func getKineStorageBackend(ctx context.Context, driver, dsn string, cfg Config)
 	switch driver {
 	case SQLiteBackend:
 		leaderElect = false
-		backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
+		backend, err = sqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
 	case DQLiteBackend:
-		backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
+		backend, err = dqlite.New(ctx, dsn, cfg.ConnectionPoolConfig)
 	case PostgresBackend:
-		backend, err = pgsql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
+		backend, err = pgsql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig)
 	case MySQLBackend:
-		backend, err = mysql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig, cfg.MetricsRegisterer)
+		backend, err = mysql.New(ctx, dsn, cfg.BackendTLSConfig, cfg.ConnectionPoolConfig)
 	case JetStreamBackend:
 		backend, err = nats.NewLegacy(ctx, dsn, cfg.BackendTLSConfig)
 	case NATSBackend: