Add metric collection for the number of goroutines attached in the Ba…

…ckground task (#963)

This PR adds a metric to collect the number of background routines.

Prometheus provides its own thread-safe Gauge metric, which can be
used to accurately collect the number of concurrent goroutines.

Also, although the only task running in the background so far is
PushPull, this commit implements the AttachGoroutine() method to take
a string called taskType as a parameter, since it seems to be focused
on reusability.

server/backend/backend.go

@@ -88,7 +88,7 @@ func New(
 
 	// 03. Create the background instance. The background instance is used to
 	// manage background tasks.
-	bg := background.New()
+	bg := background.New(metrics)
 
 	// 04. Create the database instance. If the MongoDB configuration is given,
 	// create a MongoDB instance. Otherwise, create a memory database instance.

server/backend/background/background.go

@@ -25,6 +25,7 @@ import (
 	"sync/atomic"
 
 	"github.com/yorkie-team/yorkie/server/logging"
+	"github.com/yorkie-team/yorkie/server/profiling/prometheus"
 )
 
 type routineID int32
@@ -49,18 +50,25 @@ type Background struct {
 
 	// routineID is used to generate routine ID.
 	routineID routineID
+
+	// metrics is used to collect metrics with prometheus.
+	metrics *prometheus.Metrics
 }
 
 // New creates a new background service.
-func New() *Background {
+func New(metrics *prometheus.Metrics) *Background {
 	return &Background{
 		closing: make(chan struct{}),
+		metrics: metrics,
 	}
 }
 
 // AttachGoroutine creates a goroutine on a given function and tracks it using
 // the background's WaitGroup.
-func (b *Background) AttachGoroutine(f func(ctx context.Context)) {
+func (b *Background) AttachGoroutine(
+	f func(ctx context.Context),
+	taskType string,
+) {
 	b.wgMu.RLock() // this blocks with ongoing close(b.closing)
 	defer b.wgMu.RUnlock()
 	select {
@@ -73,8 +81,12 @@ func (b *Background) AttachGoroutine(f func(ctx context.Context)) {
 	// now safe to add since WaitGroup wait has not started yet
 	b.wg.Add(1)
 	routineLogger := logging.New(b.routineID.next())
+	b.metrics.AddBackgroundGoroutines(taskType)
 	go func() {
-		defer b.wg.Done()
+		defer func() {
+			b.wg.Done()
+			b.metrics.RemoveBackgroundGoroutines(taskType)
+		}()
 		f(logging.With(context.Background(), routineLogger))
 	}()
 }

server/packs/packs.go

@@ -204,7 +204,7 @@ func PushPull(
 			be.Metrics.ObservePushPullSnapshotDurationSeconds(
 				gotime.Since(start).Seconds(),
 			)
-		})
+		}, "pushpull")
 	}
 
 	return respPack, nil

server/profiling/prometheus/metrics.go

@@ -36,6 +36,7 @@ const (
 	projectIDLabel   = "project_id"
 	projectNameLabel = "project_name"
 	hostnameLabel    = "hostname"
+	taskTypeLabel    = "task_type"
 )
 
 var (
@@ -61,6 +62,8 @@ type Metrics struct {
 	pushPullSnapshotDurationSeconds prometheus.Histogram
 	pushPullSnapshotBytesTotal      prometheus.Counter
 
+	backgroundGoroutinesTotal *prometheus.GaugeVec
+
 	userAgentTotal *prometheus.CounterVec
 }
 
@@ -134,6 +137,12 @@ func NewMetrics() (*Metrics, error) {
 			Name:      "snapshot_bytes_total",
 			Help:      "The total bytes of snapshots for response packs in PushPull.",
 		}),
+		backgroundGoroutinesTotal: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: "background",
+			Name:      "goroutines_total",
+			Help:      "The total number of goroutines attached by a particular background task.",
+		}, []string{taskTypeLabel}),
 		userAgentTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: "user_agent",
@@ -234,6 +243,20 @@ func (m *Metrics) AddServerHandledCounter(
 	}).Inc()
 }
 
+// AddBackgroundGoroutines adds the number of goroutines attached by a particular background task.
+func (m *Metrics) AddBackgroundGoroutines(taskType string) {
+	m.backgroundGoroutinesTotal.With(prometheus.Labels{
+		taskTypeLabel: taskType,
+	}).Inc()
+}
+
+// RemoveBackgroundGoroutines removes the number of goroutines attached by a particular background task.
+func (m *Metrics) RemoveBackgroundGoroutines(taskType string) {
+	m.backgroundGoroutinesTotal.With(prometheus.Labels{
+		taskTypeLabel: taskType,
+	}).Dec()
+}
+
 // Registry returns the registry of this metrics.
 func (m *Metrics) Registry() *prometheus.Registry {
 	return m.registry

test/integration/retention_test.go

@@ -44,7 +44,11 @@ func TestRetention(t *testing.T) {
 	patch, err := monkey.PatchInstanceMethodByName(
 		reflect.TypeOf(b),
 		"AttachGoroutine",
-		func(_ *background.Background, f func(c context.Context)) {
+		func(
+			_ *background.Background,
+			f func(c context.Context),
+			_ string,
+		) {
 			f(context.Background())
 		},
 	)

test/integration/snapshot_test.go

@@ -40,7 +40,11 @@ func TestSnapshot(t *testing.T) {
 	patch, err := monkey.PatchInstanceMethodByName(
 		reflect.TypeOf(b),
 		"AttachGoroutine",
-		func(_ *background.Background, f func(c context.Context)) {
+		func(
+			_ *background.Background,
+			f func(c context.Context),
+			_ string,
+		) {
 			f(context.Background())
 		},
 	)