Add a metric to collect the number of background routines

server/backend/backend.go

@@ -88,7 +88,7 @@ func New(
 
 	// 03. Create the background instance. The background instance is used to
 	// manage background tasks.
-	bg := background.New()
+	bg := background.New(metrics)
 
 	// 04. Create the database instance. If the MongoDB configuration is given,
 	// create a MongoDB instance. Otherwise, create a memory database instance.

server/backend/background/background.go

@@ -25,6 +25,7 @@ import (
 	"sync/atomic"
 
 	"github.com/yorkie-team/yorkie/server/logging"
+	"github.com/yorkie-team/yorkie/server/profiling/prometheus"
 )
 
 type routineID int32
@@ -49,18 +50,25 @@ type Background struct {
 
 	// routineID is used to generate routine ID.
 	routineID routineID
+
+	// metrics is used to collect metrics with prometheus.
+	metrics *prometheus.Metrics
 }
 
 // New creates a new background service.
-func New() *Background {
+func New(metrics *prometheus.Metrics) *Background {
 	return &Background{
 		closing: make(chan struct{}),
+		metrics: metrics,
 	}
 }
 
 // AttachGoroutine creates a goroutine on a given function and tracks it using
 // the background's WaitGroup.
-func (b *Background) AttachGoroutine(f func(ctx context.Context)) {
+func (b *Background) AttachGoroutine(
+	f func(ctx context.Context),
+	taskType string,
+) {
 	b.wgMu.RLock() // this blocks with ongoing close(b.closing)
 	defer b.wgMu.RUnlock()
 	select {
@@ -73,8 +81,12 @@ func (b *Background) AttachGoroutine(f func(ctx context.Context)) {
 	// now safe to add since WaitGroup wait has not started yet
 	b.wg.Add(1)
 	routineLogger := logging.New(b.routineID.next())
+	b.metrics.AddBackgroundGoroutines(taskType)
 	go func() {
-		defer b.wg.Done()
+		defer func() {
+			b.wg.Done()
+			b.metrics.RemoveBackgroundGoroutines(taskType)
+		}()
 		f(logging.With(context.Background(), routineLogger))
 	}()
 }

server/packs/packs.go

@@ -204,7 +204,7 @@ func PushPull(
 			be.Metrics.ObservePushPullSnapshotDurationSeconds(
 				gotime.Since(start).Seconds(),
 			)
-		})
+		}, "pushpull")
 	}
 
 	return respPack, nil

server/profiling/prometheus/metrics.go

@@ -36,6 +36,7 @@ const (
 	projectIDLabel   = "project_id"
 	projectNameLabel = "project_name"
 	hostnameLabel    = "hostname"
+	taskTypeLabel    = "task_type"
 )
 
 var (
@@ -61,6 +62,8 @@ type Metrics struct {
 	pushPullSnapshotDurationSeconds prometheus.Histogram
 	pushPullSnapshotBytesTotal      prometheus.Counter
 
+	backgroundGoroutinesTotal *prometheus.GaugeVec
+
 	userAgentTotal *prometheus.CounterVec
 }
 
@@ -134,6 +137,12 @@ func NewMetrics() (*Metrics, error) {
 			Name:      "snapshot_bytes_total",
 			Help:      "The total bytes of snapshots for response packs in PushPull.",
 		}),
+		backgroundGoroutinesTotal: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: "background",
+			Name:      "goroutines_total",
+			Help:      "The total number of goroutines attached by a particular background task.",
+		}, []string{taskTypeLabel}),
 		userAgentTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
 			Namespace: namespace,
 			Subsystem: "user_agent",
@@ -234,6 +243,20 @@ func (m *Metrics) AddServerHandledCounter(
 	}).Inc()
 }
 
+// AddBackgroundGoroutines adds the number of goroutines attached by a particular background task.
+func (m *Metrics) AddBackgroundGoroutines(taskType string) {
+	m.backgroundGoroutinesTotal.With(prometheus.Labels{
+		taskTypeLabel: taskType,
+	}).Inc()
+}
+
+// RemoveBackgroundGoroutines removes the number of goroutines attached by a particular background task.
+func (m *Metrics) RemoveBackgroundGoroutines(taskType string) {
+	m.backgroundGoroutinesTotal.With(prometheus.Labels{
+		taskTypeLabel: taskType,
+	}).Dec()
+}
+
 // Registry returns the registry of this metrics.
 func (m *Metrics) Registry() *prometheus.Registry {
 	return m.registry

test/integration/retention_test.go

@@ -44,7 +44,11 @@ func TestRetention(t *testing.T) {
 	patch, err := monkey.PatchInstanceMethodByName(
 		reflect.TypeOf(b),
 		"AttachGoroutine",
-		func(_ *background.Background, f func(c context.Context)) {
+		func(
+			_ *background.Background,
+			f func(c context.Context),
+			_ string,
+		) {
 			f(context.Background())
 		},
 	)

test/integration/snapshot_test.go

@@ -40,7 +40,11 @@ func TestSnapshot(t *testing.T) {
 	patch, err := monkey.PatchInstanceMethodByName(
 		reflect.TypeOf(b),
 		"AttachGoroutine",
-		func(_ *background.Background, f func(c context.Context)) {
+		func(
+			_ *background.Background,
+			f func(c context.Context),
+			_ string,
+		) {
 			f(context.Background())
 		},
 	)