diff --git a/server/backend/backend.go b/server/backend/backend.go index a11e630c2..de4052829 100644 --- a/server/backend/backend.go +++ b/server/backend/backend.go @@ -88,7 +88,7 @@ func New( // 03. Create the background instance. The background instance is used to // manage background tasks. - bg := background.New() + bg := background.New(metrics) // 04. Create the database instance. If the MongoDB configuration is given, // create a MongoDB instance. Otherwise, create a memory database instance. diff --git a/server/backend/background/background.go b/server/backend/background/background.go index 7216d2c52..677010350 100644 --- a/server/backend/background/background.go +++ b/server/backend/background/background.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "github.com/yorkie-team/yorkie/server/logging" + "github.com/yorkie-team/yorkie/server/profiling/prometheus" ) type routineID int32 @@ -49,18 +50,25 @@ type Background struct { // routineID is used to generate routine ID. routineID routineID + + // metrics is used to collect metrics with prometheus. + metrics *prometheus.Metrics } // New creates a new background service. -func New() *Background { +func New(metrics *prometheus.Metrics) *Background { return &Background{ closing: make(chan struct{}), + metrics: metrics, } } // AttachGoroutine creates a goroutine on a given function and tracks it using // the background's WaitGroup. -func (b *Background) AttachGoroutine(f func(ctx context.Context)) { +func (b *Background) AttachGoroutine( + f func(ctx context.Context), + taskType string, +) { b.wgMu.RLock() // this blocks with ongoing close(b.closing) defer b.wgMu.RUnlock() select { @@ -73,8 +81,12 @@ func (b *Background) AttachGoroutine(f func(ctx context.Context)) { // now safe to add since WaitGroup wait has not started yet b.wg.Add(1) routineLogger := logging.New(b.routineID.next()) + b.metrics.AddBackgroundGoroutines(taskType) go func() { - defer b.wg.Done() + defer func() { + b.wg.Done() + b.metrics.RemoveBackgroundGoroutines(taskType) + }() f(logging.With(context.Background(), routineLogger)) }() } diff --git a/server/packs/packs.go b/server/packs/packs.go index f37d94655..ce76a3f9c 100644 --- a/server/packs/packs.go +++ b/server/packs/packs.go @@ -204,7 +204,7 @@ func PushPull( be.Metrics.ObservePushPullSnapshotDurationSeconds( gotime.Since(start).Seconds(), ) - }) + }, "pushpull") } return respPack, nil diff --git a/server/profiling/prometheus/metrics.go b/server/profiling/prometheus/metrics.go index 5f3383af2..1f2c2fe76 100644 --- a/server/profiling/prometheus/metrics.go +++ b/server/profiling/prometheus/metrics.go @@ -36,6 +36,7 @@ const ( projectIDLabel = "project_id" projectNameLabel = "project_name" hostnameLabel = "hostname" + taskTypeLabel = "task_type" ) var ( @@ -61,6 +62,8 @@ type Metrics struct { pushPullSnapshotDurationSeconds prometheus.Histogram pushPullSnapshotBytesTotal prometheus.Counter + backgroundGoroutinesTotal *prometheus.GaugeVec + userAgentTotal *prometheus.CounterVec } @@ -134,6 +137,12 @@ func NewMetrics() (*Metrics, error) { Name: "snapshot_bytes_total", Help: "The total bytes of snapshots for response packs in PushPull.", }), + backgroundGoroutinesTotal: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "background", + Name: "goroutines_total", + Help: "The total number of goroutines attached by a particular background task.", + }, []string{taskTypeLabel}), userAgentTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: "user_agent", @@ -234,6 +243,20 @@ func (m *Metrics) AddServerHandledCounter( }).Inc() } +// AddBackgroundGoroutines adds the number of goroutines attached by a particular background task. +func (m *Metrics) AddBackgroundGoroutines(taskType string) { + m.backgroundGoroutinesTotal.With(prometheus.Labels{ + taskTypeLabel: taskType, + }).Inc() +} + +// RemoveBackgroundGoroutines removes the number of goroutines attached by a particular background task. +func (m *Metrics) RemoveBackgroundGoroutines(taskType string) { + m.backgroundGoroutinesTotal.With(prometheus.Labels{ + taskTypeLabel: taskType, + }).Dec() +} + // Registry returns the registry of this metrics. func (m *Metrics) Registry() *prometheus.Registry { return m.registry diff --git a/test/integration/retention_test.go b/test/integration/retention_test.go index 5864a9d3c..a8662bc98 100644 --- a/test/integration/retention_test.go +++ b/test/integration/retention_test.go @@ -44,7 +44,11 @@ func TestRetention(t *testing.T) { patch, err := monkey.PatchInstanceMethodByName( reflect.TypeOf(b), "AttachGoroutine", - func(_ *background.Background, f func(c context.Context)) { + func( + _ *background.Background, + f func(c context.Context), + _ string, + ) { f(context.Background()) }, ) diff --git a/test/integration/snapshot_test.go b/test/integration/snapshot_test.go index 7e3cc3a73..25ea67589 100644 --- a/test/integration/snapshot_test.go +++ b/test/integration/snapshot_test.go @@ -40,7 +40,11 @@ func TestSnapshot(t *testing.T) { patch, err := monkey.PatchInstanceMethodByName( reflect.TypeOf(b), "AttachGoroutine", - func(_ *background.Background, f func(c context.Context)) { + func( + _ *background.Background, + f func(c context.Context), + _ string, + ) { f(context.Background()) }, )