From d73a0f0dbc6471c582a997826209256bb3b15a47 Mon Sep 17 00:00:00 2001 From: Seungyong Lee Date: Mon, 19 Aug 2024 10:48:40 +0900 Subject: [PATCH] Add metric collection for the number of goroutines attached in the Background task (#963) This PR adds a metric to collect the number of background routines. Prometheus provides its own thread-safe Gauge metric, which can be used to accurately collect the number of concurrent goroutines. Also, although the only task running in the background so far is PushPull, this commit implements the AttachGoroutine() method to take a string called taskType as a parameter, since it seems to be focused on reusability. --- server/backend/backend.go | 2 +- server/backend/background/background.go | 18 +++++++++++++++--- server/packs/packs.go | 2 +- server/profiling/prometheus/metrics.go | 23 +++++++++++++++++++++++ test/integration/retention_test.go | 6 +++++- test/integration/snapshot_test.go | 6 +++++- 6 files changed, 50 insertions(+), 7 deletions(-) diff --git a/server/backend/backend.go b/server/backend/backend.go index a11e630c2..de4052829 100644 --- a/server/backend/backend.go +++ b/server/backend/backend.go @@ -88,7 +88,7 @@ func New( // 03. Create the background instance. The background instance is used to // manage background tasks. - bg := background.New() + bg := background.New(metrics) // 04. Create the database instance. If the MongoDB configuration is given, // create a MongoDB instance. Otherwise, create a memory database instance. diff --git a/server/backend/background/background.go b/server/backend/background/background.go index 7216d2c52..677010350 100644 --- a/server/backend/background/background.go +++ b/server/backend/background/background.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "github.com/yorkie-team/yorkie/server/logging" + "github.com/yorkie-team/yorkie/server/profiling/prometheus" ) type routineID int32 @@ -49,18 +50,25 @@ type Background struct { // routineID is used to generate routine ID. routineID routineID + + // metrics is used to collect metrics with prometheus. + metrics *prometheus.Metrics } // New creates a new background service. -func New() *Background { +func New(metrics *prometheus.Metrics) *Background { return &Background{ closing: make(chan struct{}), + metrics: metrics, } } // AttachGoroutine creates a goroutine on a given function and tracks it using // the background's WaitGroup. -func (b *Background) AttachGoroutine(f func(ctx context.Context)) { +func (b *Background) AttachGoroutine( + f func(ctx context.Context), + taskType string, +) { b.wgMu.RLock() // this blocks with ongoing close(b.closing) defer b.wgMu.RUnlock() select { @@ -73,8 +81,12 @@ func (b *Background) AttachGoroutine(f func(ctx context.Context)) { // now safe to add since WaitGroup wait has not started yet b.wg.Add(1) routineLogger := logging.New(b.routineID.next()) + b.metrics.AddBackgroundGoroutines(taskType) go func() { - defer b.wg.Done() + defer func() { + b.wg.Done() + b.metrics.RemoveBackgroundGoroutines(taskType) + }() f(logging.With(context.Background(), routineLogger)) }() } diff --git a/server/packs/packs.go b/server/packs/packs.go index f37d94655..ce76a3f9c 100644 --- a/server/packs/packs.go +++ b/server/packs/packs.go @@ -204,7 +204,7 @@ func PushPull( be.Metrics.ObservePushPullSnapshotDurationSeconds( gotime.Since(start).Seconds(), ) - }) + }, "pushpull") } return respPack, nil diff --git a/server/profiling/prometheus/metrics.go b/server/profiling/prometheus/metrics.go index 5f3383af2..1f2c2fe76 100644 --- a/server/profiling/prometheus/metrics.go +++ b/server/profiling/prometheus/metrics.go @@ -36,6 +36,7 @@ const ( projectIDLabel = "project_id" projectNameLabel = "project_name" hostnameLabel = "hostname" + taskTypeLabel = "task_type" ) var ( @@ -61,6 +62,8 @@ type Metrics struct { pushPullSnapshotDurationSeconds prometheus.Histogram pushPullSnapshotBytesTotal prometheus.Counter + backgroundGoroutinesTotal *prometheus.GaugeVec + userAgentTotal *prometheus.CounterVec } @@ -134,6 +137,12 @@ func NewMetrics() (*Metrics, error) { Name: "snapshot_bytes_total", Help: "The total bytes of snapshots for response packs in PushPull.", }), + backgroundGoroutinesTotal: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "background", + Name: "goroutines_total", + Help: "The total number of goroutines attached by a particular background task.", + }, []string{taskTypeLabel}), userAgentTotal: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: "user_agent", @@ -234,6 +243,20 @@ func (m *Metrics) AddServerHandledCounter( }).Inc() } +// AddBackgroundGoroutines adds the number of goroutines attached by a particular background task. +func (m *Metrics) AddBackgroundGoroutines(taskType string) { + m.backgroundGoroutinesTotal.With(prometheus.Labels{ + taskTypeLabel: taskType, + }).Inc() +} + +// RemoveBackgroundGoroutines removes the number of goroutines attached by a particular background task. +func (m *Metrics) RemoveBackgroundGoroutines(taskType string) { + m.backgroundGoroutinesTotal.With(prometheus.Labels{ + taskTypeLabel: taskType, + }).Dec() +} + // Registry returns the registry of this metrics. func (m *Metrics) Registry() *prometheus.Registry { return m.registry diff --git a/test/integration/retention_test.go b/test/integration/retention_test.go index 5864a9d3c..a8662bc98 100644 --- a/test/integration/retention_test.go +++ b/test/integration/retention_test.go @@ -44,7 +44,11 @@ func TestRetention(t *testing.T) { patch, err := monkey.PatchInstanceMethodByName( reflect.TypeOf(b), "AttachGoroutine", - func(_ *background.Background, f func(c context.Context)) { + func( + _ *background.Background, + f func(c context.Context), + _ string, + ) { f(context.Background()) }, ) diff --git a/test/integration/snapshot_test.go b/test/integration/snapshot_test.go index 7e3cc3a73..25ea67589 100644 --- a/test/integration/snapshot_test.go +++ b/test/integration/snapshot_test.go @@ -40,7 +40,11 @@ func TestSnapshot(t *testing.T) { patch, err := monkey.PatchInstanceMethodByName( reflect.TypeOf(b), "AttachGoroutine", - func(_ *background.Background, f func(c context.Context)) { + func( + _ *background.Background, + f func(c context.Context), + _ string, + ) { f(context.Background()) }, )