Skip to content

Commit

Permalink
PMM-9870 fix collstats indexSizes metrics.
Browse files Browse the repository at this point in the history
  • Loading branch information
@BupycHuk
BupycHuk committed Nov 11, 2024
1 parent 6907552 commit 9e7bf83
Show file tree
Hide file tree
Showing 4 changed files with 109 additions and 48 deletions.
13 changes: 3 additions & 10 deletions exporter/collstats_collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -99,23 +99,16 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {

aggregation := bson.D{
{
Key: "$collStats", Value: bson.M{
Key: "$collStats",
Value: bson.M{
// TODO: PMM-9568 : Add support to handle histogram metrics
"latencyStats": bson.M{"histograms": false},
"storageStats": bson.M{"scale": 1},
},
},
}
project := bson.D{
{
Key: "$project", Value: bson.M{
"storageStats.wiredTiger": 0,
"storageStats.indexDetails": 0,
},
},
}

cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation, project})
cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, mongo.Pipeline{aggregation})
if err != nil {
logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err)

Expand Down
21 changes: 11 additions & 10 deletions exporter/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,10 @@ func listCollections(ctx context.Context, client *mongo.Client, database string,
//
// - exclude: List of databases to be excluded. Useful to ignore system databases.
func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []string, exclude []string) ([]string, error) {
opts := &options.ListDatabasesOptions{NameOnly: pointer.ToBool(true), AuthorizedDatabases: pointer.ToBool(true)}
opts := &options.ListDatabasesOptions{
NameOnly: pointer.ToBool(true),
AuthorizedDatabases: pointer.ToBool(true),
}

filter := bson.D{}

Expand All @@ -100,35 +103,33 @@ func databases(ctx context.Context, client *mongo.Client, filterInNamespaces []s
}

func makeExcludeFilter(exclude []string) *primitive.E {
filterExpressions := []bson.D{}
if len(exclude) == 0 {
return nil
}
var filterExpressions []bson.D
for _, dbname := range exclude {
filterExpressions = append(filterExpressions,
bson.D{{Key: "name", Value: bson.D{{Key: "$ne", Value: dbname}}}},
)
}

if len(filterExpressions) == 0 {
return nil
}

return &primitive.E{Key: "$and", Value: filterExpressions}
}

func makeDBsFilter(filterInNamespaces []string) *primitive.E {
filterExpressions := []bson.D{}

nss := removeEmptyStrings(filterInNamespaces)
if len(nss) == 0 {
return nil
}
for _, namespace := range nss {
parts := strings.Split(namespace, ".")
filterExpressions = append(filterExpressions,
bson.D{{Key: "name", Value: bson.D{{Key: "$eq", Value: parts[0]}}}},
)
}

if len(filterExpressions) == 0 {
return nil
}

return &primitive.E{Key: "$or", Value: filterExpressions}
}

Expand Down
111 changes: 85 additions & 26 deletions exporter/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,25 +93,69 @@ var (
// mongodb_ss_opcounters{legacy_op_type="command"} 67923
//
nodeToPDMetrics = map[string]string{
"collStats.storageStats.indexDetails.": "index_name",
"globalLock.activeQueue.": "count_type",
"globalLock.locks.": "lock_type",
"serverStatus.asserts.": "assert_type",
"serverStatus.connections.": "conn_type",
"serverStatus.globalLock.currentQueue.": "count_type",
"serverStatus.metrics.commands.": "cmd_name",
"serverStatus.metrics.cursor.open.": "csr_type",
"serverStatus.metrics.document.": "doc_op_type",
"serverStatus.opLatencies.": "op_type",
"serverStatus.opReadConcernCounters.": "concern_type",
"serverStatus.opcounters.": "legacy_op_type",
"serverStatus.opcountersRepl.": "legacy_op_type",
"serverStatus.transactions.commitTypes.": "commit_type",
"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
"serverStatus.queues.execution.": "txn_rw_type",
"serverStatus.wiredTiger.perf.": "perf_bucket",
"systemMetrics.disks.": "device_name",
"collstats.storageStats.indexSizes.": "index_name",
"collStats.storageStats.indexDetails.": "index_name",
"globalLock.activeQueue.": "count_type",
"globalLock.locks.": "lock_type",
"serverStatus.asserts.": "assert_type",
"serverStatus.connections.": "conn_type",
"serverStatus.globalLock.currentQueue.": "count_type",
"serverStatus.metrics.commands.": "cmd_name",
"serverStatus.metrics.cursor.open.": "csr_type",
"serverStatus.metrics.document.": "doc_op_type",
"serverStatus.opLatencies.": "op_type",
"serverStatus.opReadConcernCounters.": "concern_type",
"serverStatus.opcounters.": "legacy_op_type",
"serverStatus.opcountersRepl.": "legacy_op_type",
"serverStatus.transactions.commitTypes.": "commit_type",
"serverStatus.wiredTiger.concurrentTransactions.": "txn_rw_type",
"serverStatus.queues.execution.": "txn_rw_type",
"serverStatus.wiredTiger.perf.": "perf_bucket",
"systemMetrics.disks.": "device_name",
"collstats.storageStats.indexSizes.": "index_name",
"config.transactions.stats.storageStats.indexSizes.": "index_name",
"config.image_collection.stats.storageStats.indexSizes.": "index_name",
}

// This map is used to add labels to some specific metrics.
// The difference from the case above that it works with middle nodes in the structure.
// For example, the fields under the storageStats.indexDetails. structure have this
// signature:
//
// "storageStats": primitive.M{
// "indexDetails": primitive.M{
// "_id_": primitive.M{
// "LSM": primitive.M{
// "bloom filter false positives": int32(0),
// "bloom filter hits": int32(0),
// "bloom filter misses": int32(0),
// ...
// },
// "block-manager": primitive.M{
// "allocations requiring file extension": int32(0),
// ...
// },
// ...
// },
// "name_1": primitive.M{
// ...
// },
// ...
// },
// },
//
// Applying the renaming rules, storageStats will become storageStats but instead of having metrics
// with the form storageStats.indexDetails.<index_name>.<metric_name> where index_name is each one of
// the fields inside the structure (_id_, name_1, etc), those keys will become labels for the same
// metric name. The label name is defined as the value for each metric name in the map and the value
// the label will have is the field name in the structure. Example.
//
// mongodb_storageStats_indexDetails_index_name_LSM_bloom_filter_false_positives{index_name="_id_"} 0
keyNodesToLabels = map[string]string{
"storageStats.indexDetails.": "index_name",
"config.image_collection.stats.storageStats.indexDetails.": "index_name",
"config.transactions.stats.storageStats.indexDetails.": "index_name",
"config.image_collection.stats.storageStats.indexSizes.": "index_name",
"collstats.storageStats.indexDetails.": "index_name",
}

// Regular expressions used to make the metric name Prometheus-compatible
Expand Down Expand Up @@ -237,9 +281,12 @@ func rawToPrometheusMetric(rm *rawMetric) (prometheus.Metric, error) {
// by prometheus. For first level metrics, there is no prefix so we should use the metric name or
// the help would be empty.
func metricHelp(prefix, name string) string {
if prefix != "" {
if _, ok := nodeToPDMetrics[prefix]; ok {
return prefix
}
if prefix != "" {
return prefix + name
}

return name
}
Expand All @@ -252,17 +299,29 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo
}

for k, val := range m {
nextPrefix := prefix + k

var l = make(map[string]string)
if label, ok := keyNodesToLabels[prefix]; ok {
for k, v := range labels {
l[k] = v
}
l[label] = k
nextPrefix = prefix + label
} else {
l = labels
}
switch v := val.(type) {
case bson.M:
res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...)
res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...)
case map[string]interface{}:
res = append(res, makeMetrics(prefix+k, v, labels, compatibleMode)...)
res = append(res, makeMetrics(nextPrefix, v, l, compatibleMode)...)
case primitive.A:
res = append(res, processSlice(prefix, k, v, labels, compatibleMode)...)
res = append(res, processSlice(nextPrefix, v, l, compatibleMode)...)
case []interface{}:
continue
default:
rm, err := makeRawMetric(prefix, k, v, labels)
rm, err := makeRawMetric(prefix, k, v, l)
if err != nil {
invalidMetric := prometheus.NewInvalidMetric(prometheus.NewInvalidDesc(err), err)
res = append(res, invalidMetric)
Expand Down Expand Up @@ -303,7 +362,7 @@ func makeMetrics(prefix string, m bson.M, labels map[string]string, compatibleMo

// Extract maps from arrays. Only some structures like replicasets have arrays of members
// and each member is represented by a map[string]interface{}.
func processSlice(prefix, k string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric {
func processSlice(prefix string, v []interface{}, commonLabels map[string]string, compatibleMode bool) []prometheus.Metric {
metrics := make([]prometheus.Metric, 0)
labels := make(map[string]string)
for name, value := range commonLabels {
Expand All @@ -330,7 +389,7 @@ func processSlice(prefix, k string, v []interface{}, commonLabels map[string]str
labels["member_state"] = state
}

metrics = append(metrics, makeMetrics(prefix+k, s, labels, compatibleMode)...)
metrics = append(metrics, makeMetrics(prefix, s, labels, compatibleMode)...)
}

return metrics
Expand Down
12 changes: 10 additions & 2 deletions main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,11 +150,19 @@ func buildExporter(opts GlobalFlags, uri string, log *logrus.Logger) *exporter.E
nodeName = uriParsed.Host
}

collStatsNamespaces := []string{}
if opts.CollStatsNamespaces != "" {
collStatsNamespaces = strings.Split(opts.CollStatsNamespaces, ",")
}
indexStatsCollections := []string{}
if opts.IndexStatsCollections != "" {
indexStatsCollections = strings.Split(opts.IndexStatsCollections, ",")
}
exporterOpts := &exporter.Opts{
CollStatsNamespaces: strings.Split(opts.CollStatsNamespaces, ","),
CollStatsNamespaces: collStatsNamespaces,
CompatibleMode: opts.CompatibleMode,
DiscoveringMode: opts.DiscoveringMode,
IndexStatsCollections: strings.Split(opts.IndexStatsCollections, ","),
IndexStatsCollections: indexStatsCollections,
Logger: log,
URI: uri,
NodeName: nodeName,
Expand Down

0 comments on commit 9e7bf83

Please sign in to comment.