From d8eeb3b236e3eddd0bbf0f0f4170fc1cbc4c46a5 Mon Sep 17 00:00:00 2001 From: Timo Sirainen Date: Tue, 17 Sep 2024 14:31:38 +0300 Subject: [PATCH] Convert metric_group_by setting --- data/settings.js | 138 ++++++++++++++++++++++++++++++++- docs/core/config/statistics.md | 120 ++++++++++++++++++---------- 2 files changed, 215 insertions(+), 43 deletions(-) diff --git a/data/settings.js b/data/settings.js index 69516d973..ec8fa746d 100644 --- a/data/settings.js +++ b/data/settings.js @@ -8607,7 +8607,8 @@ setting.` tags: [ 'metrics' ], values: setting_types.STRING, text: ` -Name of the metric. It is visible in statistics outputs.` +Name of the metric. It is visible in statistics outputs. The [[setting,metric]] +filter name refers to this setting.` }, metric_fields: { @@ -8624,10 +8625,141 @@ field that does not need to be listed explicitly.` metric_group_by: { tags: [ 'metrics' ], - values: setting_types.BOOLLIST, + values: setting_types.NAMED_LIST_FILTER, + seealso: [ '[[link,stats_group_by]]' ], + text: ` +Creates a new [[link,stats_group_by,group_by]] for dynamically generating +sub-metrics based on the specified field's values. The filter name refers to +the [[setting,metric_group_by_field]].` + }, + + metric_group_by_field: { + tags: [ 'metrics' ], + values: setting_types.STRING, + seealso: [ '[[link,stats_group_by]]' ], + text: ` +Generate sub-metrics based on this event field name. The +[[setting,metric_group_by]] filter name refers to this setting.` + }, + + metric_group_by_method: { + tags: [ 'metrics' ], + values: setting_types.NAMED_LIST_FILTER, + seealso: [ '[[link,stats_group_by]]' ], + text: ` +Configures [[setting,metric_group_by_method_method,the aggregation method]] for +the [[setting,metric_group_by]]. Only a single method can be specified for +a group_by. The filter name refers to the +[[setting,metric_group_by_method_method]].` + }, + + metric_group_by_method_method: { + tags: [ 'metrics' ], + values: setting_types.ENUM, + values_enum: [ 'discrete', 'exponential', 'linear' ], + default: 'discrete', + seealso: [ '[[link,stats_group_by]]' ], + text: ` +Configures the aggregation method for the +[[setting,metric_group_by]]. The [[setting,metric_group_by_method]] filter name +refers to this setting.` + }, + + metric_group_by_method_discrete_modifier: { + tags: [ 'metrics' ], + values: setting_types.STRING_NOVAR, seealso: [ '[[link,stats_group_by]]' ], text: ` -This can be used to dynamically generate sub-metrics based on fields' values.` +Configures a modifier string for values grouped by the +[[link,stats_group_by_discrete,discrete method]]. +[[link,settings_variables,%variables]] and their functions can be used: + +\`%{value}\` +: The original value. + +\`%{domain}\` +: If the value is in \`user@domain\` format, this contains the \`domain\` + text. Otherwise empty.` + }, + + metric_group_by_method_exponential_min_magnitude: { + tags: [ 'metrics' ], + values: setting_types.UINT, + seealso: [ + '[[link,stats_group_by]]', + 'metric_group_by_method_exponential_max_magnitude', + 'metric_group_by_method_exponential_base' + ], + text: ` +Configures the minimum magnitude for values grouped by the +[[link,stats_group_by_exponential,exponential method]].` + }, + + metric_group_by_method_exponential_max_magnitude: { + tags: [ 'metrics' ], + values: setting_types.UINT, + seealso: [ + '[[link,stats_group_by]]', + 'metric_group_by_method_exponential_min_magnitude', + 'metric_group_by_method_exponential_base' + ], + text: ` +Configures the maximum magnitude for values grouped by the +[[link,stats_group_by_exponential,exponential method]].` + }, + + metric_group_by_method_exponential_base: { + tags: [ 'metrics' ], + values: setting_types.UINT, + default: 10, + seealso: [ + '[[link,stats_group_by]]', + 'metric_group_by_method_exponential_min_magnitude', + 'metric_group_by_method_exponential_max_magnitude' + ], + text: ` +Configures the base for values grouped by the +[[link,stats_group_by_exponential,exponential method]]. Only 2 and 10 are +supported.` + }, + + metric_group_by_method_linear_min: { + tags: [ 'metrics' ], + values: setting_types.UINT, + seealso: [ + '[[link,stats_group_by]]', + 'metric_group_by_method_linear_max', + 'metric_group_by_method_linear_step' + ], + text: ` +Configures the minimum for values grouped by the +[[link,stats_group_by_linear,linear method]].` + }, + + metric_group_by_method_linear_max: { + tags: [ 'metrics' ], + values: setting_types.UINT, + seealso: [ + '[[link,stats_group_by]]', + 'metric_group_by_method_linear_min', + 'metric_group_by_method_linear_step' + ], + text: ` +Configures the maximum for values grouped by the +[[link,stats_group_by_linear,linear method]].` + }, + + metric_group_by_method_linear_step: { + tags: [ 'metrics' ], + values: setting_types.UINT, + seealso: [ + '[[link,stats_group_by]]', + 'metric_group_by_method_linear_min', + 'metric_group_by_method_linear_max' + ], + text: ` +Configures the step for values grouped by the +[[link,stats_group_by_linear,linear method]].` }, metric_filter: { diff --git a/docs/core/config/statistics.md b/docs/core/config/statistics.md index 553f14526..9c4620c9e 100644 --- a/docs/core/config/statistics.md +++ b/docs/core/config/statistics.md @@ -9,6 +9,15 @@ dovecotlinks: stats_group_by: hash: group-by text: "Statistics: Group By" + stats_group_by_discrete: + hash: discrete + text: "Statistics: Group By: Discrete" + stats_group_by_exponential: + hash: exponential + text: "Statistics: Group By: Exponential" + stats_group_by_linear: + hash: linear + text: "Statistics: Group By: Linear" --- # Statistics @@ -36,34 +45,34 @@ filter setting. You can also use `event=*` to match all named events. ## Group By -The `group_by` metric setting allows dynamic hierarchical metric +The [[setting,metric_group_by]] setting allows dynamic hierarchical metric generation based on event fields' values. -Each field listed in the `group_by` generates one level of "sub-metrics". +Each listed [[setting,metric_group_by]] generates one level of "sub-metrics". These automatically generated metrics are indistinguishable from those statically defined in the config file. "sub-metric" names can be up to 256 bytes in total. -Dovecot supports a number of aggregation functions that can be used to -quantize a field's value before it is used to generate a metric. - -The format is always the same: the field name, a colon, the aggregation -function name, and optionally a colon followed by colon delimited parameters -to the aggregation function. +Dovecot supports a number of aggregation methods that can be used to quantize +a field's value before it is used to generate a metric. ### `discrete` -The simplest aggregation function is to use the value as is. Because this -is a very common use case, not specifying an aggregation function is treated -as an alias for discrete aggregation. In other words, `field` and -`field:discrete` produce the same behavior. +The simplest aggregation method is to use the value as is. Because this +is a very common use case, this is the default aggregation method. + +The value can be further modified by +[[setting,metric_group_by_method_discrete_modifier]]. Example: ```[dovecot.conf] metric imap_command { filter = event=imap_command_finished - group_by = cmd_name tagged_reply_state + group_by cmd_name { + } + group_by tagged_reply_state { + } } ``` @@ -89,21 +98,24 @@ when first observed. The field's integer value is quantized into exponentially sized ranges. -The exponential aggregation function takes three colon delimited integer -arguments that define the set of ranges used: the minimum magnitude, the -maximum magnitude, and the base. The exact configuration syntax is: -`field:exponential:min:max:base` +The exponential aggregation method uses three settings: -Note: Currently, only base 2 and base 10 are supported. + * [[setting,metric_group_by_method_exponential_min_magnitude]] + * [[setting,metric_group_by_method_exponential_max_magnitude]] + * [[setting,metric_group_by_method_exponential_base]] + Note: Currently, only base 2 and base 10 are supported. -The first range starts at negative infinity and ends at `pow(base, min)`. -The second range begins at `pow(base, min) + 1` and ends at -`pow(base, min + 1)`, the next covers `pow(base, min + 1) + 1` to -`pow(base, min + 2)`, and so on. The last range covers -`pow(base, max) + 1` to positive infinity. +The first range starts at negative infinity and ends at +`pow(base, min_magnitude)`. The second range begins at +`pow(base, min_magnitude) + 1` and ends at `pow(base, min_magnitude + 1)`, +the next covers `pow(base, min_magnitude + 1) + 1` to +`pow(base, min_magnitude + 2)`, and so on. The last range covers +`pow(base, max_magnitude) + 1` to positive infinity. -For example, given the specification `duration:exponential:1:5:10`, the -ranges would be: +For example, given the settings +[[setting,metric_group_by_method_exponential_min_magnitude,1]], +[[setting,metric_group_by_method_exponential_max_magnitude,5]] and +[[setting,metric_group_by_method_exponential_base,10]], the ranges would be: * (-inf, 10] * [11, 100] @@ -112,8 +124,8 @@ ranges would be: * [10001, 100000] * [100001, +inf) -Much like the metric names generated with the `discrete` aggregation -function, the ones generated by the `exponential` function include +Much like the metric names generated with the [discrete](#discrete) aggregation +method, the ones generated by the `exponential` method include information about the value of the field. However, in this case it is the range the value belongs to. @@ -125,7 +137,15 @@ Example: ```[dovecot.conf] metric imap_command { filter = event=imap_command_finished - group_by = cmd_name duration:exponential:1:5:10 + group_by cmd_name { + } + group_by duration { + method exponential { + min_magnitude = 1 + max_magnitude = 5 + base = 10 + } + } } ``` @@ -144,7 +164,7 @@ for a `SELECT` IMAP command, the possible generated metric names are: Note: Since the metric names cannot contain `-`, the string `ninf` is used to denote negative infinity. -Note: Much like in the `discrete` case, the metrics are allocated only +Note: Much like in the [discrete](#discrete) case, the metrics are allocated only when first observed. Finally, because all intermediate level metrics are generated as well. The @@ -157,18 +177,20 @@ above example, will also generate all of the following metrics: The field's integer value is quantized into linearly sized ranges. -The linear aggregation function takes three colon delimited integer -arguments that define the set of ranges used: the minimum value, the -maximum value, and the range step size. The exact configuration syntax is: -`field:linear:min:max:step` +The linear aggregation method uses three settings: + + * [[setting,metric_group_by_method_linear_min]] + * [[setting,metric_group_by_method_linear_max]] + * [[setting,metric_group_by_method_linear_step]] The first range starts at negative infinity and ends at `min`. The second range begins at `min + 1` and ends at `min + step`, the next covers `min + step + 1` to `min + (2 * step)`, and so on. The last range covers `max + 1` to positive infinity. -For example, given the specification `net_out_bytes:linear:0:5000:1000`, the -ranges would be: +For example, given settings [[setting,metric_group_by_method_linear_min,0]], +[[setting,metric_group_by_method_linear_max,5000]] and +[[setting,metric_group_by_method_linear_step,1000]], the ranges would be: * (-inf, 0] * [1, 1000] @@ -178,7 +200,7 @@ ranges would be: * [4001, 5000] * [5001, +inf) -See the description of the [exponential](#exponential) aggregation function +See the description of the [exponential](#exponential) aggregation method for how metric names are formed from these ranges. ## Listing Statistics @@ -285,7 +307,8 @@ metric storage_http_gets { # generate per-command metrics on successful commands metric imap_command { filter = event=imap_command_finished AND tagged_reply_state=OK - group_by = cmd_name + group_by cmd_name { + } } ``` @@ -364,17 +387,34 @@ metric auth_success { metric imap_command { filter = event=imap_command_finished - group_by = cmd_name tagged_reply_state + group_by cmd_name { + } + group_by tagged_reply_state { + } } metric smtp_command { filter = event=smtp_server_command_finished - group_by = cmd_name status_code duration:exponential:1:5:10 + group_by cmd_name { + } + group_by status_code { + } + group_by duration { + method exponential { + min_magnitude = 1 + max_magnitude = 5 + } + } } metric mail_delivery { filter = event=mail_delivery_finished - group_by = duration:exponential:1:5:10 + group_by duration { + method exponential { + min_magnitude = 1 + max_magnitude = 5 + } + } } ```