Skip to content

Commit

Permalink
Config metrics and check keys as strings (#607)
Browse files Browse the repository at this point in the history 
* go mod tidy

* include config metrics

* record string check keys as labels
  • Loading branch information
@oliver006
oliver006 authored Feb 2, 2022
1 parent 1f480f5 commit 8bfc4e0
Show file tree
Hide file tree
Showing 9 changed files with 136 additions and 74 deletions.
5 changes: 4 additions & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ connection-timeout | REDIS_EXPORTER_CONNECTION_TIMEOUT | Timeo
web.listen-address | REDIS_EXPORTER_WEB_LISTEN_ADDRESS | Address to listen on for web interface and telemetry, defaults to `0.0.0.0:9121`.
web.telemetry-path | REDIS_EXPORTER_WEB_TELEMETRY_PATH | Path under which to expose metrics, defaults to `/metrics`.
redis-only-metrics | REDIS_EXPORTER_REDIS_ONLY_METRICS | Whether to also export go runtime metrics, defaults to false.
include-config-metrics | REDIS_EXPORTER_INCL_CONFIG_METRICS | Whether to include all config settings as metrics, defaults to false.
include-system-metrics | REDIS_EXPORTER_INCL_SYSTEM_METRICS | Whether to include system metrics like `total_system_memory_bytes`, defaults to false.
ping-on-connect | REDIS_EXPORTER_PING_ON_CONNECT | Whether to ping the redis instance after connecting and record the duration as a metric, defaults to false.
is-tile38 | REDIS_EXPORTER_IS_TILE38 | Whether to scrape Tile38 specific metrics, defaults to false.
Expand Down Expand Up @@ -243,7 +244,9 @@ To enable Tile38 support, run the exporter with `--is-tile38=true`.
Most items from the INFO command are exported,
see [Redis documentation](https://redis.io/commands/info) for details.\
In addition, for every database there are metrics for total keys, expiring keys and the average TTL for keys in the database.\
You can also export values of keys if they're in numeric format by using the `-check-keys` flag. The exporter will also export the size (or, depending on the data type, the length) of the key. This can be used to export the number of elements in (sorted) sets, hashes, lists, streams, etc.
You can also export values of keys by using the `-check-keys` (or related) flag. The exporter will also export the size (or, depending on the data type, the length) of the key.
This can be used to export the number of elements in (sorted) sets, hashes, lists, streams, etc.
If a key is in string format and matches with `--check-keys` (or related) then its string value will be exported as a label in the `key_value_as_string` metric.

If you require custom metric collection, you can provide a [Redis Lua script](https://redis.io/commands/eval) using the `-script` flag. An example can be found [in the contrib folder](./contrib/sample_collect_script.lua).

Expand Down
64 changes: 36 additions & 28 deletions exporter/exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ type Options struct {
ClientCertFile string
ClientKeyFile string
CaCertFile string
InclConfigMetrics bool
InclSystemMetrics bool
SkipTLSVerification bool
SetClientName bool
Expand Down Expand Up @@ -321,58 +322,61 @@ func NewRedisExporter(redisURI string, opts Options) (*Exporter, error) {
lbls []string
}{
"commands_duration_seconds_total": {txt: `Total amount of time in seconds spent per command`, lbls: []string{"cmd"}},
"commands_total": {txt: `Total number of calls per command`, lbls: []string{"cmd"}},
"commands_failed_calls_total": {txt: `Total number of errors prior command execution per command`, lbls: []string{"cmd"}},
"commands_rejected_calls_total": {txt: `Total number of errors within command execution per command`, lbls: []string{"cmd"}},
"errors_total": {txt: `Total number of errors per error type`, lbls: []string{"err"}},
"commands_total": {txt: `Total number of calls per command`, lbls: []string{"cmd"}},
"config_key_value": {txt: `Config key and value`, lbls: []string{"key", "value"}},
"config_value": {txt: `Config key and value as metric`, lbls: []string{"key"}},
"connected_clients_details": {txt: "Details about connected clients", lbls: connectedClientsLabels},
"connected_slave_lag_seconds": {txt: "Lag of connected slave", lbls: []string{"slave_ip", "slave_port", "slave_state"}},
"connected_slave_offset_bytes": {txt: "Offset of connected slave", lbls: []string{"slave_ip", "slave_port", "slave_state"}},
"db_avg_ttl_seconds": {txt: "Avg TTL in seconds", lbls: []string{"db"}},
"db_keys": {txt: "Total number of keys by DB", lbls: []string{"db"}},
"db_keys_expiring": {txt: "Total number of expiring keys by DB", lbls: []string{"db"}},
"errors_total": {txt: `Total number of errors per error type`, lbls: []string{"err"}},
"exporter_last_scrape_error": {txt: "The last scrape error status.", lbls: []string{"err"}},
"instance_info": {txt: "Information about the Redis instance", lbls: []string{"role", "redis_version", "redis_build_id", "redis_mode", "os", "maxmemory_policy", "tcp_port", "run_id", "process_id"}},
"key_group_count": {txt: `Count of keys in key group`, lbls: []string{"db", "key_group"}},
"key_group_memory_usage_bytes": {txt: `Total memory usage of key group in bytes`, lbls: []string{"db", "key_group"}},
"key_size": {txt: `The length or size of "key"`, lbls: []string{"db", "key"}},
"key_value": {txt: `The value of "key"`, lbls: []string{"db", "key"}},
"key_value_as_string": {txt: `The value of "key" as a string`, lbls: []string{"db", "key", "val"}},
"keys_count": {txt: `Count of keys`, lbls: []string{"db", "key"}},
"number_of_distinct_key_groups": {txt: `Number of distinct key groups`, lbls: []string{"db"}},
"last_key_groups_scrape_duration_milliseconds": {txt: `Duration of the last key group metrics scrape in milliseconds`},
"last_slow_execution_duration_seconds": {txt: `The amount of time needed for last slow execution, in seconds`},
"latency_spike_last": {txt: `When the latency spike last occurred`, lbls: []string{"event_name"}},
"latency_spike_duration_seconds": {txt: `Length of the last latency spike in seconds`, lbls: []string{"event_name"}},
"latency_spike_last": {txt: `When the latency spike last occurred`, lbls: []string{"event_name"}},
"master_last_io_seconds_ago": {txt: "Master last io seconds ago", lbls: []string{"master_host", "master_port"}},
"master_link_up": {txt: "Master link status on Redis slave", lbls: []string{"master_host", "master_port"}},
"master_sync_in_progress": {txt: "Master sync in progress", lbls: []string{"master_host", "master_port"}},
"master_last_io_seconds_ago": {txt: "Master last io seconds ago", lbls: []string{"master_host", "master_port"}},
"number_of_distinct_key_groups": {txt: `Number of distinct key groups`, lbls: []string{"db"}},
"script_values": {txt: "Values returned by the collect script", lbls: []string{"key"}},
"sentinel_tilt": {txt: "Sentinel is in TILT mode"},
"sentinel_master_ok_sentinels": {txt: "The number of okay sentinels monitoring this master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_ok_slaves": {txt: "The number of okay slaves of the master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_sentinels": {txt: "The number of sentinels monitoring this master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_slaves": {txt: "The number of slaves of the master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_status": {txt: "Master status on Sentinel", lbls: []string{"master_name", "master_address", "master_status"}},
"sentinel_masters": {txt: "The number of masters this sentinel is watching"},
"sentinel_running_scripts": {txt: "Number of scripts in execution right now"},
"sentinel_scripts_queue_length": {txt: "Queue of user scripts to execute"},
"sentinel_simulate_failure_flags": {txt: "Failures simulations"},
"sentinel_master_status": {txt: "Master status on Sentinel", lbls: []string{"master_name", "master_address", "master_status"}},
"sentinel_master_slaves": {txt: "The number of slaves of the master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_ok_slaves": {txt: "The number of okay slaves of the master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_sentinels": {txt: "The number of sentinels monitoring this master", lbls: []string{"master_name", "master_address"}},
"sentinel_master_ok_sentinels": {txt: "The number of okay sentinels monitoring this master", lbls: []string{"master_name", "master_address"}},
"slave_repl_offset": {txt: "Slave replication offset", lbls: []string{"master_host", "master_port"}},
"sentinel_tilt": {txt: "Sentinel is in TILT mode"},
"slave_info": {txt: "Information about the Redis slave", lbls: []string{"master_host", "master_port", "read_only"}},
"slave_repl_offset": {txt: "Slave replication offset", lbls: []string{"master_host", "master_port"}},
"slowlog_last_id": {txt: `Last id of slowlog`},
"slowlog_length": {txt: `Total slowlog`},
"start_time_seconds": {txt: "Start time of the Redis instance since unix epoch in seconds."},
"stream_group_consumer_idle_seconds": {txt: `Consumer idle time in seconds`, lbls: []string{"db", "stream", "group", "consumer"}},
"stream_group_consumer_messages_pending": {txt: `Pending number of messages for this specific consumer`, lbls: []string{"db", "stream", "group", "consumer"}},
"stream_group_consumers": {txt: `Consumers count of stream group`, lbls: []string{"db", "stream", "group"}},
"stream_group_last_delivered_id": {txt: `The epoch timestamp (ms) of the last delivered message`, lbls: []string{"db", "stream", "group"}},
"stream_group_messages_pending": {txt: `Pending number of messages in that stream group`, lbls: []string{"db", "stream", "group"}},
"stream_groups": {txt: `Groups count of stream`, lbls: []string{"db", "stream"}},
"stream_last_generated_id": {txt: `The epoch timestamp (ms) of the latest message on the stream`, lbls: []string{"db", "stream"}},
"stream_length": {txt: `The number of elements of the stream`, lbls: []string{"db", "stream"}},
"stream_radix_tree_keys": {txt: `Radix tree keys count"`, lbls: []string{"db", "stream"}},
"stream_radix_tree_nodes": {txt: `Radix tree nodes count`, lbls: []string{"db", "stream"}},
"stream_last_generated_id": {txt: `The epoch timestamp (ms) of the latest message on the stream`, lbls: []string{"db", "stream"}},
"stream_groups": {txt: `Groups count of stream`, lbls: []string{"db", "stream"}},
"stream_group_consumers": {txt: `Consumers count of stream group`, lbls: []string{"db", "stream", "group"}},
"stream_group_messages_pending": {txt: `Pending number of messages in that stream group`, lbls: []string{"db", "stream", "group"}},
"stream_group_last_delivered_id": {txt: `The epoch timestamp (ms) of the last delivered message`, lbls: []string{"db", "stream", "group"}},
"stream_group_consumer_messages_pending": {txt: `Pending number of messages for this specific consumer`, lbls: []string{"db", "stream", "group", "consumer"}},
"stream_group_consumer_idle_seconds": {txt: `Consumer idle time in seconds`, lbls: []string{"db", "stream", "group", "consumer"}},
"up": {txt: "Information about the Redis instance"},
"connected_clients_details": {txt: "Details about connected clients", lbls: connectedClientsLabels},
} {
e.metricDescriptions[k] = newMetricDescr(opts.Namespace, k, desc.txt, desc.lbls)
}
Expand Down Expand Up @@ -469,18 +473,22 @@ func (e *Exporter) extractConfigMetrics(ch chan<- prometheus.Metric, config []st
}
}

// todo: we can add more configs to this map if there's interest
if !map[string]bool{
if e.options.InclConfigMetrics {
e.registerConstMetricGauge(ch, "config_key_value", 1.0, strKey, strVal)
if val, err := strconv.ParseFloat(strVal, 64); err == nil {
e.registerConstMetricGauge(ch, "config_value", val, strKey)
}
}

if map[string]bool{
"io-threads": true,
"maxclients": true,
"maxmemory": true,
}[strKey] {
continue
}

if val, err := strconv.ParseFloat(strVal, 64); err == nil {
strKey = strings.ReplaceAll(strKey, "-", "_")
e.registerConstMetricGauge(ch, fmt.Sprintf("config_%s", strKey), val)
if val, err := strconv.ParseFloat(strVal, 64); err == nil {
strKey = strings.ReplaceAll(strKey, "-", "_")
e.registerConstMetricGauge(ch, fmt.Sprintf("config_%s", strKey), val)
}
}
}
return
Expand Down
40 changes: 32 additions & 8 deletions exporter/exporter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,11 @@ const (
)

var (
keys = []string{}
keysExpiring = []string{}
listKeys = []string{}
ts = int32(time.Now().Unix())
keys = []string{}
keysExpiring = []string{}
listKeys = []string{}
singleStringKey string
ts = int32(time.Now().Unix())

dbNumStr = "11"
altDBNumStr = "12"
Expand Down Expand Up @@ -102,6 +103,8 @@ func setupKeys(t *testing.T, c redis.Conn, dbNumStr string) error {
c.Do("SADD", TestSetName, "test-val-1")
c.Do("SADD", TestSetName, "test-val-2")

c.Do("SET", singleStringKey, "this-is-a-string")

// Create test streams
c.Do("XGROUP", "CREATE", TestStreamName, "test_group_1", "$", "MKSTREAM")
c.Do("XGROUP", "CREATE", TestStreamName, "test_group_2", "$", "MKSTREAM")
Expand Down Expand Up @@ -135,6 +138,7 @@ func deleteKeys(c redis.Conn, dbNumStr string) {

c.Do("DEL", TestSetName)
c.Do("DEL", TestStreamName)
c.Do("DEL", singleStringKey)
}

func setupDBKeys(t *testing.T, uri string) error {
Expand Down Expand Up @@ -221,6 +225,26 @@ func TestIncludeSystemMemoryMetric(t *testing.T) {
}
}

func TestIncludeConfigMetrics(t *testing.T) {
for _, inc := range []bool{false, true} {
r := prometheus.NewRegistry()
ts := httptest.NewServer(promhttp.HandlerFor(r, promhttp.HandlerOpts{}))
e, _ := NewRedisExporter(os.Getenv("TEST_REDIS_URI"), Options{Namespace: "test", InclConfigMetrics: inc})
r.Register(e)

what := `test_config_key_value{key="appendonly",value="no"}`

body := downloadURL(t, ts.URL+"/metrics")
if inc && !strings.Contains(body, what) {
t.Errorf("want metrics to include test_config_key_value, have:\n%s", body)
} else if !inc && strings.Contains(body, what) {
t.Errorf("did NOT want metrics to include test_config_key_value, have:\n%s", body)
}

ts.Close()
}
}

func TestNonExistingHost(t *testing.T) {
e, _ := NewRedisExporter("unix:///tmp/doesnt.exist", Options{Namespace: "test"})

Expand Down Expand Up @@ -345,14 +369,14 @@ func init() {
}

for _, n := range []string{"john", "paul", "ringo", "george"} {
key := fmt.Sprintf("key_%s_%d", n, ts)
keys = append(keys, key)
keys = append(keys, fmt.Sprintf("key_%s_%d", n, ts))
}

singleStringKey = fmt.Sprintf("key_string_%d", ts)

listKeys = append(listKeys, "beatles_list")

for _, n := range []string{"A.J.", "Howie", "Nick", "Kevin", "Brian"} {
key := fmt.Sprintf("key_exp_%s_%d", n, ts)
keysExpiring = append(keysExpiring, key)
keysExpiring = append(keysExpiring, fmt.Sprintf("key_exp_%s_%d", n, ts))
}
}
6 changes: 3 additions & 3 deletions exporter/key_groups_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func TestKeyGroupMetrics(t *testing.T) {
wantedCount: map[string]int{
"key_ringo": 1,
"key_paul": 1,
"unclassified": 5,
"unclassified": 6,
"key_exp": 5,
},
wantedMemory: map[string]bool{
Expand All @@ -92,13 +92,13 @@ func TestKeyGroupMetrics(t *testing.T) {
// updates of the init() function
wantedCount: map[string]int{
"test-stream": 1,
"overflow": 11,
"overflow": 12,
},
wantedMemory: map[string]bool{
"test-stream": true,
"overflow": true,
},
wantedDistintKeyGroups: 12,
wantedDistintKeyGroups: 13,
},
}

Expand Down
13 changes: 9 additions & 4 deletions exporter/keys.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.C
log.Debugf("allKeys: %#v", allKeys)
for _, k := range allKeys {
if e.options.IsCluster {
//Cluster mode only has one db
// Cluster mode only has one db
k.db = "0"
} else {
if _, err := doRedisCmd(c, "SELECT", k.db); err != nil {
Expand All @@ -116,9 +116,14 @@ func (e *Exporter) extractCheckKeyMetrics(ch chan<- prometheus.Metric, c redis.C

// Only run on single value strings
if info.keyType == "string" {
// Only record value metric if value is float-y
if val, err := redis.Float64(doRedisCmd(c, "GET", k.key)); err == nil {
e.registerConstMetricGauge(ch, "key_value", val, dbLabel, k.key)
if strVal, err := redis.String(doRedisCmd(c, "GET", k.key)); err == nil {
if val, err := strconv.ParseFloat(strVal, 64); err == nil {
// Only record value metric if value is float-y
e.registerConstMetricGauge(ch, "key_value", val, dbLabel, k.key)
} else {
// if it's not float-y then we'll record the value as a string label
e.registerConstMetricGauge(ch, "key_value_as_string", 1.0, dbLabel, k.key, strVal)
}
}
}
default:
Expand Down
58 changes: 46 additions & 12 deletions exporter/keys_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,30 +24,64 @@ const (
func TestKeyValuesAndSizes(t *testing.T) {
e, _ := NewRedisExporter(
os.Getenv("TEST_REDIS_URI"),
Options{Namespace: "test", CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0])},
Options{
Namespace: "test",
CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(keys[0]),
Registry: prometheus.NewRegistry()},
)
ts := httptest.NewServer(e)
defer ts.Close()

setupDBKeys(t, os.Getenv("TEST_REDIS_URI"))
defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI"))

chM := make(chan prometheus.Metric)
chM := make(chan prometheus.Metric, 10000)
go func() {
e.Collect(chM)
close(chM)
}()

want := map[string]bool{"test_key_size": false, "test_key_value": false}

for m := range chM {
for k := range want {
if strings.Contains(m.Desc().String(), k) {
want[k] = true
}
body := downloadURL(t, ts.URL+"/metrics")
for _, want := range []string{
"test_key_size",
"test_key_value",
} {
if !strings.Contains(body, want) {
t.Fatalf("didn't find %s, body: %s", want, body)
return
}
}
for k, found := range want {
if !found {
t.Errorf("didn't find %s", k)
}

func TestKeyValuesAsLabel(t *testing.T) {
e, _ := NewRedisExporter(
os.Getenv("TEST_REDIS_URI"),
Options{
Namespace: "test",
CheckSingleKeys: dbNumStrFull + "=" + url.QueryEscape(singleStringKey),
Registry: prometheus.NewRegistry()},
)
ts := httptest.NewServer(e)
defer ts.Close()

setupDBKeys(t, os.Getenv("TEST_REDIS_URI"))
defer deleteKeysFromDB(t, os.Getenv("TEST_REDIS_URI"))

chM := make(chan prometheus.Metric, 10000)
go func() {
e.Collect(chM)
close(chM)
}()

body := downloadURL(t, ts.URL+"/metrics")
for _, want := range []string{
"test_key_size",
"test_key_value",
"key_value_as_string",
} {
if !strings.Contains(body, want) {
t.Fatalf("didn't find %s, body: %s", want, body)
return
}
}
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ go 1.13

require (
github.com/gomodule/redigo v1.8.8
github.com/prometheus/client_golang v1.12.1
github.com/mna/redisc v1.3.2
github.com/prometheus/client_golang v1.12.1
github.com/prometheus/client_model v0.2.0
github.com/sirupsen/logrus v1.8.1
)
Loading

0 comments on commit 8bfc4e0

Please sign in to comment.