Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tsdb: update vm to v1.76.1; expose more params (#213) #214

Merged
merged 2 commits into from
Nov 30, 2023
Merged

tsdb: update vm to v1.76.1; expose more params (#213) #214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
cebe853
This is an automated cherry-pick of #213
mornyx Nov 30, 2023
b4c1a06
resolve conflicts
mornyx Nov 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ const (
DefProfileSeconds = 10
DefProfilingTimeoutSeconds = 120
DefProfilingDataRetentionSeconds = 3 * 24 * 60 * 60 // 3 days
DefTSDBRetentionPeriod = "1" // 1 month
DefTSDBSearchMaxUniqueTimeseries = 300000
)

type Config struct {
Expand All @@ -39,6 +41,7 @@ type Config struct {
Storage Storage `toml:"storage" json:"storage"`
ContinueProfiling ContinueProfilingConfig `toml:"-" json:"continuous_profiling"`
Security Security `toml:"security" json:"security"`
TSDB TSDB `toml:"tsdb" json:"tsdb"`
}

var defaultConfig = Config{
Expand All @@ -60,6 +63,10 @@ var defaultConfig = Config{
TimeoutSeconds: DefProfilingTimeoutSeconds,
DataRetentionSeconds: DefProfilingDataRetentionSeconds,
},
TSDB: TSDB{
RetentionPeriod: DefTSDBRetentionPeriod,
SearchMaxUniqueTimeseries: DefTSDBSearchMaxUniqueTimeseries,
},
}

func GetDefaultConfig() Config {
Expand Down Expand Up @@ -388,6 +395,11 @@ func buildTLSConfig(caPath, keyPath, certPath string) *tls.Config {
return tlsConfig
}

type TSDB struct {
RetentionPeriod string `toml:"retention-period" json:"retention_period"`
SearchMaxUniqueTimeseries int64 `toml:"search-max-unique-timeseries" json:"search_max_unique_timeseries"`
}

type ContinueProfilingConfig struct {
Enable bool `json:"enable"`
ProfileSeconds int `json:"profile_seconds"`
Expand Down
11 changes: 11 additions & 0 deletions config/config.toml.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,14 @@ path = "data"
ca-path = ""
cert-path = ""
key-path = ""

[tsdb]
# Data with timestamps outside the retentionPeriod is automatically deleted
# The following optional suffixes are supported: h (hour), d (day), w (week), y (year).
# If suffix isn't set, then the duration is counted in months.
retention-period = "1"
# `search-max-unique-timeseries` limits the number of unique time series a single query can find and process.
# VictoriaMetrics(tsdb) keeps in memory some metainformation about the time series located by each query
# and spends some CPU time for processing the found time series. This means that the maximum memory usage
# and CPU usage a single query can use is proportional to `search-max-unique-timeseries`.
search-max-unique-timeseries = 300000
6 changes: 3 additions & 3 deletions config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ key-path = "ngm.key"`
data, err := json.Marshal(cfg)
require.NoError(t, err)
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))

ctx, cancel := context.WithCancel(context.Background())
defer cancel()
Expand Down Expand Up @@ -114,7 +114,7 @@ path = "data1"`
data, err = json.Marshal(globalCfg)
require.NoError(t, err)
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))

cfgData = ``
err = ioutil.WriteFile(cfgFileName, []byte(cfgData), 0666)
Expand All @@ -125,7 +125,7 @@ path = "data1"`
data, err = json.Marshal(GetGlobalConfig())
require.NoError(t, err)
// TODO(mornyx): Rollback when tiflash#5687 is fixed.
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"}}`, string(data))
require.Equal(t, `{"address":"0.0.0.0:12020","advertise_address":"10.0.1.8:12020","pd":{"endpoints":["10.0.1.8:2378","10.0.1.9:2379"]},"log":{"path":"log","level":"INFO"},"storage":{"path":"data"},"continuous_profiling":{"enable":false,"profile_seconds":10,"interval_seconds":60,"timeout_seconds":120,"data_retention_seconds":259200},"security":{"ca_path":"ngm.ca","cert_path":"ngm.cert","key_path":"ngm.key"},"tsdb":{"retention_period":"1","search_max_unique_timeseries":300000}}`, string(data))
}

func TestConfigValid(t *testing.T) {
Expand Down
11 changes: 3 additions & 8 deletions database/timeseries/vm.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package timeseries

import (
"flag"
"fmt"
"os"
"path"
"time"
Expand All @@ -14,30 +15,24 @@ import (
"github.com/VictoriaMetrics/VictoriaMetrics/app/vmstorage"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/fs"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
"github.com/VictoriaMetrics/VictoriaMetrics/lib/storage"
"github.com/pingcap/log"
"github.com/spf13/pflag"
"go.uber.org/zap"
)

var (
retentionPeriod = pflag.String("retention-period", "1", "Data with timestamps outside the retentionPeriod is automatically deleted\nThe following optional suffixes are supported: h (hour), d (day), w (week), y (year). If suffix isn't set, then the duration is counted in months")
)

func Init(cfg *config.Config) {
if err := initLogger(cfg); err != nil {
log.Fatal("Failed to open log file", zap.Error(err))
}
initDataDir(path.Join(cfg.Storage.Path, "tsdb"))

_ = flag.Set("retentionPeriod", *retentionPeriod)
_ = flag.Set("retentionPeriod", cfg.TSDB.RetentionPeriod)
_ = flag.Set("search.maxStepForPointsAdjustment", "1s")
_ = flag.Set("search.maxUniqueTimeseries", fmt.Sprintf("%d", cfg.TSDB.SearchMaxUniqueTimeseries))

// Some components in VictoriaMetrics want parsed arguments, i.e. assert `flag.Parsed()`. Make them happy.
_ = flag.CommandLine.Parse(nil)

startTime := time.Now()
storage.SetMinScrapeIntervalForDeduplication(0)
vmstorage.Init(promql.ResetRollupResultCacheIfNeeded)
vmselect.Init()
vminsert.Init()
Expand Down
18 changes: 9 additions & 9 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ go 1.19

require (
github.com/BurntSushi/toml v1.2.1
github.com/VictoriaMetrics/VictoriaMetrics v1.65.0
github.com/VictoriaMetrics/VictoriaMetrics v1.76.1
github.com/dgraph-io/badger/v3 v3.2103.1
github.com/genjidb/genji v0.13.0
github.com/genjidb/genji/engine/badgerengine v0.13.0
Expand All @@ -25,7 +25,7 @@ require (
github.com/prometheus/common v0.39.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.3
github.com/valyala/gozstd v1.14.2
github.com/valyala/gozstd v1.16.0
github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f
go.etcd.io/etcd/api/v3 v3.5.7
go.etcd.io/etcd/client/pkg/v3 v3.5.10
Expand All @@ -42,10 +42,10 @@ require (
cloud.google.com/go/compute v1.23.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
github.com/ReneKroon/ttlcache/v2 v2.3.0 // indirect
github.com/VictoriaMetrics/fastcache v1.6.0 // indirect
github.com/VictoriaMetrics/fasthttp v1.0.16 // indirect
github.com/VictoriaMetrics/metrics v1.17.3 // indirect
github.com/VictoriaMetrics/metricsql v0.21.0 // indirect
github.com/VictoriaMetrics/fastcache v1.10.0 // indirect
github.com/VictoriaMetrics/fasthttp v1.1.0 // indirect
github.com/VictoriaMetrics/metrics v1.18.1 // indirect
github.com/VictoriaMetrics/metricsql v0.41.0 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
Expand Down Expand Up @@ -135,10 +135,10 @@ require (
github.com/ugorji/go/codec v1.2.11 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect
github.com/valyala/fastjson v1.6.3 // indirect
github.com/valyala/fastrand v1.0.0 // indirect
github.com/valyala/fastrand v1.1.0 // indirect
github.com/valyala/fasttemplate v1.2.1 // indirect
github.com/valyala/histogram v1.1.2 // indirect
github.com/valyala/quicktemplate v1.6.3 // indirect
github.com/valyala/histogram v1.2.0 // indirect
github.com/valyala/quicktemplate v1.7.0 // indirect
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
Expand Down
Loading