diff --git a/cmd/jaeger/config-elasticsearch.yaml b/cmd/jaeger/config-elasticsearch.yaml index 308b7d11c5f..46a8f6a07b4 100644 --- a/cmd/jaeger/config-elasticsearch.yaml +++ b/cmd/jaeger/config-elasticsearch.yaml @@ -20,19 +20,21 @@ extensions: backends: some_storage: elasticsearch: - index_prefix: "jaeger-main" indices: spans: + index_prefix: "jaeger-main" date_layout: "2006-01-02" rollover_frequency: "day" shards: 5 replicas: 1 services: + index_prefix: "jaeger-main" date_layout: "2006-01-02" rollover_frequency: "day" shards: 5 replicas: 1 dependencies: + index_prefix: "jaeger-main" date_layout: "2006-01-02" rollover_frequency: "day" shards: 5 @@ -44,7 +46,9 @@ extensions: replicas: 1 another_storage: elasticsearch: - index_prefix: "jaeger-archive" + indices: + spans: + index_prefix: "jaeger-archive" receivers: otlp: diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 5a2b9a2d18a..03097da32aa 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -69,7 +69,6 @@ type Configuration struct { BulkWorkers int `mapstructure:"-"` BulkActions int `mapstructure:"-"` BulkFlushInterval time.Duration `mapstructure:"-"` - IndexPrefix string `mapstructure:"index_prefix"` Indices Indices `mapstructure:"indices"` ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"` AdaptiveSamplingLookback time.Duration `mapstructure:"-"` diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index af35be59ef1..2ca216c065b 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -254,18 +254,17 @@ func createSpanWriter( } writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{ - Client: clientFn, - IndexPrefix: cfg.IndexPrefix, - SpanIndexDateLayout: cfg.Indices.Spans.DateLayout, - ServiceIndexDateLayout: cfg.Indices.Services.DateLayout, - AllTagsAsFields: cfg.Tags.AllAsFields, - TagKeysAsFields: tags, - TagDotReplacement: cfg.Tags.DotReplacement, - Archive: archive, - UseReadWriteAliases: cfg.UseReadWriteAliases, - Logger: logger, - MetricsFactory: mFactory, - ServiceCacheTTL: cfg.ServiceCacheTTL, + Client: clientFn, + SpanIndex: cfg.Indices.Spans, + ServiceIndex: cfg.Indices.Services, + AllTagsAsFields: cfg.Tags.AllAsFields, + TagKeysAsFields: tags, + TagDotReplacement: cfg.Tags.DotReplacement, + Archive: archive, + UseReadWriteAliases: cfg.UseReadWriteAliases, + Logger: logger, + MetricsFactory: mFactory, + ServiceCacheTTL: cfg.ServiceCacheTTL, }) // Creating a template here would conflict with the one created for ILM resulting to no index rollover @@ -275,7 +274,7 @@ func createSpanWriter( if err != nil { return nil, err } - if err := writer.CreateTemplates(spanMapping, serviceMapping, cfg.IndexPrefix); err != nil { + if err := writer.CreateTemplates(spanMapping, serviceMapping, cfg.Indices.Spans.Prefix); err != nil { return nil, err } } @@ -286,7 +285,7 @@ func (f *Factory) CreateSamplingStore(int /* maxBuckets */) (samplingstore.Store params := esSampleStore.Params{ Client: f.getPrimaryClient, Logger: f.logger, - IndexPrefix: f.primaryConfig.IndexPrefix, + IndexPrefix: f.primaryConfig.Indices.Sampling.Prefix, IndexDateLayout: f.primaryConfig.Indices.Sampling.DateLayout, IndexRolloverFrequency: config.RolloverFrequencyAsNegativeDuration(f.primaryConfig.Indices.Sampling.RolloverFrequency), Lookback: f.primaryConfig.AdaptiveSamplingLookback, @@ -325,7 +324,7 @@ func createDependencyReader( reader := esDepStore.NewDependencyStore(esDepStore.Params{ Client: clientFn, Logger: logger, - IndexPrefix: cfg.IndexPrefix, + IndexPrefix: cfg.Indices.Dependencies.Prefix, IndexDateLayout: cfg.Indices.Dependencies.RolloverFrequency, MaxDocCount: cfg.MaxDocCount, UseReadWriteAliases: cfg.UseReadWriteAliases, diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index 9afa5da6771..854c190e017 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -67,6 +67,7 @@ const ( defaultIndexRolloverFrequency = "day" defaultSendGetBodyAs = "" + defaultIndexPrefix = "" ) var defaultIndexOptions = config.IndexOptions{ @@ -208,7 +209,7 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { "A time.Duration after which bulk requests are committed, regardless of other thresholds. Set to zero to disable. By default, this is disabled.") flagSet.String( nsConfig.namespace+suffixIndexPrefix, - nsConfig.IndexPrefix, + defaultIndexPrefix, "Optional prefix of Jaeger indices. For example \"production\" creates \"production-jaeger-*\".") flagSet.String( nsConfig.namespace+suffixIndexDateSeparator, @@ -341,13 +342,13 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.BulkFlushInterval = v.GetDuration(cfg.namespace + suffixBulkFlushInterval) cfg.Timeout = v.GetDuration(cfg.namespace + suffixTimeout) cfg.ServiceCacheTTL = v.GetDuration(cfg.namespace + suffixServiceCacheTTL) - cfg.IndexPrefix = v.GetString(cfg.namespace + suffixIndexPrefix) + indexPrefix := v.GetString(cfg.namespace + suffixIndexPrefix) // TODO cfg.Indices does not have a separate flag - cfg.Indices.Spans.Prefix = cfg.IndexPrefix - cfg.Indices.Services.Prefix = cfg.IndexPrefix - cfg.Indices.Sampling.Prefix = cfg.IndexPrefix - cfg.Indices.Dependencies.Prefix = cfg.IndexPrefix + cfg.Indices.Spans.Prefix = indexPrefix + cfg.Indices.Services.Prefix = indexPrefix + cfg.Indices.Sampling.Prefix = indexPrefix + cfg.Indices.Dependencies.Prefix = indexPrefix cfg.Tags.AllAsFields = v.GetBool(cfg.namespace + suffixTagsAsFieldsAll) cfg.Tags.Include = v.GetString(cfg.namespace + suffixTagsAsFieldsInclude) diff --git a/plugin/storage/es/spanstore/writer.go b/plugin/storage/es/spanstore/writer.go index 74fd361171d..036a4ad9ff9 100644 --- a/plugin/storage/es/spanstore/writer.go +++ b/plugin/storage/es/spanstore/writer.go @@ -15,6 +15,7 @@ import ( "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/cache" "github.com/jaegertracing/jaeger/pkg/es" + cfg "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/metrics" "github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel" storageMetrics "github.com/jaegertracing/jaeger/storage/spanstore/metrics" @@ -46,18 +47,20 @@ type SpanWriter struct { // SpanWriterParams holds constructor parameters for NewSpanWriter type SpanWriterParams struct { - Client func() es.Client - Logger *zap.Logger - MetricsFactory metrics.Factory - IndexPrefix string - SpanIndexDateLayout string - ServiceIndexDateLayout string - AllTagsAsFields bool - TagKeysAsFields []string - TagDotReplacement string - Archive bool - UseReadWriteAliases bool - ServiceCacheTTL time.Duration + Client func() es.Client + Logger *zap.Logger + MetricsFactory metrics.Factory + SpanIndex cfg.IndexOptions + ServiceIndex cfg.IndexOptions + // IndexPrefix string + // SpanIndexDateLayout string + // ServiceIndexDateLayout string + AllTagsAsFields bool + TagKeysAsFields []string + TagDotReplacement string + Archive bool + UseReadWriteAliases bool + ServiceCacheTTL time.Duration } // NewSpanWriter creates a new SpanWriter for use @@ -76,7 +79,7 @@ func NewSpanWriter(p SpanWriterParams) *SpanWriter { }, serviceWriter: serviceOperationStorage.Write, spanConverter: dbmodel.NewFromDomain(p.AllTagsAsFields, p.TagKeysAsFields, p.TagDotReplacement), - spanServiceIndex: getSpanAndServiceIndexFn(p.Archive, p.UseReadWriteAliases, p.IndexPrefix, p.SpanIndexDateLayout, p.ServiceIndexDateLayout), + spanServiceIndex: getSpanAndServiceIndexFn(p.Archive, p.UseReadWriteAliases, p.SpanIndex, p.ServiceIndex), } } @@ -99,12 +102,15 @@ func (s *SpanWriter) CreateTemplates(spanTemplate, serviceTemplate, indexPrefix // spanAndServiceIndexFn returns names of span and service indices type spanAndServiceIndexFn func(spanTime time.Time) (string, string) -func getSpanAndServiceIndexFn(archive, useReadWriteAliases bool, prefix, spanDateLayout string, serviceDateLayout string) spanAndServiceIndexFn { - if prefix != "" { - prefix += indexPrefixSeparator +func getSpanAndServiceIndexFn(archive, useReadWriteAliases bool, spanIndexOpts, serviceIndexOpts cfg.IndexOptions) spanAndServiceIndexFn { + if spanIndexOpts.Prefix != "" { + spanIndexOpts.Prefix += indexPrefixSeparator } - spanIndexPrefix := prefix + spanIndexBaseName - serviceIndexPrefix := prefix + serviceIndexBaseName + if serviceIndexOpts.Prefix != "" { + serviceIndexOpts.Prefix += indexPrefixSeparator + } + spanIndexPrefix := spanIndexOpts.Prefix + spanIndexBaseName + serviceIndexPrefix := serviceIndexOpts.Prefix + serviceIndexBaseName if archive { return func(_ time.Time) (string, string) { if useReadWriteAliases { @@ -120,7 +126,7 @@ func getSpanAndServiceIndexFn(archive, useReadWriteAliases bool, prefix, spanDat } } return func(date time.Time) (string, string) { - return indexWithDate(spanIndexPrefix, spanDateLayout, date), indexWithDate(serviceIndexPrefix, serviceDateLayout, date) + return indexWithDate(spanIndexPrefix, spanIndexOpts.DateLayout, date), indexWithDate(serviceIndexPrefix, serviceIndexOpts.DateLayout, date) } } diff --git a/plugin/storage/es/spanstore/writer_test.go b/plugin/storage/es/spanstore/writer_test.go index 17749dd30bb..aaaec3c0bf0 100644 --- a/plugin/storage/es/spanstore/writer_test.go +++ b/plugin/storage/es/spanstore/writer_test.go @@ -19,6 +19,7 @@ import ( "github.com/jaegertracing/jaeger/internal/metricstest" "github.com/jaegertracing/jaeger/model" "github.com/jaegertracing/jaeger/pkg/es" + "github.com/jaegertracing/jaeger/pkg/es/config" "github.com/jaegertracing/jaeger/pkg/es/mocks" "github.com/jaegertracing/jaeger/pkg/testutils" "github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel" @@ -40,7 +41,7 @@ func withSpanWriter(fn func(w *spanWriterTest)) { client: client, logger: logger, logBuffer: logBuffer, - writer: NewSpanWriter(SpanWriterParams{Client: func() es.Client { return client }, Logger: logger, MetricsFactory: metricsFactory, SpanIndexDateLayout: "2006-01-02", ServiceIndexDateLayout: "2006-01-02"}), + writer: NewSpanWriter(SpanWriterParams{Client: func() es.Client { return client }, Logger: logger, MetricsFactory: metricsFactory, SpanIndex: config.IndexOptions{DateLayout: "2006-01-02"}, ServiceIndex: config.IndexOptions{DateLayout: "2006-01-02"}}), } fn(w) } @@ -57,6 +58,16 @@ func TestSpanWriterIndices(t *testing.T) { serviceDataLayout := "2006-01-02" spanDataLayoutFormat := date.UTC().Format(spanDataLayout) serviceDataLayoutFormat := date.UTC().Format(serviceDataLayout) + + spanIndexOpts := config.IndexOptions{DateLayout: spanDataLayout} + serviceIndexOpts := config.IndexOptions{DateLayout: serviceDataLayout} + + serviceIndexOptsWithFoo := serviceIndexOpts + serviceIndexOptsWithFoo.Prefix = "foo:" + + spanIndexOptsWithFoo := spanIndexOpts + spanIndexOptsWithFoo.Prefix = "foo:" + testCases := []struct { indices []string params SpanWriterParams @@ -64,49 +75,49 @@ func TestSpanWriterIndices(t *testing.T) { { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: false, + SpanIndex: spanIndexOpts, ServiceIndex: serviceIndexOpts, Archive: false, }, indices: []string{spanIndexBaseName + spanDataLayoutFormat, serviceIndexBaseName + serviceDataLayoutFormat}, }, { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, UseReadWriteAliases: true, + SpanIndex: spanIndexOpts, ServiceIndex: serviceIndexOpts, UseReadWriteAliases: true, }, indices: []string{spanIndexBaseName + "write", serviceIndexBaseName + "write"}, }, { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: false, + SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, Archive: false, }, indices: []string{"foo:" + indexPrefixSeparator + spanIndexBaseName + spanDataLayoutFormat, "foo:" + indexPrefixSeparator + serviceIndexBaseName + serviceDataLayoutFormat}, }, { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, UseReadWriteAliases: true, + SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, UseReadWriteAliases: true, }, indices: []string{"foo:-" + spanIndexBaseName + "write", "foo:-" + serviceIndexBaseName + "write"}, }, { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: true, + SpanIndex: spanIndexOpts, ServiceIndex: serviceIndexOpts, Archive: true, }, indices: []string{spanIndexBaseName + archiveIndexSuffix, ""}, }, { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: true, + SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, Archive: true, }, indices: []string{"foo:" + indexPrefixSeparator + spanIndexBaseName + archiveIndexSuffix, ""}, }, { params: SpanWriterParams{ Client: clientFn, Logger: logger, MetricsFactory: metricsFactory, - IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: true, UseReadWriteAliases: true, + SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, Archive: true, UseReadWriteAliases: true, }, indices: []string{"foo:" + indexPrefixSeparator + spanIndexBaseName + archiveWriteIndexSuffix, ""}, },