remove indexprefix

Signed-off-by: Jared Tan <jian.tan@daocloud.io>

cmd/jaeger/config-elasticsearch.yaml

@@ -20,19 +20,21 @@ extensions:
     backends:
       some_storage:
         elasticsearch:
-          index_prefix: "jaeger-main"
           indices:
             spans:
+              index_prefix: "jaeger-main"
               date_layout: "2006-01-02"
               rollover_frequency: "day"
               shards: 5
               replicas: 1
             services:
+              index_prefix: "jaeger-main"
               date_layout: "2006-01-02"
               rollover_frequency: "day"
               shards: 5
               replicas: 1
             dependencies:
+              index_prefix: "jaeger-main"
               date_layout: "2006-01-02"
               rollover_frequency: "day"
               shards: 5
@@ -44,7 +46,9 @@ extensions:
               replicas: 1
       another_storage:
         elasticsearch:
-          index_prefix: "jaeger-archive"
+          indices:
+            spans:
+              index_prefix: "jaeger-archive"
 
 receivers:
   otlp:

pkg/es/config/config.go

@@ -69,7 +69,6 @@ type Configuration struct {
 	BulkWorkers              int            `mapstructure:"-"`
 	BulkActions              int            `mapstructure:"-"`
 	BulkFlushInterval        time.Duration  `mapstructure:"-"`
-	IndexPrefix              string         `mapstructure:"index_prefix"`
 	Indices                  Indices        `mapstructure:"indices"`
 	ServiceCacheTTL          time.Duration  `mapstructure:"service_cache_ttl"`
 	AdaptiveSamplingLookback time.Duration  `mapstructure:"-"`

plugin/storage/es/factory.go

@@ -254,18 +254,17 @@ func createSpanWriter(
 	}
 
 	writer := esSpanStore.NewSpanWriter(esSpanStore.SpanWriterParams{
-		Client:                 clientFn,
-		IndexPrefix:            cfg.IndexPrefix,
-		SpanIndexDateLayout:    cfg.Indices.Spans.DateLayout,
-		ServiceIndexDateLayout: cfg.Indices.Services.DateLayout,
-		AllTagsAsFields:        cfg.Tags.AllAsFields,
-		TagKeysAsFields:        tags,
-		TagDotReplacement:      cfg.Tags.DotReplacement,
-		Archive:                archive,
-		UseReadWriteAliases:    cfg.UseReadWriteAliases,
-		Logger:                 logger,
-		MetricsFactory:         mFactory,
-		ServiceCacheTTL:        cfg.ServiceCacheTTL,
+		Client:              clientFn,
+		SpanIndex:           cfg.Indices.Spans,
+		ServiceIndex:        cfg.Indices.Services,
+		AllTagsAsFields:     cfg.Tags.AllAsFields,
+		TagKeysAsFields:     tags,
+		TagDotReplacement:   cfg.Tags.DotReplacement,
+		Archive:             archive,
+		UseReadWriteAliases: cfg.UseReadWriteAliases,
+		Logger:              logger,
+		MetricsFactory:      mFactory,
+		ServiceCacheTTL:     cfg.ServiceCacheTTL,
 	})
 
 	// Creating a template here would conflict with the one created for ILM resulting to no index rollover
@@ -275,7 +274,7 @@ func createSpanWriter(
 		if err != nil {
 			return nil, err
 		}
-		if err := writer.CreateTemplates(spanMapping, serviceMapping, cfg.IndexPrefix); err != nil {
+		if err := writer.CreateTemplates(spanMapping, serviceMapping, cfg.Indices.Spans.Prefix); err != nil {
 			return nil, err
 		}
 	}
@@ -286,7 +285,7 @@ func (f *Factory) CreateSamplingStore(int /* maxBuckets */) (samplingstore.Store
 	params := esSampleStore.Params{
 		Client:                 f.getPrimaryClient,
 		Logger:                 f.logger,
-		IndexPrefix:            f.primaryConfig.IndexPrefix,
+		IndexPrefix:            f.primaryConfig.Indices.Sampling.Prefix,
 		IndexDateLayout:        f.primaryConfig.Indices.Sampling.DateLayout,
 		IndexRolloverFrequency: config.RolloverFrequencyAsNegativeDuration(f.primaryConfig.Indices.Sampling.RolloverFrequency),
 		Lookback:               f.primaryConfig.AdaptiveSamplingLookback,
@@ -325,7 +324,7 @@ func createDependencyReader(
 	reader := esDepStore.NewDependencyStore(esDepStore.Params{
 		Client:              clientFn,
 		Logger:              logger,
-		IndexPrefix:         cfg.IndexPrefix,
+		IndexPrefix:         cfg.Indices.Dependencies.Prefix,
 		IndexDateLayout:     cfg.Indices.Dependencies.RolloverFrequency,
 		MaxDocCount:         cfg.MaxDocCount,
 		UseReadWriteAliases: cfg.UseReadWriteAliases,

plugin/storage/es/options.go

@@ -67,6 +67,7 @@ const (
 
 	defaultIndexRolloverFrequency = "day"
 	defaultSendGetBodyAs          = ""
+	defaultIndexPrefix            = ""
 )
 
 var defaultIndexOptions = config.IndexOptions{
@@ -208,7 +209,7 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) {
 		"A time.Duration after which bulk requests are committed, regardless of other thresholds. Set to zero to disable. By default, this is disabled.")
 	flagSet.String(
 		nsConfig.namespace+suffixIndexPrefix,
-		nsConfig.IndexPrefix,
+		defaultIndexPrefix,
 		"Optional prefix of Jaeger indices. For example \"production\" creates \"production-jaeger-*\".")
 	flagSet.String(
 		nsConfig.namespace+suffixIndexDateSeparator,
@@ -341,13 +342,13 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) {
 	cfg.BulkFlushInterval = v.GetDuration(cfg.namespace + suffixBulkFlushInterval)
 	cfg.Timeout = v.GetDuration(cfg.namespace + suffixTimeout)
 	cfg.ServiceCacheTTL = v.GetDuration(cfg.namespace + suffixServiceCacheTTL)
-	cfg.IndexPrefix = v.GetString(cfg.namespace + suffixIndexPrefix)
+	indexPrefix := v.GetString(cfg.namespace + suffixIndexPrefix)
 
 	// TODO cfg.Indices does not have a separate flag
-	cfg.Indices.Spans.Prefix = cfg.IndexPrefix
-	cfg.Indices.Services.Prefix = cfg.IndexPrefix
-	cfg.Indices.Sampling.Prefix = cfg.IndexPrefix
-	cfg.Indices.Dependencies.Prefix = cfg.IndexPrefix
+	cfg.Indices.Spans.Prefix = indexPrefix
+	cfg.Indices.Services.Prefix = indexPrefix
+	cfg.Indices.Sampling.Prefix = indexPrefix
+	cfg.Indices.Dependencies.Prefix = indexPrefix
 
 	cfg.Tags.AllAsFields = v.GetBool(cfg.namespace + suffixTagsAsFieldsAll)
 	cfg.Tags.Include = v.GetString(cfg.namespace + suffixTagsAsFieldsInclude)

plugin/storage/es/spanstore/writer.go

@@ -15,6 +15,7 @@ import (
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/pkg/cache"
 	"github.com/jaegertracing/jaeger/pkg/es"
+	cfg "github.com/jaegertracing/jaeger/pkg/es/config"
 	"github.com/jaegertracing/jaeger/pkg/metrics"
 	"github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel"
 	storageMetrics "github.com/jaegertracing/jaeger/storage/spanstore/metrics"
@@ -46,18 +47,20 @@ type SpanWriter struct {
 
 // SpanWriterParams holds constructor parameters for NewSpanWriter
 type SpanWriterParams struct {
-	Client                 func() es.Client
-	Logger                 *zap.Logger
-	MetricsFactory         metrics.Factory
-	IndexPrefix            string
-	SpanIndexDateLayout    string
-	ServiceIndexDateLayout string
-	AllTagsAsFields        bool
-	TagKeysAsFields        []string
-	TagDotReplacement      string
-	Archive                bool
-	UseReadWriteAliases    bool
-	ServiceCacheTTL        time.Duration
+	Client         func() es.Client
+	Logger         *zap.Logger
+	MetricsFactory metrics.Factory
+	SpanIndex      cfg.IndexOptions
+	ServiceIndex   cfg.IndexOptions
+	// IndexPrefix            string
+	// SpanIndexDateLayout    string
+	// ServiceIndexDateLayout string
+	AllTagsAsFields     bool
+	TagKeysAsFields     []string
+	TagDotReplacement   string
+	Archive             bool
+	UseReadWriteAliases bool
+	ServiceCacheTTL     time.Duration
 }
 
 // NewSpanWriter creates a new SpanWriter for use
@@ -76,7 +79,7 @@ func NewSpanWriter(p SpanWriterParams) *SpanWriter {
 		},
 		serviceWriter:    serviceOperationStorage.Write,
 		spanConverter:    dbmodel.NewFromDomain(p.AllTagsAsFields, p.TagKeysAsFields, p.TagDotReplacement),
-		spanServiceIndex: getSpanAndServiceIndexFn(p.Archive, p.UseReadWriteAliases, p.IndexPrefix, p.SpanIndexDateLayout, p.ServiceIndexDateLayout),
+		spanServiceIndex: getSpanAndServiceIndexFn(p.Archive, p.UseReadWriteAliases, p.SpanIndex, p.ServiceIndex),
 	}
 }
 
@@ -99,12 +102,15 @@ func (s *SpanWriter) CreateTemplates(spanTemplate, serviceTemplate, indexPrefix
 // spanAndServiceIndexFn returns names of span and service indices
 type spanAndServiceIndexFn func(spanTime time.Time) (string, string)
 
-func getSpanAndServiceIndexFn(archive, useReadWriteAliases bool, prefix, spanDateLayout string, serviceDateLayout string) spanAndServiceIndexFn {
-	if prefix != "" {
-		prefix += indexPrefixSeparator
+func getSpanAndServiceIndexFn(archive, useReadWriteAliases bool, spanIndexOpts, serviceIndexOpts cfg.IndexOptions) spanAndServiceIndexFn {
+	if spanIndexOpts.Prefix != "" {
+		spanIndexOpts.Prefix += indexPrefixSeparator
 	}
-	spanIndexPrefix := prefix + spanIndexBaseName
-	serviceIndexPrefix := prefix + serviceIndexBaseName
+	if serviceIndexOpts.Prefix != "" {
+		serviceIndexOpts.Prefix += indexPrefixSeparator
+	}
+	spanIndexPrefix := spanIndexOpts.Prefix + spanIndexBaseName
+	serviceIndexPrefix := serviceIndexOpts.Prefix + serviceIndexBaseName
 	if archive {
 		return func(_ time.Time) (string, string) {
 			if useReadWriteAliases {
@@ -120,7 +126,7 @@ func getSpanAndServiceIndexFn(archive, useReadWriteAliases bool, prefix, spanDat
 		}
 	}
 	return func(date time.Time) (string, string) {
-		return indexWithDate(spanIndexPrefix, spanDateLayout, date), indexWithDate(serviceIndexPrefix, serviceDateLayout, date)
+		return indexWithDate(spanIndexPrefix, spanIndexOpts.DateLayout, date), indexWithDate(serviceIndexPrefix, serviceIndexOpts.DateLayout, date)
 	}
 }
 

plugin/storage/es/spanstore/writer_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/jaegertracing/jaeger/internal/metricstest"
 	"github.com/jaegertracing/jaeger/model"
 	"github.com/jaegertracing/jaeger/pkg/es"
+	"github.com/jaegertracing/jaeger/pkg/es/config"
 	"github.com/jaegertracing/jaeger/pkg/es/mocks"
 	"github.com/jaegertracing/jaeger/pkg/testutils"
 	"github.com/jaegertracing/jaeger/plugin/storage/es/spanstore/dbmodel"
@@ -40,7 +41,7 @@ func withSpanWriter(fn func(w *spanWriterTest)) {
 		client:    client,
 		logger:    logger,
 		logBuffer: logBuffer,
-		writer:    NewSpanWriter(SpanWriterParams{Client: func() es.Client { return client }, Logger: logger, MetricsFactory: metricsFactory, SpanIndexDateLayout: "2006-01-02", ServiceIndexDateLayout: "2006-01-02"}),
+		writer:    NewSpanWriter(SpanWriterParams{Client: func() es.Client { return client }, Logger: logger, MetricsFactory: metricsFactory, SpanIndex: config.IndexOptions{DateLayout: "2006-01-02"}, ServiceIndex: config.IndexOptions{DateLayout: "2006-01-02"}}),
 	}
 	fn(w)
 }
@@ -57,56 +58,66 @@ func TestSpanWriterIndices(t *testing.T) {
 	serviceDataLayout := "2006-01-02"
 	spanDataLayoutFormat := date.UTC().Format(spanDataLayout)
 	serviceDataLayoutFormat := date.UTC().Format(serviceDataLayout)
+
+	spanIndexOpts := config.IndexOptions{DateLayout: spanDataLayout}
+	serviceIndexOpts := config.IndexOptions{DateLayout: serviceDataLayout}
+
+	serviceIndexOptsWithFoo := serviceIndexOpts
+	serviceIndexOptsWithFoo.Prefix = "foo:"
+
+	spanIndexOptsWithFoo := spanIndexOpts
+	spanIndexOptsWithFoo.Prefix = "foo:"
+
 	testCases := []struct {
 		indices []string
 		params  SpanWriterParams
 	}{
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: false,
+				SpanIndex: spanIndexOpts, ServiceIndex: serviceIndexOpts, Archive: false,
 			},
 			indices: []string{spanIndexBaseName + spanDataLayoutFormat, serviceIndexBaseName + serviceDataLayoutFormat},
 		},
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, UseReadWriteAliases: true,
+				SpanIndex: spanIndexOpts, ServiceIndex: serviceIndexOpts, UseReadWriteAliases: true,
 			},
 			indices: []string{spanIndexBaseName + "write", serviceIndexBaseName + "write"},
 		},
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: false,
+				SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, Archive: false,
 			},
 			indices: []string{"foo:" + indexPrefixSeparator + spanIndexBaseName + spanDataLayoutFormat, "foo:" + indexPrefixSeparator + serviceIndexBaseName + serviceDataLayoutFormat},
 		},
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, UseReadWriteAliases: true,
+				SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, UseReadWriteAliases: true,
 			},
 			indices: []string{"foo:-" + spanIndexBaseName + "write", "foo:-" + serviceIndexBaseName + "write"},
 		},
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: true,
+				SpanIndex: spanIndexOpts, ServiceIndex: serviceIndexOpts, Archive: true,
 			},
 			indices: []string{spanIndexBaseName + archiveIndexSuffix, ""},
 		},
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: true,
+				SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, Archive: true,
 			},
 			indices: []string{"foo:" + indexPrefixSeparator + spanIndexBaseName + archiveIndexSuffix, ""},
 		},
 		{
 			params: SpanWriterParams{
 				Client: clientFn, Logger: logger, MetricsFactory: metricsFactory,
-				IndexPrefix: "foo:", SpanIndexDateLayout: spanDataLayout, ServiceIndexDateLayout: serviceDataLayout, Archive: true, UseReadWriteAliases: true,
+				SpanIndex: spanIndexOptsWithFoo, ServiceIndex: serviceIndexOptsWithFoo, Archive: true, UseReadWriteAliases: true,
 			},
 			indices: []string{"foo:" + indexPrefixSeparator + spanIndexBaseName + archiveWriteIndexSuffix, ""},
 		},