diff --git a/cmd/jaeger/config-elasticsearch.yaml b/cmd/jaeger/config-elasticsearch.yaml index b6417e0a368..a77f1f7797d 100644 --- a/cmd/jaeger/config-elasticsearch.yaml +++ b/cmd/jaeger/config-elasticsearch.yaml @@ -17,19 +17,19 @@ extensions: some_storage: elasticsearch: index_prefix: "jaeger-main" -# indices: -# spans: -# date_layout: "2006-01-02" -# rollover_frequency: "hour" -# services: -# date_layout: "2006-01-02" -# rollover_frequency: "hour" -# dependencies: -# date_layout: "2006-01-02" -# rollover_frequency: "hour" -# sampling: -# date_layout: "2006-01-02" -# rollover_frequency: "hour" + indices: + spans: + date_layout: "2006-01-02" + rollover_frequency: "day" + services: + date_layout: "2006-01-02" + rollover_frequency: "day" + dependencies: + date_layout: "2006-01-02" + rollover_frequency: "day" + sampling: + date_layout: "2006-01-02" + rollover_frequency: "day" another_storage: elasticsearch: index_prefix: "jaeger-archive" diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go index 1b74911b141..7c525fc4e97 100644 --- a/pkg/es/config/config.go +++ b/pkg/es/config/config.go @@ -44,10 +44,17 @@ import ( storageMetrics "github.com/jaegertracing/jaeger/storage/spanstore/metrics" ) +type TemplateOptions struct { + Priority int64 `mapstructure:"priority"` + NumShards int64 `mapstructure:"num_shards"` + NumReplicas int64 `mapstructure:"num_replicas"` +} + // IndexOptions describes the index format and rollover frequency type IndexOptions struct { - DateLayout string `mapstructure:"date_layout"` - RolloverFrequency string `mapstructure:"rollover_frequency"` + DateLayout string `mapstructure:"date_layout"` + RolloverFrequency string `mapstructure:"rollover_frequency"` + TemplateOptions TemplateOptions `mapstructure:"template"` } // Indices describes different configuration options for each index type @@ -60,40 +67,35 @@ type Indices struct { // Configuration describes the configuration properties needed to connect to an ElasticSearch cluster type Configuration struct { - Servers []string `mapstructure:"server_urls" valid:"required,url"` - RemoteReadClusters []string `mapstructure:"remote_read_clusters"` - Username string `mapstructure:"username"` - Password string `mapstructure:"password" json:"-"` - TokenFilePath string `mapstructure:"token_file"` - PasswordFilePath string `mapstructure:"password_file"` - AllowTokenFromContext bool `mapstructure:"-"` - Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing - SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"` - MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query - MaxSpanAge time.Duration `mapstructure:"-"` // configures the maximum lookback on span reads - NumShards int64 `mapstructure:"num_shards"` - NumReplicas int64 `mapstructure:"num_replicas"` - PrioritySpanTemplate int64 `mapstructure:"priority_span_template"` - PriorityServiceTemplate int64 `mapstructure:"priority_service_template"` - PriorityDependenciesTemplate int64 `mapstructure:"priority_dependencies_template"` - Timeout time.Duration `mapstructure:"-"` - BulkSize int `mapstructure:"-"` - BulkWorkers int `mapstructure:"-"` - BulkActions int `mapstructure:"-"` - BulkFlushInterval time.Duration `mapstructure:"-"` - IndexPrefix string `mapstructure:"index_prefix"` - Indices Indices `mapstructure:"indices"` - ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"` - AdaptiveSamplingLookback time.Duration `mapstructure:"-"` - Tags TagsAsFields `mapstructure:"tags_as_fields"` - Enabled bool `mapstructure:"-"` - TLS tlscfg.Options `mapstructure:"tls"` - UseReadWriteAliases bool `mapstructure:"use_aliases"` - CreateIndexTemplates bool `mapstructure:"create_mappings"` - UseILM bool `mapstructure:"use_ilm"` - Version uint `mapstructure:"version"` - LogLevel string `mapstructure:"log_level"` - SendGetBodyAs string `mapstructure:"send_get_body_as"` + Servers []string `mapstructure:"server_urls" valid:"required,url"` + RemoteReadClusters []string `mapstructure:"remote_read_clusters"` + Username string `mapstructure:"username"` + Password string `mapstructure:"password" json:"-"` + TokenFilePath string `mapstructure:"token_file"` + PasswordFilePath string `mapstructure:"password_file"` + AllowTokenFromContext bool `mapstructure:"-"` + Sniffer bool `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing + SnifferTLSEnabled bool `mapstructure:"sniffer_tls_enabled"` + MaxDocCount int `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query + MaxSpanAge time.Duration `mapstructure:"-"` // configures the maximum lookback on span reads + Timeout time.Duration `mapstructure:"-"` + BulkSize int `mapstructure:"-"` + BulkWorkers int `mapstructure:"-"` + BulkActions int `mapstructure:"-"` + BulkFlushInterval time.Duration `mapstructure:"-"` + IndexPrefix string `mapstructure:"index_prefix"` + Indices Indices `mapstructure:"indices"` + ServiceCacheTTL time.Duration `mapstructure:"service_cache_ttl"` + AdaptiveSamplingLookback time.Duration `mapstructure:"-"` + Tags TagsAsFields `mapstructure:"tags_as_fields"` + Enabled bool `mapstructure:"-"` + TLS tlscfg.Options `mapstructure:"tls"` + UseReadWriteAliases bool `mapstructure:"use_aliases"` + CreateIndexTemplates bool `mapstructure:"create_mappings"` + UseILM bool `mapstructure:"use_ilm"` + Version uint `mapstructure:"version"` + LogLevel string `mapstructure:"log_level"` + SendGetBodyAs string `mapstructure:"send_get_body_as"` } // TagsAsFields holds configuration for tag schema. @@ -246,20 +248,20 @@ func (c *Configuration) ApplyDefaults(source *Configuration) { if c.AdaptiveSamplingLookback == 0 { c.AdaptiveSamplingLookback = source.AdaptiveSamplingLookback } - if c.NumShards == 0 { - c.NumShards = source.NumShards + if c.Indices.Dependencies.TemplateOptions.NumShards == 0 { + c.Indices.Dependencies.TemplateOptions.NumShards = source.Indices.Dependencies.TemplateOptions.NumShards } - if c.NumReplicas == 0 { - c.NumReplicas = source.NumReplicas + if c.Indices.Dependencies.TemplateOptions.NumReplicas == 0 { + c.Indices.Dependencies.TemplateOptions.NumReplicas = source.Indices.Dependencies.TemplateOptions.NumReplicas } - if c.PrioritySpanTemplate == 0 { - c.PrioritySpanTemplate = source.PrioritySpanTemplate + if c.Indices.Spans.TemplateOptions.Priority == 0 { + c.Indices.Spans.TemplateOptions.Priority = source.Indices.Spans.TemplateOptions.Priority } - if c.PriorityServiceTemplate == 0 { - c.PriorityServiceTemplate = source.PriorityServiceTemplate + if c.Indices.Services.TemplateOptions.Priority == 0 { + c.Indices.Services.TemplateOptions.Priority = source.Indices.Services.TemplateOptions.Priority } - if c.PrioritySpanTemplate == 0 { - c.PriorityDependenciesTemplate = source.PriorityDependenciesTemplate + if c.Indices.Dependencies.TemplateOptions.Priority == 0 { + c.Indices.Dependencies.TemplateOptions.Priority = source.Indices.Dependencies.TemplateOptions.Priority } if c.BulkSize == 0 { c.BulkSize = source.BulkSize diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go index cc662c00f87..5670f5097b1 100644 --- a/plugin/storage/es/factory.go +++ b/plugin/storage/es/factory.go @@ -325,14 +325,20 @@ func (f *Factory) CreateSamplingStore(int /* maxBuckets */) (samplingstore.Store func mappingBuilderFromConfig(cfg *config.Configuration) mappings.MappingBuilder { return mappings.MappingBuilder{ TemplateBuilder: es.TextTemplateBuilder{}, - Shards: cfg.NumShards, - Replicas: cfg.NumReplicas, + ShardsSpan: cfg.Indices.Spans.TemplateOptions.NumShards, + ReplicasSpan: cfg.Indices.Spans.TemplateOptions.NumReplicas, + ShardsService: cfg.Indices.Services.TemplateOptions.NumShards, + ReplicasService: cfg.Indices.Services.TemplateOptions.NumShards, + ShardsSampling: cfg.Indices.Sampling.TemplateOptions.NumShards, + ReplicasSampling: cfg.Indices.Sampling.TemplateOptions.NumShards, + ShardsDependencies: cfg.Indices.Dependencies.TemplateOptions.NumShards, + ReplicasDependencies: cfg.Indices.Dependencies.TemplateOptions.NumShards, EsVersion: cfg.Version, IndexPrefix: cfg.IndexPrefix, UseILM: cfg.UseILM, - PrioritySpanTemplate: cfg.PrioritySpanTemplate, - PriorityServiceTemplate: cfg.PriorityServiceTemplate, - PriorityDependenciesTemplate: cfg.PriorityDependenciesTemplate, + PrioritySpanTemplate: cfg.Indices.Spans.TemplateOptions.Priority, + PriorityServiceTemplate: cfg.Indices.Services.TemplateOptions.Priority, + PriorityDependenciesTemplate: cfg.Indices.Dependencies.TemplateOptions.Priority, } } diff --git a/plugin/storage/es/mappings/jaeger-dependencies-6.json b/plugin/storage/es/mappings/jaeger-dependencies-6.json index cdfa62d9547..ab28d332435 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-6.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-dependencies-*", "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsDependencies }}, + "index.number_of_replicas": {{ .ReplicasDependencies }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true }, diff --git a/plugin/storage/es/mappings/jaeger-dependencies-7.json b/plugin/storage/es/mappings/jaeger-dependencies-7.json index 18afe1b056e..b86e3aee97a 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-7.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsDependencies }}, + "index.number_of_replicas": {{ .ReplicasDependencies }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-dependencies-8.json b/plugin/storage/es/mappings/jaeger-dependencies-8.json index 57767866284..3ae30e8e74a 100644 --- a/plugin/storage/es/mappings/jaeger-dependencies-8.json +++ b/plugin/storage/es/mappings/jaeger-dependencies-8.json @@ -8,8 +8,8 @@ }, {{- end }} "settings": { - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsDependencies }}, + "index.number_of_replicas": {{ .ReplicasDependencies }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": true {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/jaeger-sampling-6.json b/plugin/storage/es/mappings/jaeger-sampling-6.json index 458d490a357..d67e179a3bd 100644 --- a/plugin/storage/es/mappings/jaeger-sampling-6.json +++ b/plugin/storage/es/mappings/jaeger-sampling-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-sampling-*", "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsSampling }}, + "index.number_of_replicas": {{ .ReplicasSampling }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":false }, diff --git a/plugin/storage/es/mappings/jaeger-sampling-7.json b/plugin/storage/es/mappings/jaeger-sampling-7.json index 167c1d47928..6b23538808b 100644 --- a/plugin/storage/es/mappings/jaeger-sampling-7.json +++ b/plugin/storage/es/mappings/jaeger-sampling-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsSampling }}, + "index.number_of_replicas": {{ .ReplicasSampling }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":false {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-sampling-8.json b/plugin/storage/es/mappings/jaeger-sampling-8.json index 0667520803a..7eefb9748bc 100644 --- a/plugin/storage/es/mappings/jaeger-sampling-8.json +++ b/plugin/storage/es/mappings/jaeger-sampling-8.json @@ -8,8 +8,8 @@ }, {{- end }} "settings": { - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsSampling }}, + "index.number_of_replicas": {{ .ReplicasSampling }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": false {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/jaeger-service-6.json b/plugin/storage/es/mappings/jaeger-service-6.json index 1adb8e0bf38..8311a6c317a 100644 --- a/plugin/storage/es/mappings/jaeger-service-6.json +++ b/plugin/storage/es/mappings/jaeger-service-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-service-*", "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsService }}, + "index.number_of_replicas": {{ .ReplicasService }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json index 0ca2d186319..ae1d4eed049 100644 --- a/plugin/storage/es/mappings/jaeger-service-7.json +++ b/plugin/storage/es/mappings/jaeger-service-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsService }}, + "index.number_of_replicas": {{ .ReplicasService }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-service-8.json b/plugin/storage/es/mappings/jaeger-service-8.json index 97ab02f573d..70e0fcde2da 100644 --- a/plugin/storage/es/mappings/jaeger-service-8.json +++ b/plugin/storage/es/mappings/jaeger-service-8.json @@ -8,8 +8,8 @@ }, {{- end }} "settings": { - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsService }}, + "index.number_of_replicas": {{ .ReplicasService }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": true {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/jaeger-span-6.json b/plugin/storage/es/mappings/jaeger-span-6.json index 3c73f923c4c..0e75be5701c 100644 --- a/plugin/storage/es/mappings/jaeger-span-6.json +++ b/plugin/storage/es/mappings/jaeger-span-6.json @@ -1,8 +1,8 @@ { "template": "*jaeger-span-*", "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsSpan }}, + "index.number_of_replicas": {{ .ReplicasSpan }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true, "index.mapper.dynamic":false diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json index 3d8bbae95dc..6a5bcd099f6 100644 --- a/plugin/storage/es/mappings/jaeger-span-7.json +++ b/plugin/storage/es/mappings/jaeger-span-7.json @@ -6,8 +6,8 @@ }, {{- end }} "settings":{ - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsSpan }}, + "index.number_of_replicas": {{ .ReplicasSpan }}, "index.mapping.nested_fields.limit":50, "index.requests.cache.enable":true {{- if .UseILM }} diff --git a/plugin/storage/es/mappings/jaeger-span-8.json b/plugin/storage/es/mappings/jaeger-span-8.json index 60bc5eaa910..e111d2b0de0 100644 --- a/plugin/storage/es/mappings/jaeger-span-8.json +++ b/plugin/storage/es/mappings/jaeger-span-8.json @@ -9,8 +9,8 @@ }, {{- end}} "settings": { - "index.number_of_shards": {{ .Shards }}, - "index.number_of_replicas": {{ .Replicas }}, + "index.number_of_shards": {{ .ShardsSpan }}, + "index.number_of_replicas": {{ .ReplicasSpan }}, "index.mapping.nested_fields.limit": 50, "index.requests.cache.enable": true {{- if .UseILM }}, diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go index d5da6ccb97f..9cb8dba8644 100644 --- a/plugin/storage/es/mappings/mapping.go +++ b/plugin/storage/es/mappings/mapping.go @@ -30,8 +30,14 @@ var MAPPINGS embed.FS // MappingBuilder holds parameters required to render an elasticsearch index template type MappingBuilder struct { TemplateBuilder es.TemplateBuilder - Shards int64 - Replicas int64 + ShardsSpan int64 + ReplicasSpan int64 + ShardsService int64 + ReplicasService int64 + ShardsDependencies int64 + ReplicasDependencies int64 + ShardsSampling int64 + ReplicasSampling int64 PrioritySpanTemplate int64 PriorityServiceTemplate int64 PriorityDependenciesTemplate int64 diff --git a/plugin/storage/es/mappings/mapping_test.go b/plugin/storage/es/mappings/mapping_test.go index a3b35f1dcca..1994b5c723f 100644 --- a/plugin/storage/es/mappings/mapping_test.go +++ b/plugin/storage/es/mappings/mapping_test.go @@ -59,8 +59,14 @@ func TestMappingBuilderGetMapping(t *testing.T) { t.Run(tt.mapping, func(t *testing.T) { mb := &MappingBuilder{ TemplateBuilder: es.TextTemplateBuilder{}, - Shards: 3, - Replicas: 3, + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, PrioritySpanTemplate: 500, PriorityServiceTemplate: 501, PriorityDependenciesTemplate: 502, @@ -149,13 +155,19 @@ func TestMappingBuilderFixMapping(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { mappingBuilder := MappingBuilder{ - TemplateBuilder: test.templateBuilderMockFunc(), - Shards: 3, - Replicas: 5, - EsVersion: 7, - IndexPrefix: "test", - UseILM: true, - ILMPolicyName: "jaeger-test-policy", + TemplateBuilder: test.templateBuilderMockFunc(), + ShardsSpan: 3, + ReplicasSpan: 5, + ShardsService: 3, + ReplicasService: 5, + ShardsDependencies: 3, + ReplicasDependencies: 5, + ShardsSampling: 3, + ReplicasSampling: 5, + EsVersion: 7, + IndexPrefix: "test", + UseILM: true, + ILMPolicyName: "jaeger-test-policy", } _, err := mappingBuilder.fixMapping("test") if test.err != "" { @@ -169,12 +181,18 @@ func TestMappingBuilderFixMapping(t *testing.T) { func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { type args struct { - shards int64 - replicas int64 - esVersion uint - indexPrefix string - useILM bool - ilmPolicyName string + ShardsSpan int64 + ReplicasSpan int64 + ShardsService int64 + ReplicasService int64 + ShardsDependencies int64 + ReplicasDependencies int64 + ShardsSampling int64 + ReplicasSampling int64 + esVersion uint + indexPrefix string + useILM bool + ilmPolicyName string } tests := []struct { name string @@ -185,12 +203,18 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { { name: "ES Version 7", args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - indexPrefix: "test", - useILM: true, - ilmPolicyName: "jaeger-test-policy", + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -204,12 +228,18 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { { name: "ES Version 7 Service Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - indexPrefix: "test", - useILM: true, - ilmPolicyName: "jaeger-test-policy", + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -225,12 +255,18 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { { name: "ES Version < 7", args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - indexPrefix: "test", - useILM: true, - ilmPolicyName: "jaeger-test-policy", + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -244,12 +280,18 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { { name: "ES Version < 7 Service Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - indexPrefix: "test", - useILM: true, - ilmPolicyName: "jaeger-test-policy", + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -264,12 +306,18 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { { name: "ES Version < 7 Span Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 6, - indexPrefix: "test", - useILM: true, - ilmPolicyName: "jaeger-test-policy", + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, + esVersion: 6, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -283,12 +331,18 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { { name: "ES Version 7 Span Error", args: args{ - shards: 3, - replicas: 3, - esVersion: 7, - indexPrefix: "test", - useILM: true, - ilmPolicyName: "jaeger-test-policy", + ShardsSpan: 3, + ReplicasSpan: 3, + ShardsService: 3, + ReplicasService: 3, + ShardsDependencies: 3, + ReplicasDependencies: 3, + ShardsSampling: 3, + ReplicasSampling: 3, + esVersion: 7, + indexPrefix: "test", + useILM: true, + ilmPolicyName: "jaeger-test-policy", }, mockNewTextTemplateBuilder: func() es.TemplateBuilder { tb := mocks.TemplateBuilder{} @@ -303,13 +357,19 @@ func TestMappingBuilderGetSpanServiceMappings(t *testing.T) { for _, test := range tests { t.Run(test.name, func(t *testing.T) { mappingBuilder := MappingBuilder{ - TemplateBuilder: test.mockNewTextTemplateBuilder(), - Shards: test.args.shards, - Replicas: test.args.replicas, - EsVersion: test.args.esVersion, - IndexPrefix: test.args.indexPrefix, - UseILM: test.args.useILM, - ILMPolicyName: test.args.ilmPolicyName, + TemplateBuilder: test.mockNewTextTemplateBuilder(), + ShardsSpan: test.args.ShardsSpan, + ReplicasSpan: test.args.ReplicasSpan, + ShardsService: test.args.ShardsService, + ReplicasService: test.args.ReplicasService, + ShardsDependencies: test.args.ShardsDependencies, + ReplicasDependencies: test.args.ReplicasDependencies, + ShardsSampling: test.args.ShardsSampling, + ReplicasSampling: test.args.ReplicasSampling, + EsVersion: test.args.esVersion, + IndexPrefix: test.args.indexPrefix, + UseILM: test.args.useILM, + ILMPolicyName: test.args.ilmPolicyName, } _, _, err := mappingBuilder.GetSpanServiceMappings() if test.err != "" { diff --git a/plugin/storage/es/options.go b/plugin/storage/es/options.go index f9d96d7b6ad..965795e7459 100644 --- a/plugin/storage/es/options.go +++ b/plugin/storage/es/options.go @@ -29,45 +29,54 @@ import ( ) const ( - suffixUsername = ".username" - suffixPassword = ".password" - suffixSniffer = ".sniffer" - suffixSnifferTLSEnabled = ".sniffer-tls-enabled" - suffixTokenPath = ".token-file" - suffixPasswordPath = ".password-file" - suffixServerURLs = ".server-urls" - suffixRemoteReadClusters = ".remote-read-clusters" - suffixMaxSpanAge = ".max-span-age" - suffixAdaptiveSamplingLookback = ".adaptive-sampling.lookback" - suffixNumShards = ".num-shards" - suffixNumReplicas = ".num-replicas" - suffixPrioritySpanTemplate = ".prioirity-span-template" - suffixPriorityServiceTemplate = ".prioirity-service-template" - suffixPriorityDependenciesTemplate = ".prioirity-dependencies-template" - suffixBulkSize = ".bulk.size" - suffixBulkWorkers = ".bulk.workers" - suffixBulkActions = ".bulk.actions" - suffixBulkFlushInterval = ".bulk.flush-interval" - suffixTimeout = ".timeout" - suffixIndexPrefix = ".index-prefix" - suffixIndexDateSeparator = ".index-date-separator" - suffixIndexRolloverFrequencySpans = ".index-rollover-frequency-spans" - suffixIndexRolloverFrequencyServices = ".index-rollover-frequency-services" - suffixIndexRolloverFrequencySampling = ".index-rollover-frequency-adaptive-sampling" - suffixServiceCacheTTL = ".service-cache-ttl" - suffixTagsAsFields = ".tags-as-fields" - suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" - suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" - suffixTagsFile = suffixTagsAsFields + ".config-file" - suffixTagDeDotChar = suffixTagsAsFields + ".dot-replacement" - suffixReadAlias = ".use-aliases" - suffixUseILM = ".use-ilm" - suffixCreateIndexTemplate = ".create-index-templates" - suffixEnabled = ".enabled" - suffixVersion = ".version" - suffixMaxDocCount = ".max-doc-count" - suffixLogLevel = ".log-level" - suffixSendGetBodyAs = ".send-get-body-as" + suffixUsername = ".username" + suffixPassword = ".password" + suffixSniffer = ".sniffer" + suffixSnifferTLSEnabled = ".sniffer-tls-enabled" + suffixTokenPath = ".token-file" + suffixPasswordPath = ".password-file" + suffixServerURLs = ".server-urls" + suffixRemoteReadClusters = ".remote-read-clusters" + suffixMaxSpanAge = ".max-span-age" + suffixAdaptiveSamplingLookback = ".adaptive-sampling.lookback" + suffixNumShards = ".num-shards" + suffixNumReplicas = ".num-replicas" + suffixNumShardsSpan = ".num-shards-spans" + suffixNumShardsServices = ".num-shards-services" + suffixNumShardsSampling = ".num-shards-sampling" + suffixNumShardsDependencies = ".num-shards-dependencies" + suffixNumReplicasSpan = ".num-replicas-spans" + suffixNumReplicasServices = ".num-replicas-services" + suffixNumReplicasSampling = ".num-replicas-sampling" + suffixNumReplicasDependencies = ".num-replicas-dependencies" + suffixPrioritySpanTemplate = ".prioirity-span-template" + suffixPriorityServiceTemplate = ".prioirity-service-template" + suffixPriorityDependenciesTemplate = ".prioirity-dependencies-template" + suffixBulkSize = ".bulk.size" + suffixBulkWorkers = ".bulk.workers" + suffixBulkActions = ".bulk.actions" + suffixBulkFlushInterval = ".bulk.flush-interval" + suffixTimeout = ".timeout" + suffixIndexPrefix = ".index-prefix" + suffixIndexDateSeparator = ".index-date-separator" + suffixIndexRolloverFrequencySpans = ".index-rollover-frequency-spans" + suffixIndexRolloverFrequencyServices = ".index-rollover-frequency-services" + suffixIndexRolloverFrequencySampling = ".index-rollover-frequency-adaptive-sampling" + suffixIndexRolloverFrequencyDependencies = ".index-rollover-frequency-adaptive-dependencies" + suffixServiceCacheTTL = ".service-cache-ttl" + suffixTagsAsFields = ".tags-as-fields" + suffixTagsAsFieldsAll = suffixTagsAsFields + ".all" + suffixTagsAsFieldsInclude = suffixTagsAsFields + ".include" + suffixTagsFile = suffixTagsAsFields + ".config-file" + suffixTagDeDotChar = suffixTagsAsFields + ".dot-replacement" + suffixReadAlias = ".use-aliases" + suffixUseILM = ".use-ilm" + suffixCreateIndexTemplate = ".create-index-templates" + suffixEnabled = ".enabled" + suffixVersion = ".version" + suffixMaxDocCount = ".max-doc-count" + suffixLogLevel = ".log-level" + suffixSendGetBodyAs = ".send-get-body-as" // default number of documents to return from a query (elasticsearch allowed limit) // see search.max_buckets and index.max_result_window defaultMaxDocCount = 10_000 @@ -84,6 +93,11 @@ var ( defaultIndexOptions = config.IndexOptions{ DateLayout: initDateLayout(defaultIndexRolloverFrequency, defaultIndexDateSeparator), RolloverFrequency: defaultIndexRolloverFrequency, + TemplateOptions: config.TemplateOptions{ + Priority: 0, + NumShards: 5, + NumReplicas: 1, + }, } ) @@ -175,30 +189,63 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { nsConfig.namespace+suffixTimeout, nsConfig.Timeout, "Timeout used for queries. A Timeout of zero means no timeout") + // TODO deprecated flag to be removed flagSet.Int64( nsConfig.namespace+suffixNumShards, - nsConfig.NumShards, - "The number of shards per index in Elasticsearch") + defaultIndexOptions.TemplateOptions.NumShards, + "(deprecated, will be removed in the future, use .num-shards-spans or .num-shards-services or .num-shards-sampling or .num-shards-dependencies instead) The number of shards per index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumReplicas, + defaultIndexOptions.TemplateOptions.NumReplicas, + "(deprecated, will be removed in the future, use .num-replicas-spans or .num-replicas-services or .num-replicas-sampling or .num-replicas-dependencies instead) The number of replicas per index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumShardsSpan, + nsConfig.Indices.Spans.TemplateOptions.NumShards, + "The number of shards per span index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumShardsServices, + nsConfig.Indices.Services.TemplateOptions.NumShards, + "The number of shards per service index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumShardsSampling, + nsConfig.Indices.Sampling.TemplateOptions.NumShards, + "The number of shards per sampling index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumShardsDependencies, + nsConfig.Indices.Dependencies.TemplateOptions.NumShards, + "The number of shards per dependencies index in Elasticsearch") flagSet.Duration( nsConfig.namespace+suffixServiceCacheTTL, nsConfig.ServiceCacheTTL, "The TTL for the cache of known service names", ) flagSet.Int64( - nsConfig.namespace+suffixNumReplicas, - nsConfig.NumReplicas, - "The number of replicas per index in Elasticsearch") + nsConfig.namespace+suffixNumReplicasSpan, + nsConfig.Indices.Spans.TemplateOptions.NumReplicas, + "The number of replicas per span index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumReplicasServices, + nsConfig.Indices.Services.TemplateOptions.NumReplicas, + "The number of replicas per service index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumReplicasSampling, + nsConfig.Indices.Sampling.TemplateOptions.NumReplicas, + "The number of replicas per sampling index in Elasticsearch") + flagSet.Int64( + nsConfig.namespace+suffixNumReplicasDependencies, + nsConfig.Indices.Dependencies.TemplateOptions.NumReplicas, + "The number of replicas per dependencies index in Elasticsearch") flagSet.Int64( nsConfig.namespace+suffixPrioritySpanTemplate, - nsConfig.PrioritySpanTemplate, + nsConfig.Indices.Spans.TemplateOptions.Priority, "Priority of jaeger-span index template (ESv8 only)") flagSet.Int64( nsConfig.namespace+suffixPriorityServiceTemplate, - nsConfig.PriorityServiceTemplate, + nsConfig.Indices.Services.TemplateOptions.Priority, "Priority of jaeger-service index template (ESv8 only)") flagSet.Int64( nsConfig.namespace+suffixPriorityDependenciesTemplate, - nsConfig.PriorityDependenciesTemplate, + nsConfig.Indices.Dependencies.TemplateOptions.Priority, "Priority of jaeger-dependecies index template (ESv8 only)") flagSet.Int( nsConfig.namespace+suffixBulkSize, @@ -239,6 +286,11 @@ func addFlags(flagSet *flag.FlagSet, nsConfig *namespaceConfig) { defaultIndexRolloverFrequency, "Rotates jaeger-sampling indices over the given period. For example \"day\" creates \"jaeger-sampling-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ "This does not delete old indices. For details on complete index management solutions supported by Jaeger, refer to: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover") + flagSet.String( + nsConfig.namespace+suffixIndexRolloverFrequencyDependencies, + defaultIndexRolloverFrequency, + "Rotates jaeger-dependencies indices over the given period. For example \"day\" creates \"jaeger-dependencies-yyyy-MM-dd\" every day after UTC 12AM. Valid options: [hour, day]. "+ + "This does not delete old indices. For details on complete index management solutions supported by Jaeger, refer to: https://www.jaegertracing.io/docs/deployment/#elasticsearch-rollover") flagSet.Bool( nsConfig.namespace+suffixTagsAsFieldsAll, nsConfig.Tags.AllAsFields, @@ -320,6 +372,13 @@ func (opt *Options) InitFromViper(v *viper.Viper) { } func initFromViper(cfg *namespaceConfig, v *viper.Viper) { + overrideIndexShardsNums := func(deprecatedShards, newShards int64) int64 { + if deprecatedShards > 0 { + return deprecatedShards + } else { + return newShards + } + } cfg.Username = v.GetString(cfg.namespace + suffixUsername) cfg.Password = v.GetString(cfg.namespace + suffixPassword) cfg.TokenFilePath = v.GetString(cfg.namespace + suffixTokenPath) @@ -329,11 +388,17 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.Servers = strings.Split(stripWhiteSpace(v.GetString(cfg.namespace+suffixServerURLs)), ",") cfg.MaxSpanAge = v.GetDuration(cfg.namespace + suffixMaxSpanAge) cfg.AdaptiveSamplingLookback = v.GetDuration(cfg.namespace + suffixAdaptiveSamplingLookback) - cfg.NumShards = v.GetInt64(cfg.namespace + suffixNumShards) - cfg.NumReplicas = v.GetInt64(cfg.namespace + suffixNumReplicas) - cfg.PrioritySpanTemplate = v.GetInt64(cfg.namespace + suffixPrioritySpanTemplate) - cfg.PriorityServiceTemplate = v.GetInt64(cfg.namespace + suffixPriorityServiceTemplate) - cfg.PriorityDependenciesTemplate = v.GetInt64(cfg.namespace + suffixPriorityDependenciesTemplate) + cfg.Indices.Spans.TemplateOptions.NumShards = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumShards), v.GetInt64(cfg.namespace+suffixNumShardsSpan)) + cfg.Indices.Services.TemplateOptions.NumShards = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumShards), v.GetInt64(cfg.namespace+suffixNumShardsServices)) + cfg.Indices.Sampling.TemplateOptions.NumShards = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumShards), v.GetInt64(cfg.namespace+suffixNumShardsSampling)) + cfg.Indices.Dependencies.TemplateOptions.NumShards = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumShards), v.GetInt64(cfg.namespace+suffixNumShardsDependencies)) + cfg.Indices.Spans.TemplateOptions.NumReplicas = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumReplicas), v.GetInt64(cfg.namespace+suffixNumReplicasSpan)) + cfg.Indices.Services.TemplateOptions.NumReplicas = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumReplicas), v.GetInt64(cfg.namespace+suffixNumReplicasServices)) + cfg.Indices.Sampling.TemplateOptions.NumReplicas = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumReplicas), v.GetInt64(cfg.namespace+suffixNumReplicasSampling)) + cfg.Indices.Dependencies.TemplateOptions.NumReplicas = overrideIndexShardsNums(v.GetInt64(cfg.namespace+suffixNumReplicas), v.GetInt64(cfg.namespace+suffixNumReplicasDependencies)) + cfg.Indices.Spans.TemplateOptions.Priority = v.GetInt64(cfg.namespace + suffixPrioritySpanTemplate) + cfg.Indices.Services.TemplateOptions.Priority = v.GetInt64(cfg.namespace + suffixPriorityServiceTemplate) + cfg.Indices.Dependencies.TemplateOptions.Priority = v.GetInt64(cfg.namespace + suffixPriorityDependenciesTemplate) cfg.BulkSize = v.GetInt(cfg.namespace + suffixBulkSize) cfg.BulkWorkers = v.GetInt(cfg.namespace + suffixBulkWorkers) cfg.BulkActions = v.GetInt(cfg.namespace + suffixBulkActions) @@ -372,8 +437,8 @@ func initFromViper(cfg *namespaceConfig, v *viper.Viper) { cfg.Indices.Services.DateLayout = initDateLayout(cfg.Indices.Services.RolloverFrequency, separator) cfg.Indices.Sampling.DateLayout = initDateLayout(cfg.Indices.Sampling.RolloverFrequency, separator) - // Dependencies calculation should be daily, and this index size is very small - cfg.Indices.Dependencies.DateLayout = initDateLayout(defaultIndexRolloverFrequency, separator) + // Daily is recommended for dependencies calculation, and this index size is very small + cfg.Indices.Dependencies.DateLayout = initDateLayout(cfg.Indices.Dependencies.RolloverFrequency, separator) var err error cfg.TLS, err = cfg.getTLSFlagsConfig().InitFromViper(v) if err != nil { @@ -417,20 +482,15 @@ func initDateLayout(rolloverFreq, sep string) string { func DefaultConfig() config.Configuration { return config.Configuration{ - Username: "", - Password: "", - Sniffer: false, - MaxSpanAge: 72 * time.Hour, - AdaptiveSamplingLookback: 72 * time.Hour, - NumShards: 5, - NumReplicas: 1, - PrioritySpanTemplate: 0, - PriorityServiceTemplate: 0, - PriorityDependenciesTemplate: 0, - BulkSize: 5 * 1000 * 1000, - BulkWorkers: 1, - BulkActions: 1000, - BulkFlushInterval: time.Millisecond * 200, + Username: "", + Password: "", + Sniffer: false, + MaxSpanAge: 72 * time.Hour, + AdaptiveSamplingLookback: 72 * time.Hour, + BulkSize: 5 * 1000 * 1000, + BulkWorkers: 1, + BulkActions: 1000, + BulkFlushInterval: time.Millisecond * 200, Tags: config.TagsAsFields{ DotReplacement: "@", },