polish

Signed-off-by: Jared Tan <jian.tan@daocloud.io>
jaegertracing · Aug 2, 2024 · 05d35bc · 05d35bc
1 parent 40ae1fb
commit 05d35bc
Show file tree

Hide file tree

Showing 18 changed files with 356 additions and 216 deletions.
diff --git a/cmd/jaeger/config-elasticsearch.yaml b/cmd/jaeger/config-elasticsearch.yaml
@@ -17,19 +17,19 @@ extensions:
       some_storage:
         elasticsearch:
           index_prefix: "jaeger-main"
-#          indices:
-#            spans:
-#              date_layout: "2006-01-02"
-#              rollover_frequency: "hour"
-#            services:
-#              date_layout: "2006-01-02"
-#              rollover_frequency: "hour"
-#            dependencies:
-#              date_layout: "2006-01-02"
-#              rollover_frequency: "hour"
-#            sampling:
-#              date_layout: "2006-01-02"
-#              rollover_frequency: "hour"
+          indices:
+            spans:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+            services:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+            dependencies:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
+            sampling:
+              date_layout: "2006-01-02"
+              rollover_frequency: "day"
       another_storage:
         elasticsearch:
           index_prefix: "jaeger-archive"

diff --git a/pkg/es/config/config.go b/pkg/es/config/config.go
@@ -44,10 +44,17 @@ import (
 	storageMetrics "github.com/jaegertracing/jaeger/storage/spanstore/metrics"
 )
 
+type TemplateOptions struct {
+	Priority    int64 `mapstructure:"priority"`
+	NumShards   int64 `mapstructure:"num_shards"`
+	NumReplicas int64 `mapstructure:"num_replicas"`
+}
+
 // IndexOptions describes the index format and rollover frequency
 type IndexOptions struct {
-	DateLayout        string `mapstructure:"date_layout"`
-	RolloverFrequency string `mapstructure:"rollover_frequency"`
+	DateLayout        string          `mapstructure:"date_layout"`
+	RolloverFrequency string          `mapstructure:"rollover_frequency"`
+	TemplateOptions   TemplateOptions `mapstructure:"template"`
 }
 
 // Indices describes different configuration options for each index type
@@ -60,40 +67,35 @@ type Indices struct {
 
 // Configuration describes the configuration properties needed to connect to an ElasticSearch cluster
 type Configuration struct {
-	Servers                      []string       `mapstructure:"server_urls" valid:"required,url"`
-	RemoteReadClusters           []string       `mapstructure:"remote_read_clusters"`
-	Username                     string         `mapstructure:"username"`
-	Password                     string         `mapstructure:"password" json:"-"`
-	TokenFilePath                string         `mapstructure:"token_file"`
-	PasswordFilePath             string         `mapstructure:"password_file"`
-	AllowTokenFromContext        bool           `mapstructure:"-"`
-	Sniffer                      bool           `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing
-	SnifferTLSEnabled            bool           `mapstructure:"sniffer_tls_enabled"`
-	MaxDocCount                  int            `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query
-	MaxSpanAge                   time.Duration  `mapstructure:"-"` // configures the maximum lookback on span reads
-	NumShards                    int64          `mapstructure:"num_shards"`
-	NumReplicas                  int64          `mapstructure:"num_replicas"`
-	PrioritySpanTemplate         int64          `mapstructure:"priority_span_template"`
-	PriorityServiceTemplate      int64          `mapstructure:"priority_service_template"`
-	PriorityDependenciesTemplate int64          `mapstructure:"priority_dependencies_template"`
-	Timeout                      time.Duration  `mapstructure:"-"`
-	BulkSize                     int            `mapstructure:"-"`
-	BulkWorkers                  int            `mapstructure:"-"`
-	BulkActions                  int            `mapstructure:"-"`
-	BulkFlushInterval            time.Duration  `mapstructure:"-"`
-	IndexPrefix                  string         `mapstructure:"index_prefix"`
-	Indices                      Indices        `mapstructure:"indices"`
-	ServiceCacheTTL              time.Duration  `mapstructure:"service_cache_ttl"`
-	AdaptiveSamplingLookback     time.Duration  `mapstructure:"-"`
-	Tags                         TagsAsFields   `mapstructure:"tags_as_fields"`
-	Enabled                      bool           `mapstructure:"-"`
-	TLS                          tlscfg.Options `mapstructure:"tls"`
-	UseReadWriteAliases          bool           `mapstructure:"use_aliases"`
-	CreateIndexTemplates         bool           `mapstructure:"create_mappings"`
-	UseILM                       bool           `mapstructure:"use_ilm"`
-	Version                      uint           `mapstructure:"version"`
-	LogLevel                     string         `mapstructure:"log_level"`
-	SendGetBodyAs                string         `mapstructure:"send_get_body_as"`
+	Servers                  []string       `mapstructure:"server_urls" valid:"required,url"`
+	RemoteReadClusters       []string       `mapstructure:"remote_read_clusters"`
+	Username                 string         `mapstructure:"username"`
+	Password                 string         `mapstructure:"password" json:"-"`
+	TokenFilePath            string         `mapstructure:"token_file"`
+	PasswordFilePath         string         `mapstructure:"password_file"`
+	AllowTokenFromContext    bool           `mapstructure:"-"`
+	Sniffer                  bool           `mapstructure:"sniffer"` // https://github.com/olivere/elastic/wiki/Sniffing
+	SnifferTLSEnabled        bool           `mapstructure:"sniffer_tls_enabled"`
+	MaxDocCount              int            `mapstructure:"-"` // Defines maximum number of results to fetch from storage per query
+	MaxSpanAge               time.Duration  `mapstructure:"-"` // configures the maximum lookback on span reads
+	Timeout                  time.Duration  `mapstructure:"-"`
+	BulkSize                 int            `mapstructure:"-"`
+	BulkWorkers              int            `mapstructure:"-"`
+	BulkActions              int            `mapstructure:"-"`
+	BulkFlushInterval        time.Duration  `mapstructure:"-"`
+	IndexPrefix              string         `mapstructure:"index_prefix"`
+	Indices                  Indices        `mapstructure:"indices"`
+	ServiceCacheTTL          time.Duration  `mapstructure:"service_cache_ttl"`
+	AdaptiveSamplingLookback time.Duration  `mapstructure:"-"`
+	Tags                     TagsAsFields   `mapstructure:"tags_as_fields"`
+	Enabled                  bool           `mapstructure:"-"`
+	TLS                      tlscfg.Options `mapstructure:"tls"`
+	UseReadWriteAliases      bool           `mapstructure:"use_aliases"`
+	CreateIndexTemplates     bool           `mapstructure:"create_mappings"`
+	UseILM                   bool           `mapstructure:"use_ilm"`
+	Version                  uint           `mapstructure:"version"`
+	LogLevel                 string         `mapstructure:"log_level"`
+	SendGetBodyAs            string         `mapstructure:"send_get_body_as"`
 }
 
 // TagsAsFields holds configuration for tag schema.
@@ -246,20 +248,20 @@ func (c *Configuration) ApplyDefaults(source *Configuration) {
 	if c.AdaptiveSamplingLookback == 0 {
 		c.AdaptiveSamplingLookback = source.AdaptiveSamplingLookback
 	}
-	if c.NumShards == 0 {
-		c.NumShards = source.NumShards
+	if c.Indices.Dependencies.TemplateOptions.NumShards == 0 {
+		c.Indices.Dependencies.TemplateOptions.NumShards = source.Indices.Dependencies.TemplateOptions.NumShards
 	}
-	if c.NumReplicas == 0 {
-		c.NumReplicas = source.NumReplicas
+	if c.Indices.Dependencies.TemplateOptions.NumReplicas == 0 {
+		c.Indices.Dependencies.TemplateOptions.NumReplicas = source.Indices.Dependencies.TemplateOptions.NumReplicas
 	}
-	if c.PrioritySpanTemplate == 0 {
-		c.PrioritySpanTemplate = source.PrioritySpanTemplate
+	if c.Indices.Spans.TemplateOptions.Priority == 0 {
+		c.Indices.Spans.TemplateOptions.Priority = source.Indices.Spans.TemplateOptions.Priority
 	}
-	if c.PriorityServiceTemplate == 0 {
-		c.PriorityServiceTemplate = source.PriorityServiceTemplate
+	if c.Indices.Services.TemplateOptions.Priority == 0 {
+		c.Indices.Services.TemplateOptions.Priority = source.Indices.Services.TemplateOptions.Priority
 	}
-	if c.PrioritySpanTemplate == 0 {
-		c.PriorityDependenciesTemplate = source.PriorityDependenciesTemplate
+	if c.Indices.Dependencies.TemplateOptions.Priority == 0 {
+		c.Indices.Dependencies.TemplateOptions.Priority = source.Indices.Dependencies.TemplateOptions.Priority
 	}
 	if c.BulkSize == 0 {
 		c.BulkSize = source.BulkSize

diff --git a/plugin/storage/es/factory.go b/plugin/storage/es/factory.go
@@ -325,14 +325,20 @@ func (f *Factory) CreateSamplingStore(int /* maxBuckets */) (samplingstore.Store
 func mappingBuilderFromConfig(cfg *config.Configuration) mappings.MappingBuilder {
 	return mappings.MappingBuilder{
 		TemplateBuilder:              es.TextTemplateBuilder{},
-		Shards:                       cfg.NumShards,
-		Replicas:                     cfg.NumReplicas,
+		ShardsSpan:                   cfg.Indices.Spans.TemplateOptions.NumShards,
+		ReplicasSpan:                 cfg.Indices.Spans.TemplateOptions.NumReplicas,
+		ShardsService:                cfg.Indices.Services.TemplateOptions.NumShards,
+		ReplicasService:              cfg.Indices.Services.TemplateOptions.NumShards,
+		ShardsSampling:               cfg.Indices.Sampling.TemplateOptions.NumShards,
+		ReplicasSampling:             cfg.Indices.Sampling.TemplateOptions.NumShards,
+		ShardsDependencies:           cfg.Indices.Dependencies.TemplateOptions.NumShards,
+		ReplicasDependencies:         cfg.Indices.Dependencies.TemplateOptions.NumShards,
 		EsVersion:                    cfg.Version,
 		IndexPrefix:                  cfg.IndexPrefix,
 		UseILM:                       cfg.UseILM,
-		PrioritySpanTemplate:         cfg.PrioritySpanTemplate,
-		PriorityServiceTemplate:      cfg.PriorityServiceTemplate,
-		PriorityDependenciesTemplate: cfg.PriorityDependenciesTemplate,
+		PrioritySpanTemplate:         cfg.Indices.Spans.TemplateOptions.Priority,
+		PriorityServiceTemplate:      cfg.Indices.Services.TemplateOptions.Priority,
+		PriorityDependenciesTemplate: cfg.Indices.Dependencies.TemplateOptions.Priority,
 	}
 }
 

diff --git a/plugin/storage/es/mappings/jaeger-dependencies-6.json b/plugin/storage/es/mappings/jaeger-dependencies-6.json
@@ -1,8 +1,8 @@
 {
   "template": "*jaeger-dependencies-*",
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsDependencies }},
+    "index.number_of_replicas": {{ .ReplicasDependencies }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true
   },

diff --git a/plugin/storage/es/mappings/jaeger-dependencies-7.json b/plugin/storage/es/mappings/jaeger-dependencies-7.json
@@ -6,8 +6,8 @@
   },
   {{- end }}
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsDependencies }},
+    "index.number_of_replicas": {{ .ReplicasDependencies }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true
   {{- if .UseILM }}

diff --git a/plugin/storage/es/mappings/jaeger-dependencies-8.json b/plugin/storage/es/mappings/jaeger-dependencies-8.json
@@ -8,8 +8,8 @@
     },
     {{- end }}
     "settings": {
-      "index.number_of_shards": {{ .Shards }},
-      "index.number_of_replicas": {{ .Replicas }},
+      "index.number_of_shards": {{ .ShardsDependencies }},
+      "index.number_of_replicas": {{ .ReplicasDependencies }},
       "index.mapping.nested_fields.limit": 50,
       "index.requests.cache.enable": true
       {{- if .UseILM }},

diff --git a/plugin/storage/es/mappings/jaeger-sampling-6.json b/plugin/storage/es/mappings/jaeger-sampling-6.json
@@ -1,8 +1,8 @@
 {
   "template": "*jaeger-sampling-*",
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsSampling }},
+    "index.number_of_replicas": {{ .ReplicasSampling }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":false
   },

diff --git a/plugin/storage/es/mappings/jaeger-sampling-7.json b/plugin/storage/es/mappings/jaeger-sampling-7.json
@@ -6,8 +6,8 @@
   },
   {{- end }}
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsSampling }},
+    "index.number_of_replicas": {{ .ReplicasSampling }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":false
   {{- if .UseILM }}

diff --git a/plugin/storage/es/mappings/jaeger-sampling-8.json b/plugin/storage/es/mappings/jaeger-sampling-8.json
@@ -8,8 +8,8 @@
     },
     {{- end }}
     "settings": {
-      "index.number_of_shards": {{ .Shards }},
-      "index.number_of_replicas": {{ .Replicas }},
+      "index.number_of_shards": {{ .ShardsSampling }},
+      "index.number_of_replicas": {{ .ReplicasSampling }},
       "index.mapping.nested_fields.limit": 50,
       "index.requests.cache.enable": false
       {{- if .UseILM }},

diff --git a/plugin/storage/es/mappings/jaeger-service-6.json b/plugin/storage/es/mappings/jaeger-service-6.json
@@ -1,8 +1,8 @@
 {
   "template": "*jaeger-service-*",
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsService }},
+    "index.number_of_replicas": {{ .ReplicasService }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true,
     "index.mapper.dynamic":false

diff --git a/plugin/storage/es/mappings/jaeger-service-7.json b/plugin/storage/es/mappings/jaeger-service-7.json
@@ -6,8 +6,8 @@
   },
   {{- end }}
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{  .Replicas }},
+    "index.number_of_shards": {{ .ShardsService }},
+    "index.number_of_replicas": {{  .ReplicasService }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true
   {{- if .UseILM }}

diff --git a/plugin/storage/es/mappings/jaeger-service-8.json b/plugin/storage/es/mappings/jaeger-service-8.json
@@ -8,8 +8,8 @@
     },
     {{- end }}
     "settings": {
-      "index.number_of_shards": {{ .Shards }},
-      "index.number_of_replicas": {{  .Replicas }},
+      "index.number_of_shards": {{ .ShardsService }},
+      "index.number_of_replicas": {{  .ReplicasService }},
       "index.mapping.nested_fields.limit": 50,
       "index.requests.cache.enable": true
       {{- if .UseILM }},

diff --git a/plugin/storage/es/mappings/jaeger-span-6.json b/plugin/storage/es/mappings/jaeger-span-6.json
@@ -1,8 +1,8 @@
 {
   "template": "*jaeger-span-*",
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsSpan }},
+    "index.number_of_replicas": {{ .ReplicasSpan }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true,
     "index.mapper.dynamic":false

diff --git a/plugin/storage/es/mappings/jaeger-span-7.json b/plugin/storage/es/mappings/jaeger-span-7.json
@@ -6,8 +6,8 @@
   },
   {{- end }}
   "settings":{
-    "index.number_of_shards": {{ .Shards }},
-    "index.number_of_replicas": {{ .Replicas }},
+    "index.number_of_shards": {{ .ShardsSpan }},
+    "index.number_of_replicas": {{ .ReplicasSpan }},
     "index.mapping.nested_fields.limit":50,
     "index.requests.cache.enable":true
     {{- if .UseILM }}

diff --git a/plugin/storage/es/mappings/jaeger-span-8.json b/plugin/storage/es/mappings/jaeger-span-8.json
@@ -9,8 +9,8 @@
     },
     {{- end}}
     "settings": {
-      "index.number_of_shards": {{ .Shards }},
-      "index.number_of_replicas": {{ .Replicas }},
+      "index.number_of_shards": {{ .ShardsSpan }},
+      "index.number_of_replicas": {{ .ReplicasSpan }},
       "index.mapping.nested_fields.limit": 50,
       "index.requests.cache.enable": true
       {{- if .UseILM }},

diff --git a/plugin/storage/es/mappings/mapping.go b/plugin/storage/es/mappings/mapping.go
@@ -30,8 +30,14 @@ var MAPPINGS embed.FS
 // MappingBuilder holds parameters required to render an elasticsearch index template
 type MappingBuilder struct {
 	TemplateBuilder              es.TemplateBuilder
-	Shards                       int64
-	Replicas                     int64
+	ShardsSpan                   int64
+	ReplicasSpan                 int64
+	ShardsService                int64
+	ReplicasService              int64
+	ShardsDependencies           int64
+	ReplicasDependencies         int64
+	ShardsSampling               int64
+	ReplicasSampling             int64
 	PrioritySpanTemplate         int64
 	PriorityServiceTemplate      int64
 	PriorityDependenciesTemplate int64