From 5314518c748c721ba6ac54eb30791bacab345b19 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 30 Nov 2022 09:32:31 +0530 Subject: [PATCH 01/24] feat: opamp server application --- go.mod | 8 +- go.sum | 14 +- pkg/query-service/app/opamp/config.yaml | 76 ++++++ pkg/query-service/app/opamp/logger.go | 15 ++ pkg/query-service/app/opamp/model/agent.go | 280 ++++++++++++++++++++ pkg/query-service/app/opamp/model/agents.go | 128 +++++++++ pkg/query-service/app/opamp/opamp_server.go | 86 ++++++ pkg/query-service/app/server.go | 23 ++ 8 files changed, 622 insertions(+), 8 deletions(-) create mode 100644 pkg/query-service/app/opamp/config.yaml create mode 100644 pkg/query-service/app/opamp/logger.go create mode 100644 pkg/query-service/app/opamp/model/agent.go create mode 100644 pkg/query-service/app/opamp/model/agents.go create mode 100644 pkg/query-service/app/opamp/opamp_server.go diff --git a/go.mod b/go.mod index b25925512c..132fbac4e4 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,7 @@ require ( github.com/minio/minio-go/v6 v6.0.57 github.com/mitchellh/mapstructure v1.5.0 github.com/oklog/oklog v0.3.2 + github.com/open-telemetry/opamp-go v0.5.0 github.com/pkg/errors v0.9.1 github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 @@ -35,7 +36,9 @@ require ( require ( github.com/beevik/etree v1.1.0 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/google/go-cmp v0.5.8 // indirect + github.com/gorilla/websocket v1.4.2 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/cpuid v1.2.3 // indirect @@ -63,7 +66,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/felixge/httpsnoop v1.0.1 // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/ghodss/yaml v1.0.0 // indirect github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c github.com/go-logfmt/logfmt v0.5.0 // indirect @@ -139,9 +141,9 @@ require ( google.golang.org/api v0.51.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20211013025323-ce878158c4d4 // indirect - google.golang.org/grpc v1.41.0 + google.golang.org/grpc v1.42.0 google.golang.org/grpc/examples v0.0.0-20210803221256-6ba56c814be7 // indirect - google.golang.org/protobuf v1.27.1 // indirect + google.golang.org/protobuf v1.27.1 gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 diff --git a/go.sum b/go.sum index 0782dacd7d..2b888df26c 100644 --- a/go.sum +++ b/go.sum @@ -48,7 +48,6 @@ github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSW github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.3 h1:Vok8zUb/wlqc9u8oEqQzBMBRDoFd8NxPRqgYEqMnV88= github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/ClickHouse/clickhouse-go/v2 v2.0.12 h1:Nbl/NZwoM6LGJm7smNBgvtdr/rxjlIssSW3eG/Nmb9E= github.com/ClickHouse/clickhouse-go/v2 v2.0.12/go.mod h1:u4RoNQLLM2W6hNSPYrIESLJqaWSInZVmfM+MlaAhXcg= @@ -91,8 +90,11 @@ github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f/go.mod h1:xeT/CQ0qZHangbYbWShlCGAx31aV4AjGswDUjhKS6HQ= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -247,6 +249,8 @@ github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosimple/slug v1.10.0 h1:3XbiQua1IpCdrvuntWvGBxVm+K99wCSxJjlxkP49GGQ= github.com/gosimple/slug v1.10.0/go.mod h1:MICb3w495l9KNdZm+Xn5b6T2Hn831f9DMxiJ1r+bAjw= github.com/gosimple/unidecode v1.0.0 h1:kPdvM+qy0tnk4/BrnkrbdJ82xe88xn7c9hcaipDz4dQ= @@ -369,6 +373,8 @@ github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v0.3.1-0.20170117200651-66bb6560562f h1:tt7Qj+4Pic1KiUqT7XNMnbAE3TLJAGH+5LMuX4roYbE= github.com/oklog/ulid v0.3.1-0.20170117200651-66bb6560562f/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/open-telemetry/opamp-go v0.5.0 h1:2YFbb6G4qBkq3yTRdVb5Nfz9hKHW/ldUyex352e1J7g= +github.com/open-telemetry/opamp-go v0.5.0/go.mod h1:IMdeuHGVc5CjKSu5/oNV0o+UmiXuahoHvoZ4GOmAI9M= github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/opentracing-go v1.0.1/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= @@ -381,7 +387,6 @@ github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible h1:FhnA4iH8T/yYW+AolPONZjGE897wxj3MAzfEbrZkSYw= github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -644,7 +649,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211110154304-99a53858aa08 h1:WecRHqgE09JBkh/584XIE6PMz5KKE/vER4izNUi30AQ= @@ -841,8 +845,8 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0 h1:f+PlOh7QV4iIJkPrx5NQ7qaNGFQ3OTse67yaDHfju4E= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= +google.golang.org/grpc v1.42.0 h1:XT2/MFpuPFsEX2fWh3YQtHkZ+WYZFQRfaUgLZYj/p6A= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/grpc/examples v0.0.0-20210803221256-6ba56c814be7 h1:k3XsiLoPLXhNlZJy1sHKvkgGEfpMk8bsdJVHKKhTdrc= google.golang.org/grpc/examples v0.0.0-20210803221256-6ba56c814be7/go.mod h1:bF8wuZSAZTcbF7ZPKrDI/qY52toTP/yxLpRRY4Eu9Js= diff --git a/pkg/query-service/app/opamp/config.yaml b/pkg/query-service/app/opamp/config.yaml new file mode 100644 index 0000000000..d5ef74e00f --- /dev/null +++ b/pkg/query-service/app/opamp/config.yaml @@ -0,0 +1,76 @@ +receivers: + otlp/spanmetrics: + protocols: + grpc: + endpoint: "localhost:12345" + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: + hostmetrics: + collection_interval: 30s + scrapers: + cpu: + load: + memory: + disk: + filesystem: + network: +processors: + batch: + send_batch_size: 1000 + timeout: 10s + signozspanmetrics/prometheus: + metrics_exporter: prometheus + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + dimensions_cache_size: 10000 + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + # memory_limiter: + # # 80% of maximum memory up to 2G + # limit_mib: 1500 + # # 25% of limit up to 2G + # spike_limit_mib: 512 + # check_interval: 5s + # + # # 50% of the maximum memory + # limit_percentage: 50 + # # 20% of max memory usage spike expected + # spike_limit_percentage: 20 + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + zpages: {} +exporters: + clickhousetraces: + datasource: tcp://localhost:9000/?database=signoz_traces + migrations: exporter/clickhousetracesexporter/migrations + clickhousemetricswrite: + endpoint: tcp://localhost:9000/?database=signoz_metrics + resource_to_telemetry_conversion: + enabled: true + prometheus: + endpoint: "0.0.0.0:8889" +service: + extensions: [zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhousetraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] + metrics/spanmetrics: + receivers: [otlp/spanmetrics] + exporters: [prometheus] \ No newline at end of file diff --git a/pkg/query-service/app/opamp/logger.go b/pkg/query-service/app/opamp/logger.go new file mode 100644 index 0000000000..49f9e630fc --- /dev/null +++ b/pkg/query-service/app/opamp/logger.go @@ -0,0 +1,15 @@ +package opamp + +import "log" + +type Logger struct { + logger *log.Logger +} + +func (l *Logger) Debugf(format string, v ...interface{}) { + l.logger.Printf(format, v...) +} + +func (l *Logger) Errorf(format string, v ...interface{}) { + l.logger.Printf(format, v...) +} diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go new file mode 100644 index 0000000000..7e96fc1caf --- /dev/null +++ b/pkg/query-service/app/opamp/model/agent.go @@ -0,0 +1,280 @@ +package data + +import ( + "bytes" + "context" + "crypto/sha256" + "sync" + "time" + + "google.golang.org/protobuf/proto" + + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/open-telemetry/opamp-go/server/types" +) + +type InstanceId string + +type Agent struct { + Id InstanceId `json:"instanceId" yaml:"instanceId" db:"instance_id"` + Status *protobufs.AgentToServer `json:"status" yaml:"status" db:"status"` + StartedAt time.Time `json:"startedAt" yaml:"startedAt" db:"started_at"` + EffectiveConfig string `json:"effectiveConfig" yaml:"effectiveConfig" db:"effective_config"` + remoteConfig *protobufs.AgentRemoteConfig + + conn types.Connection + connMutex sync.Mutex + mux sync.RWMutex +} + +func NewAgent( + instanceId InstanceId, + conn types.Connection, +) *Agent { + result, err := db.Exec("INSERT INTO agents (instance_id, status, started_at, effective_config) VALUES (?, ?, ?, ?)", instanceId, nil, time.Now(), "") + if err != nil { + panic(err) + } + rowsAffected, err := result.RowsAffected() + if err != nil { + panic(err) + } + if rowsAffected != 1 { + panic("expected 1 row to be affected") + } + return &Agent{Id: instanceId, conn: conn} +} + +func (agent *Agent) UpdateStatus( + statusMsg *protobufs.AgentToServer, + response *protobufs.ServerToAgent, +) { + // TODO: update status in the database. + agent.mux.Lock() + defer agent.mux.Unlock() + agent.processStatusUpdate(statusMsg, response) +} + +func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) (agentDescrChanged bool) { + prevStatus := agent.Status + + if agent.Status == nil { + // First time this Agent reports a status, remember it. + agent.Status = newStatus + agentDescrChanged = true + } else { + // Not a new Agent. Update the Status. + agent.Status.SequenceNum = newStatus.SequenceNum + + // Check what's changed in the AgentDescription. + if newStatus.AgentDescription != nil { + // If the AgentDescription field is set it means the Agent tells us + // something is changed in the field since the last status report + // (or this is the first report). + // Make full comparison of previous and new descriptions to see if it + // really is different. + if prevStatus != nil && proto.Equal(prevStatus.AgentDescription, newStatus.AgentDescription) { + // Agent description didn't change. + agentDescrChanged = false + } else { + // Yes, the description is different, update it. + agent.Status.AgentDescription = newStatus.AgentDescription + agentDescrChanged = true + } + } else { + // AgentDescription field is not set, which means description didn't change. + agentDescrChanged = false + } + + // Update remote config status if it is included and is different from what we have. + if newStatus.RemoteConfigStatus != nil && + !proto.Equal(agent.Status.RemoteConfigStatus, newStatus.RemoteConfigStatus) { + agent.Status.RemoteConfigStatus = newStatus.RemoteConfigStatus + } + } + return agentDescrChanged +} + +func (agent *Agent) updateHealth(newStatus *protobufs.AgentToServer) { + if newStatus.Health == nil { + return + } + + agent.Status.Health = newStatus.Health + + if agent.Status != nil && agent.Status.Health != nil && agent.Status.Health.Healthy { + agent.StartedAt = time.Unix(0, int64(agent.Status.Health.StartTimeUnixNano)).UTC() + } +} + +func (agent *Agent) updateRemoteConfigStatus(newStatus *protobufs.AgentToServer) { + // Update remote config status if it is included and is different from what we have. + if newStatus.RemoteConfigStatus != nil { + agent.Status.RemoteConfigStatus = newStatus.RemoteConfigStatus + } +} + +func (agent *Agent) updateStatusField(newStatus *protobufs.AgentToServer) (agentDescrChanged bool) { + if agent.Status == nil { + // First time this Agent reports a status, remember it. + agent.Status = newStatus + agentDescrChanged = true + } + + agentDescrChanged = agent.updateAgentDescription(newStatus) || agentDescrChanged + agent.updateRemoteConfigStatus(newStatus) + agent.updateHealth(newStatus) + + // TODO: update status in the database. + + return agentDescrChanged +} + +func (agent *Agent) updateEffectiveConfig( + newStatus *protobufs.AgentToServer, + response *protobufs.ServerToAgent, +) { + // Update effective config if provided. + if newStatus.EffectiveConfig != nil { + if newStatus.EffectiveConfig.ConfigMap != nil { + agent.Status.EffectiveConfig = newStatus.EffectiveConfig + + // Convert to string for displaying purposes. + agent.EffectiveConfig = "" + for _, cfg := range newStatus.EffectiveConfig.ConfigMap.ConfigMap { + // TODO: we just concatenate parts of effective config as a single + // blob to show in the UI. A proper approach is to keep the effective + // config as a set and show the set in the UI. + agent.EffectiveConfig = agent.EffectiveConfig + string(cfg.Body) + } + } + } +} + +func (agent *Agent) hasCapability(capability protobufs.AgentCapabilities) bool { + return agent.Status.Capabilities&uint64(capability) != 0 +} + +func (agent *Agent) processStatusUpdate( + newStatus *protobufs.AgentToServer, + response *protobufs.ServerToAgent, +) { + // We don't have any status for this Agent, or we lost the previous status update from the Agent, so our + // current status is not up-to-date. + lostPreviousUpdate := (agent.Status == nil) || (agent.Status != nil && agent.Status.SequenceNum+1 != newStatus.SequenceNum) + + agentDescrChanged := agent.updateStatusField(newStatus) + + // Check if any fields were omitted in the status report. + effectiveConfigOmitted := newStatus.EffectiveConfig == nil && + agent.hasCapability(protobufs.AgentCapabilities_AgentCapabilities_ReportsEffectiveConfig) + + remoteConfigStatusOmitted := newStatus.RemoteConfigStatus == nil && + agent.hasCapability(protobufs.AgentCapabilities_AgentCapabilities_ReportsRemoteConfig) + + healthOmitted := newStatus.Health == nil && + agent.hasCapability(protobufs.AgentCapabilities_AgentCapabilities_ReportsHealth) + + // True if the status was not fully reported. + statusIsCompressed := effectiveConfigOmitted || remoteConfigStatusOmitted || healthOmitted + + if statusIsCompressed && lostPreviousUpdate { + // The status message is not fully set in the message that we received, but we lost the previous + // status update. Request full status update from the agent. + response.Flags |= uint64(protobufs.ServerToAgentFlags_ServerToAgentFlags_ReportFullState) + } + + configChanged := false + if agentDescrChanged { + // Agent description is changed. + + // We need to recalculate the config. + configChanged = agent.updateRemoteConfig() + } + + // If remote config is changed and different from what the Agent has then + // send the new remote config to the Agent. + if configChanged || + (agent.Status.RemoteConfigStatus != nil && + bytes.Compare(agent.Status.RemoteConfigStatus.LastRemoteConfigHash, agent.remoteConfig.ConfigHash) != 0) { + // The new status resulted in a change in the config of the Agent or the Agent + // does not have this config (hash is different). Send the new config the Agent. + response.RemoteConfig = agent.remoteConfig + } + + agent.updateEffectiveConfig(newStatus, response) +} + +func (agent *Agent) updateRemoteConfig() bool { + hash := sha256.New() + + cfg := protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{}, + }, + } + + // Calculate the hash. + for k, v := range cfg.Config.ConfigMap { + hash.Write([]byte(k)) + hash.Write(v.Body) + hash.Write([]byte(v.ContentType)) + } + + cfg.ConfigHash = hash.Sum(nil) + + configChanged := !isEqualRemoteConfig(agent.remoteConfig, &cfg) + + agent.remoteConfig = &cfg + + return configChanged +} + +func isEqualRemoteConfig(c1, c2 *protobufs.AgentRemoteConfig) bool { + if c1 == c2 { + return true + } + if c1 == nil || c2 == nil { + return false + } + return isEqualConfigSet(c1.Config, c2.Config) +} + +func isEqualConfigSet(c1, c2 *protobufs.AgentConfigMap) bool { + if c1 == c2 { + return true + } + if c1 == nil || c2 == nil { + return false + } + if len(c1.ConfigMap) != len(c2.ConfigMap) { + return false + } + for k, v1 := range c1.ConfigMap { + v2, ok := c2.ConfigMap[k] + if !ok { + return false + } + if !isEqualConfigFile(v1, v2) { + return false + } + } + return true +} + +func isEqualConfigFile(f1, f2 *protobufs.AgentConfigFile) bool { + if f1 == f2 { + return true + } + if f1 == nil || f2 == nil { + return false + } + return bytes.Compare(f1.Body, f2.Body) == 0 && f1.ContentType == f2.ContentType +} + +func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { + agent.connMutex.Lock() + defer agent.connMutex.Unlock() + + agent.conn.Send(context.Background(), msg) +} diff --git a/pkg/query-service/app/opamp/model/agents.go b/pkg/query-service/app/opamp/model/agents.go new file mode 100644 index 0000000000..52967c1f04 --- /dev/null +++ b/pkg/query-service/app/opamp/model/agents.go @@ -0,0 +1,128 @@ +package data + +import ( + "fmt" + "sync" + + "github.com/jmoiron/sqlx" + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/open-telemetry/opamp-go/server/types" +) + +var db *sqlx.DB + +func InitDB(dataSourceName string) (*sqlx.DB, error) { + var err error + + db, err = sqlx.Open("sqlite3", dataSourceName) + if err != nil { + return nil, err + } + + table_schema := `CREATE TABLE IF NOT EXISTS agents ( + instance_id INTEGER PRIMARY KEY UNIQUE, + status TEXT NOT NULL, + effective_config TEXT NOT NULL + );` + + _, err = db.Exec(table_schema) + if err != nil { + return nil, fmt.Errorf("Error in creating agents table: %s", err.Error()) + } + + rows, err := db.Query("SELECT * FROM agents") + if err != nil { + return nil, fmt.Errorf("Error in querying agents table: %s", err.Error()) + } + defer rows.Close() + + agents := &Agents{ + agentsById: map[InstanceId]*Agent{}, + connections: map[types.Connection]map[InstanceId]bool{}, + mux: sync.RWMutex{}, + } + + for rows.Next() { + var agent Agent + err = rows.Scan(&agent.Id, &agent.Status, &agent.EffectiveConfig) + if err != nil { + return nil, fmt.Errorf("Error in scanning agents table: %s", err.Error()) + } + agents.agentsById[agent.Id] = &agent + } + + AllAgents = *agents + + return db, nil +} + +type Agents struct { + mux sync.RWMutex + agentsById map[InstanceId]*Agent + connections map[types.Connection]map[InstanceId]bool +} + +// RemoveConnection removes the connection all Agent instances associated with the +// connection. +func (agents *Agents) RemoveConnection(conn types.Connection) { + agents.mux.Lock() + defer agents.mux.Unlock() + + for instanceId := range agents.connections[conn] { + delete(agents.agentsById, instanceId) + } + delete(agents.connections, conn) + + // TODO: remove agent from database? +} + +func (agents *Agents) FindAgent(agentId InstanceId) *Agent { + agents.mux.RLock() + defer agents.mux.RUnlock() + return agents.agentsById[agentId] +} + +func (agents *Agents) FindOrCreateAgent(agentId InstanceId, conn types.Connection) *Agent { + agents.mux.Lock() + defer agents.mux.Unlock() + + // Ensure the Agent is in the agentsById map. + agent := agents.agentsById[agentId] + if agent == nil { + agent = NewAgent(agentId, conn) + agents.agentsById[agentId] = agent + + // Ensure the Agent's instance id is associated with the connection. + if agents.connections[conn] == nil { + agents.connections[conn] = map[InstanceId]bool{} + } + agents.connections[conn][agentId] = true + } + + return agent +} + +func (agents *Agents) UpdateAgent(agentId InstanceId, status *protobufs.AgentToServer, effectiveConfig string) error { + agents.mux.Lock() + defer agents.mux.Unlock() + + agent := agents.agentsById[agentId] + if agent == nil { + return fmt.Errorf("Agent with id %s not found", agentId) + } + + agent.Status = status + agent.EffectiveConfig = effectiveConfig + + _, err := db.Exec("UPDATE agents SET status = ?, effective_config = ? WHERE instance_id = ?", status, effectiveConfig, agentId) + if err != nil { + return fmt.Errorf("Error in updating agents table: %s", err.Error()) + } + + return nil +} + +var AllAgents = Agents{ + agentsById: map[InstanceId]*Agent{}, + connections: map[types.Connection]map[InstanceId]bool{}, +} diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go new file mode 100644 index 0000000000..d3d2889b95 --- /dev/null +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -0,0 +1,86 @@ +package opamp + +import ( + "context" + "log" + "os" + + "github.com/open-telemetry/opamp-go/protobufs" + "github.com/open-telemetry/opamp-go/server" + "github.com/open-telemetry/opamp-go/server/types" + model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" +) + +type Server struct { + server server.OpAMPServer + agents *model.Agents + cnt int +} + +func NewServer(agents *model.Agents) *Server { + srv := &Server{ + agents: agents, + } + + srv.server = server.New(&Logger{log.New(os.Stdout, "opamp: ", log.LstdFlags)}) + + return srv +} + +func (srv *Server) Start() { + settings := server.StartSettings{ + Settings: server.Settings{ + Callbacks: server.CallbacksStruct{ + OnMessageFunc: srv.onMessage, + OnConnectionCloseFunc: srv.onDisconnect, + }, + }, + ListenEndpoint: "127.0.0.1:4320", + } + + srv.server.Start(settings) +} + +func (srv *Server) Stop() { + srv.server.Stop(context.Background()) +} + +func (srv *Server) onDisconnect(conn types.Connection) { + srv.agents.RemoveConnection(conn) +} + +func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { + instanceId := model.InstanceId(msg.InstanceUid) + + agent := srv.agents.FindOrCreateAgent(instanceId, conn) + + // FIXME: This is a hack to get the agent to send the initial config + f, err := os.ReadFile("./server/config.yaml") + if err != nil { + panic(err) + } + var response *protobufs.ServerToAgent + response = &protobufs.ServerToAgent{} + if srv.cnt <= 2 { + + // Start building the response. + response = &protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: f, + }, + }, + }, + ConfigHash: []byte("123"), + }, + } + srv.cnt++ + } + // Update the agent's status. + + agent.UpdateStatus(msg, response) + + return response +} diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index b79b7d011f..4d3807ef95 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -17,6 +17,9 @@ import ( "github.com/soheilhy/cmux" "go.signoz.io/signoz/pkg/query-service/app/clickhouseReader" "go.signoz.io/signoz/pkg/query-service/app/dashboards" + opamp "go.signoz.io/signoz/pkg/query-service/app/opamp" + opAmpModel "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/dao" "go.signoz.io/signoz/pkg/query-service/featureManager" @@ -57,6 +60,9 @@ type Server struct { privateHTTP *http.Server unavailableChannel chan healthcheck.Status + + // opamp server + opampServer *opamp.Server } // HealthCheckStatus returns health check status channel a client can subscribe to @@ -135,9 +141,24 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { s.privateHTTP = privateServer + localDB, err = opAmpModel.InitDB(constants.RELATIONAL_DATASOURCE_PATH) + if err != nil { + return nil, err + } + + opampServer, err := s.createOpampServer() + if err != nil { + return nil, err + } + s.opampServer = opampServer + return s, nil } +func (s *Server) createOpampServer() (*opamp.Server, error) { + return opamp.NewServer(&opAmpModel.Agents{}), nil +} + func (s *Server) createPrivateServer(api *APIHandler) (*http.Server, error) { r := NewRouter() @@ -362,6 +383,8 @@ func (s *Server) Start() error { }() + s.opampServer.Start() + return nil } From 7ef38ec8381c3710eca3402b8c553286ed63e75a Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Sat, 24 Dec 2022 00:43:10 +0530 Subject: [PATCH 02/24] chore: opamp --- go.mod | 45 ++-- go.sum | 243 +++++++++++++++--- .../app/clickhouseReader/reader.go | 2 +- pkg/query-service/app/opamp/model/agent.go | 27 +- pkg/query-service/app/opamp/model/agents.go | 37 +-- pkg/query-service/app/opamp/opamp_server.go | 84 ++++-- pkg/query-service/app/server.go | 2 +- pkg/query-service/pqlEngine/engine.go | 2 +- 8 files changed, 325 insertions(+), 117 deletions(-) diff --git a/go.mod b/go.mod index 9b782983f7..4865601a07 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/gosimple/slug v1.10.0 github.com/jmoiron/sqlx v1.3.4 github.com/json-iterator/go v1.1.12 + github.com/knadh/koanf v1.4.4 github.com/mailru/easyjson v0.7.7 github.com/mattn/go-sqlite3 v1.14.8 github.com/minio/minio-go/v6 v6.0.57 @@ -21,7 +22,7 @@ require ( github.com/open-telemetry/opamp-go v0.5.0 github.com/pkg/errors v0.9.1 github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f - github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 + github.com/prometheus/common v0.26.0 github.com/prometheus/prometheus v2.5.0+incompatible github.com/rs/cors v1.7.0 github.com/russellhaering/gosaml2 v0.8.0 @@ -29,7 +30,8 @@ require ( github.com/sethvargo/go-password v0.2.0 github.com/smartystreets/goconvey v1.6.4 github.com/soheilhy/cmux v0.1.4 - go.uber.org/zap v1.16.0 + go.opentelemetry.io/collector/confmap v0.68.0 + go.uber.org/zap v1.17.0 gopkg.in/segmentio/analytics-go.v3 v3.1.0 k8s.io/apimachinery v0.0.0-20180621070125-103fd098999d ) @@ -37,22 +39,26 @@ require ( require ( cloud.google.com/go/compute v1.7.0 // indirect github.com/beevik/etree v1.1.0 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/form3tech-oss/jwt-go v3.2.2+incompatible // indirect - github.com/fsnotify/fsnotify v1.4.9 // indirect github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect github.com/klauspost/cpuid v1.2.3 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/minio/md5-simd v1.1.0 // indirect github.com/minio/sha256-simd v0.1.1 // indirect + github.com/mitchellh/copystructure v1.2.0 // indirect + github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v0.9.0-pre1.0.20181001174001-0a8115f42e03 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20 // indirect + go.opentelemetry.io/collector/featuregate v0.68.0 // indirect gopkg.in/ini.v1 v1.42.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( @@ -69,9 +75,8 @@ require ( github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/ghodss/yaml v1.0.0 // indirect - github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c + github.com/go-kit/kit v0.9.0 github.com/go-logfmt/logfmt v0.5.0 // indirect - github.com/go-stack/stack v1.8.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect @@ -88,50 +93,44 @@ require ( github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/consul v1.1.1-0.20180615161029-bed22a81e9fd // indirect - github.com/hashicorp/go-cleanhttp v0.5.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-msgpack v1.1.5 // indirect - github.com/hashicorp/go-multierror v1.1.0 // indirect - github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect - github.com/hashicorp/go-sockaddr v1.0.0 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/golang-lru v0.5.1 // indirect - github.com/hashicorp/memberlist v0.1.0 // indirect - github.com/hashicorp/serf v0.8.1-0.20161007004122-1d4fa605f6ff // indirect + github.com/hashicorp/serf v0.9.6 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/lib/pq v1.10.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect - github.com/miekg/dns v1.0.4 // indirect + github.com/miekg/dns v1.1.41 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 // indirect + github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect github.com/oklog/run v1.1.0 // indirect github.com/oklog/ulid v0.3.1-0.20170117200651-66bb6560562f // indirect github.com/opentracing/opentracing-go v1.1.0 - github.com/pascaldekloe/goe v0.1.0 // indirect github.com/paulmach/orb v0.4.0 // indirect github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.14 // indirect - github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect - github.com/prometheus/procfs v0.0.8 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/procfs v0.6.0 // indirect github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da // indirect - github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/segmentio/backo-go v1.0.0 // indirect github.com/shopspring/decimal v1.3.1 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/smartystreets/assertions v1.1.0 github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/pflag v1.0.3 // indirect - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.8.1 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect go.opencensus.io v0.23.0 // indirect go.opentelemetry.io/otel v1.4.1 // indirect go.opentelemetry.io/otel/trace v1.4.1 // indirect - go.uber.org/atomic v1.6.0 // indirect - go.uber.org/multierr v1.5.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + go.uber.org/multierr v1.9.0 // indirect golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094 diff --git a/go.sum b/go.sum index 7d92ec2e19..c2388574e4 100644 --- a/go.sum +++ b/go.sum @@ -27,7 +27,6 @@ cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= -cloud.google.com/go v0.102.0 h1:DAq3r8y4mDgyB/ZPJ9v/5VJNqjgJAxTn6ZYLlUywOu8= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= @@ -60,10 +59,8 @@ github.com/Azure/azure-sdk-for-go v5.0.0-beta.0.20161028183111-bd73d950fa44+inco github.com/Azure/azure-sdk-for-go v5.0.0-beta.0.20161028183111-bd73d950fa44+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-autorest v10.8.1+incompatible h1:u0jVQf+a6k6x8A+sT60l6EY9XZu+kHdnZVPAYqpVRo0= github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ClickHouse/clickhouse-go v1.5.3 h1:Vok8zUb/wlqc9u8oEqQzBMBRDoFd8NxPRqgYEqMnV88= github.com/ClickHouse/clickhouse-go v1.5.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= github.com/ClickHouse/clickhouse-go/v2 v2.0.12 h1:Nbl/NZwoM6LGJm7smNBgvtdr/rxjlIssSW3eG/Nmb9E= github.com/ClickHouse/clickhouse-go/v2 v2.0.12/go.mod h1:u4RoNQLLM2W6hNSPYrIESLJqaWSInZVmfM+MlaAhXcg= @@ -76,28 +73,47 @@ github.com/SigNoz/prometheus v1.9.77 h1:We3pxbCnCRWQE7tSELBkcmneiXRtcpOlXrGtLPVQ github.com/SigNoz/prometheus v1.9.77/go.mod h1:Y4J9tGDmacMC+EcOTp+EIAn2C1sN+9kE+idyVKadiVM= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= github.com/auth0/go-jwt-middleware v1.0.1 h1:/fsQ4vRr4zod1wKReUH+0A3ySRjGiT9G34kypO/EKwI= github.com/auth0/go-jwt-middleware v1.0.1/go.mod h1:YSeUX3z6+TF2H+7padiEqNJ73Zy9vXW72U//IgN0BIM= github.com/aws/aws-sdk-go v1.13.44-0.20180507225419-00862f899353/go.mod h1:ZRmQr0FajVIyZ4ZzBYKG5P3ZqPz9IHG41ZoMu1ADI3k= github.com/aws/aws-sdk-go v1.27.0 h1:0xphMHGMLBrPMfxR2AmVjZKcMEESEgWF8Kru94BNByk= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go-v2 v1.9.2/go.mod h1:cK/D0BBs0b/oWPIcX/Z/obahJK1TT7IPVjy53i/mX/4= +github.com/aws/aws-sdk-go-v2/config v1.8.3/go.mod h1:4AEiLtAb8kLs7vgw2ZV3p2VZ1+hBavOc84hqxVNpCyw= +github.com/aws/aws-sdk-go-v2/credentials v1.4.3/go.mod h1:FNNC6nQZQUuyhq5aE5c7ata8o9e4ECGmS4lAXC7o1mQ= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.6.0/go.mod h1:gqlclDEZp4aqJOancXK6TN24aKhT0W0Ae9MHk3wzTMM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.4/go.mod h1:ZcBrrI3zBKlhGFNYWvju0I3TR93I7YIgAfy82Fh4lcQ= +github.com/aws/aws-sdk-go-v2/service/appconfig v1.4.2/go.mod h1:FZ3HkCe+b10uFZZkFdvf98LHW21k49W8o8J366lqVKY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72HRZDLMtmVQiLG2tLfQcaWLCssELvGl+Zf2WVxMmR8= +github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= +github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= +github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/beorn7/perks v0.0.0-20160229213445-3ac7bf7a47d1/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v0.0.0-20161118035902-4a94f899c20b/go.mod h1:fX/lfQBkSCDXZSUgv6jVIu/EVA3/JNseAX5asI4c4T4= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -117,6 +133,8 @@ github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68m github.com/cockroachdb/cockroach v0.0.0-20170608034007-84bc9597164f/go.mod h1:xeT/CQ0qZHangbYbWShlCGAx31aV4AjGswDUjhKS6HQ= github.com/coreos/go-oidc/v3 v3.4.0 h1:xz7elHb/LDwm/ERpwHd+5nb7wFHL32rsr6bBOgaeu6g= github.com/coreos/go-oidc/v3 v3.4.0/go.mod h1:eHUXhZtXPQLgEaDrOVTgwbgmz1xGOkJNye6h3zkD2Pw= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -137,6 +155,9 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.m github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= @@ -149,11 +170,15 @@ github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-ini/ini v1.21.1/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= -github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c h1:lGtNy7NU/+ytYPPneoErOaNrYkF5DOVCYViUK/7t7XA= github.com/go-kit/kit v0.4.1-0.20170517165212-6964666de57c/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0 h1:wDJmvq38kDhkVxi50ni9ykkdUr1PKgqKOoi01fa0Mdk= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-ldap/ldap v3.0.2+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -165,6 +190,9 @@ github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LB github.com/go-stack/stack v1.5.4/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= @@ -205,6 +233,7 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20160529050041-d9eb7a3d35ec/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -287,36 +316,64 @@ github.com/gosimple/unidecode v1.0.0/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOn github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA= github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul v1.1.1-0.20180615161029-bed22a81e9fd h1:u6o+bd6FHxDKoCSa8PJ5vrHhAYSKgJtAHQtLO1EYgos= github.com/hashicorp/consul v1.1.1-0.20180615161029-bed22a81e9fd/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/consul/api v1.13.0/go.mod h1:ZlVrynguJKcYr54zGaDbaL3fOvKC9m72FhPvA8T35KQ= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.0.0-20160407174126-ad28ea4487f0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= +github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= -github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 h1:VBj0QYQ0u2MCJzBfeYXGexnAl17GsH1yidnoxCqqD9E= +github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= +github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= -github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/golang-lru v0.0.0-20180201235237-0fb14efe8c47/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8= -github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= -github.com/hashicorp/serf v0.8.1-0.20161007004122-1d4fa605f6ff h1:epPiU3hEuHbpThFTQSGbdBBJemXM7aNQIU1thmpucTU= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.3.0 h1:8+567mCcFDnS5ADl7lrpxPMWiFCElyUEeW0gtj34fMA= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/hashicorp/serf v0.8.1-0.20161007004122-1d4fa605f6ff/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= +github.com/hashicorp/serf v0.9.6 h1:uuEX1kLR6aoda1TBttmJQKDLZE1Ob7KN0NPdE7EtCDc= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= +github.com/hashicorp/vault/api v1.0.4/go.mod h1:gDcqh3WGcR1cpF5AJz/B1UFheUEneMoIospckxBxk6Q= +github.com/hashicorp/vault/sdk v0.1.13/go.mod h1:B+hVj7TpuQY1Y/GPbCpffmgd+tSEwvhkWnjtSYCaS2M= +github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hashicorp/yamux v0.0.0-20181012175058-2f1d1f20f75d/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM= +github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= +github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/influxdata/influxdb v1.2.3-0.20170331210902-15e594fc09f1/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= @@ -329,12 +386,19 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -342,13 +406,19 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.1.1-0.20150905172533-109e267447e9/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/knadh/koanf v1.4.4 h1:d2jY5nCCeoaiqvEKSBW9rEc93EfNy/XWgWsSB3j7JEA= +github.com/knadh/koanf v1.4.4/go.mod h1:Hgyjp4y8v44hpZtPzs7JZfRAW5AhN7KfZcwv1RYggDs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -364,84 +434,129 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= github.com/mattermost/xml-roundtrip-validator v0.1.0/go.mod h1:qccnGMcpgwcNaBnxqpJpWWUiPNr5H3O8eDgGV9gT5To= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.8 h1:gDp86IdQsN/xWjIEmr9MF6o9mpksUgh0fu+9ByFxzIU= github.com/mattn/go-sqlite3 v1.14.8/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v0.0.0-20150406173934-fc2b8d3a73c4/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/miekg/dns v1.0.4 h1:Ec3LTJwwzqT1++63P12fhtdEbQhtPE7TBdD6rlhqrMM= github.com/miekg/dns v1.0.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= github.com/minio/minio-go/v6 v6.0.57 h1:ixPkbKkyD7IhnluRgQpGSpHdpvNVaW6OD5R9IAO/9Tw= github.com/minio/minio-go/v6 v6.0.57/go.mod h1:5+R/nM9Pwrh0vqF+HbYYDQ84wdUFPyXHkrdT4AIkifM= github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= +github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= +github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s= github.com/mitchellh/go-homedir v0.0.0-20180523094522-3864e76763d9/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= +github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= +github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mkevac/debugcharts v0.0.0-20191222103121-ae1c48aa8615/go.mod h1:Ad7oeElCZqA1Ufj0U9/liOF4BtVepxRcTvr2ey7zTvM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnuG+zWp9L0Uk= github.com/oklog/oklog v0.2.3-0.20170918173356-f857583a70c3/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/oklog v0.3.2 h1:wVfs8F+in6nTBMkA7CbRw+zZMIB7nNM825cM1wuzoTk= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= +github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v0.3.1-0.20170117200651-66bb6560562f h1:tt7Qj+4Pic1KiUqT7XNMnbAE3TLJAGH+5LMuX4roYbE= github.com/oklog/ulid v0.3.1-0.20170117200651-66bb6560562f/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/oklog/ulid/v2 v2.0.2/go.mod h1:mtBL0Qe/0HAx6/a4Z30qxVIAL1eQDweXq5lxOEiwQ68= github.com/open-telemetry/opamp-go v0.5.0 h1:2YFbb6G4qBkq3yTRdVb5Nfz9hKHW/ldUyex352e1J7g= github.com/open-telemetry/opamp-go v0.5.0/go.mod h1:IMdeuHGVc5CjKSu5/oNV0o+UmiXuahoHvoZ4GOmAI9M= github.com/opentracing-contrib/go-stdlib v0.0.0-20170113013457-1de4cc2120e7/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing/opentracing-go v1.0.1/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulmach/orb v0.4.0 h1:ilp1MQjRapLJ1+qcays1nZpe0mvkCY+b8JU/qBKRZ1A= github.com/paulmach/orb v0.4.0/go.mod h1:FkcWtplUAIVqAuhAOV2d3rpbnQyliDOjOcLW9dUrfdU= github.com/paulmach/protoscan v0.2.1-0.20210522164731-4e53c6875432/go.mod h1:2sV+uZ/oQh66m4XJVZm5iqUZ62BN88Ex1E+TTS0nLzI= -github.com/pborman/getopt v0.0.0-20170112200414-7148bc3a4c30/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= +github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= +github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible h1:FhnA4iH8T/yYW+AolPONZjGE897wxj3MAzfEbrZkSYw= github.com/peterbourgon/diskv v2.0.2-0.20180312054125-0646ccaebea1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= -github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE= github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20161029093637-248dadf4e906/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f h1:h0p1aZ9F5d6IXOygysob3g4B07b+HuVUQC0VJKD8wA4= github.com/posthog/posthog-go v0.0.0-20220817142604-0b0bbf0f9c0f/go.mod h1:oa2sAs9tGai3VldabTV0eWejt/O4/OOD7azP8GaikqU= -github.com/prometheus/client_golang v0.9.0-pre1.0.20181001174001-0a8115f42e03 h1:hqNopISksxji/N5zEy1xMN7TrnSyVG/LymiwnkXi6/Q= github.com/prometheus/client_golang v0.9.0-pre1.0.20181001174001-0a8115f42e03/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= +github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1 h1:osmNoEW2SCW3L7EX0km2LYM8HKpNWRiouxjE3XHkyGc= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20180518154759-7600349dcfe1/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= +github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/procfs v0.0.0-20160411190841-abf152e5f3e9/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= -github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20 h1:Jh/eKJuru9z9u3rUGdQ8gYc3aZmCGkjXT3gmy0Ex8W8= github.com/prometheus/tsdb v0.0.0-20181003080831-0ce41118ed20/go.mod h1:lFf/o1J2a31WmWQbxYXfY1azJK5Xp5D8hwKMnVMBTGU= +github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -454,6 +569,9 @@ github.com/russellhaering/gosaml2 v0.8.0/go.mod h1:byViER/1YPUa0Puj9ROZblpoq2jsE github.com/russellhaering/goxmldsig v1.2.0 h1:Y6GTTc9Un5hCxSzVz4UIWQ/zuVwDvzJk80guqzwx6Vg= github.com/russellhaering/goxmldsig v1.2.0/go.mod h1:gM4MDENBQf7M+V824SGfyIUVFWydB7n0KkEubVJl+Tw= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/samuel/go-zookeeper v0.0.0-20161028232340-1d7be4effb13/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da h1:p3Vo3i64TCLY7gIfzeQaUJ+kppEO5WQG3cL8iE8tGHU= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= @@ -471,7 +589,10 @@ github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFR github.com/shurcooL/httpfs v0.0.0-20171119174359-809beceb2371/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20180711163814-62bca832be04/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.5.0/go.mod h1:+F7Ogzej0PZc/94MaYx/nvG9jOFMD2osvC3s+Squfpo= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= @@ -486,17 +607,22 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= @@ -507,6 +633,9 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= +go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -515,23 +644,28 @@ go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/collector/confmap v0.68.0 h1:TYXfjHDQct9R7RW8Cfe0BOi50o7R1bnC5rlwOxa+JP0= +go.opentelemetry.io/collector/confmap v0.68.0/go.mod h1:gvG/g1eSLVHGKbRmn91g7ukfLuBx36hbuMf/ecpSzlE= +go.opentelemetry.io/collector/featuregate v0.68.0 h1:weK0hx7VTDChvIXO7/BUUiJvOlQpB8wMCeUc0XNYJ/Y= +go.opentelemetry.io/collector/featuregate v0.68.0/go.mod h1:tewuFKJYalWBU0bmNKg++MC1ipINXUr6szYzOw2p1GI= go.opentelemetry.io/otel v1.4.1 h1:QbINgGDDcoQUoMJa2mMaWno49lja9sHwp6aoa2n3a4g= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel/trace v1.4.1 h1:O+16qcdTrT7zxv2J6GejTPFinSwA++cYerC5iSiF8EQ= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= -go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= -go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee h1:0mgffUl7nfd+FpvXMVz4IDEaUSmT1ysygQC7qYo7sG4= -go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= -go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= -go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= +go.uber.org/zap v1.17.0 h1:MTjgFu6ZLKvY6Pvaqk97GlxNBuMpV4Hy/3P6tRGlI2U= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= @@ -559,7 +693,6 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= @@ -571,10 +704,10 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.4.2 h1:Gz96sIWK3OalVv/I/qNygP42zyoKp3xptRVCWRFEBvo= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -583,9 +716,11 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -608,6 +743,7 @@ golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -653,9 +789,15 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -663,14 +805,20 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220220014-0732a990476f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -681,6 +829,8 @@ golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -688,14 +838,18 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -724,6 +878,7 @@ golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuX golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -752,10 +907,9 @@ golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -793,7 +947,6 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -855,6 +1008,7 @@ google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/cloud v0.0.0-20160622021550-0a83eba2cadb/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -933,9 +1087,11 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90 h1:4SPz2GL2CXJt28MTF8V6Ap/9ZiVbQlJeGSd9qtA7DLs= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= @@ -984,8 +1140,11 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.5/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= @@ -998,23 +1157,28 @@ gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/segmentio/analytics-go.v3 v3.1.0 h1:UzxH1uaGZRpMKDhJyBz0pexz6yUoBU3x8bJsRk/HV6U= gopkg.in/segmentio/analytics-go.v3 v3.1.0/go.mod h1:4QqqlTlSSpVlWA9/9nDcPw+FkM2yv1NQoYjUbL9/JAw= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.0.0-20180628040859-072894a440bd h1:HzgYeLDS1jLxw8DGr68KJh9cdQ5iZJizG0HZWstIhfQ= k8s.io/api v0.0.0-20180628040859-072894a440bd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= @@ -1027,3 +1191,4 @@ k8s.io/kube-openapi v0.0.0-20180629012420-d83b052f768a/go.mod h1:BXM9ceUBTj2QnfH rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/pkg/query-service/app/clickhouseReader/reader.go b/pkg/query-service/app/clickhouseReader/reader.go index ab07dbbd23..14fc54210c 100644 --- a/pkg/query-service/app/clickhouseReader/reader.go +++ b/pkg/query-service/app/clickhouseReader/reader.go @@ -164,7 +164,7 @@ func (r *ClickHouseReader) Start(readerReady chan bool) { // Format: &allowedFormat, // } - logger := promlog.New(logLevel) + logger := promlog.New(&promlog.Config{}) startTime := func() (int64, error) { return int64(promModel.Latest), nil diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go index 7e96fc1caf..22508cf67c 100644 --- a/pkg/query-service/app/opamp/model/agent.go +++ b/pkg/query-service/app/opamp/model/agent.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "fmt" "sync" "time" @@ -27,29 +28,36 @@ type Agent struct { mux sync.RWMutex } +func isUniqueConstraintViolation(err error) bool { + errStr := err.Error() + return bytes.Contains([]byte(errStr), []byte("UNIQUE constraint failed")) +} + func NewAgent( instanceId InstanceId, conn types.Connection, -) *Agent { - result, err := db.Exec("INSERT INTO agents (instance_id, status, started_at, effective_config) VALUES (?, ?, ?, ?)", instanceId, nil, time.Now(), "") +) (*Agent, error) { + result, err := db.Exec("INSERT INTO agents (instance_id, status, effective_config) VALUES (?, ?, ?)", instanceId, nil, "") if err != nil { - panic(err) + if isUniqueConstraintViolation(err) { + return &Agent{Id: instanceId, conn: conn}, nil + } + return nil, err } rowsAffected, err := result.RowsAffected() if err != nil { - panic(err) + return nil, err } if rowsAffected != 1 { - panic("expected 1 row to be affected") + return nil, fmt.Errorf("Expected 1 row to be affected, got %d", rowsAffected) } - return &Agent{Id: instanceId, conn: conn} + return &Agent{Id: instanceId, conn: conn}, nil } func (agent *Agent) UpdateStatus( statusMsg *protobufs.AgentToServer, response *protobufs.ServerToAgent, ) { - // TODO: update status in the database. agent.mux.Lock() defer agent.mux.Unlock() agent.processStatusUpdate(statusMsg, response) @@ -124,9 +132,6 @@ func (agent *Agent) updateStatusField(newStatus *protobufs.AgentToServer) (agent agentDescrChanged = agent.updateAgentDescription(newStatus) || agentDescrChanged agent.updateRemoteConfigStatus(newStatus) agent.updateHealth(newStatus) - - // TODO: update status in the database. - return agentDescrChanged } @@ -200,6 +205,7 @@ func (agent *Agent) processStatusUpdate( // The new status resulted in a change in the config of the Agent or the Agent // does not have this config (hash is different). Send the new config the Agent. response.RemoteConfig = agent.remoteConfig + agent.SendToAgent(response) } agent.updateEffectiveConfig(newStatus, response) @@ -275,6 +281,7 @@ func isEqualConfigFile(f1, f2 *protobufs.AgentConfigFile) bool { func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { agent.connMutex.Lock() defer agent.connMutex.Unlock() + fmt.Print("agent.conn", agent.conn) agent.conn.Send(context.Background(), msg) } diff --git a/pkg/query-service/app/opamp/model/agents.go b/pkg/query-service/app/opamp/model/agents.go index 52967c1f04..817e75ae01 100644 --- a/pkg/query-service/app/opamp/model/agents.go +++ b/pkg/query-service/app/opamp/model/agents.go @@ -20,8 +20,8 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { } table_schema := `CREATE TABLE IF NOT EXISTS agents ( - instance_id INTEGER PRIMARY KEY UNIQUE, - status TEXT NOT NULL, + instance_id TEXT PRIMARY KEY UNIQUE, + status TEXT, effective_config TEXT NOT NULL );` @@ -37,19 +37,19 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { defer rows.Close() agents := &Agents{ - agentsById: map[InstanceId]*Agent{}, - connections: map[types.Connection]map[InstanceId]bool{}, + agentsById: make(map[InstanceId]*Agent), + connections: make(map[types.Connection]map[InstanceId]bool), mux: sync.RWMutex{}, } - for rows.Next() { - var agent Agent - err = rows.Scan(&agent.Id, &agent.Status, &agent.EffectiveConfig) - if err != nil { - return nil, fmt.Errorf("Error in scanning agents table: %s", err.Error()) - } - agents.agentsById[agent.Id] = &agent - } + // for rows.Next() { + // var agent Agent + // err = rows.Scan(&agent.Id, &agent.Status, &agent.EffectiveConfig) + // if err != nil { + // return nil, fmt.Errorf("Error in scanning agents table: %s", err.Error()) + // } + // agents.agentsById[agent.Id] = &agent + // } AllAgents = *agents @@ -72,8 +72,6 @@ func (agents *Agents) RemoveConnection(conn types.Connection) { delete(agents.agentsById, instanceId) } delete(agents.connections, conn) - - // TODO: remove agent from database? } func (agents *Agents) FindAgent(agentId InstanceId) *Agent { @@ -82,14 +80,18 @@ func (agents *Agents) FindAgent(agentId InstanceId) *Agent { return agents.agentsById[agentId] } -func (agents *Agents) FindOrCreateAgent(agentId InstanceId, conn types.Connection) *Agent { +func (agents *Agents) FindOrCreateAgent(agentId InstanceId, conn types.Connection) (*Agent, error) { agents.mux.Lock() defer agents.mux.Unlock() // Ensure the Agent is in the agentsById map. agent := agents.agentsById[agentId] + var err error if agent == nil { - agent = NewAgent(agentId, conn) + agent, err = NewAgent(agentId, conn) + if err != nil { + return nil, err + } agents.agentsById[agentId] = agent // Ensure the Agent's instance id is associated with the connection. @@ -98,8 +100,7 @@ func (agents *Agents) FindOrCreateAgent(agentId InstanceId, conn types.Connectio } agents.connections[conn][agentId] = true } - - return agent + return agent, nil } func (agents *Agents) UpdateAgent(agentId InstanceId, status *protobufs.AgentToServer, effectiveConfig string) error { diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go index d3d2889b95..da0175715f 100644 --- a/pkg/query-service/app/opamp/opamp_server.go +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -2,19 +2,24 @@ package opamp import ( "context" + "crypto/sha256" + "fmt" "log" + "math/rand" "os" + "time" + "github.com/knadh/koanf/parsers/yaml" "github.com/open-telemetry/opamp-go/protobufs" "github.com/open-telemetry/opamp-go/server" "github.com/open-telemetry/opamp-go/server/types" + "go.opentelemetry.io/collector/confmap" model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" ) type Server struct { server server.OpAMPServer agents *model.Agents - cnt int } func NewServer(agents *model.Agents) *Server { @@ -39,6 +44,58 @@ func (srv *Server) Start() { } srv.server.Start(settings) + // TODO: remove this + go func() { + ticker := time.NewTicker(60 * time.Second) + for range ticker.C { + agent := srv.agents.FindAgent(model.InstanceId("00000000000000000000000000")) + if agent == nil { + continue + } + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + agentConf := confmap.NewFromStringMap(c) + + configs := []string{ + ` +processors: + batch: + timeout: 2s +`, + `processors: + batch: + timeout: 1s`, + } + + // random choice between 2 configs + config2 := configs[rand.Intn(len(configs))] + c2, err := yaml.Parser().Unmarshal([]byte(config2)) + fmt.Println("config2 err", err) + conf2 := confmap.NewFromStringMap(c2) + + err = agentConf.Merge(conf2) + fmt.Println("merging", err) + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + fmt.Println(conf2.ToStringMap()) + fmt.Println("sending new config", string(configR), err) + // hash of configR + hash := sha256.New() + hash.Write(configR) + agent.SendToAgent(&protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: configR, + ContentType: "application/x-yaml", + }, + }, + }, + ConfigHash: hash.Sum(nil), + }, + }) + } + }() } func (srv *Server) Stop() { @@ -52,33 +109,12 @@ func (srv *Server) onDisconnect(conn types.Connection) { func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { instanceId := model.InstanceId(msg.InstanceUid) - agent := srv.agents.FindOrCreateAgent(instanceId, conn) - - // FIXME: This is a hack to get the agent to send the initial config - f, err := os.ReadFile("./server/config.yaml") + agent, err := srv.agents.FindOrCreateAgent(instanceId, conn) if err != nil { - panic(err) + // Handle this } var response *protobufs.ServerToAgent response = &protobufs.ServerToAgent{} - if srv.cnt <= 2 { - - // Start building the response. - response = &protobufs.ServerToAgent{ - RemoteConfig: &protobufs.AgentRemoteConfig{ - Config: &protobufs.AgentConfigMap{ - ConfigMap: map[string]*protobufs.AgentConfigFile{ - "collector.yaml": { - Body: f, - }, - }, - }, - ConfigHash: []byte("123"), - }, - } - srv.cnt++ - } - // Update the agent's status. agent.UpdateStatus(msg, response) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 4d3807ef95..3375bca330 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -156,7 +156,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } func (s *Server) createOpampServer() (*opamp.Server, error) { - return opamp.NewServer(&opAmpModel.Agents{}), nil + return opamp.NewServer(&opAmpModel.AllAgents), nil } func (s *Server) createPrivateServer(api *APIHandler) (*http.Server, error) { diff --git a/pkg/query-service/pqlEngine/engine.go b/pkg/query-service/pqlEngine/engine.go index 659683986d..31082e2518 100644 --- a/pkg/query-service/pqlEngine/engine.go +++ b/pkg/query-service/pqlEngine/engine.go @@ -42,7 +42,7 @@ func NewPqlEngine(config *pconfig.Config) (*PqlEngine, error) { logLevel := plog.AllowedLevel{} logLevel.Set("debug") - logger := plog.New(logLevel) + logger := plog.New(&plog.Config{}) opts := pql.EngineOpts{ Logger: log.With(logger, "component", "promql evaluator"), From c8509e6961bb311ac6bc1d21273048c57a364811 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Wed, 28 Dec 2022 10:27:06 +0530 Subject: [PATCH 03/24] chore: refactor server implementation --- pkg/query-service/app/opamp/model/agent.go | 77 ++++++------ pkg/query-service/app/opamp/model/agents.go | 108 ++++++---------- pkg/query-service/app/opamp/opamp_server.go | 132 +++++++++++--------- 3 files changed, 149 insertions(+), 168 deletions(-) diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go index 22508cf67c..fdf70e886c 100644 --- a/pkg/query-service/app/opamp/model/agent.go +++ b/pkg/query-service/app/opamp/model/agent.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "crypto/sha256" - "fmt" "sync" "time" @@ -14,50 +13,56 @@ import ( "github.com/open-telemetry/opamp-go/server/types" ) -type InstanceId string +type AgentStatus int + +const ( + AgentStatusUnknown AgentStatus = iota + AgentStatusConnected + AgentStatusDisconnected +) type Agent struct { - Id InstanceId `json:"instanceId" yaml:"instanceId" db:"instance_id"` - Status *protobufs.AgentToServer `json:"status" yaml:"status" db:"status"` - StartedAt time.Time `json:"startedAt" yaml:"startedAt" db:"started_at"` - EffectiveConfig string `json:"effectiveConfig" yaml:"effectiveConfig" db:"effective_config"` + ID string `json:"agentId" yaml:"agentId" db:"agent_id"` + StartedAt time.Time `json:"startedAt" yaml:"startedAt" db:"started_at"` + TerminatedAt time.Time `json:"terminatedAt" yaml:"terminatedAt" db:"terminated_at"` + EffectiveConfig string `json:"effectiveConfig" yaml:"effectiveConfig" db:"effective_config"` + CurrentStatus AgentStatus `json:"currentStatus" yaml:"currentStatus" db:"current_status"` remoteConfig *protobufs.AgentRemoteConfig + Status *protobufs.AgentToServer conn types.Connection connMutex sync.Mutex mux sync.RWMutex } -func isUniqueConstraintViolation(err error) bool { - errStr := err.Error() - return bytes.Contains([]byte(errStr), []byte("UNIQUE constraint failed")) +func New(ID string, conn types.Connection) *Agent { + return &Agent{ID: ID, StartedAt: time.Now(), CurrentStatus: AgentStatusConnected, conn: conn} } -func NewAgent( - instanceId InstanceId, - conn types.Connection, -) (*Agent, error) { - result, err := db.Exec("INSERT INTO agents (instance_id, status, effective_config) VALUES (?, ?, ?)", instanceId, nil, "") - if err != nil { - if isUniqueConstraintViolation(err) { - return &Agent{Id: instanceId, conn: conn}, nil - } - return nil, err - } - rowsAffected, err := result.RowsAffected() +// Upsert inserts or updates the agent in the database. +func (agent *Agent) Upsert() error { + agent.mux.Lock() + defer agent.mux.Unlock() + + _, err := db.NamedExec(`INSERT OR REPLACE INTO agents ( + agent_id, + started_at, + effective_config, + current_status + ) VALUES ( + :agent_id, + :started_at, + :effective_config, + :current_status + )`, agent) if err != nil { - return nil, err + return err } - if rowsAffected != 1 { - return nil, fmt.Errorf("Expected 1 row to be affected, got %d", rowsAffected) - } - return &Agent{Id: instanceId, conn: conn}, nil + + return nil } -func (agent *Agent) UpdateStatus( - statusMsg *protobufs.AgentToServer, - response *protobufs.ServerToAgent, -) { +func (agent *Agent) UpdateStatus(statusMsg *protobufs.AgentToServer, response *protobufs.ServerToAgent) { agent.mux.Lock() defer agent.mux.Unlock() agent.processStatusUpdate(statusMsg, response) @@ -135,10 +140,7 @@ func (agent *Agent) updateStatusField(newStatus *protobufs.AgentToServer) (agent return agentDescrChanged } -func (agent *Agent) updateEffectiveConfig( - newStatus *protobufs.AgentToServer, - response *protobufs.ServerToAgent, -) { +func (agent *Agent) updateEffectiveConfig(newStatus *protobufs.AgentToServer, response *protobufs.ServerToAgent) { // Update effective config if provided. if newStatus.EffectiveConfig != nil { if newStatus.EffectiveConfig.ConfigMap != nil { @@ -146,11 +148,9 @@ func (agent *Agent) updateEffectiveConfig( // Convert to string for displaying purposes. agent.EffectiveConfig = "" + // There should be only one config in the map. for _, cfg := range newStatus.EffectiveConfig.ConfigMap.ConfigMap { - // TODO: we just concatenate parts of effective config as a single - // blob to show in the UI. A proper approach is to keep the effective - // config as a set and show the set in the UI. - agent.EffectiveConfig = agent.EffectiveConfig + string(cfg.Body) + agent.EffectiveConfig = string(cfg.Body) } } } @@ -281,7 +281,6 @@ func isEqualConfigFile(f1, f2 *protobufs.AgentConfigFile) bool { func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { agent.connMutex.Lock() defer agent.connMutex.Unlock() - fmt.Print("agent.conn", agent.conn) agent.conn.Send(context.Background(), msg) } diff --git a/pkg/query-service/app/opamp/model/agents.go b/pkg/query-service/app/opamp/model/agents.go index 817e75ae01..7879f5811b 100644 --- a/pkg/query-service/app/opamp/model/agents.go +++ b/pkg/query-service/app/opamp/model/agents.go @@ -3,14 +3,26 @@ package data import ( "fmt" "sync" + "time" "github.com/jmoiron/sqlx" - "github.com/open-telemetry/opamp-go/protobufs" "github.com/open-telemetry/opamp-go/server/types" ) var db *sqlx.DB +var AllAgents = Agents{ + agentsById: map[string]*Agent{}, + connections: map[types.Connection]map[string]bool{}, +} + +type Agents struct { + mux sync.RWMutex + agentsById map[string]*Agent + connections map[types.Connection]map[string]bool +} + +// InitDB initializes the database and creates the agents table. func InitDB(dataSourceName string) (*sqlx.DB, error) { var err error @@ -19,49 +31,27 @@ func InitDB(dataSourceName string) (*sqlx.DB, error) { return nil, err } - table_schema := `CREATE TABLE IF NOT EXISTS agents ( - instance_id TEXT PRIMARY KEY UNIQUE, - status TEXT, + tableSchema := `CREATE TABLE IF NOT EXISTS agents ( + agent_id TEXT PRIMARY KEY UNIQUE, + started_at datetime NOT NULL, + terminated_at datetime, + current_status TEXT NOT NULL, effective_config TEXT NOT NULL );` - _, err = db.Exec(table_schema) + _, err = db.Exec(tableSchema) if err != nil { return nil, fmt.Errorf("Error in creating agents table: %s", err.Error()) } - rows, err := db.Query("SELECT * FROM agents") - if err != nil { - return nil, fmt.Errorf("Error in querying agents table: %s", err.Error()) - } - defer rows.Close() - - agents := &Agents{ - agentsById: make(map[InstanceId]*Agent), - connections: make(map[types.Connection]map[InstanceId]bool), + AllAgents = Agents{ + agentsById: make(map[string]*Agent), + connections: make(map[types.Connection]map[string]bool), mux: sync.RWMutex{}, } - - // for rows.Next() { - // var agent Agent - // err = rows.Scan(&agent.Id, &agent.Status, &agent.EffectiveConfig) - // if err != nil { - // return nil, fmt.Errorf("Error in scanning agents table: %s", err.Error()) - // } - // agents.agentsById[agent.Id] = &agent - // } - - AllAgents = *agents - return db, nil } -type Agents struct { - mux sync.RWMutex - agentsById map[InstanceId]*Agent - connections map[types.Connection]map[InstanceId]bool -} - // RemoveConnection removes the connection all Agent instances associated with the // connection. func (agents *Agents) RemoveConnection(conn types.Connection) { @@ -69,61 +59,43 @@ func (agents *Agents) RemoveConnection(conn types.Connection) { defer agents.mux.Unlock() for instanceId := range agents.connections[conn] { + agent := agents.agentsById[instanceId] + agent.CurrentStatus = AgentStatusDisconnected + agent.TerminatedAt = time.Now() + agent.Upsert() delete(agents.agentsById, instanceId) } delete(agents.connections, conn) } -func (agents *Agents) FindAgent(agentId InstanceId) *Agent { +// FindAgent returns the Agent instance associated with the given agentID. +func (agents *Agents) FindAgent(agentID string) *Agent { agents.mux.RLock() defer agents.mux.RUnlock() - return agents.agentsById[agentId] + return agents.agentsById[agentID] } -func (agents *Agents) FindOrCreateAgent(agentId InstanceId, conn types.Connection) (*Agent, error) { +// FindOrCreateAgent returns the Agent instance associated with the given agentID. +// If the Agent instance does not exist, it is created and added to the list of +// Agent instances. +func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection) (*Agent, error) { agents.mux.Lock() defer agents.mux.Unlock() - // Ensure the Agent is in the agentsById map. - agent := agents.agentsById[agentId] + agent, ok := agents.agentsById[agentID] var err error - if agent == nil { - agent, err = NewAgent(agentId, conn) + if !ok || agent == nil { + agent = New(agentID, conn) + err = agent.Upsert() if err != nil { return nil, err } - agents.agentsById[agentId] = agent + agents.agentsById[agentID] = agent - // Ensure the Agent's instance id is associated with the connection. if agents.connections[conn] == nil { - agents.connections[conn] = map[InstanceId]bool{} + agents.connections[conn] = map[string]bool{} } - agents.connections[conn][agentId] = true + agents.connections[conn][agentID] = true } return agent, nil } - -func (agents *Agents) UpdateAgent(agentId InstanceId, status *protobufs.AgentToServer, effectiveConfig string) error { - agents.mux.Lock() - defer agents.mux.Unlock() - - agent := agents.agentsById[agentId] - if agent == nil { - return fmt.Errorf("Agent with id %s not found", agentId) - } - - agent.Status = status - agent.EffectiveConfig = effectiveConfig - - _, err := db.Exec("UPDATE agents SET status = ?, effective_config = ? WHERE instance_id = ?", status, effectiveConfig, agentId) - if err != nil { - return fmt.Errorf("Error in updating agents table: %s", err.Error()) - } - - return nil -} - -var AllAgents = Agents{ - agentsById: map[InstanceId]*Agent{}, - connections: map[types.Connection]map[InstanceId]bool{}, -} diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go index da0175715f..d3ecb105a1 100644 --- a/pkg/query-service/app/opamp/opamp_server.go +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -4,9 +4,7 @@ import ( "context" "crypto/sha256" "fmt" - "log" "math/rand" - "os" "time" "github.com/knadh/koanf/parsers/yaml" @@ -15,20 +13,26 @@ import ( "github.com/open-telemetry/opamp-go/server/types" "go.opentelemetry.io/collector/confmap" model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.uber.org/zap" ) type Server struct { - server server.OpAMPServer - agents *model.Agents + server server.OpAMPServer + agents *model.Agents + logger *zap.Logger + capabilities int32 } +const capabilities = protobufs.ServerCapabilities_ServerCapabilities_AcceptsEffectiveConfig | + protobufs.ServerCapabilities_ServerCapabilities_OffersRemoteConfig | + protobufs.ServerCapabilities_ServerCapabilities_AcceptsStatus + func NewServer(agents *model.Agents) *Server { srv := &Server{ agents: agents, } - srv.server = server.New(&Logger{log.New(os.Stdout, "opamp: ", log.LstdFlags)}) - + srv.server = server.New(zap.S()) return srv } @@ -45,57 +49,7 @@ func (srv *Server) Start() { srv.server.Start(settings) // TODO: remove this - go func() { - ticker := time.NewTicker(60 * time.Second) - for range ticker.C { - agent := srv.agents.FindAgent(model.InstanceId("00000000000000000000000000")) - if agent == nil { - continue - } - config := agent.EffectiveConfig - c, err := yaml.Parser().Unmarshal([]byte(config)) - agentConf := confmap.NewFromStringMap(c) - - configs := []string{ - ` -processors: - batch: - timeout: 2s -`, - `processors: - batch: - timeout: 1s`, - } - - // random choice between 2 configs - config2 := configs[rand.Intn(len(configs))] - c2, err := yaml.Parser().Unmarshal([]byte(config2)) - fmt.Println("config2 err", err) - conf2 := confmap.NewFromStringMap(c2) - - err = agentConf.Merge(conf2) - fmt.Println("merging", err) - configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) - fmt.Println(conf2.ToStringMap()) - fmt.Println("sending new config", string(configR), err) - // hash of configR - hash := sha256.New() - hash.Write(configR) - agent.SendToAgent(&protobufs.ServerToAgent{ - RemoteConfig: &protobufs.AgentRemoteConfig{ - Config: &protobufs.AgentConfigMap{ - ConfigMap: map[string]*protobufs.AgentConfigFile{ - "collector.yaml": { - Body: configR, - ContentType: "application/x-yaml", - }, - }, - }, - ConfigHash: hash.Sum(nil), - }, - }) - } - }() + go srv.dummy() } func (srv *Server) Stop() { @@ -107,16 +61,72 @@ func (srv *Server) onDisconnect(conn types.Connection) { } func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { - instanceId := model.InstanceId(msg.InstanceUid) + agentID := msg.InstanceUid - agent, err := srv.agents.FindOrCreateAgent(instanceId, conn) + agent, err := srv.agents.FindOrCreateAgent(agentID, conn) if err != nil { - // Handle this + zap.S().Errorf("Failed to find or create agent %q: %v", agentID, err) + // TODO: handle error } var response *protobufs.ServerToAgent - response = &protobufs.ServerToAgent{} + response = &protobufs.ServerToAgent{ + InstanceUid: agentID, + Capabilities: uint64(capabilities), + } agent.UpdateStatus(msg, response) return response } + +func (srv *Server) dummy() { + ticker := time.NewTicker(60 * time.Second) + for range ticker.C { + agent := srv.agents.FindAgent("00000000000000000000000000") + if agent == nil { + continue + } + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + agentConf := confmap.NewFromStringMap(c) + + configs := []string{ + ` +processors: + batch: + timeout: 2s +`, + `processors: + batch: + timeout: 1s`, + } + + // random choice between 2 configs + config2 := configs[rand.Intn(len(configs))] + c2, err := yaml.Parser().Unmarshal([]byte(config2)) + fmt.Println("config2 err", err) + conf2 := confmap.NewFromStringMap(c2) + + err = agentConf.Merge(conf2) + fmt.Println("merging", err) + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + fmt.Println(conf2.ToStringMap()) + fmt.Println("sending new config", string(configR), err) + // hash of configR + hash := sha256.New() + hash.Write(configR) + agent.SendToAgent(&protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: configR, + ContentType: "application/x-yaml", + }, + }, + }, + ConfigHash: hash.Sum(nil), + }, + }) + } +} From e1116a3b48e418b167f2226503dceee78a363dd4 Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Tue, 3 Jan 2023 00:48:00 +0530 Subject: [PATCH 04/24] chore: add Stop --- pkg/query-service/app/server.go | 24 ++++++++++++++++++++++++ pkg/query-service/main.go | 8 +++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 3375bca330..b6b039128f 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -388,6 +388,30 @@ func (s *Server) Start() error { return nil } +func (s *Server) Stop() error { + if s.httpServer != nil { + if err := s.httpServer.Shutdown(context.Background()); err != nil { + return err + } + } + + if s.privateHTTP != nil { + if err := s.privateHTTP.Shutdown(context.Background()); err != nil { + return err + } + } + + if s.opampServer != nil { + s.opampServer.Stop() + } + + if s.ruleManager != nil { + s.ruleManager.Stop() + } + + return nil +} + func makeRulesManager( promConfigPath, alertManagerURL string, diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index 24e9d06cb8..10bfe67306 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -84,7 +84,13 @@ func main() { case status := <-server.HealthCheckStatus(): logger.Info("Received HealthCheck status: ", zap.Int("status", int(status))) case <-signalsChannel: - logger.Fatal("Received OS Interrupt Signal ... ") + logger.Info("Received OS Interrupt Signal ... ") + err := server.Stop() + if err != nil { + logger.Fatal("Failed to stop server", zap.Error(err)) + } + logger.Info("Server stopped") + return } } From 70794a97bb15f1a2d25239dd35c06b2a5cc7c831 Mon Sep 17 00:00:00 2001 From: mindhash Date: Thu, 9 Mar 2023 15:05:00 +0530 Subject: [PATCH 05/24] chore: merged opamp updates --- ee/query-service/app/server.go | 69 ++++- go.mod | 1 + go.sum | 2 + pkg/query-service/agentConf/db.go | 224 ++++++++++++++ pkg/query-service/agentConf/manager.go | 217 ++++++++++++++ pkg/query-service/agentConf/sqlite/init.go | 65 ++++ pkg/query-service/agentConf/version.go | 63 ++++ .../app/opamp/configure_ingestionRules.go | 228 ++++++++++++++ pkg/query-service/app/opamp/configure_lb.go | 277 ++++++++++++++++++ .../app/opamp/configure_lb_test.go | 103 +++++++ pkg/query-service/app/opamp/model/agent.go | 50 +++- pkg/query-service/app/opamp/model/agents.go | 26 +- .../app/opamp/model/coordinator.go | 66 +++++ pkg/query-service/app/opamp/opamp_server.go | 152 +++++++--- .../otelconfig/filterprocessor/config.go | 11 + .../opamp/otelconfig/otlpreceiver/config.go | 6 + .../otelconfig/otlpreceiver/grpcSettings.go | 14 + .../otelconfig/otlpreceiver/httpSettings.go | 9 + .../app/opamp/otelconfig/otlpreceiver/tls.go | 38 +++ .../opamp/otelconfig/tailsampler/config.go | 82 ++++++ .../app/opamp/pipeline_builder.go | 196 +++++++++++++ pkg/query-service/app/opamp/signal.go | 9 + pkg/query-service/app/server.go | 28 +- pkg/query-service/constants/constants.go | 1 + 24 files changed, 1862 insertions(+), 75 deletions(-) create mode 100644 pkg/query-service/agentConf/db.go create mode 100644 pkg/query-service/agentConf/manager.go create mode 100644 pkg/query-service/agentConf/sqlite/init.go create mode 100644 pkg/query-service/agentConf/version.go create mode 100644 pkg/query-service/app/opamp/configure_ingestionRules.go create mode 100644 pkg/query-service/app/opamp/configure_lb.go create mode 100644 pkg/query-service/app/opamp/configure_lb_test.go create mode 100644 pkg/query-service/app/opamp/model/coordinator.go create mode 100644 pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go create mode 100644 pkg/query-service/app/opamp/otelconfig/tailsampler/config.go create mode 100644 pkg/query-service/app/opamp/pipeline_builder.go create mode 100644 pkg/query-service/app/opamp/signal.go diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index af4a90f885..83412fdb42 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -25,16 +25,18 @@ import ( licensepkg "go.signoz.io/signoz/ee/query-service/license" "go.signoz.io/signoz/ee/query-service/usage" + "go.signoz.io/signoz/pkg/query-service/agentConf" baseapp "go.signoz.io/signoz/pkg/query-service/app" "go.signoz.io/signoz/pkg/query-service/app/dashboards" - "go.signoz.io/signoz/pkg/query-service/app/explorer" + baseexplorer "go.signoz.io/signoz/pkg/query-service/app/explorer" + "go.signoz.io/signoz/pkg/query-service/app/opamp" + opAmpModel "go.signoz.io/signoz/pkg/query-service/app/opamp/model" baseauth "go.signoz.io/signoz/pkg/query-service/auth" - "go.signoz.io/signoz/pkg/query-service/constants" baseconst "go.signoz.io/signoz/pkg/query-service/constants" "go.signoz.io/signoz/pkg/query-service/healthcheck" basealm "go.signoz.io/signoz/pkg/query-service/integrations/alertManager" baseint "go.signoz.io/signoz/pkg/query-service/interfaces" - "go.signoz.io/signoz/pkg/query-service/model" + basemodel "go.signoz.io/signoz/pkg/query-service/model" pqle "go.signoz.io/signoz/pkg/query-service/pqlEngine" rules "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/telemetry" @@ -42,6 +44,8 @@ import ( "go.uber.org/zap" ) +const AppDbEngine = "sqlite" + type ServerOptions struct { PromConfigPath string HTTPHostPort string @@ -80,13 +84,14 @@ func (s Server) HealthCheckStatus() chan healthcheck.Status { // NewServer creates and initializes Server func NewServer(serverOptions *ServerOptions) (*Server, error) { - modelDao, err := dao.InitDao("sqlite", baseconst.RELATIONAL_DATASOURCE_PATH) + modelDao, err := dao.InitDao(AppDbEngine, baseconst.RELATIONAL_DATASOURCE_PATH) if err != nil { return nil, err } + baseexplorer.InitWithDSN(baseconst.RELATIONAL_DATASOURCE_PATH) + localDB, err := dashboards.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH) - explorer.InitWithDSN(constants.RELATIONAL_DATASOURCE_PATH) if err != nil { return nil, err @@ -127,8 +132,19 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { return nil, err } + // initiate opamp + _, err = opAmpModel.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH) + if err != nil { + return nil, err + } + + // initiate agent config handler + if err := agentConf.Initiate(localDB, AppDbEngine); err != nil { + return nil, err + } + // start the usagemanager - usageManager, err := usage.New("sqlite", localDB, lm.GetRepo(), reader.GetConn()) + usageManager, err := usage.New(AppDbEngine, localDB, lm.GetRepo(), reader.GetConn()) if err != nil { return nil, err } @@ -193,7 +209,7 @@ func (s *Server) createPrivateServer(apiHandler *api.APIHandler) (*http.Server, // ip here for alert manager AllowedOrigins: []string{"*"}, AllowedMethods: []string{"GET", "DELETE", "POST", "PUT", "PATCH"}, - AllowedHeaders: []string{"Accept", "Authorization", "Content-Type", "SIGNOZ-API-KEY"}, + AllowedHeaders: []string{"Accept", "Authorization", "Content-Type"}, }) handler := c.Handler(r) @@ -208,7 +224,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e r := mux.NewRouter() - getUserFromRequest := func(r *http.Request) (*model.UserPayload, error) { + getUserFromRequest := func(r *http.Request) (*basemodel.UserPayload, error) { patToken := r.Header.Get("SIGNOZ-API-KEY") if len(patToken) > 0 { zap.S().Debugf("Received a non-zero length PAT token") @@ -226,6 +242,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e } return baseauth.GetUserFromRequest(r) } + am := baseapp.NewAuthMiddleware(getUserFromRequest) r.Use(setTimeoutMiddleware) r.Use(s.analyticsMiddleware) @@ -234,7 +251,6 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e apiHandler.RegisterRoutes(r, am) apiHandler.RegisterMetricsRoutes(r, am) apiHandler.RegisterLogsRoutes(r, am) - apiHandler.RegisterQueryRangeV3Routes(r, am) c := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, @@ -299,7 +315,7 @@ func extractDashboardMetaData(path string, r *http.Request) (map[string]interfac pathToExtractBodyFrom := "/api/v2/metrics/query_range" data := map[string]interface{}{} - var postData *model.QueryRangeParamsV2 + var postData *basemodel.QueryRangeParamsV2 if path == pathToExtractBodyFrom && (r.Method == "POST") { if r.Body != nil { @@ -472,7 +488,7 @@ func (s *Server) Start() error { if port, err := utils.GetPort(s.privateConn.Addr()); err == nil { privatePort = port } - fmt.Println("starting private http") + go func() { zap.S().Info("Starting Private HTTP server", zap.Int("port", privatePort), zap.String("addr", s.serverOptions.PrivateHostPort)) @@ -488,6 +504,37 @@ func (s *Server) Start() error { }() + go func() { + zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", baseconst.OpAmpWsEndpoint)) + err := opamp.InitalizeServer(baseconst.OpAmpWsEndpoint, &opAmpModel.AllAgents) + if err != nil { + zap.S().Info("opamp ws server failed to start", err) + s.unavailableChannel <- healthcheck.Unavailable + } + }() + + return nil +} + +func (s *Server) Stop() error { + if s.httpServer != nil { + if err := s.httpServer.Shutdown(context.Background()); err != nil { + return err + } + } + + if s.privateHTTP != nil { + if err := s.privateHTTP.Shutdown(context.Background()); err != nil { + return err + } + } + + opamp.StopServer() + + if s.ruleManager != nil { + s.ruleManager.Stop() + } + return nil } diff --git a/go.mod b/go.mod index 41be686da2..ac1ab45631 100644 --- a/go.mod +++ b/go.mod @@ -66,6 +66,7 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/armon/go-metrics v0.4.0 // indirect + github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df // indirect github.com/beevik/etree v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect diff --git a/go.sum b/go.sum index b43909d537..a435711f2f 100644 --- a/go.sum +++ b/go.sum @@ -110,6 +110,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72H github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= +github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go new file mode 100644 index 0000000000..159bf920f1 --- /dev/null +++ b/pkg/query-service/agentConf/db.go @@ -0,0 +1,224 @@ +package agentConf + +import ( + "context" + "database/sql" + "fmt" + "math/rand" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/agentConf/sqlite" + "go.signoz.io/signoz/pkg/query-service/model" + "go.uber.org/zap" +) + +func init() { + rand.Seed(2000) +} + +// Repo handles DDL and DML ops on ingestion rules +type Repo struct { + db *sqlx.DB +} + +func (r *Repo) initDB(engine string) error { + switch engine { + case "sqlite3", "sqlite": + return sqlite.InitDB(r.db) + default: + return fmt.Errorf("unsupported db") + } +} + +func (r *Repo) GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]ConfigVersion, error) { + var c []ConfigVersion + err := r.db.SelectContext(ctx, &c, `SELECT + id, + version, + element_type, + COALESCE(created_by, -1) as created_by, + active, + is_valid, + disabled, + deploy_status, + deploy_result + FROM agent_config_versions + WHERE element_type = $1`, typ) + + return c, err +} + +func (r *Repo) GetConfigVersion(ctx context.Context, typ ElementTypeDef, v int) (*ConfigVersion, error) { + var c ConfigVersion + err := r.db.GetContext(ctx, &c, `SELECT + id, + version, + element_type, + COALESCE(created_by, -1) as created_by, + COALESCE((SELECT NAME FROM users + WHERE id = v.created_by), "unknown") created_by_name, + active, + is_valid, + disabled, + deploy_status, + deploy_result, + last_hash, + last_config + FROM agent_config_versions v + WHERE element_type = $1 + AND version = $2`, typ, v) + + return &c, err + +} + +func (r *Repo) GetLatestVersion(ctx context.Context, typ ElementTypeDef) (*ConfigVersion, error) { + var c ConfigVersion + err := r.db.GetContext(ctx, &c, `SELECT + id, + version, + element_type, + COALESCE(created_by, -1) as created_by, + active, + is_valid, + disabled, + deploy_status, + deploy_result + FROM agent_config_versions + WHERE element_type = $1 + AND version = ( + SELECT MAX(version) + FROM agent_config_versions + WHERE element_type=$2)`, typ, typ) + if err != nil { + zap.S().Errorf("failed get latest config version for element:", typ, err) + } + return &c, err +} + +func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []string) (fnerr error) { + + if string(c.ElementType) == "" { + return fmt.Errorf("element type is required for creating agent config version") + } + + if len(elements) == 0 { + zap.S().Errorf("insert config called with no elements", c.ElementType) + return fmt.Errorf("config must have atleast one element") + } + + if c.Version != 0 { + zap.S().Errorf("invalid version assignment while inserting agent config", c.Version, c.ElementType) + return fmt.Errorf("user defined versions are not supported in the agent config") + } + + configVersion, err := r.GetLatestVersion(ctx, c.ElementType) + if err != nil { + if err != sql.ErrNoRows { + zap.S().Errorf("failed to fetch latest config version", err) + return fmt.Errorf("failed to fetch latest config version") + } + } + + c.Version = updateVersion(configVersion.Version) + + defer func() { + if fnerr != nil { + // remove all the damage (invalid rows from db) + r.db.Exec("DELETE FROM agent_config_versions WHERE id = $1", c.ID) + r.db.Exec("DELETE FROM agent_config_elements WHERE version_id=$1", c.ID) + } + }() + + // insert config + configQuery := `INSERT INTO agent_config_versions( + id, + version, + element_type, + active, + is_valid, + disabled, + deploy_status, + deploy_result) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)` + + _, err = r.db.ExecContext(ctx, + configQuery, + c.ID, + c.Version, + c.ElementType, + false, + false, + false, + c.DeployStatus, + c.DeployResult) + + if err != nil { + zap.S().Errorf("error in inserting config version: ", zap.Error(err)) + return fmt.Errorf("failed to insert ingestion rule") + } + + elementsQuery := `INSERT INTO agent_config_elements( + id, + version_id, + element_type, + element_id) + VALUES ($1, $2, $3, $4)` + + for _, e := range elements { + + _, err = r.db.ExecContext(ctx, + elementsQuery, + uuid.NewString(), + c.ID, + c.ElementType, + e) + if err != nil { + return err + } + } + + return nil +} + +func (r *Repo) updateDeployStatus(ctx context.Context, + elementType ElementTypeDef, + version int, + status string, + result string, + lastHash string, + lastconf string) error { + + updateQuery := `UPDATE agent_config_versions + set deploy_status = $1, + deploy_result = $2, + last_hash = COALESCE($3, last_hash), + last_config = $4 + WHERE version=$5 + AND element_type = $6` + + _, err := r.db.ExecContext(ctx, updateQuery, status, result, lastHash, lastconf, version, string(elementType)) + if err != nil { + zap.S().Errorf("failed to get ingestion rule from db", err) + return model.BadRequestStr("failed to get ingestion rule from db") + } + + return nil +} + +func (r *Repo) updateDeployStatusByHash(ctx context.Context, confighash string, status string, result string) error { + + updateQuery := `UPDATE agent_config_versions + set deploy_status = $1, + deploy_result = $2 + WHERE last_hash=$4` + + _, err := r.db.ExecContext(ctx, updateQuery, status, result, confighash) + if err != nil { + zap.S().Errorf("failed to get ingestion rule from db", err) + return model.BadRequestStr("failed to get ingestion rule from db") + } + + return nil +} diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go new file mode 100644 index 0000000000..c560200d8a --- /dev/null +++ b/pkg/query-service/agentConf/manager.go @@ -0,0 +1,217 @@ +package agentConf + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/pkg/query-service/app/opamp" + filterprocessor "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/filterprocessor" + tsp "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/tailsampler" + "go.uber.org/zap" + yaml "gopkg.in/yaml.v3" +) + +var m *Manager + +func init() { + m = &Manager{} +} + +type Manager struct { + Repo + // lock to make sure only one update is sent to remote agents at a time + lock uint32 +} + +// Ready indicates if Manager can accept new config update requests +func (mgr *Manager) Ready() bool { + if mgr.lock != 0 { + return false + } + return opamp.Ready() +} + +func Initiate(db *sqlx.DB, engine string) error { + m.Repo = Repo{db} + return m.initDB(engine) +} + +// Ready indicates if Manager can accept new config update requests +func Ready() bool { + return m.Ready() +} + +func GetLatestVersion(ctx context.Context, elementType ElementTypeDef) (*ConfigVersion, error) { + return m.GetLatestVersion(ctx, elementType) +} + +func GetConfigVersion(ctx context.Context, elementType ElementTypeDef, version int) (*ConfigVersion, error) { + return m.GetConfigVersion(ctx, elementType, version) +} + +func GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]ConfigVersion, error) { + return m.GetConfigHistory(ctx, typ) +} + +// StartNewVersion launches a new config version for given set of elements +func StartNewVersion(ctx context.Context, eleType ElementTypeDef, elementIds []string) (*ConfigVersion, error) { + + if !m.Ready() { + // agent is already being updated, ask caller to wait and re-try after sometime + return nil, fmt.Errorf("agent updater is busy") + } + + // create a new version + cfg := NewConfigversion(eleType) + + // insert new config and elements into database + err := m.insertConfig(ctx, cfg, elementIds) + if err != nil { + return nil, err + } + + return cfg, nil +} + +func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { + if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { + return fmt.Errorf("agent updater is busy") + } + + defer atomic.StoreUint32(&m.lock, 0) + + configVersion, err := GetConfigVersion(ctx, typ, version) + if err != nil { + zap.S().Debugf("failed to fetch config version during redeploy", err) + return fmt.Errorf("failed to fetch details of the config version") + } + + if configVersion == nil || (configVersion != nil && configVersion.LastConf == "") { + zap.S().Debugf("config version has no conf yaml", configVersion) + return fmt.Errorf("the config version can not be redeployed") + } + switch typ { + case ElementTypeSamplingRules: + var config *tsp.Config + if err := yaml.Unmarshal([]byte(configVersion.LastConf), &config); err != nil { + zap.S().Errorf("failed to read last conf correctly", err) + return fmt.Errorf("failed to read the stored config correctly") + } + + // merge current config with new filter params + processorConf := map[string]interface{}{ + "signoz_tail_sampling": config, + } + + opamp.AddToTracePipelineSpec("signoz_tail_sampling") + configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Errorf("failed to call agent config update for trace processor:", err) + return fmt.Errorf("failed to deploy the config") + } + + m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf) + case ElementTypeDropRules: + var filterConfig *filterprocessor.Config + if err := yaml.Unmarshal([]byte(configVersion.LastConf), &filterConfig); err != nil { + zap.S().Errorf("failed to read last conf correctly", err) + return fmt.Errorf("failed to read the stored config correctly") + } + processorConf := map[string]interface{}{ + "filter": filterConfig, + } + + opamp.AddToMetricsPipelineSpec("filter") + configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Errorf("failed to call agent config update for trace processor:", err) + return err + } + + m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf) + } + + return nil +} + +// UpsertFilterProcessor updates the agent config with new filter processor params +func UpsertFilterProcessor(ctx context.Context, version int, config *filterprocessor.Config) error { + if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { + return fmt.Errorf("agent updater is busy") + } + defer atomic.StoreUint32(&m.lock, 0) + + // merge current config with new filter params + // merge current config with new filter params + processorConf := map[string]interface{}{ + "filter": config, + } + + opamp.AddToMetricsPipelineSpec("filter") + configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Errorf("failed to call agent config update for trace processor:", err) + return err + } + + processorConfYaml, err := yaml.Marshal(config) + if err != nil { + zap.S().Warnf("unexpected error while transforming processor config to yaml", err) + } + + m.updateDeployStatus(ctx, ElementTypeDropRules, version, string(DeployInitiated), "Deployment started", configHash, string(processorConfYaml)) + return nil +} + +// OnConfigUpdate is a callback function passed to opamp server. +// It receives a config hash with error status. We assume +// successful deployment if no error is received. +// this method is currently expected to be called only once in the lifecycle +// but can be improved in future to accept continuous request status updates from opamp +func (m *Manager) OnConfigUpdate(agentId string, hash string, err error) { + + status := string(Deployed) + + message := "deploy successful" + + defer func() { + zap.S().Errorf(status, zap.String("agentId", agentId), zap.String("agentResponse", message)) + }() + + if err != nil { + status = string(DeployFailed) + message = fmt.Sprintf("%s: %s", agentId, err.Error()) + } + + m.updateDeployStatusByHash(context.Background(), hash, status, message) +} + +// UpsertSamplingProcessor updates the agent config with new filter processor params +func UpsertSamplingProcessor(ctx context.Context, version int, config *tsp.Config) error { + if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { + return fmt.Errorf("agent updater is busy") + } + defer atomic.StoreUint32(&m.lock, 0) + + // merge current config with new filter params + processorConf := map[string]interface{}{ + "signoz_tail_sampling": config, + } + + opamp.AddToTracePipelineSpec("signoz_tail_sampling") + configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) + if err != nil { + zap.S().Errorf("failed to call agent config update for trace processor:", err) + return err + } + + processorConfYaml, err := yaml.Marshal(config) + if err != nil { + zap.S().Warnf("unexpected error while transforming processor config to yaml", err) + } + + m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, string(processorConfYaml)) + return nil +} diff --git a/pkg/query-service/agentConf/sqlite/init.go b/pkg/query-service/agentConf/sqlite/init.go new file mode 100644 index 0000000000..b844fc6a62 --- /dev/null +++ b/pkg/query-service/agentConf/sqlite/init.go @@ -0,0 +1,65 @@ +package sqlite + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/jmoiron/sqlx" +) + +func InitDB(db *sqlx.DB) error { + var err error + if db == nil { + return fmt.Errorf("invalid db connection") + } + + table_schema := `CREATE TABLE IF NOT EXISTS agent_config_versions( + id TEXT PRIMARY KEY, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_by TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + version INTEGER DEFAULT 1, + active int, + is_valid int, + disabled int, + element_type VARCHAR(120) NOT NULL, + deploy_status VARCHAR(80) NOT NULL DEFAULT 'DIRTY', + deploy_sequence INTEGER, + deploy_result TEXT, + last_hash TEXT, + last_config TEXT, + UNIQUE(element_type, version) + ); + + + CREATE UNIQUE INDEX IF NOT EXISTS agent_config_versions_u1 + ON agent_config_versions(element_type, version); + + CREATE INDEX IF NOT EXISTS agent_config_versions_nu1 + ON agent_config_versions(last_hash); + + + CREATE TABLE IF NOT EXISTS agent_config_elements( + id TEXT PRIMARY KEY, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_by TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + element_id TEXT NOT NULL, + element_type VARCHAR(120) NOT NULL, + version_id TEXT NOT NULL + ); + + CREATE UNIQUE INDEX IF NOT EXISTS agent_config_elements_u1 + ON agent_config_elements(version_id, element_id, element_type); + + ` + + _, err = db.Exec(table_schema) + if err != nil { + return errors.Wrap(err, "Error in creating agent config tables") + } + return nil +} diff --git a/pkg/query-service/agentConf/version.go b/pkg/query-service/agentConf/version.go new file mode 100644 index 0000000000..d568c01d06 --- /dev/null +++ b/pkg/query-service/agentConf/version.go @@ -0,0 +1,63 @@ +package agentConf + +import "github.com/google/uuid" + +type ElementTypeDef string + +const ( + ElementTypeSamplingRules ElementTypeDef = "sampling_rules" + ElementTypeDropRules ElementTypeDef = "drop_rules" + ElementTypeLogPipelines ElementTypeDef = "log_pipelines" + ElementTypeLbExporter ElementTypeDef = "lb_exporter" +) + +type DeployStatus string + +const ( + PendingDeploy DeployStatus = "DIRTY" + Deploying DeployStatus = "DEPLOYING" + Deployed DeployStatus = "DEPLOYED" + DeployInitiated DeployStatus = "IN_PROGRESS" + DeployFailed DeployStatus = "FAILED" +) + +type ConfigVersion struct { + ID string `json:"id" db:"id"` + Version int `json:"version" db:"version"` + ElementType ElementTypeDef `json:"elementType" db:"element_type"` + CreatedBy string `json:"createdBy" db:"created_by"` + CreatedByName string `json:"createdByName" db:"created_by_name"` + Active bool `json:"active" db:"active"` + IsValid bool `json:"is_valid" db:"is_valid"` + Disabled bool `json:"disabled" db:"disabled"` + + DeployStatus DeployStatus `json:"deployStatus" db:"deploy_status"` + DeployResult string `json:"deployResult" db:"deploy_result"` + + LastHash string `json:"lastHash" db:"last_hash"` + LastConf string `json:"lastConf" db:"last_config"` +} + +func NewConfigversion(typeDef ElementTypeDef) *ConfigVersion { + return &ConfigVersion{ + ID: uuid.NewString(), + ElementType: typeDef, + Active: false, + IsValid: false, + Disabled: false, + DeployStatus: PendingDeploy, + // todo: get user id from context? + // CreatedBy + } +} + +func updateVersion(v int) int { + return v + 1 +} + +type ConfigElements struct { + VersionID string + Version int + ElementType ElementTypeDef + ElementId string +} diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go new file mode 100644 index 0000000000..2c48aef96b --- /dev/null +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -0,0 +1,228 @@ +package opamp + +import ( + "context" + "crypto/sha256" + "fmt" + + "github.com/knadh/koanf/parsers/yaml" + "github.com/open-telemetry/opamp-go/protobufs" + "go.opentelemetry.io/collector/confmap" + model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.uber.org/zap" +) + +// inserts or updates ingestion controller processors depending +// on the signal (metrics or traces) +func UpsertControlProcessors(ctx context.Context, signal string, processors map[string]interface{}, callback model.OnChangeCallback) (hash string, fnerr error) { + // note: only processors enabled through tracesPipelinePlan will be added + // to pipeline. To enable or disable processors from pipeline, call + // AddToTracePipeline() or RemoveFromTracesPipeline() prior to calling + // this method + zap.S().Debugf("initiating deployment config", signal, processors) + + if signal != string(Metrics) && signal != string(Traces) { + zap.S().Errorf("received invalid signal int UpsertControllProcessors", signal) + fnerr = fmt.Errorf("invalid kind of target to deploy processor") + return + } + + if opAmpServer == nil { + fnerr = fmt.Errorf("opamp server is down, unable to push config to agent at this moment") + return + } + + agents := opAmpServer.agents.GetAllAgents() + if len(agents) == 0 { + fnerr = fmt.Errorf("no agents available at the moment") + return + } + + // flag to indicate if LB needs to be setup + withLB := false + if len(agents) > 1 { + withLB = true + } + + for _, agent := range agents { + + agenthash, err := addIngestionControlToAgent(agent, signal, processors, withLB) + if err != nil { + zap.S().Errorf("failed to push ingestion rules config to agent", agent.ID, err) + continue + } + + if agenthash != "" { + // subscribe callback + model.ListenToConfigUpdate(agent.ID, agenthash, callback) + } + + hash = agenthash + } + + return hash, nil +} + +func checkPipelineExists(agentConf *confmap.Conf, name string) bool { + service := agentConf.Get("service") + + pipeline := service.(map[string]interface{})["pipelines"].(map[string]interface{}) + if _, ok := pipeline[name]; !ok { + return false + } + return true +} + +func checkExporterExists(agentConf *confmap.Conf, signal string, name string) bool { + service := agentConf.Get("service") + signalSpec := service.(map[string]interface{})["pipelines"].(map[string]interface{})[signal] + exporters := signalSpec.(map[string]interface{})["exporters"].([]interface{}) + var found bool + for _, e := range exporters { + if e == name { + found = true + } + } + + return found +} + +// addIngestionControlToAgent adds ingestion contorl rules to agent config +func addIngestionControlToAgent(agent *model.Agent, signal string, processors map[string]interface{}, withLB bool) (string, error) { + confHash := "" + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + if err != nil { + return confHash, err + } + + agentConf := confmap.NewFromStringMap(c) + + // check if LB needs to be configured + if signal == string(Traces) && withLB { + + // check if lb pipeline is setup for lb agents, here we assume + // that otlp_internal will exist implicitly if traces/lb pipeline exists + if agent.CanLB && !checkPipelineExists(agentConf, TracesLbPipelineName) { + zap.S().Debugf("LB not configured for agent. Preparing the lb exporter spec", agent.ID) + serviceConf, err := prepareLbAgentSpec(agentConf) + if err != nil { + zap.S().Errorf("failed to prepare lb exporter spec for agent", agent.ID, err) + return confHash, err + } + if err := agentConf.Merge(serviceConf); err != nil { + zap.S().Errorf("failed to merge lb exporter spec for agent", agent.ID, err) + return confHash, err + } + } + + // check if otlp_internal is setup for non-lb agents + if !agent.CanLB && !checkExporterExists(agentConf, string(Traces), OtlpInternalReceiver) { + zap.S().Debugf("OTLP internal not configured for agent. Preparing the lb exporter spec", agent.ID) + serviceConf, err := prepareNonLbAgentSpec(agentConf) + if err != nil { + zap.S().Errorf("failed to prepare OTLP internal receiver spec for agent", agent.ID, err) + return confHash, err + } + if err := agentConf.Merge(serviceConf); err != nil { + zap.S().Errorf("failed to merge OTLP internal receiver spec for agent", agent.ID, err) + return confHash, err + } + } + } + + // add ingestion control + serviceConf, err := prepareControlProcesorsSpec(agentConf, Signal(signal), processors) + if err != nil { + zap.S().Errorf("failed to prepare ingestion control processors for agent ", agent.ID, err) + return confHash, err + } + + err = agentConf.Merge(serviceConf) + if err != nil { + return confHash, err + } + + // ------ complete adding processor + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + if err != nil { + return confHash, err + } + + zap.S().Debugf("sending new config", string(configR)) + hash := sha256.New() + _, err = hash.Write(configR) + if err != nil { + return confHash, err + } + confHash = string(hash.Sum(nil)) + agent.EffectiveConfig = string(configR) + err = agent.Upsert() + if err != nil { + return confHash, err + } + + agent.SendToAgent(&protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: configR, + ContentType: "application/x-yaml", + }, + }, + }, + ConfigHash: []byte(confHash), + }, + }) + + return string(confHash), nil +} + +// prepare spec to introduce ingestion control in agent conf +func prepareControlProcesorsSpec(agentConf *confmap.Conf, signal Signal, processors map[string]interface{}) (*confmap.Conf, error) { + agentProcessors := agentConf.Get("processors").(map[string]interface{}) + + for key, params := range processors { + agentProcessors[key] = params + } + + updatedProcessors := map[string]interface{}{ + "processors": agentProcessors, + } + + updatedProcessorConf := confmap.NewFromStringMap(updatedProcessors) + + // upsert changed processor parameters in config + err := agentConf.Merge(updatedProcessorConf) + if err != nil { + return agentConf, err + } + + // edit pipeline if processor is missing + service := agentConf.Get("service") + + signalSpec := service.(map[string]interface{})["pipelines"].(map[string]interface{})[string(signal)] + currentPipeline := signalSpec.(map[string]interface{})["processors"].([]interface{}) + + // merge tracesPipelinePlan with current pipeline + mergedPipeline, err := buildPipeline(signal, currentPipeline) + if err != nil { + zap.S().Errorf("failed to build pipeline", signal, err) + return agentConf, err + } + + // add merged pipeline to the service + // todo(): check if this overwrites other processors + serviceConf := map[string]interface{}{ + "service": map[string]interface{}{ + "pipelines": map[string]interface{}{ + string(signal): map[string]interface{}{ + "processors": mergedPipeline, + }, + }, + }, + } + + return confmap.NewFromStringMap(serviceConf), nil +} diff --git a/pkg/query-service/app/opamp/configure_lb.go b/pkg/query-service/app/opamp/configure_lb.go new file mode 100644 index 0000000000..de2ba6a420 --- /dev/null +++ b/pkg/query-service/app/opamp/configure_lb.go @@ -0,0 +1,277 @@ +package opamp + +import ( + "fmt" + + deepcopy "github.com/barkimedes/go-deepcopy" + + "github.com/knadh/koanf/parsers/yaml" + "go.opentelemetry.io/collector/confmap" + model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/otlpreceiver" +) + +const ( + TracesLbPipelineName = "traces/lb" + TracesDefaultPipelineName = "traces" + OtlpInternalReceiver = "otlp/internal" + LbExporterName = "loadbalancing" +) + +func ConfigureLbExporter() error { + // acquire lock on config updater + agents := opAmpServer.agents.GetAllAgents() + var lbAgents []*model.Agent + var nonLbreceivers []*model.Agent + + for _, agent := range agents { + if agent.CanLB { + lbAgents = append(lbAgents, agent) + } else { + nonLbreceivers = append(nonLbreceivers, agent) + } + } + + if len(lbAgents) == 0 { + return fmt.Errorf("at least one agent with LB exporter support required") + } + + process := func(dryRun bool) error { + // todo: build a worker gorup and call agent updates in parallel + + for _, agent := range lbAgents { + configureLbAgents(agent, dryRun) + } + + for _, agent := range nonLbreceivers { + configureLbAgents(agent, dryRun) + } + + return nil + } + // todo(): allow dry run support in collecto rfirst + // if err := process(true); err != nil { + // return err + // } + + process(false) + + // if any of theagents fail, do not apply the config + return nil +} + +// configureLbAgents deploys lb exporter specific config to agent +func configureLbAgents(agent *model.Agent, dryRun bool) error { + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + if err != nil { + return err + } + agentConf := confmap.NewFromStringMap(c) + err = prepareLbSpec(agentConf, agent.CanLB, dryRun) + if err != nil { + return err + } + // apply uodated config to agent + return nil +} + +// prepareLbSpec deploys lb exporter specific config to agent +func prepareLbSpec(agentConf *confmap.Conf, canLB bool, dryRun bool) error { + + // fetch agent config + var serviceConf *confmap.Conf + var err error + if canLB { + serviceConf, err = prepareLbAgentSpec(agentConf) + } else { + serviceConf, err = prepareNonLbAgentSpec(agentConf) + } + + if err != nil { + return err + } + + err = agentConf.Merge(serviceConf) + if err != nil { + return err + } + + return nil +} + +// prepareLbAgentSpec creates LB exporter agents +func prepareLbAgentSpec(agentConf *confmap.Conf) (serviceConf *confmap.Conf, fnerr error) { + + // add a new otlp receiver otlp_internal at 0.0.0.0:4949 + // this receiver will enable collecting traces re-routed by lb exporter + + // add a new pipeline + // traces/lb: + // receivers: [otlp, jaeger] + // processors: [] + // exporters: [lbExporter] + + // update receiver in service > pipelines > traces + // traces: + // receivers: [otlp_internal] + // processors: [signoz_tail_sampling, batch] + // exporters: [clickhousetraceexporter] + + // apply updated config + + // perform above in dry-run mode and then final mode + // if all agents succeed exit success else fail + + receivers := agentConf.Get("receivers").(map[string]interface{}) + + // add otlp internal receiver to receive traces from lb exporter + // at 0.0.0.0:4949 + receivers[OtlpInternalReceiver] = map[string]interface{}{ + "protocols": otlpreceiver.Protocols{ + GRPC: &otlpreceiver.GRPCServerSettings{ + Endpoint: "0.0.0.0:4949", + }, + HTTP: &otlpreceiver.HTTPServerSettings{ + Endpoint: "0.0.0.0:4949", + TLSSetting: nil, + }, + }} + + exporters := agentConf.Get("exporters").(map[string]interface{}) + + // load balancing settings from here: settings here https://pkg.go.dev/go.opentelemetry.io/collector/config/ + // more description of config: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/loadbalancingexporter#configuration + + exporters["loadbalancing"] = map[string]interface{}{ + "protocol": map[string]interface{}{ + "otlp": map[string]interface{}{ + // Timeout is the timeout for every attempt to send data to the backend. + "timeout": "1s", + }, + }, + "resolver": map[string]interface{}{ + "static": map[string]interface{}{ + "hostnames": []interface{}{ + "0.0.0.0:4949", + }, + }, + // "dns": map[string]interface{}{ + // // todo(): can we get this from agent? + // "hostname": constants.GetOrDefaultEnv("OTLP_ENDPOINT_HOST", "otlp.example.com"), + // "port": constants.GetOrDefaultEnv("OTLP_ENDPOINT_PORT", "4317"), + // }, + }, + } + + updatedReceivers := map[string]interface{}{ + "receivers": receivers, + "exporters": exporters, + } + + updatedReceiverConf := confmap.NewFromStringMap(updatedReceivers) + + err := agentConf.Merge(updatedReceiverConf) + if err != nil { + return nil, err + } + + // remove [jaegar, otlp] from pipelines >> traces + // change pipeline to + // traces: + // receivers: [otlp_internal] + // .... + + service := agentConf.Get("service").(map[string]interface{}) + pipelines := service["pipelines"].(map[string]interface{}) + + preUpdateTraces := service["pipelines"].(map[string]interface{})["traces"].(map[string]interface{}) + + // capture existing receiver list in case lb config needs to be applied + preUpdateRcvrs := preUpdateTraces["receivers"].([]interface{}) + lbreceivers, err := deepcopy.Anything(preUpdateRcvrs) + if err != nil { + return nil, err + } + pipelines[TracesLbPipelineName] = map[string]interface{}{ + "receivers": lbreceivers, + "processors": []string{}, + "exporters": []string{LbExporterName}, + } + preUpdateTraces["receivers"] = []string{OtlpInternalReceiver} + pipelines[TracesDefaultPipelineName], _ = deepcopy.Anything(preUpdateTraces) + + service["pipelines"] = pipelines + + // todo(amol): try updating the traces keys directly but if that + // destroys the rest of the pipeline then update full pipelines[] + s := map[string]interface{}{ + "service": service, + } + + return confmap.NewFromStringMap(s), nil + +} + +// prepareNonLbAgentSpec creates non-LB exporter agents that handle otlp receiver +// only. +func prepareNonLbAgentSpec(agentConf *confmap.Conf) (serviceConf *confmap.Conf, fnerr error) { + + receivers := agentConf.Get("receivers").(map[string]interface{}) + + // add otlp internal receiver to receive traces from lb exporter + // at 0.0.0.0:4949 + receivers["otlp/internal"] = otlpreceiver.Protocols{ + HTTP: &otlpreceiver.HTTPServerSettings{ + Endpoint: "0.0.0.0:4949", + }, + } + + updatedReceivers := map[string]interface{}{ + "receivers": receivers, + } + + updatedReceiverConf := confmap.NewFromStringMap(updatedReceivers) + + err := agentConf.Merge(updatedReceiverConf) + if err != nil { + return nil, err + } + + // remove [jaegar, otlp] from pipelines >> traces + // change pipeline to + // traces: + // receivers: [otlp_internal] + // .... + + service := agentConf.Get("service").(map[string]interface{}) + pipelines := service["pipelines"].(map[string]interface{}) + + preUpdateTraces := service["pipelines"].(map[string]interface{})["traces"].(map[string]interface{}) + + preUpdateTraces["receivers"] = []string{"otlp_internal"} + updatedTraces, err := deepcopy.Anything(preUpdateTraces) + if err != nil { + return nil, err + } + pipelines[TracesDefaultPipelineName] = updatedTraces + service["pipelines"] = pipelines + + // todo(amol): try updating the traces keys directly but if that + // destroys the rest of the pipeline then update full pipelines[] + s := map[string]interface{}{ + "service": service, + } + + return confmap.NewFromStringMap(s), nil + +} + +// DisableLbExporter in a given agent +func DisableLbExporter(agent *model.Agent, dryRun bool) error { + // reverse the steps from EnableLbExporter + // remove otlp_internal from pipelines >> traces + // move receivers from pipelines>>traces/lb to pipelines >> traces + // remove pipeline traces/lb + return nil +} diff --git a/pkg/query-service/app/opamp/configure_lb_test.go b/pkg/query-service/app/opamp/configure_lb_test.go new file mode 100644 index 0000000000..124bfea8df --- /dev/null +++ b/pkg/query-service/app/opamp/configure_lb_test.go @@ -0,0 +1,103 @@ +package opamp + +import ( + "fmt" + "testing" + + "github.com/knadh/koanf/parsers/yaml" + "go.opentelemetry.io/collector/confmap" +) + +func TestPrepareLbSpec(t *testing.T) { + c, err := yaml.Parser().Unmarshal([]byte(`--- +receivers: + otlp: + protocols: + grpc: +processors: + batch: + send_batch_size: 1000 + timeout: 10s +extensions: + zpages: {} +exporters: + logging: + debug: +service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [logging] +`)) + + if err != nil { + fmt.Println("error loading yaml:", err) + t.Fail() + return + } + + agentConf := confmap.NewFromStringMap(c) + serviceConf, err := prepareLbAgentSpec(agentConf) + if err != nil { + fmt.Println("prepareLbSpec error:", err) + } + _ = agentConf.Merge(serviceConf) + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + if err != nil { + fmt.Println("failed to marshal agent conf", err) + } + fmt.Println("TestPrepareLbSpec merged config:", string(configR)) + if len(string(configR)) == 0 { + t.Fail() + return + } + +} + +func TestPrepareNonLbSpec(t *testing.T) { + c, err := yaml.Parser().Unmarshal([]byte(`--- +receivers: + otlp: + protocols: + grpc: +processors: + batch: + send_batch_size: 1000 + timeout: 10s +extensions: + zpages: {} +exporters: + logging: + debug: +service: + pipelines: + traces: + receivers: [otlp] + processors: [] + exporters: [logging] +`)) + + if err != nil { + fmt.Println("error loading yaml:", err) + t.Fail() + return + } + + agentConf := confmap.NewFromStringMap(c) + serviceConf, err := prepareNonLbAgentSpec(agentConf) + if err != nil { + fmt.Println("prepareLbSpec error:", err) + } + _ = agentConf.Merge(serviceConf) + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + if err != nil { + fmt.Println("failed to marshal agent conf", err) + } + fmt.Println("TestPrepareNonLbSpec merged config:", string(configR)) + if len(string(configR)) == 0 { + t.Fail() + return + } + +} diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go index fdf70e886c..ba2ecfcddc 100644 --- a/pkg/query-service/app/opamp/model/agent.go +++ b/pkg/query-service/app/opamp/model/agent.go @@ -1,4 +1,4 @@ -package data +package model import ( "bytes" @@ -21,6 +21,10 @@ const ( AgentStatusDisconnected ) +// set in agent description when agent is capable of supporting +// lb exporter configuration. values: 1 (true) or 0 (false) +const lbExporterFlag = "capabilities.lbexporter" + type Agent struct { ID string `json:"agentId" yaml:"agentId" db:"agent_id"` StartedAt time.Time `json:"startedAt" yaml:"startedAt" db:"started_at"` @@ -30,6 +34,12 @@ type Agent struct { remoteConfig *protobufs.AgentRemoteConfig Status *protobufs.AgentToServer + // can this agent be load balancer + CanLB bool + + // is this agent setup as load balancer + IsLb bool + conn types.Connection connMutex sync.Mutex mux sync.RWMutex @@ -68,6 +78,29 @@ func (agent *Agent) UpdateStatus(statusMsg *protobufs.AgentToServer, response *p agent.processStatusUpdate(statusMsg, response) } +// extracts lb exporter support flag from agent description. the flag +// is used to decide if lb exporter can be enabled on the agent. +func ExtractLbFlag(agentDescr *protobufs.AgentDescription) bool { + + if agentDescr == nil { + return false + } + + if len(agentDescr.NonIdentifyingAttributes) > 0 { + for _, kv := range agentDescr.NonIdentifyingAttributes { + anyvalue, ok := kv.Value.Value.(*protobufs.AnyValue_StringValue) + if !ok { + continue + } + if kv.Key == lbExporterFlag && anyvalue.StringValue == "1" { + // agent can support load balancer config + return true + } + } + } + return false +} + func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) (agentDescrChanged bool) { prevStatus := agent.Status @@ -103,8 +136,23 @@ func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) ( if newStatus.RemoteConfigStatus != nil && !proto.Equal(agent.Status.RemoteConfigStatus, newStatus.RemoteConfigStatus) { agent.Status.RemoteConfigStatus = newStatus.RemoteConfigStatus + + // todo: need to address multiple agent scenario here + // for now, the first response will be sent back to the UI + if agent.Status.RemoteConfigStatus.Status == protobufs.RemoteConfigStatuses_RemoteConfigStatuses_APPLIED { + onConfigSuccess(agent.ID, string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash)) + } + + if agent.Status.RemoteConfigStatus.Status == protobufs.RemoteConfigStatuses_RemoteConfigStatuses_FAILED { + onConfigFailure(agent.ID, string(agent.Status.RemoteConfigStatus.LastRemoteConfigHash), agent.Status.RemoteConfigStatus.ErrorMessage) + } } } + + if agentDescrChanged { + agent.CanLB = ExtractLbFlag(newStatus.AgentDescription) + } + return agentDescrChanged } diff --git a/pkg/query-service/app/opamp/model/agents.go b/pkg/query-service/app/opamp/model/agents.go index 7879f5811b..18faddb48b 100644 --- a/pkg/query-service/app/opamp/model/agents.go +++ b/pkg/query-service/app/opamp/model/agents.go @@ -1,4 +1,4 @@ -package data +package model import ( "fmt" @@ -22,6 +22,10 @@ type Agents struct { connections map[types.Connection]map[string]bool } +func (a *Agents) Count() int { + return len(a.connections) +} + // InitDB initializes the database and creates the agents table. func InitDB(dataSourceName string) (*sqlx.DB, error) { var err error @@ -78,17 +82,17 @@ func (agents *Agents) FindAgent(agentID string) *Agent { // FindOrCreateAgent returns the Agent instance associated with the given agentID. // If the Agent instance does not exist, it is created and added to the list of // Agent instances. -func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection) (*Agent, error) { +func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection) (*Agent, bool, error) { agents.mux.Lock() defer agents.mux.Unlock() - + var created bool agent, ok := agents.agentsById[agentID] var err error if !ok || agent == nil { agent = New(agentID, conn) err = agent.Upsert() if err != nil { - return nil, err + return nil, created, err } agents.agentsById[agentID] = agent @@ -96,6 +100,18 @@ func (agents *Agents) FindOrCreateAgent(agentID string, conn types.Connection) ( agents.connections[conn] = map[string]bool{} } agents.connections[conn][agentID] = true + created = true + } + return agent, created, nil +} + +func (agents *Agents) GetAllAgents() []*Agent { + agents.mux.RLock() + defer agents.mux.RUnlock() + + allAgents := []*Agent{} + for _, v := range agents.agentsById { + allAgents = append(allAgents, v) } - return agent, nil + return allAgents } diff --git a/pkg/query-service/app/opamp/model/coordinator.go b/pkg/query-service/app/opamp/model/coordinator.go new file mode 100644 index 0000000000..a1f17f43a2 --- /dev/null +++ b/pkg/query-service/app/opamp/model/coordinator.go @@ -0,0 +1,66 @@ +package model + +import ( + "fmt" + "sync" +) + +// communicates with calling apis when config is applied or fails +var coordinator *Coordinator + +func init() { + subscribers := make(map[string][]OnChangeCallback, 0) + coordinator = &Coordinator{ + subscribers: subscribers, + } +} + +type OnChangeCallback func(agentId string, hash string, err error) + +// responsible for managing subscribers on config change +type Coordinator struct { + mutex sync.Mutex + + // hash wise list of subscribers + subscribers map[string][]OnChangeCallback +} + +func onConfigSuccess(agentId string, hash string) { + notifySubscribers(agentId, hash, nil) +} + +func onConfigFailure(agentId string, hash string, errorMessage string) { + notifySubscribers(agentId, hash, fmt.Errorf(errorMessage)) +} + +// OnSuccess listens to config changes and notifies subscribers +func notifySubscribers(agentId string, hash string, err error) { + // this method currently does not handle multi-agent scenario. + // as soon as a message is delivered, we release all the subscribers + // for a given hash + subs, ok := coordinator.subscribers[hash] + if !ok { + return + } + + for _, s := range subs { + s(agentId, hash, err) + } + + // delete all subscribers for this hash, assume future + // notifies will be disabled. the first response is processed + delete(coordinator.subscribers, hash) +} + +// callers subscribe to this function to listen on config change requests +func ListenToConfigUpdate(agentId string, hash string, ss OnChangeCallback) { + coordinator.mutex.Lock() + defer coordinator.mutex.Unlock() + + if subs, ok := coordinator.subscribers[hash]; ok { + subs = append(subs, ss) + coordinator.subscribers[hash] = subs + } else { + coordinator.subscribers[hash] = []OnChangeCallback{ss} + } +} diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go index d3ecb105a1..93456f7e1a 100644 --- a/pkg/query-service/app/opamp/opamp_server.go +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -3,9 +3,8 @@ package opamp import ( "context" "crypto/sha256" - "fmt" - "math/rand" - "time" + + "strings" "github.com/knadh/koanf/parsers/yaml" "github.com/open-telemetry/opamp-go/protobufs" @@ -13,9 +12,12 @@ import ( "github.com/open-telemetry/opamp-go/server/types" "go.opentelemetry.io/collector/confmap" model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.uber.org/zap" ) +var opAmpServer *Server + type Server struct { server server.OpAMPServer agents *model.Agents @@ -27,16 +29,27 @@ const capabilities = protobufs.ServerCapabilities_ServerCapabilities_AcceptsEffe protobufs.ServerCapabilities_ServerCapabilities_OffersRemoteConfig | protobufs.ServerCapabilities_ServerCapabilities_AcceptsStatus -func NewServer(agents *model.Agents) *Server { - srv := &Server{ +func InitalizeServer(listener string, agents *model.Agents) error { + + if agents == nil { + agents = &model.AllAgents + } + + opAmpServer = &Server{ agents: agents, } + opAmpServer.server = server.New(zap.S()) - srv.server = server.New(zap.S()) - return srv + return opAmpServer.Start(listener) } -func (srv *Server) Start() { +func StopServer() { + if opAmpServer != nil { + opAmpServer.Stop() + } +} + +func (srv *Server) Start(listener string) error { settings := server.StartSettings{ Settings: server.Settings{ Callbacks: server.CallbacksStruct{ @@ -44,12 +57,10 @@ func (srv *Server) Start() { OnConnectionCloseFunc: srv.onDisconnect, }, }, - ListenEndpoint: "127.0.0.1:4320", + ListenEndpoint: listener, } - srv.server.Start(settings) - // TODO: remove this - go srv.dummy() + return srv.server.Start(settings) } func (srv *Server) Stop() { @@ -63,11 +74,17 @@ func (srv *Server) onDisconnect(conn types.Connection) { func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { agentID := msg.InstanceUid - agent, err := srv.agents.FindOrCreateAgent(agentID, conn) + agent, created, err := srv.agents.FindOrCreateAgent(agentID, conn) if err != nil { zap.S().Errorf("Failed to find or create agent %q: %v", agentID, err) // TODO: handle error } + + if created { + agent.CanLB = model.ExtractLbFlag(msg.AgentDescription) + zap.S().Debugf("New agent added:", zap.Bool("canLb", agent.CanLB), zap.String("ID", agent.ID), zap.Any("status", agent.CurrentStatus)) + } + var response *protobufs.ServerToAgent response = &protobufs.ServerToAgent{ InstanceUid: agentID, @@ -79,42 +96,95 @@ func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer return response } -func (srv *Server) dummy() { - ticker := time.NewTicker(60 * time.Second) - for range ticker.C { - agent := srv.agents.FindAgent("00000000000000000000000000") - if agent == nil { - continue - } +// global var methods to support singleton pattern. we want to discourage +// allow multiple servers in one installation +func Ready() bool { + if opAmpServer == nil { + return false + } + if opAmpServer.agents.Count() == 0 { + zap.S().Warnf("no agents available, all agent config requests will be rejected") + return false + } + return true +} + +func Subscribe(agentId string, hash string, f model.OnChangeCallback) { + model.ListenToConfigUpdate(agentId, hash, f) +} + +func UpsertProcessor(ctx context.Context, processors map[string]interface{}, names []interface{}) error { + x := map[string]interface{}{ + "processors": processors, + } + + newConf := confmap.NewFromStringMap(x) + + agents := opAmpServer.agents.GetAllAgents() + for _, agent := range agents { config := agent.EffectiveConfig c, err := yaml.Parser().Unmarshal([]byte(config)) + if err != nil { + return err + } agentConf := confmap.NewFromStringMap(c) - configs := []string{ - ` -processors: - batch: - timeout: 2s -`, - `processors: - batch: - timeout: 1s`, + err = agentConf.Merge(newConf) + if err != nil { + return err + } + + service := agentConf.Get("service") + + logs := service.(map[string]interface{})["pipelines"].(map[string]interface{})["logs"] + processors := logs.(map[string]interface{})["processors"].([]interface{}) + userProcessors := []interface{}{} + // remove old ones + for _, v := range processors { + if !strings.HasPrefix(v.(string), "logstransform/pipeline_") { + userProcessors = append(userProcessors, v) + } + } + // all user processors are pushed after pipelines + processors = append(names, userProcessors...) + + service.(map[string]interface{})["pipelines"].(map[string]interface{})["logs"].(map[string]interface{})["processors"] = processors + + s := map[string]interface{}{ + "service": map[string]interface{}{ + "pipelines": map[string]interface{}{ + "logs": map[string]interface{}{ + "processors": processors, + }, + }, + }, } - // random choice between 2 configs - config2 := configs[rand.Intn(len(configs))] - c2, err := yaml.Parser().Unmarshal([]byte(config2)) - fmt.Println("config2 err", err) - conf2 := confmap.NewFromStringMap(c2) + serviceC := confmap.NewFromStringMap(s) - err = agentConf.Merge(conf2) - fmt.Println("merging", err) + err = agentConf.Merge(serviceC) + if err != nil { + return err + } + + // ------ complete adding processor configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) - fmt.Println(conf2.ToStringMap()) - fmt.Println("sending new config", string(configR), err) - // hash of configR + if err != nil { + return err + } + + zap.S().Infof("sending new config", string(configR)) hash := sha256.New() - hash.Write(configR) + _, err = hash.Write(configR) + if err != nil { + return err + } + agent.EffectiveConfig = string(configR) + err = agent.Upsert() + if err != nil { + return err + } + agent.SendToAgent(&protobufs.ServerToAgent{ RemoteConfig: &protobufs.AgentRemoteConfig{ Config: &protobufs.AgentConfigMap{ @@ -129,4 +199,6 @@ processors: }, }) } + + return nil } diff --git a/pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go b/pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go new file mode 100644 index 0000000000..1d4d93a959 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/filterprocessor/config.go @@ -0,0 +1,11 @@ +package filterprocessor + +type Config struct { + Metrics MetricFilters `mapstructure:"metrics"` +} + +// MetricFilters filters by Metric properties. +type MetricFilters struct { + MetricConditions []string `mapstructure:"metric" yaml:"metric,omitempty"` + DataPointConditions []string `mapstructure:"datapoint" yaml:"datapoint,omitempty"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go new file mode 100644 index 0000000000..869afc9f43 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go @@ -0,0 +1,6 @@ +package otlpreceiver + +type Protocols struct { + GRPC *GRPCServerSettings `mapstructure:"grpc"` + HTTP *HTTPServerSettings `mapstructure:"http"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go new file mode 100644 index 0000000000..f0f55eda10 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go @@ -0,0 +1,14 @@ +package otlpreceiver + +type GRPCServerSettings struct { + // Endpoint configures the address for this network connection. + // For TCP and UDP networks, the address has the form "host:port". The host must be a literal IP address, + // or a host name that can be resolved to IP addresses. The port must be a literal port number or a service name. + // If the host is a literal IPv6 address it must be enclosed in square brackets, as in "[2001:db8::1]:80" or + // "[fe80::1%zone]:80". The zone specifies the scope of the literal IPv6 address as defined in RFC 4007. + Endpoint string `mapstructure:"endpoint"` + + // Transport to use. Known protocols are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only), "udp", "udp4" (IPv4-only), + // "udp6" (IPv6-only), "ip", "ip4" (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and "unixpacket". + Transport string `mapstructure:"transport"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go new file mode 100644 index 0000000000..64c9a16fe5 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go @@ -0,0 +1,9 @@ +package otlpreceiver + +type HTTPServerSettings struct { + // Endpoint configures the listening address for the server. + Endpoint string `mapstructure:"endpoint" yaml:"endpoint"` + + // TLSSetting struct exposes TLS client configuration. + TLSSetting *TLSServerSetting `mapstructure:"tls" yaml:"tls"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go new file mode 100644 index 0000000000..39978fcb29 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go @@ -0,0 +1,38 @@ +package otlpreceiver + +import "time" + +type TLSSetting struct { + // Path to the CA cert. For a client this verifies the server certificate. + // For a server this verifies client certificates. If empty uses system root CA. + // (optional) + CAFile string `mapstructure:"ca_file"` + + // Path to the TLS cert to use for TLS required connections. (optional) + CertFile string `mapstructure:"cert_file"` + + // Path to the TLS key to use for TLS required connections. (optional) + KeyFile string `mapstructure:"key_file"` + + // MinVersion sets the minimum TLS version that is acceptable. + // If not set, TLS 1.2 will be used. (optional) + MinVersion string `mapstructure:"min_version"` + + // MaxVersion sets the maximum TLS version that is acceptable. + // If not set, refer to crypto/tls for defaults. (optional) + MaxVersion string `mapstructure:"max_version"` + + // ReloadInterval specifies the duration after which the certificate will be reloaded + // If not set, it will never be reloaded (optional) + ReloadInterval time.Duration `mapstructure:"reload_interval"` +} + +type TLSServerSetting struct { + // squash ensures fields are correctly decoded in embedded struct. + TLSSetting `mapstructure:",squash"` + + // Path to the TLS cert to use by the server to verify a client certificate. (optional) + // This sets the ClientCAs and ClientAuth to RequireAndVerifyClientCert in the TLSConfig. Please refer to + // https://godoc.org/crypto/tls#Config for more information. (optional) + ClientCAFile string `mapstructure:"client_ca_file"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/tailsampler/config.go b/pkg/query-service/app/opamp/otelconfig/tailsampler/config.go new file mode 100644 index 0000000000..d43a8eacb4 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/tailsampler/config.go @@ -0,0 +1,82 @@ +package tailsampler + +import "time" + +type PolicyType string + +type Config struct { + DecisionWait time.Duration `mapstructure:"decision_wait" yaml:"decision_wait"` + NumTraces uint64 `mapstructure:"num_traces" yaml:"num_traces"` + ExpectedNewTracesPerSec uint64 `mapstructure:"expected_new_traces_per_sec" yaml:"expected_new_traces_per_sec"` + PolicyCfgs []PolicyCfg `mapstructure:"policies" yaml:"policies"` + + // read only version number (optional) + Version int +} + +type ProbabilisticCfg struct { + // HashSalt allows one to configure the hashing salts. This is important in scenarios where multiple layers of collectors + // have different sampling rates: if they use the same salt all passing one layer may pass the other even if they have + // different sampling rates, configuring different salts avoids that. + HashSalt string `mapstructure:"hash_salt" yaml:"hash_salt"` + // SamplingPercentage is the percentage rate at which traces are going to be sampled. Defaults to zero, i.e.: no sample. + // Values greater or equal 100 are treated as "sample all traces". + SamplingPercentage float64 `mapstructure:"sampling_percentage" yaml:"sampling_percentage"` +} + +type NumericAttributeCfg struct { + // Tag that the filter is going to be matching against. + Key string `mapstructure:"key" yaml:"key"` + // MinValue is the minimum value of the attribute to be considered a match. + MinValue int64 `mapstructure:"min_value" yaml:"min_value"` + // MaxValue is the maximum value of the attribute to be considered a match. + MaxValue int64 `mapstructure:"max_value" yaml:"max_value"` +} + +type StringAttributeCfg struct { + // Tag that the filter is going to be matching against. + Key string `mapstructure:"key" yaml:"key"` + // Values indicate the set of values or regular expressions to use when matching against attribute values. + // StringAttribute Policy will apply exact value match on Values unless EnabledRegexMatching is true. + Values []string `mapstructure:"values" yaml:"values"` + // EnabledRegexMatching determines whether match attribute values by regexp string. + EnabledRegexMatching bool `mapstructure:"enabled_regex_matching" yaml:"enabled_regex_matching"` + // CacheMaxSize is the maximum number of attribute entries of LRU Cache that stores the matched result + // from the regular expressions defined in Values. + // CacheMaxSize will not be used if EnabledRegexMatching is set to false. + CacheMaxSize int `mapstructure:"cache_max_size" yaml:"cache_max_size"` + // InvertMatch indicates that values or regular expressions must not match against attribute values. + // If InvertMatch is true and Values is equal to 'acme', all other values will be sampled except 'acme'. + // Also, if the specified Key does not match on any resource or span attributes, data will be sampled. + InvertMatch bool `mapstructure:"invert_match" yaml:"invert_match"` +} + +type PolicyFilterCfg struct { + // values: AND | OR + FilterOp string `mapstructure:"filter_op" yaml:"filter_op"` + + StringAttributeCfgs []StringAttributeCfg `mapstructure:"string_attributes" yaml:"string_attributes"` + NumericAttributeCfgs []NumericAttributeCfg `mapstructure:"numeric_attributes" yaml:"numeric_attributes"` +} + +// PolicyCfg identifies policy rules in policy group +type PolicyCfg struct { + // name of the policy + Name string `mapstructure:"name" yaml:"name"` + + // Type of the policy this will be used to match the proper configuration of the policy. + Type PolicyType `mapstructure:"type" yaml:"type"` + + // Set to true for sampling rule (root) and false for conditions + Root bool `mapstructure:"root" yaml:"root"` + + Priority int `mapstructure:"priority" yaml:"priority"` + + // sampling applied when PolicyFilter matches + ProbabilisticCfg `mapstructure:",squash" yaml:"sampling"` + + // filter to activate policy + PolicyFilterCfg `mapstructure:",squash" yaml:"policy_filter"` + + SubPolicies []PolicyCfg `mapstructure:"sub_policies" yaml:"sub_policies"` +} diff --git a/pkg/query-service/app/opamp/pipeline_builder.go b/pkg/query-service/app/opamp/pipeline_builder.go new file mode 100644 index 0000000000..841a9ce5c6 --- /dev/null +++ b/pkg/query-service/app/opamp/pipeline_builder.go @@ -0,0 +1,196 @@ +package opamp + +import ( + "fmt" + "sync" + + "go.uber.org/zap" +) + +var lockTracesPipelineSpec sync.RWMutex +var lockMetricsPipelineSpec sync.RWMutex + +type pipelineStatus struct { + Name string + Enabled bool +} + +var tracesPipelineSpec = map[int]pipelineStatus{ + 0: { + Name: "signoz_tail_sampling", + Enabled: false, + }, + 1: { + Name: "batch", + Enabled: true, + }, +} + +var metricsPipelineSpec = map[int]pipelineStatus{ + 0: { + Name: "filter", + Enabled: false, + }, + 1: { + Name: "batch", + Enabled: true, + }, +} + +func updatePipelineSpec(signal string, name string, enabled bool) { + switch signal { + case "metrics": + lockMetricsPipelineSpec.Lock() + defer lockMetricsPipelineSpec.Unlock() + + for i := 0; i < len(metricsPipelineSpec); i++ { + p := metricsPipelineSpec[i] + if p.Name == name { + p.Enabled = enabled + metricsPipelineSpec[i] = p + } + } + case "traces": + lockTracesPipelineSpec.Lock() + defer lockTracesPipelineSpec.Unlock() + + for i := 0; i < len(tracesPipelineSpec); i++ { + p := tracesPipelineSpec[i] + if p.Name == name { + p.Enabled = enabled + tracesPipelineSpec[i] = p + } + } + default: + return + } + +} + +// AddToTracePipeline to enable processor in traces pipeline +func AddToTracePipelineSpec(processor string) { + updatePipelineSpec("traces", processor, true) +} + +// RemoveFromTracePipeline to remove processor from traces pipeline +func RemoveFromTracePipelineSpec(name string) { + updatePipelineSpec("traces", name, false) +} + +// AddToMetricsPipeline to enable processor in traces pipeline +func AddToMetricsPipelineSpec(processor string) { + updatePipelineSpec("metrics", processor, true) +} + +// RemoveFromMetricsPipeline to remove processor from traces pipeline +func RemoveFromMetricsPipelineSpec(name string) { + updatePipelineSpec("metrics", name, false) +} + +func checkDuplicates(pipeline []interface{}) bool { + exists := make(map[string]bool, len(pipeline)) + zap.S().Debugf("checking duplicate processors in the pipeline:", pipeline) + for _, processor := range pipeline { + name := processor.(string) + if _, ok := exists[name]; ok { + return true + } + + exists[name] = true + } + return false +} + +func buildPipeline(signal Signal, current []interface{}) ([]interface{}, error) { + var spec map[int]pipelineStatus + + switch signal { + case Metrics: + spec = metricsPipelineSpec + lockMetricsPipelineSpec.Lock() + defer lockMetricsPipelineSpec.Unlock() + case Traces: + spec = tracesPipelineSpec + lockTracesPipelineSpec.Lock() + defer lockTracesPipelineSpec.Unlock() + default: + return nil, fmt.Errorf("invalid signal") + } + + pipeline := current + // create a reverse map of existing config processors and their position + existing := map[string]int{} + for i, p := range current { + name := p.(string) + existing[name] = i + } + + // create mapping from our tracesPipelinePlan (processors managed by us) to position in existing processors (from current config) + // this means, if "batch" holds position 3 in the current effective config, and 2 in our config, the map will be [2]: 3 + specVsExistingMap := map[int]int{} + + // go through plan and map its elements to current positions in effective config + for i, m := range spec { + if loc, ok := existing[m.Name]; ok { + specVsExistingMap[i] = loc + } + } + + lastMatched := -1 + inserts := 0 + + // go through plan again in the increasing order + for i := 0; i < len(spec); i++ { + m := spec[i] + + if loc, ok := specVsExistingMap[i]; ok { + // element from plan already exists in current effective config. + + currentPos := loc + inserts + // if disabled then remove from the pipeline + if !m.Enabled { + zap.S().Debugf("build_pipeline: found a disabled item, removing from pipeline at position", currentPos-1, " ", m.Name) + if currentPos-1 <= 0 { + pipeline = pipeline[currentPos+1:] + } else { + pipeline = append(pipeline[:currentPos-1], pipeline[currentPos+1:]...) + } + } + + // capture last position where match was found, this will be used + // to insert missing elements next to it + lastMatched = currentPos + + } else { + if m.Enabled { + // track inserts as they shift the elements in pipeline + inserts++ + + // we use last matched to insert new item. This means, we keep inserting missing processors + // right after last matched processsor (e.g. insert filters after tail_sampling for existing list of [batch, tail_sampling]) + + if lastMatched <= 0 { + zap.S().Debugf("build_pipeline: found a new item to be inserted, inserting at position 0:", m.Name) + pipeline = append([]interface{}{m.Name}, pipeline[lastMatched+1:]...) + } else { + zap.S().Debugf("build_pipeline: found a new item to be inserted, inserting at position :", lastMatched, " ", m.Name) + prior := make([]interface{}, len(pipeline[:lastMatched])) + next := make([]interface{}, len(pipeline[lastMatched:])) + copy(prior, pipeline[:lastMatched]) + copy(next, pipeline[lastMatched:]) + + pipeline = append(prior, m.Name) + pipeline = append(pipeline, next...) + } + } + } + } + + if checkDuplicates(pipeline) { + // duplicates are most likely because the processor sequence in effective config conflicts + // with the planned sequence as per planned pipeline + return pipeline, fmt.Errorf("the effective config has an unexpected processor sequence: %v", pipeline) + } + + return pipeline, nil +} diff --git a/pkg/query-service/app/opamp/signal.go b/pkg/query-service/app/opamp/signal.go new file mode 100644 index 0000000000..754dfc33a8 --- /dev/null +++ b/pkg/query-service/app/opamp/signal.go @@ -0,0 +1,9 @@ +package opamp + +type Signal string + +const ( + Metrics Signal = "metrics" + Traces Signal = "traces" + Logs Signal = "logs" +) diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 607d57fe65..f737b3925c 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -66,9 +66,6 @@ type Server struct { privateHTTP *http.Server unavailableChannel chan healthcheck.Status - - // opamp server - opampServer *opamp.Server } // HealthCheckStatus returns health check status channel a client can subscribe to @@ -148,24 +145,14 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { s.privateHTTP = privateServer - localDB, err = opAmpModel.InitDB(constants.RELATIONAL_DATASOURCE_PATH) - if err != nil { - return nil, err - } - - opampServer, err := s.createOpampServer() + _, err = opAmpModel.InitDB(constants.RELATIONAL_DATASOURCE_PATH) if err != nil { return nil, err } - s.opampServer = opampServer return s, nil } -func (s *Server) createOpampServer() (*opamp.Server, error) { - return opamp.NewServer(&opAmpModel.AllAgents), nil -} - func (s *Server) createPrivateServer(api *APIHandler) (*http.Server, error) { r := NewRouter() @@ -460,7 +447,14 @@ func (s *Server) Start() error { }() - s.opampServer.Start() + go func() { + zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", constants.OpAmpWsEndpoint)) + err := opamp.InitalizeServer(constants.OpAmpWsEndpoint, &opAmpModel.AllAgents) + if err != nil { + zap.S().Info("opamp ws server failed to start", err) + s.unavailableChannel <- healthcheck.Unavailable + } + }() return nil } @@ -478,9 +472,7 @@ func (s *Server) Stop() error { } } - if s.opampServer != nil { - s.opampServer.Stop() - } + opamp.StopServer() if s.ruleManager != nil { s.ruleManager.Stop() diff --git a/pkg/query-service/constants/constants.go b/pkg/query-service/constants/constants.go index 5bdc147c60..a18a69ac9a 100644 --- a/pkg/query-service/constants/constants.go +++ b/pkg/query-service/constants/constants.go @@ -11,6 +11,7 @@ const ( HTTPHostPort = "0.0.0.0:8080" // Address to serve http (query service) PrivateHostPort = "0.0.0.0:8085" // Address to server internal services like alert manager DebugHttpPort = "0.0.0.0:6060" // Address to serve http (pprof) + OpAmpWsEndpoint = "0.0.0.0:4320" // address for opamp websocket ) var ConfigSignozIo = "https://config.signoz.io/api/v1" From a3cad2b05b13300de4ec0e700d0e2ace7165a570 Mon Sep 17 00:00:00 2001 From: mindhash Date: Thu, 9 Mar 2023 15:15:56 +0530 Subject: [PATCH 06/24] chore: removed all errorf --- pkg/query-service/agentConf/db.go | 14 +++++++------- pkg/query-service/agentConf/manager.go | 16 ++++++++-------- .../app/opamp/configure_ingestionRules.go | 16 ++++++++-------- pkg/query-service/app/opamp/opamp_server.go | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 159bf920f1..2e67a2210c 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -92,7 +92,7 @@ func (r *Repo) GetLatestVersion(ctx context.Context, typ ElementTypeDef) (*Confi FROM agent_config_versions WHERE element_type=$2)`, typ, typ) if err != nil { - zap.S().Errorf("failed get latest config version for element:", typ, err) + zap.S().Error("failed get latest config version for element:", typ, err) } return &c, err } @@ -104,19 +104,19 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st } if len(elements) == 0 { - zap.S().Errorf("insert config called with no elements", c.ElementType) + zap.S().Error("insert config called with no elements", c.ElementType) return fmt.Errorf("config must have atleast one element") } if c.Version != 0 { - zap.S().Errorf("invalid version assignment while inserting agent config", c.Version, c.ElementType) + zap.S().Error("invalid version assignment while inserting agent config", c.Version, c.ElementType) return fmt.Errorf("user defined versions are not supported in the agent config") } configVersion, err := r.GetLatestVersion(ctx, c.ElementType) if err != nil { if err != sql.ErrNoRows { - zap.S().Errorf("failed to fetch latest config version", err) + zap.S().Error("failed to fetch latest config version", err) return fmt.Errorf("failed to fetch latest config version") } } @@ -155,7 +155,7 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st c.DeployResult) if err != nil { - zap.S().Errorf("error in inserting config version: ", zap.Error(err)) + zap.S().Error("error in inserting config version: ", zap.Error(err)) return fmt.Errorf("failed to insert ingestion rule") } @@ -200,7 +200,7 @@ func (r *Repo) updateDeployStatus(ctx context.Context, _, err := r.db.ExecContext(ctx, updateQuery, status, result, lastHash, lastconf, version, string(elementType)) if err != nil { - zap.S().Errorf("failed to get ingestion rule from db", err) + zap.S().Error("failed to get ingestion rule from db", err) return model.BadRequestStr("failed to get ingestion rule from db") } @@ -216,7 +216,7 @@ func (r *Repo) updateDeployStatusByHash(ctx context.Context, confighash string, _, err := r.db.ExecContext(ctx, updateQuery, status, result, confighash) if err != nil { - zap.S().Errorf("failed to get ingestion rule from db", err) + zap.S().Error("failed to get ingestion rule from db", err) return model.BadRequestStr("failed to get ingestion rule from db") } diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index c560200d8a..660a4eb3a0 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -27,7 +27,7 @@ type Manager struct { // Ready indicates if Manager can accept new config update requests func (mgr *Manager) Ready() bool { - if mgr.lock != 0 { + if atomic.LoadUint32(&mgr.lock) != 0 { return false } return opamp.Ready() @@ -96,7 +96,7 @@ func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { case ElementTypeSamplingRules: var config *tsp.Config if err := yaml.Unmarshal([]byte(configVersion.LastConf), &config); err != nil { - zap.S().Errorf("failed to read last conf correctly", err) + zap.S().Error("failed to read last conf correctly", err) return fmt.Errorf("failed to read the stored config correctly") } @@ -108,7 +108,7 @@ func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { opamp.AddToTracePipelineSpec("signoz_tail_sampling") configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) if err != nil { - zap.S().Errorf("failed to call agent config update for trace processor:", err) + zap.S().Error("failed to call agent config update for trace processor:", err) return fmt.Errorf("failed to deploy the config") } @@ -116,7 +116,7 @@ func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { case ElementTypeDropRules: var filterConfig *filterprocessor.Config if err := yaml.Unmarshal([]byte(configVersion.LastConf), &filterConfig); err != nil { - zap.S().Errorf("failed to read last conf correctly", err) + zap.S().Error("failed to read last conf correctly", err) return fmt.Errorf("failed to read the stored config correctly") } processorConf := map[string]interface{}{ @@ -126,7 +126,7 @@ func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { opamp.AddToMetricsPipelineSpec("filter") configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) if err != nil { - zap.S().Errorf("failed to call agent config update for trace processor:", err) + zap.S().Error("failed to call agent config update for trace processor:", err) return err } @@ -152,7 +152,7 @@ func UpsertFilterProcessor(ctx context.Context, version int, config *filterproce opamp.AddToMetricsPipelineSpec("filter") configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) if err != nil { - zap.S().Errorf("failed to call agent config update for trace processor:", err) + zap.S().Error("failed to call agent config update for trace processor:", err) return err } @@ -177,7 +177,7 @@ func (m *Manager) OnConfigUpdate(agentId string, hash string, err error) { message := "deploy successful" defer func() { - zap.S().Errorf(status, zap.String("agentId", agentId), zap.String("agentResponse", message)) + zap.S().Error(status, zap.String("agentId", agentId), zap.String("agentResponse", message)) }() if err != nil { @@ -203,7 +203,7 @@ func UpsertSamplingProcessor(ctx context.Context, version int, config *tsp.Confi opamp.AddToTracePipelineSpec("signoz_tail_sampling") configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) if err != nil { - zap.S().Errorf("failed to call agent config update for trace processor:", err) + zap.S().Error("failed to call agent config update for trace processor:", err) return err } diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go index 2c48aef96b..6a2bf7b39d 100644 --- a/pkg/query-service/app/opamp/configure_ingestionRules.go +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -22,7 +22,7 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ zap.S().Debugf("initiating deployment config", signal, processors) if signal != string(Metrics) && signal != string(Traces) { - zap.S().Errorf("received invalid signal int UpsertControllProcessors", signal) + zap.S().Error("received invalid signal int UpsertControllProcessors", signal) fnerr = fmt.Errorf("invalid kind of target to deploy processor") return } @@ -48,7 +48,7 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ agenthash, err := addIngestionControlToAgent(agent, signal, processors, withLB) if err != nil { - zap.S().Errorf("failed to push ingestion rules config to agent", agent.ID, err) + zap.S().Error("failed to push ingestion rules config to agent", agent.ID, err) continue } @@ -107,11 +107,11 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma zap.S().Debugf("LB not configured for agent. Preparing the lb exporter spec", agent.ID) serviceConf, err := prepareLbAgentSpec(agentConf) if err != nil { - zap.S().Errorf("failed to prepare lb exporter spec for agent", agent.ID, err) + zap.S().Error("failed to prepare lb exporter spec for agent", agent.ID, err) return confHash, err } if err := agentConf.Merge(serviceConf); err != nil { - zap.S().Errorf("failed to merge lb exporter spec for agent", agent.ID, err) + zap.S().Error("failed to merge lb exporter spec for agent", agent.ID, err) return confHash, err } } @@ -121,11 +121,11 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma zap.S().Debugf("OTLP internal not configured for agent. Preparing the lb exporter spec", agent.ID) serviceConf, err := prepareNonLbAgentSpec(agentConf) if err != nil { - zap.S().Errorf("failed to prepare OTLP internal receiver spec for agent", agent.ID, err) + zap.S().Error("failed to prepare OTLP internal receiver spec for agent", agent.ID, err) return confHash, err } if err := agentConf.Merge(serviceConf); err != nil { - zap.S().Errorf("failed to merge OTLP internal receiver spec for agent", agent.ID, err) + zap.S().Error("failed to merge OTLP internal receiver spec for agent", agent.ID, err) return confHash, err } } @@ -134,7 +134,7 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma // add ingestion control serviceConf, err := prepareControlProcesorsSpec(agentConf, Signal(signal), processors) if err != nil { - zap.S().Errorf("failed to prepare ingestion control processors for agent ", agent.ID, err) + zap.S().Error("failed to prepare ingestion control processors for agent ", agent.ID, err) return confHash, err } @@ -208,7 +208,7 @@ func prepareControlProcesorsSpec(agentConf *confmap.Conf, signal Signal, process // merge tracesPipelinePlan with current pipeline mergedPipeline, err := buildPipeline(signal, currentPipeline) if err != nil { - zap.S().Errorf("failed to build pipeline", signal, err) + zap.S().Error("failed to build pipeline", signal, err) return agentConf, err } diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go index 93456f7e1a..237b07f121 100644 --- a/pkg/query-service/app/opamp/opamp_server.go +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -76,7 +76,7 @@ func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer agent, created, err := srv.agents.FindOrCreateAgent(agentID, conn) if err != nil { - zap.S().Errorf("Failed to find or create agent %q: %v", agentID, err) + zap.S().Error("Failed to find or create agent %q: %v", agentID, err) // TODO: handle error } From 0fe8932a6fc577f6b3d47f5f2301d9ea6e1e4367 Mon Sep 17 00:00:00 2001 From: mindhash Date: Thu, 9 Mar 2023 15:20:23 +0530 Subject: [PATCH 07/24] chore: added a comment about zero version --- pkg/query-service/agentConf/db.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 2e67a2210c..4c732e96fa 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -109,6 +109,9 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st } if c.Version != 0 { + // the version can not be set by the user, we want to auto-assign the versions + // in a monotonically increasing order starting with 1. hence, we reject insert + // requests with version anything other than 0. here, 0 indicates un-assigned zap.S().Error("invalid version assignment while inserting agent config", c.Version, c.ElementType) return fmt.Errorf("user defined versions are not supported in the agent config") } From 9df43caecdd4be7576815478b24df55456f232b4 Mon Sep 17 00:00:00 2001 From: mindhash Date: Mon, 13 Mar 2023 11:37:05 +0530 Subject: [PATCH 08/24] feat: added user context for created by --- pkg/query-service/agentConf/db.go | 19 ++++++++++++++----- pkg/query-service/auth/jwt.go | 12 ++++++++++++ 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 4c732e96fa..5d556bdec5 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -9,6 +9,7 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" "go.signoz.io/signoz/pkg/query-service/agentConf/sqlite" + "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/model" "go.uber.org/zap" ) @@ -134,22 +135,30 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st } }() + userPayload, err := auth.ExtractUserFromContext(ctx) + if err != nil || userPayload == nil { + zap.S().Error("failed to find user in the context", err) + return fmt.Errorf("failed to identify user of the request") + } + // insert config configQuery := `INSERT INTO agent_config_versions( id, version, + created_by, element_type, active, is_valid, disabled, deploy_status, deploy_result) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)` + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)` _, err = r.db.ExecContext(ctx, configQuery, c.ID, c.Version, + userPayload.User.Id, c.ElementType, false, false, @@ -203,8 +212,8 @@ func (r *Repo) updateDeployStatus(ctx context.Context, _, err := r.db.ExecContext(ctx, updateQuery, status, result, lastHash, lastconf, version, string(elementType)) if err != nil { - zap.S().Error("failed to get ingestion rule from db", err) - return model.BadRequestStr("failed to get ingestion rule from db") + zap.S().Error("failed to update deploy status", err) + return model.BadRequestStr("failed to update deploy status") } return nil @@ -219,8 +228,8 @@ func (r *Repo) updateDeployStatusByHash(ctx context.Context, confighash string, _, err := r.db.ExecContext(ctx, updateQuery, status, result, confighash) if err != nil { - zap.S().Error("failed to get ingestion rule from db", err) - return model.BadRequestStr("failed to get ingestion rule from db") + zap.S().Error("failed to update deploy status", err) + return model.BadRequestStr("failed to update deploy status") } return nil diff --git a/pkg/query-service/auth/jwt.go b/pkg/query-service/auth/jwt.go index 705b7892ad..35a76838f2 100644 --- a/pkg/query-service/auth/jwt.go +++ b/pkg/query-service/auth/jwt.go @@ -92,3 +92,15 @@ func ExtractJwtFromContext(ctx context.Context) (string, error) { func ExtractJwtFromRequest(r *http.Request) (string, error) { return jwtmiddleware.FromAuthHeader(r) } + +// ExtractUserFromContext extracts user from context.Context using accssJwt. here, +// we do not validate if user exists as it would be done so while adding +// user to the context +func ExtractUserFromContext(ctx context.Context) (*model.UserPayload, error) { + jwt, err := ExtractJwtFromContext(ctx) + if err != nil { + return nil, err + } + + return validateUser(jwt) +} From efa1d7d270c2be1da71c68c68e2143e15659e1f7 Mon Sep 17 00:00:00 2001 From: mindhash Date: Mon, 13 Mar 2023 11:39:25 +0530 Subject: [PATCH 09/24] chore: changed debugf to debug --- pkg/query-service/agentConf/manager.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index 660a4eb3a0..49717cf83f 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -84,12 +84,12 @@ func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { configVersion, err := GetConfigVersion(ctx, typ, version) if err != nil { - zap.S().Debugf("failed to fetch config version during redeploy", err) + zap.S().Debug("failed to fetch config version during redeploy", err) return fmt.Errorf("failed to fetch details of the config version") } if configVersion == nil || (configVersion != nil && configVersion.LastConf == "") { - zap.S().Debugf("config version has no conf yaml", configVersion) + zap.S().Debug("config version has no conf yaml", configVersion) return fmt.Errorf("the config version can not be redeployed") } switch typ { @@ -177,7 +177,7 @@ func (m *Manager) OnConfigUpdate(agentId string, hash string, err error) { message := "deploy successful" defer func() { - zap.S().Error(status, zap.String("agentId", agentId), zap.String("agentResponse", message)) + zap.S().Info(status, zap.String("agentId", agentId), zap.String("agentResponse", message)) }() if err != nil { From 56148c54dcb4f9c15aee86826c5cc27488551326 Mon Sep 17 00:00:00 2001 From: mindhash Date: Mon, 13 Mar 2023 14:53:08 +0530 Subject: [PATCH 10/24] chore: removed lb from opamp + added config parser --- .../app/opamp/configure_ingestionRules.go | 123 ++------ pkg/query-service/app/opamp/configure_lb.go | 277 ------------------ .../app/opamp/configure_lb_test.go | 103 ------- .../app/opamp/otelconfig/config_parser.go | 182 ++++++++++++ 4 files changed, 200 insertions(+), 485 deletions(-) delete mode 100644 pkg/query-service/app/opamp/configure_lb.go delete mode 100644 pkg/query-service/app/opamp/configure_lb_test.go create mode 100644 pkg/query-service/app/opamp/otelconfig/config_parser.go diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go index 6a2bf7b39d..b4f59a1d6f 100644 --- a/pkg/query-service/app/opamp/configure_ingestionRules.go +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -9,6 +9,7 @@ import ( "github.com/open-telemetry/opamp-go/protobufs" "go.opentelemetry.io/collector/confmap" model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" "go.uber.org/zap" ) @@ -19,11 +20,12 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ // to pipeline. To enable or disable processors from pipeline, call // AddToTracePipeline() or RemoveFromTracesPipeline() prior to calling // this method - zap.S().Debugf("initiating deployment config", signal, processors) + + zap.S().Debug("initiating ingestion rules deployment config", signal, processors) if signal != string(Metrics) && signal != string(Traces) { - zap.S().Error("received invalid signal int UpsertControllProcessors", signal) - fnerr = fmt.Errorf("invalid kind of target to deploy processor") + zap.S().Error("received invalid signal int UpsertControlProcessors", signal) + fnerr = fmt.Errorf("signal not supported in ingestion rules: %s", signal) return } @@ -38,10 +40,10 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ return } - // flag to indicate if LB needs to be setup - withLB := false - if len(agents) > 1 { - withLB = true + if len(agents) > 1 && signal == string(Traces) { + zap.S().Debug("found multiple agents. this feature is not supported for traces pipeline (sampling rules)") + fnerr = fmt.Errorf("multiple agents not supported in sampling rules") + return } for _, agent := range agents { @@ -63,30 +65,6 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ return hash, nil } -func checkPipelineExists(agentConf *confmap.Conf, name string) bool { - service := agentConf.Get("service") - - pipeline := service.(map[string]interface{})["pipelines"].(map[string]interface{}) - if _, ok := pipeline[name]; !ok { - return false - } - return true -} - -func checkExporterExists(agentConf *confmap.Conf, signal string, name string) bool { - service := agentConf.Get("service") - signalSpec := service.(map[string]interface{})["pipelines"].(map[string]interface{})[signal] - exporters := signalSpec.(map[string]interface{})["exporters"].([]interface{}) - var found bool - for _, e := range exporters { - if e == name { - found = true - } - } - - return found -} - // addIngestionControlToAgent adds ingestion contorl rules to agent config func addIngestionControlToAgent(agent *model.Agent, signal string, processors map[string]interface{}, withLB bool) (string, error) { confHash := "" @@ -98,51 +76,13 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma agentConf := confmap.NewFromStringMap(c) - // check if LB needs to be configured - if signal == string(Traces) && withLB { - - // check if lb pipeline is setup for lb agents, here we assume - // that otlp_internal will exist implicitly if traces/lb pipeline exists - if agent.CanLB && !checkPipelineExists(agentConf, TracesLbPipelineName) { - zap.S().Debugf("LB not configured for agent. Preparing the lb exporter spec", agent.ID) - serviceConf, err := prepareLbAgentSpec(agentConf) - if err != nil { - zap.S().Error("failed to prepare lb exporter spec for agent", agent.ID, err) - return confHash, err - } - if err := agentConf.Merge(serviceConf); err != nil { - zap.S().Error("failed to merge lb exporter spec for agent", agent.ID, err) - return confHash, err - } - } - - // check if otlp_internal is setup for non-lb agents - if !agent.CanLB && !checkExporterExists(agentConf, string(Traces), OtlpInternalReceiver) { - zap.S().Debugf("OTLP internal not configured for agent. Preparing the lb exporter spec", agent.ID) - serviceConf, err := prepareNonLbAgentSpec(agentConf) - if err != nil { - zap.S().Error("failed to prepare OTLP internal receiver spec for agent", agent.ID, err) - return confHash, err - } - if err := agentConf.Merge(serviceConf); err != nil { - zap.S().Error("failed to merge OTLP internal receiver spec for agent", agent.ID, err) - return confHash, err - } - } - } - - // add ingestion control - serviceConf, err := prepareControlProcesorsSpec(agentConf, Signal(signal), processors) + // add ingestion control spec + err = makeIngestionControlSpec(agentConf, Signal(signal), processors) if err != nil { zap.S().Error("failed to prepare ingestion control processors for agent ", agent.ID, err) return confHash, err } - err = agentConf.Merge(serviceConf) - if err != nil { - return confHash, err - } - // ------ complete adding processor configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) if err != nil { @@ -180,49 +120,22 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma } // prepare spec to introduce ingestion control in agent conf -func prepareControlProcesorsSpec(agentConf *confmap.Conf, signal Signal, processors map[string]interface{}) (*confmap.Conf, error) { - agentProcessors := agentConf.Get("processors").(map[string]interface{}) - - for key, params := range processors { - agentProcessors[key] = params - } - - updatedProcessors := map[string]interface{}{ - "processors": agentProcessors, - } - - updatedProcessorConf := confmap.NewFromStringMap(updatedProcessors) - - // upsert changed processor parameters in config - err := agentConf.Merge(updatedProcessorConf) - if err != nil { - return agentConf, err - } +func makeIngestionControlSpec(agentConf *confmap.Conf, signal Signal, processors map[string]interface{}) error { + configParser := otelconfig.NewConfigParser(agentConf) + configParser.UpdateProcessors(processors) // edit pipeline if processor is missing - service := agentConf.Get("service") - - signalSpec := service.(map[string]interface{})["pipelines"].(map[string]interface{})[string(signal)] - currentPipeline := signalSpec.(map[string]interface{})["processors"].([]interface{}) + currentPipeline := configParser.PipelineProcessors(string(signal)) // merge tracesPipelinePlan with current pipeline mergedPipeline, err := buildPipeline(signal, currentPipeline) if err != nil { zap.S().Error("failed to build pipeline", signal, err) - return agentConf, err + return err } // add merged pipeline to the service - // todo(): check if this overwrites other processors - serviceConf := map[string]interface{}{ - "service": map[string]interface{}{ - "pipelines": map[string]interface{}{ - string(signal): map[string]interface{}{ - "processors": mergedPipeline, - }, - }, - }, - } + configParser.UpdateProcsInPipeline(string(signal), mergedPipeline) - return confmap.NewFromStringMap(serviceConf), nil + return nil } diff --git a/pkg/query-service/app/opamp/configure_lb.go b/pkg/query-service/app/opamp/configure_lb.go deleted file mode 100644 index de2ba6a420..0000000000 --- a/pkg/query-service/app/opamp/configure_lb.go +++ /dev/null @@ -1,277 +0,0 @@ -package opamp - -import ( - "fmt" - - deepcopy "github.com/barkimedes/go-deepcopy" - - "github.com/knadh/koanf/parsers/yaml" - "go.opentelemetry.io/collector/confmap" - model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" - "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/otlpreceiver" -) - -const ( - TracesLbPipelineName = "traces/lb" - TracesDefaultPipelineName = "traces" - OtlpInternalReceiver = "otlp/internal" - LbExporterName = "loadbalancing" -) - -func ConfigureLbExporter() error { - // acquire lock on config updater - agents := opAmpServer.agents.GetAllAgents() - var lbAgents []*model.Agent - var nonLbreceivers []*model.Agent - - for _, agent := range agents { - if agent.CanLB { - lbAgents = append(lbAgents, agent) - } else { - nonLbreceivers = append(nonLbreceivers, agent) - } - } - - if len(lbAgents) == 0 { - return fmt.Errorf("at least one agent with LB exporter support required") - } - - process := func(dryRun bool) error { - // todo: build a worker gorup and call agent updates in parallel - - for _, agent := range lbAgents { - configureLbAgents(agent, dryRun) - } - - for _, agent := range nonLbreceivers { - configureLbAgents(agent, dryRun) - } - - return nil - } - // todo(): allow dry run support in collecto rfirst - // if err := process(true); err != nil { - // return err - // } - - process(false) - - // if any of theagents fail, do not apply the config - return nil -} - -// configureLbAgents deploys lb exporter specific config to agent -func configureLbAgents(agent *model.Agent, dryRun bool) error { - config := agent.EffectiveConfig - c, err := yaml.Parser().Unmarshal([]byte(config)) - if err != nil { - return err - } - agentConf := confmap.NewFromStringMap(c) - err = prepareLbSpec(agentConf, agent.CanLB, dryRun) - if err != nil { - return err - } - // apply uodated config to agent - return nil -} - -// prepareLbSpec deploys lb exporter specific config to agent -func prepareLbSpec(agentConf *confmap.Conf, canLB bool, dryRun bool) error { - - // fetch agent config - var serviceConf *confmap.Conf - var err error - if canLB { - serviceConf, err = prepareLbAgentSpec(agentConf) - } else { - serviceConf, err = prepareNonLbAgentSpec(agentConf) - } - - if err != nil { - return err - } - - err = agentConf.Merge(serviceConf) - if err != nil { - return err - } - - return nil -} - -// prepareLbAgentSpec creates LB exporter agents -func prepareLbAgentSpec(agentConf *confmap.Conf) (serviceConf *confmap.Conf, fnerr error) { - - // add a new otlp receiver otlp_internal at 0.0.0.0:4949 - // this receiver will enable collecting traces re-routed by lb exporter - - // add a new pipeline - // traces/lb: - // receivers: [otlp, jaeger] - // processors: [] - // exporters: [lbExporter] - - // update receiver in service > pipelines > traces - // traces: - // receivers: [otlp_internal] - // processors: [signoz_tail_sampling, batch] - // exporters: [clickhousetraceexporter] - - // apply updated config - - // perform above in dry-run mode and then final mode - // if all agents succeed exit success else fail - - receivers := agentConf.Get("receivers").(map[string]interface{}) - - // add otlp internal receiver to receive traces from lb exporter - // at 0.0.0.0:4949 - receivers[OtlpInternalReceiver] = map[string]interface{}{ - "protocols": otlpreceiver.Protocols{ - GRPC: &otlpreceiver.GRPCServerSettings{ - Endpoint: "0.0.0.0:4949", - }, - HTTP: &otlpreceiver.HTTPServerSettings{ - Endpoint: "0.0.0.0:4949", - TLSSetting: nil, - }, - }} - - exporters := agentConf.Get("exporters").(map[string]interface{}) - - // load balancing settings from here: settings here https://pkg.go.dev/go.opentelemetry.io/collector/config/ - // more description of config: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/exporter/loadbalancingexporter#configuration - - exporters["loadbalancing"] = map[string]interface{}{ - "protocol": map[string]interface{}{ - "otlp": map[string]interface{}{ - // Timeout is the timeout for every attempt to send data to the backend. - "timeout": "1s", - }, - }, - "resolver": map[string]interface{}{ - "static": map[string]interface{}{ - "hostnames": []interface{}{ - "0.0.0.0:4949", - }, - }, - // "dns": map[string]interface{}{ - // // todo(): can we get this from agent? - // "hostname": constants.GetOrDefaultEnv("OTLP_ENDPOINT_HOST", "otlp.example.com"), - // "port": constants.GetOrDefaultEnv("OTLP_ENDPOINT_PORT", "4317"), - // }, - }, - } - - updatedReceivers := map[string]interface{}{ - "receivers": receivers, - "exporters": exporters, - } - - updatedReceiverConf := confmap.NewFromStringMap(updatedReceivers) - - err := agentConf.Merge(updatedReceiverConf) - if err != nil { - return nil, err - } - - // remove [jaegar, otlp] from pipelines >> traces - // change pipeline to - // traces: - // receivers: [otlp_internal] - // .... - - service := agentConf.Get("service").(map[string]interface{}) - pipelines := service["pipelines"].(map[string]interface{}) - - preUpdateTraces := service["pipelines"].(map[string]interface{})["traces"].(map[string]interface{}) - - // capture existing receiver list in case lb config needs to be applied - preUpdateRcvrs := preUpdateTraces["receivers"].([]interface{}) - lbreceivers, err := deepcopy.Anything(preUpdateRcvrs) - if err != nil { - return nil, err - } - pipelines[TracesLbPipelineName] = map[string]interface{}{ - "receivers": lbreceivers, - "processors": []string{}, - "exporters": []string{LbExporterName}, - } - preUpdateTraces["receivers"] = []string{OtlpInternalReceiver} - pipelines[TracesDefaultPipelineName], _ = deepcopy.Anything(preUpdateTraces) - - service["pipelines"] = pipelines - - // todo(amol): try updating the traces keys directly but if that - // destroys the rest of the pipeline then update full pipelines[] - s := map[string]interface{}{ - "service": service, - } - - return confmap.NewFromStringMap(s), nil - -} - -// prepareNonLbAgentSpec creates non-LB exporter agents that handle otlp receiver -// only. -func prepareNonLbAgentSpec(agentConf *confmap.Conf) (serviceConf *confmap.Conf, fnerr error) { - - receivers := agentConf.Get("receivers").(map[string]interface{}) - - // add otlp internal receiver to receive traces from lb exporter - // at 0.0.0.0:4949 - receivers["otlp/internal"] = otlpreceiver.Protocols{ - HTTP: &otlpreceiver.HTTPServerSettings{ - Endpoint: "0.0.0.0:4949", - }, - } - - updatedReceivers := map[string]interface{}{ - "receivers": receivers, - } - - updatedReceiverConf := confmap.NewFromStringMap(updatedReceivers) - - err := agentConf.Merge(updatedReceiverConf) - if err != nil { - return nil, err - } - - // remove [jaegar, otlp] from pipelines >> traces - // change pipeline to - // traces: - // receivers: [otlp_internal] - // .... - - service := agentConf.Get("service").(map[string]interface{}) - pipelines := service["pipelines"].(map[string]interface{}) - - preUpdateTraces := service["pipelines"].(map[string]interface{})["traces"].(map[string]interface{}) - - preUpdateTraces["receivers"] = []string{"otlp_internal"} - updatedTraces, err := deepcopy.Anything(preUpdateTraces) - if err != nil { - return nil, err - } - pipelines[TracesDefaultPipelineName] = updatedTraces - service["pipelines"] = pipelines - - // todo(amol): try updating the traces keys directly but if that - // destroys the rest of the pipeline then update full pipelines[] - s := map[string]interface{}{ - "service": service, - } - - return confmap.NewFromStringMap(s), nil - -} - -// DisableLbExporter in a given agent -func DisableLbExporter(agent *model.Agent, dryRun bool) error { - // reverse the steps from EnableLbExporter - // remove otlp_internal from pipelines >> traces - // move receivers from pipelines>>traces/lb to pipelines >> traces - // remove pipeline traces/lb - return nil -} diff --git a/pkg/query-service/app/opamp/configure_lb_test.go b/pkg/query-service/app/opamp/configure_lb_test.go deleted file mode 100644 index 124bfea8df..0000000000 --- a/pkg/query-service/app/opamp/configure_lb_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package opamp - -import ( - "fmt" - "testing" - - "github.com/knadh/koanf/parsers/yaml" - "go.opentelemetry.io/collector/confmap" -) - -func TestPrepareLbSpec(t *testing.T) { - c, err := yaml.Parser().Unmarshal([]byte(`--- -receivers: - otlp: - protocols: - grpc: -processors: - batch: - send_batch_size: 1000 - timeout: 10s -extensions: - zpages: {} -exporters: - logging: - debug: -service: - pipelines: - traces: - receivers: [otlp] - processors: [] - exporters: [logging] -`)) - - if err != nil { - fmt.Println("error loading yaml:", err) - t.Fail() - return - } - - agentConf := confmap.NewFromStringMap(c) - serviceConf, err := prepareLbAgentSpec(agentConf) - if err != nil { - fmt.Println("prepareLbSpec error:", err) - } - _ = agentConf.Merge(serviceConf) - configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) - if err != nil { - fmt.Println("failed to marshal agent conf", err) - } - fmt.Println("TestPrepareLbSpec merged config:", string(configR)) - if len(string(configR)) == 0 { - t.Fail() - return - } - -} - -func TestPrepareNonLbSpec(t *testing.T) { - c, err := yaml.Parser().Unmarshal([]byte(`--- -receivers: - otlp: - protocols: - grpc: -processors: - batch: - send_batch_size: 1000 - timeout: 10s -extensions: - zpages: {} -exporters: - logging: - debug: -service: - pipelines: - traces: - receivers: [otlp] - processors: [] - exporters: [logging] -`)) - - if err != nil { - fmt.Println("error loading yaml:", err) - t.Fail() - return - } - - agentConf := confmap.NewFromStringMap(c) - serviceConf, err := prepareNonLbAgentSpec(agentConf) - if err != nil { - fmt.Println("prepareLbSpec error:", err) - } - _ = agentConf.Merge(serviceConf) - configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) - if err != nil { - fmt.Println("failed to marshal agent conf", err) - } - fmt.Println("TestPrepareNonLbSpec merged config:", string(configR)) - if len(string(configR)) == 0 { - t.Fail() - return - } - -} diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser.go b/pkg/query-service/app/opamp/otelconfig/config_parser.go new file mode 100644 index 0000000000..b08dd0b27d --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/config_parser.go @@ -0,0 +1,182 @@ +package otelconfig + +import ( + "sync" + + "go.opentelemetry.io/collector/confmap" +) + +type ConfigPart map[string]interface{} + +type ConfigList []interface{} + +type ConfigParser struct { + lock sync.Mutex + agentConf *confmap.Conf +} + +func NewConfigParser(agentConf *confmap.Conf) ConfigParser { + return ConfigParser{ + agentConf: agentConf, + } +} + +func (cp *ConfigParser) Service() ConfigPart { + service := cp.agentConf.Get("service") + if service == nil { + return ConfigPart{} + } + return service.(ConfigPart) +} + +// components gets the high level parts like receivers, exporters, processors etc +func (cp *ConfigParser) components(partName, nameOptional string) ConfigPart { + parts := cp.agentConf.Get(partName) + if parts == nil { + return ConfigPart{} + } + + parsedParts := parts.(ConfigPart) + if nameOptional != "" { + if p, ok := parsedParts[nameOptional]; ok { + return p.(ConfigPart) + } else { + return ConfigPart{} + } + } + + return parsedParts +} + +func (cp *ConfigParser) Processors() ConfigPart { + return cp.components("processors", "") +} + +func (cp *ConfigParser) Processor(name string) ConfigPart { + return cp.components("processors", name) +} + +func (cp *ConfigParser) Exporters() ConfigPart { + return cp.components("exporters", "") +} + +func (cp *ConfigParser) Exporter(name string) ConfigPart { + return cp.components("exporters", name) +} + +func (cp *ConfigParser) Receivers() ConfigPart { + return cp.components("receivers", "") +} + +func (cp *ConfigParser) Receiver(name string) ConfigPart { + return cp.components("receivers", name) +} + +func (cp *ConfigParser) Pipelines(nameOptional string) ConfigPart { + services := cp.Service() + if p, ok := services["pipelines"]; ok { + pipelines := p.(ConfigPart) + if nameOptional != "" { + if namedPipeline, ok := pipelines[nameOptional]; ok { + return namedPipeline.(ConfigPart) + } else { + return ConfigPart{} + } + + } + return pipelines + } + return ConfigPart{} +} + +// component can be "recevers", "exporter" or "processors" +func (cp *ConfigParser) PipelineComponent(pipelineName, pipelineComponent string) ConfigList { + pipeline := cp.Pipelines(pipelineName) + if exporters, ok := pipeline[pipelineComponent]; ok { + exporters := exporters.(ConfigList) + return exporters + } + return ConfigList{} +} + +func (cp *ConfigParser) PipelineExporters(pipelineName string) ConfigList { + return cp.PipelineComponent(pipelineName, "exporters") +} + +func (cp *ConfigParser) PipelineReceivers(pipelineName string) ConfigList { + return cp.PipelineComponent(pipelineName, "receivers") +} + +func (cp *ConfigParser) PipelineProcessors(pipelineName string) ConfigList { + return cp.PipelineComponent(pipelineName, "processors") +} + +func (cp *ConfigParser) CheckPipelineExists(name string) bool { + if name == "" { + return false + } + pipelines := cp.Pipelines(name) + return len(pipelines) > 0 +} + +// CheckEntryInPipeline lets you look for an entry in pipeline by receiver, processor or exporter name +func (cp *ConfigParser) CheckEntryInPipeline(pipelineName, pipelineComponent, name string) bool { + if pipelineName == "" || pipelineComponent == "" || name == "" { + return false + } + + list := cp.PipelineComponent(pipelineName, pipelineComponent) + var found bool + for _, item := range list { + if item == name { + found = true + } + } + + return found +} + +func (cp *ConfigParser) CheckExporterInPipeline(pipelineName, name string) bool { + return cp.CheckEntryInPipeline(pipelineName, "exporters", name) +} + +func (cp *ConfigParser) CheckProcessorInPipeline(pipelineName, name string) bool { + return cp.CheckEntryInPipeline(pipelineName, "processors", name) +} + +func (cp *ConfigParser) Merge(c *confmap.Conf) { + cp.lock.Lock() + defer cp.lock.Unlock() + cp.agentConf.Merge(c) +} + +func (cp *ConfigParser) UpdateProcessors(processors ConfigPart) { + updates := cp.Processors() + + for key, params := range processors { + updates[key] = params + } + + updatedProcessors := ConfigPart{ + "processors": updates, + } + + updatedProcessorConf := confmap.NewFromStringMap(updatedProcessors) + + cp.Merge(updatedProcessorConf) +} + +func (cp *ConfigParser) UpdateProcsInPipeline(pipelineName string, list ConfigList) { + + serviceConf := ConfigPart{ + "service": ConfigPart{ + "pipelines": ConfigPart{ + pipelineName: ConfigPart{ + "processors": list, + }, + }, + }, + } + + cp.Merge(confmap.NewFromStringMap(serviceConf)) +} From e72b5b4fd218c91e5e37f696400dad1bdb73762d Mon Sep 17 00:00:00 2001 From: mindhash Date: Mon, 13 Mar 2023 20:09:49 +0530 Subject: [PATCH 11/24] chore: pulled updates on ingestion rules api --- ee/query-service/app/api/api.go | 36 ++- ee/query-service/app/api/dropRules.go | 19 ++ ee/query-service/app/api/ingestionRules.go | 157 +++++++++++++ ee/query-service/app/api/samplingRules.go | 19 ++ ee/query-service/app/server.go | 20 +- ee/query-service/ingestionRules/controller.go | 222 ++++++++++++++++++ ee/query-service/ingestionRules/db.go | 187 +++++++++++++++ ee/query-service/ingestionRules/db_test.go | 103 ++++++++ .../ingestionRules/dropBuilder.go | 53 +++++ .../ingestionRules/postableRule.go | 47 ++++ .../ingestionRules/samplingBuilder.go | 131 +++++++++++ .../ingestionRules/sqlite/init.go | 38 +++ ee/query-service/model/audit.go | 20 ++ ee/query-service/model/ingestionRules.go | 219 +++++++++++++++++ pkg/query-service/agentConf/db.go | 6 +- .../app/opamp/configure_ingestionRules.go | 2 +- pkg/query-service/auth/jwt.go | 12 - pkg/query-service/auth/userContext.go | 43 ++++ pkg/query-service/model/queryParams.go | 1 + 19 files changed, 1308 insertions(+), 27 deletions(-) create mode 100644 ee/query-service/app/api/dropRules.go create mode 100644 ee/query-service/app/api/ingestionRules.go create mode 100644 ee/query-service/app/api/samplingRules.go create mode 100644 ee/query-service/ingestionRules/controller.go create mode 100644 ee/query-service/ingestionRules/db.go create mode 100644 ee/query-service/ingestionRules/db_test.go create mode 100644 ee/query-service/ingestionRules/dropBuilder.go create mode 100644 ee/query-service/ingestionRules/postableRule.go create mode 100644 ee/query-service/ingestionRules/samplingBuilder.go create mode 100644 ee/query-service/ingestionRules/sqlite/init.go create mode 100644 ee/query-service/model/audit.go create mode 100644 ee/query-service/model/ingestionRules.go create mode 100644 pkg/query-service/auth/userContext.go diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 42410b65e7..303b8a74e0 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -5,6 +5,7 @@ import ( "github.com/gorilla/mux" "go.signoz.io/signoz/ee/query-service/dao" + "go.signoz.io/signoz/ee/query-service/ingestionRules" "go.signoz.io/signoz/ee/query-service/interfaces" "go.signoz.io/signoz/ee/query-service/license" baseapp "go.signoz.io/signoz/pkg/query-service/app" @@ -15,11 +16,12 @@ import ( ) type APIHandlerOptions struct { - DataConnector interfaces.DataConnector - AppDao dao.ModelDao - RulesManager *rules.Manager - FeatureFlags baseint.FeatureLookup - LicenseManager *license.Manager + DataConnector interfaces.DataConnector + AppDao dao.ModelDao + RulesManager *rules.Manager + FeatureFlags baseint.FeatureLookup + LicenseManager *license.Manager + IngestionController *ingestionRules.IngestionController } type APIHandler struct { @@ -114,6 +116,30 @@ func (ah *APIHandler) RegisterRoutes(router *mux.Router, am *baseapp.AuthMiddlew am.AdminAccess(ah.deleteDomain)). Methods(http.MethodDelete) + router.HandleFunc("/api/v1/dropRules/{version}", + am.AdminAccess(ah.listDropRules)). + Methods(http.MethodGet) + + router.HandleFunc("/api/v1/dropRules/{version}/deploy", + am.AdminAccess(ah.deployDropRules)). + Methods(http.MethodPost) + + router.HandleFunc("/api/v1/dropRules", + am.AdminAccess(ah.createDropRule)). + Methods(http.MethodPost) + + router.HandleFunc("/api/v1/samplingRules/{version}/deploy", + am.AdminAccess(ah.deploySamplingRules)). + Methods(http.MethodPost) + + router.HandleFunc("/api/v1/samplingRules/{version}", + am.AdminAccess(ah.listSamplingRules)). + Methods(http.MethodGet) + + router.HandleFunc("/api/v1/samplingRules", + am.AdminAccess(ah.createSamplingRule)). + Methods(http.MethodPost) + // base overrides router.HandleFunc("/api/v1/version", am.OpenAccess(ah.getVersion)).Methods(http.MethodGet) router.HandleFunc("/api/v1/invite/{token}", am.OpenAccess(ah.getInvite)).Methods(http.MethodGet) diff --git a/ee/query-service/app/api/dropRules.go b/ee/query-service/app/api/dropRules.go new file mode 100644 index 0000000000..2b43d28378 --- /dev/null +++ b/ee/query-service/app/api/dropRules.go @@ -0,0 +1,19 @@ +package api + +import ( + "net/http" + + "go.signoz.io/signoz/pkg/query-service/agentConf" +) + +func (ah *APIHandler) listDropRules(w http.ResponseWriter, r *http.Request) { + ah.listIngestionRulesHandler(w, r, agentConf.ElementTypeSamplingRules) +} + +func (ah *APIHandler) createDropRule(w http.ResponseWriter, r *http.Request) { + ah.createIngestionRule(w, r, agentConf.ElementTypeDropRules) +} + +func (ah *APIHandler) deployDropRules(w http.ResponseWriter, r *http.Request) { + ah.redeployIngestionRule(w, r, agentConf.ElementTypeDropRules) +} diff --git a/ee/query-service/app/api/ingestionRules.go b/ee/query-service/app/api/ingestionRules.go new file mode 100644 index 0000000000..a18f2ba05a --- /dev/null +++ b/ee/query-service/app/api/ingestionRules.go @@ -0,0 +1,157 @@ +package api + +import ( + "context" + "encoding/json" + "net/http" + "strconv" + + "github.com/gorilla/mux" + "go.signoz.io/signoz/ee/query-service/ingestionRules" + "go.signoz.io/signoz/ee/query-service/model" + "go.signoz.io/signoz/pkg/query-service/agentConf" + baseauth "go.signoz.io/signoz/pkg/query-service/auth" + "go.uber.org/zap" +) + +// ingestion rules handler - combines common methods for drop and sampling rules + +func parseAgentConfigVersion(r *http.Request) (int, *model.ApiError) { + versionString := mux.Vars(r)["version"] + + if versionString == "latest" { + return -1, nil + } + + version64, err := strconv.ParseInt(versionString, 0, 8) + + if err != nil { + return 0, model.BadRequestStr("invalid version number") + } + + return int(version64), nil +} + +func (ah *APIHandler) listIngestionRulesHandler(w http.ResponseWriter, r *http.Request, elementType agentConf.ElementTypeDef) { + + version, err := parseAgentConfigVersion(r) + if err != nil { + RespondError(w, model.BadRequestStr("invalid version"), nil) + return + } + + var payload *ingestionRules.IngestionRulesResponse + var apierr *model.ApiError + + if version > 0 { + payload, apierr = ah.listIngestionRulesByVersion(context.Background(), version, elementType) + } else { + payload, apierr = ah.listIngestionRules(context.Background(), elementType) + } + + if apierr != nil { + RespondError(w, apierr, payload) + return + } + ah.Respond(w, payload) +} + +// listIngestionRules lists rules for latest version +func (ah *APIHandler) listIngestionRules(ctx context.Context, elementType agentConf.ElementTypeDef) (*ingestionRules.IngestionRulesResponse, *model.ApiError) { + + // get lateset agent config + lastestConfig, err := agentConf.GetLatestVersion(ctx, elementType) + if err != nil || lastestConfig == nil { + zap.S().Errorf("failed to get latest agent config version ", err) + return nil, model.InternalErrorStr("Failed to get latest agent config version") + } + + payload, apierr := ah.opts.IngestionController.GetRulesByVersion(ctx, lastestConfig.Version, elementType) + if apierr != nil { + return payload, apierr + } + + history, err := agentConf.GetConfigHistory(ctx, elementType) + if apierr != nil { + return payload, apierr + } + payload.History = history + return payload, nil +} + +// listIngestionRulesByVersion lists rules along with config version history +func (ah *APIHandler) listIngestionRulesByVersion(ctx context.Context, version int, elementType agentConf.ElementTypeDef) (*ingestionRules.IngestionRulesResponse, *model.ApiError) { + + payload, apierr := ah.opts.IngestionController.GetRulesByVersion(ctx, version, elementType) + if apierr != nil { + return payload, apierr + } + + history, err := agentConf.GetConfigHistory(ctx, elementType) + if err != nil { + zap.S().Errorf("failed to retreive config history for element type", elementType, err) + return payload, model.InternalErrorStr("failed to retrieve agent config history") + } + + payload.History = history + return payload, nil +} + +func (ah *APIHandler) createIngestionRule(w http.ResponseWriter, r *http.Request, elementType agentConf.ElementTypeDef) { + + ctx, err := baseauth.AttachUserToContext(context.Background(), r) + if err != nil { + RespondError(w, model.BadRequestStr("failed to find or attach user to the context"), nil) + return + } + + req := ingestionRules.PostableIngestionRules{} + + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + RespondError(w, model.BadRequest(err), nil) + return + } + + createRule := func(ctx context.Context, postable []ingestionRules.PostableIngestionRule) (*ingestionRules.IngestionRulesResponse, *model.ApiError) { + if len(postable) == 0 { + zap.S().Warnf("found no rules in the http request, this will delete all the rules") + } + + for _, p := range postable { + if apierr := p.IsValid(); apierr != nil { + zap.S().Debugf("received invalid dropping rule in the POST request", apierr) + return nil, apierr + } + } + + return ah.opts.IngestionController.ApplyRules(ctx, elementType, postable) + } + + ingestionRuleResponse, apierr := createRule(ctx, req.Rules) + if apierr != nil { + RespondError(w, apierr, nil) + return + } + + ah.Respond(w, ingestionRuleResponse) +} + +func (ah *APIHandler) redeployIngestionRule(w http.ResponseWriter, r *http.Request, elementType agentConf.ElementTypeDef) { + version, apierr := parseAgentConfigVersion(r) + if apierr != nil { + RespondError(w, apierr, nil) + return + } + + if version == 0 { + RespondError(w, model.BadRequestStr("config version required"), nil) + return + } + + if err := agentConf.Redeploy(context.Background(), elementType, version); err != nil { + RespondError(w, model.InternalError(err), nil) + return + } + + ah.Respond(w, "deployment started") +} diff --git a/ee/query-service/app/api/samplingRules.go b/ee/query-service/app/api/samplingRules.go new file mode 100644 index 0000000000..91c3c3e176 --- /dev/null +++ b/ee/query-service/app/api/samplingRules.go @@ -0,0 +1,19 @@ +package api + +import ( + "net/http" + + "go.signoz.io/signoz/pkg/query-service/agentConf" +) + +func (ah *APIHandler) listSamplingRules(w http.ResponseWriter, r *http.Request) { + ah.listIngestionRulesHandler(w, r, agentConf.ElementTypeSamplingRules) +} + +func (ah *APIHandler) createSamplingRule(w http.ResponseWriter, r *http.Request) { + ah.createIngestionRule(w, r, agentConf.ElementTypeSamplingRules) +} + +func (ah *APIHandler) deploySamplingRules(w http.ResponseWriter, r *http.Request) { + ah.redeployIngestionRule(w, r, agentConf.ElementTypeSamplingRules) +} diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index 83412fdb42..bd9685df9c 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -21,6 +21,7 @@ import ( "go.signoz.io/signoz/ee/query-service/app/api" "go.signoz.io/signoz/ee/query-service/app/db" "go.signoz.io/signoz/ee/query-service/dao" + "go.signoz.io/signoz/ee/query-service/ingestionRules" "go.signoz.io/signoz/ee/query-service/interfaces" licensepkg "go.signoz.io/signoz/ee/query-service/license" "go.signoz.io/signoz/ee/query-service/usage" @@ -117,7 +118,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { go qb.Start(readerReady) reader = qb } else { - return nil, fmt.Errorf("Storage type: %s is not supported in query service", storage) + return nil, fmt.Errorf("storage type: %s is not supported in query service", storage) } <-readerReady @@ -132,6 +133,12 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { return nil, err } + // ingestion rules manager + ingestionController, err := ingestionRules.NewIngestionController(localDB, AppDbEngine) + if err != nil { + return nil, err + } + // initiate opamp _, err = opAmpModel.InitDB(baseconst.RELATIONAL_DATASOURCE_PATH) if err != nil { @@ -156,11 +163,12 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { telemetry.GetInstance().SetReader(reader) apiOpts := api.APIHandlerOptions{ - DataConnector: reader, - AppDao: modelDao, - RulesManager: rm, - FeatureFlags: lm, - LicenseManager: lm, + DataConnector: reader, + AppDao: modelDao, + RulesManager: rm, + FeatureFlags: lm, + LicenseManager: lm, + IngestionController: ingestionController, } apiHandler, err := api.NewAPIHandler(apiOpts) diff --git a/ee/query-service/ingestionRules/controller.go b/ee/query-service/ingestionRules/controller.go new file mode 100644 index 0000000000..31e2bf1463 --- /dev/null +++ b/ee/query-service/ingestionRules/controller.go @@ -0,0 +1,222 @@ +package ingestionRules + +import ( + "context" + "fmt" + + "github.com/jmoiron/sqlx" + "go.signoz.io/signoz/ee/query-service/model" + agentConf "go.signoz.io/signoz/pkg/query-service/agentConf" + "go.uber.org/zap" +) + +// Controller takes care of deployment cycle of ingestion rules. +type IngestionController struct { + Repo +} + +func NewIngestionController(db *sqlx.DB, engine string) (*IngestionController, error) { + repo := NewRepo(db) + err := repo.InitDB(engine) + return &IngestionController{Repo: repo}, err +} + +// IngestionRulesResponse is used to prepare http response for rule config related requests +type IngestionRulesResponse struct { + *agentConf.ConfigVersion + + Rules []model.IngestionRule `json:"rules"` + History []agentConf.ConfigVersion `json:"history"` +} + +// ApplyRules conditionally calls applyDropRules or applySampling rules +func (ic *IngestionController) ApplyRules(ctx context.Context, elementType agentConf.ElementTypeDef, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { + switch elementType { + case agentConf.ElementTypeDropRules: + return ic.ApplyDropRules(ctx, postable) + case agentConf.ElementTypeSamplingRules: + return ic.ApplySamplingRules(ctx, postable) + } + return nil, model.BadRequestStr("unexpected element type") +} + +// ApplyDropRules stores new or changed drop rules and initiates a new config update +func (ic *IngestionController) ApplyDropRules(ctx context.Context, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { + var dropRules []model.IngestionRule + + // scan through postable rules, to select the existing rules or insert missing ones + for _, r := range postable { + + // note: we process only new and changed rules here, deleted rules are not expected + // from client. if user deletes a rule, the client should not send that rule in the update. + // in effect, the new config version will not have that rule. + + if r.Id == "" { + // looks like a new or changed rule, store it first + inserted, err := ic.insertRule(ctx, &r) + if err != nil || inserted == nil { + zap.S().Errorf("failed to insert edited ingestion rule", err) + return nil, model.BadRequestStr("failed to insert edited rule") + } else { + dropRules = append(dropRules, *inserted) + } + } else { + // we pick the latest rule instead of one sent from client (browser) + // to make sure the ID is valid and relates to an ingestion rule. + // since rules are immutable, we can also use stored version from DB + // instead of picking the update from client (browser) request + selected, err := ic.GetRule(ctx, r.Id) + if err != nil || selected == nil { + zap.S().Errorf("failed to find edited ingestion rule", err, r.Id) + return nil, model.BadRequestStr("failed to find edited rule, invalid request") + } + dropRules = append(dropRules, *selected) + } + + } + + // prepare filter config (processor) from the drop rules + filterConfig, err := PrepareDropFilter(dropRules) + if err != nil { + zap.S().Errorf("failed to generate processor config from ingestion rules for deployment", err) + return nil, model.BadRequest(err) + } + + if !agentConf.Ready() { + // todo(amol): may be good idea to new ingestion rules created in this request? + return nil, model.InternalErrorStr("Agent updater unavailable at the moment. Please try in sometime") + } + + // prepare config elements + elements := make([]string, len(dropRules)) + for i, r := range dropRules { + elements[i] = r.Id + } + + // prepare config by calling gen func + cfg, err := agentConf.StartNewVersion(ctx, agentConf.ElementTypeDropRules, elements) + if err != nil || cfg == nil { + zap.S().Errorf("failed to start a new config version for drop rules", err) + return nil, model.InternalError(err) + } + + zap.S().Info("applying drop rule config", cfg) + + // queue up the config to push to opamp + err = agentConf.UpsertFilterProcessor(ctx, cfg.Version, filterConfig) + history, _ := agentConf.GetConfigHistory(ctx, agentConf.ElementTypeDropRules) + + response := &IngestionRulesResponse{ + ConfigVersion: cfg, + Rules: dropRules, + History: history, + } + + if err != nil { + zap.S().Errorf("failed to insert drop rules into agent config", filterConfig, err) + return response, model.InternalErrorStr(fmt.Sprintf("failed to apply drop rules: %s", err.Error())) + } + + return response, nil +} + +// GetRulesByVersion responds with version info and associated drop rules +func (ic *IngestionController) GetRulesByVersion(ctx context.Context, version int, typ agentConf.ElementTypeDef) (*IngestionRulesResponse, *model.ApiError) { + rules, apierr := ic.getRulesByVersion(ctx, version) + if apierr != nil { + zap.S().Errorf("failed to get ingestion rules for version", version, apierr) + return nil, model.InternalErrorStr("failed to get drop rules for given version") + } + + configVersion, err := agentConf.GetConfigVersion(ctx, typ, version) + if err != nil || configVersion == nil { + zap.S().Errorf("failed to get version info", version, err) + return nil, model.InternalErrorStr("failed to get info on the given version") + } + + return &IngestionRulesResponse{ + ConfigVersion: configVersion, + Rules: rules, + }, nil +} + +// ApplySamplingRules stores new or changed sampling rules and initiates a new config update +func (ic *IngestionController) ApplySamplingRules(ctx context.Context, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { + var smplRules []model.IngestionRule + + // scan through postable rules, to select the existing rules or insert missing ones + for _, r := range postable { + if apierr := r.IsValid(); apierr != nil { + return nil, apierr + } + + if err := r.Config.SamplingConfig.Valid(); err != nil { + return nil, model.BadRequest(err) + } + + // note: we process only new and changed rules here, deleted rules are not expected + // from client. if user deletes a rule, the client should not send that rule in the update. + // in effect, the new config version will not have that rule. + + if r.Id == "" { + // looks like a new or changed rule, store it first + inserted, err := ic.insertRule(ctx, &r) + if err != nil || inserted == nil { + zap.S().Errorf("failed to insert edited ingestion rule", err) + return nil, model.BadRequestStr("failed to insert edited rule") + } else { + smplRules = append(smplRules, *inserted) + } + } else { + selected, err := ic.GetRule(ctx, r.Id) + if err != nil || selected == nil { + zap.S().Errorf("failed to find edited ingestion rule", err) + return nil, model.BadRequestStr("failed to find edited rule, invalid request") + } + smplRules = append(smplRules, *selected) + } + + } + + // prepare params for sampling processor from the rules + params, err := PrepareTailSamplingParams(smplRules) + if err != nil { + zap.S().Errorf("failed to generate processor config from ingestion rules for deployment", err) + return nil, model.BadRequest(err) + } + + if !agentConf.Ready() { + return nil, model.InternalErrorStr("Agent updater unavailable at the moment. Please try in sometime") + } + + // prepare config elements + elements := make([]string, len(smplRules)) + for i, r := range smplRules { + elements[i] = r.Id + } + + // prepare config by calling gen func + cfg, err := agentConf.StartNewVersion(ctx, agentConf.ElementTypeSamplingRules, elements) + if err != nil || cfg == nil { + return nil, model.InternalError(err) + } + + params.Version = cfg.Version + zap.S().Info("applying sampling rule config", cfg) + + // queue up the config to push to opamp + err = agentConf.UpsertSamplingProcessor(ctx, cfg.Version, params) + history, _ := agentConf.GetConfigHistory(ctx, agentConf.ElementTypeSamplingRules) + + response := &IngestionRulesResponse{ + ConfigVersion: cfg, + Rules: smplRules, + History: history, + } + + if err != nil { + zap.S().Errorf("failed to insert sampling rules into agent config: ", params, err) + return response, model.InternalErrorStr(fmt.Sprintf("failed to apply drop rules: %s", err.Error())) + } + return response, nil +} diff --git a/ee/query-service/ingestionRules/db.go b/ee/query-service/ingestionRules/db.go new file mode 100644 index 0000000000..7fa83b9b78 --- /dev/null +++ b/ee/query-service/ingestionRules/db.go @@ -0,0 +1,187 @@ +package ingestionRules + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + + "github.com/google/uuid" + "github.com/jmoiron/sqlx" + "github.com/pkg/errors" + "go.signoz.io/signoz/ee/query-service/ingestionRules/sqlite" + "go.signoz.io/signoz/ee/query-service/model" + baseauth "go.signoz.io/signoz/pkg/query-service/auth" + "go.uber.org/zap" +) + +// Repo handles DDL and DML ops on ingestion rules +type Repo struct { + db *sqlx.DB +} + +// NewRepo initiates a new ingestion repo +func NewRepo(db *sqlx.DB) Repo { + return Repo{ + db: db, + } +} + +func (r *Repo) InitDB(engine string) error { + switch engine { + case "sqlite3", "sqlite": + return sqlite.InitDB(r.db) + default: + return fmt.Errorf("unsupported db") + } +} + +// InsertRule stores a given postable rule to database +func (r *Repo) insertRule(ctx context.Context, postable *PostableIngestionRule) (*model.IngestionRule, error) { + userId, err := baseauth.UserIdFromContext(ctx) + if err != nil { + return nil, err + } + + if err := postable.IsValid(); err != nil { + return nil, errors.Wrap(err, "failed to validate postable ingestion rule") + } + + if postable.Priority == 0 { + // default priority of all rules is set to same number 1. so, + // all rules will have same priority to start with. + // user can chagne the priority to higher integer value to execute them first + postable.Priority = 1 + } + + rawConfig, err := json.Marshal(postable.Config) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal postable ingestion config") + } + + insertRow := &model.IngestionRule{ + Id: uuid.New().String(), + Name: postable.Name, + Source: postable.Source, + RuleType: postable.RuleType, + RuleSubType: postable.RuleSubType, + Priority: postable.Priority, + Config: postable.Config, + RawConfig: string(rawConfig), + Creator: model.Creator{CreatedBy: userId}, + } + + insertQuery := `INSERT INTO ingestion_rules + (id, name, source, rule_type, rule_subtype, priority, config_json, created_by) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)` + + _, err = r.db.ExecContext(ctx, + insertQuery, + insertRow.Id, + insertRow.Name, + insertRow.Source, + insertRow.RuleType, + insertRow.RuleSubType, + insertRow.Priority, + insertRow.RawConfig, + userId) + + if err != nil { + zap.S().Error("error in inserting ingestion rule data: ", zap.Error(err)) + return insertRow, errors.Wrap(err, "failed to insert ingestion rule") + } + + return insertRow, nil +} + +// getRulesByVersion returns rules associated with a given version +func (r *Repo) getRulesByVersion(ctx context.Context, version int) ([]model.IngestionRule, []error) { + var errors []error + rules := []model.IngestionRule{} + + versionQuery := `SELECT r.id, + source, + priority, + rule_type, + rule_subtype, + name, + config_json + FROM ingestion_rules r, + agent_config_elements e, + agent_config_versions v + WHERE r.id = e.element_id + AND v.id = e.version_id + AND v.version = $1` + + err := r.db.SelectContext(ctx, &rules, versionQuery, version) + if err != nil { + if err == sql.ErrNoRows { + return rules, nil + } + return nil, []error{fmt.Errorf("failed to get ingestion rules from db: %v", err)} + } + + if len(rules) == 0 { + return rules, nil + } + + for _, d := range rules { + if err := d.ParseRawConfig(); err != nil { + errors = append(errors, err) + } + } + + return rules, errors +} + +// GetRule returns rules and errors (if any) +func (r *Repo) GetRule(ctx context.Context, id string) (*model.IngestionRule, *model.ApiError) { + rules := []model.IngestionRule{} + + ruleQuery := `SELECT id, + source, + priority, + rule_type, + rule_subtype, + name, + config_json + FROM ingestion_rules + WHERE id = $1` + + err := r.db.SelectContext(ctx, &rules, ruleQuery, id) + if err != nil { + zap.S().Errorf("failed to get ingestion rule from db", err) + return nil, model.BadRequestStr("failed to get ingestion rule from db") + } + + if len(rules) == 0 { + zap.S().Warnf("No row found for ingestion rule id", id) + return nil, nil + } + + if len(rules) == 1 { + err := rules[0].ParseRawConfig() + if err != nil { + zap.S().Errorf("invalid rule config found", id, err) + return &rules[0], model.InternalErrorStr("found an invalid rule config ") + } + return &rules[0], nil + } + + return nil, model.InternalErrorStr("mutliple rules with same id") + +} + +func (r *Repo) DeleteRule(ctx context.Context, id string) *model.ApiError { + deleteQuery := `DELETE + FROM ingestion_rules + WHERE id = $1` + + _, err := r.db.ExecContext(ctx, deleteQuery, id) + if err != nil { + return model.BadRequest(err) + } + + return nil + +} diff --git a/ee/query-service/ingestionRules/db_test.go b/ee/query-service/ingestionRules/db_test.go new file mode 100644 index 0000000000..7f0e39df2e --- /dev/null +++ b/ee/query-service/ingestionRules/db_test.go @@ -0,0 +1,103 @@ +package ingestionRules + +import ( + "context" + "fmt" + "math/rand" + "reflect" + "testing" + "time" + + "github.com/jmoiron/sqlx" + + _ "github.com/mattn/go-sqlite3" + + "go.signoz.io/signoz/ee/query-service/ingestionRules/sqlite" + "go.signoz.io/signoz/ee/query-service/model" + basemodel "go.signoz.io/signoz/pkg/query-service/model" +) + +var TestDBPath string +var TestDBConn *sqlx.DB + +func init() { + rand.Seed(time.Now().UnixNano()) + TestDBPath = fmt.Sprintf("/var/tmp/%d.db", rand.Int()) +} + +func GetTestDB() (*sqlx.DB, error) { + if TestDBConn != nil { + return TestDBConn, nil + } + db, err := sqlx.Open("sqlite3", TestDBPath) + if err != nil { + return nil, err + } + db.SetMaxOpenConns(3) + + if err := sqlite.InitDB(db); err != nil { + return nil, err + } + + TestDBConn = db + return db, nil +} + +func TestInsertRule(t *testing.T) { + db, err := GetTestDB() + + if err != nil { + t.Fatal(err) + } + + repo := NewRepo(db) + + p1 := PostableIngestionRule{ + Name: "p1", + Source: model.IngestionSourceMetrics, + RuleType: model.IngestionRuleTypeDrop, + RuleSubType: model.IngestionRuleSubTypeAO, + Config: &model.IngestionRuleConfig{ + DropConfig: DropConfig{ + FilterSet: basemodel.FilterSet{ + Operator: "AND", + Items: []basemodel.FilterItem{ + { + Key: "name", + KeyType: "metric_name", + Operator: "==", + Value: "signoz_calls_total", + }, + { + Key: "http_status_code", + KeyType: "label", + Operator: "==", + Value: "401", + }}, + }, + }, + }, + } + + inserted, err := repo.InsertRule(context.Background(), &p1) + if err != nil { + t.Fatal(err) + } + + selected, errs := repo.GetRule(context.Background(), inserted.Id) + if !errs.IsNil() { + t.Fatal(err) + } + + if selected.Id != inserted.Id { + t.Logf("failed to insert rule") + t.Fail() + } + + if !reflect.DeepEqual(selected.Config, inserted.Config) { + t.Logf("failed to insert rule config correctly") + t.Fail() + } + + repo.DeleteRule(context.Background(), inserted.Id) +} diff --git a/ee/query-service/ingestionRules/dropBuilder.go b/ee/query-service/ingestionRules/dropBuilder.go new file mode 100644 index 0000000000..59a6086892 --- /dev/null +++ b/ee/query-service/ingestionRules/dropBuilder.go @@ -0,0 +1,53 @@ +package ingestionRules + +import ( + "go.signoz.io/signoz/ee/query-service/model" + filterprocessor "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/filterprocessor" +) + +// this file contains methods to transform ingestion rules into +// collector config components + +func PrepareDropFilter(rules []model.IngestionRule) (*filterprocessor.Config, error) { + metricRules := getMetricRules(rules) + + metricExprs, err := PrepareDropExpressions(metricRules) + if len(err) > 0 { + return nil, err[0] + } + return &filterprocessor.Config{ + Metrics: filterprocessor.MetricFilters{ + DataPointConditions: metricExprs, + }, + }, nil +} + +// PrepareDropExpression creates the final OTTL expression for filter processor +func PrepareDropExpressions(rules []model.IngestionRule) (result []string, fnerr []error) { + // result captures the final expression to be set in fitler processor + if len(rules) == 0 { + return result, nil + } + + for _, r := range rules { + if r.RuleType == model.IngestionRuleTypeDrop { + expr, err := r.Config.DropConfig.PrepareExpression() + if err != nil { + fnerr = append(fnerr, err) + } + result = append(result, expr) + } + } + + return result, nil +} + +func getMetricRules(allRules []model.IngestionRule) []model.IngestionRule { + metricRules := make([]model.IngestionRule, 0) + for _, r := range allRules { + if r.RuleType == model.IngestionRuleTypeDrop && r.Source == model.IngestionSourceMetrics { + metricRules = append(metricRules, r) + } + } + return metricRules +} diff --git a/ee/query-service/ingestionRules/postableRule.go b/ee/query-service/ingestionRules/postableRule.go new file mode 100644 index 0000000000..c19ab5e6db --- /dev/null +++ b/ee/query-service/ingestionRules/postableRule.go @@ -0,0 +1,47 @@ +package ingestionRules + +import ( + "fmt" + + "go.signoz.io/signoz/ee/query-service/model" +) + +// PostableIngestionRules are a list of user defined ingestion rules +type PostableIngestionRules struct { + Rules []PostableIngestionRule `json:"rules"` +} + +// PostableIngestionRule captures user inputs in setting the ingestion rule +type PostableIngestionRule struct { + Id string `json:"id"` + Name string `json:"name"` + Source model.IngestionSource `json:"source"` + RuleType model.IngestionRuleType `json:"ruleType"` + RuleSubType model.IngestionRuleSubtype `json:"ruleSubType"` + Priority int `json:"priority"` + Config *model.IngestionRuleConfig `json:"config"` +} + +// IsValid checks if postable rule has all the required params +func (p *PostableIngestionRule) IsValid() *model.ApiError { + if p.Name == "" { + return model.BadRequestStr("ingestion rule name is required") + } + if p.RuleType == "" { + return model.BadRequestStr("ingestion rule type is required") + } + + if p.RuleSubType == "" { + return model.BadRequestStr("ingestion rule subtype is required") + } + + if p.Source == "" { + return model.BadRequestStr("ingestion source is required") + } + + if p.Config == nil { + return model.BadRequestStr(fmt.Sprintf("invalid config found on rule: %s", p.Name)) + } + + return nil +} diff --git a/ee/query-service/ingestionRules/samplingBuilder.go b/ee/query-service/ingestionRules/samplingBuilder.go new file mode 100644 index 0000000000..e1d2bded48 --- /dev/null +++ b/ee/query-service/ingestionRules/samplingBuilder.go @@ -0,0 +1,131 @@ +package ingestionRules + +import ( + "sort" + "strings" + "time" + + "go.signoz.io/signoz/ee/query-service/model" + "go.uber.org/zap" + + tsp "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/tailsampler" +) + +func preparePolicycfg(config model.SamplingConfig) tsp.PolicyCfg { + policy := tsp.PolicyCfg{ + Root: true, + Name: config.Name, + Priority: config.Priority, + Type: "policy_group", + PolicyFilterCfg: tsp.PolicyFilterCfg{ + StringAttributeCfgs: []tsp.StringAttributeCfg{}, + NumericAttributeCfgs: []tsp.NumericAttributeCfg{}, + }, + } + + // assign root filter operator + if strings.ToLower(config.FilterSet.Operator) == "and" { + policy.FilterOp = "AND" + } + if strings.ToLower(config.FilterSet.Operator) == "or" { + policy.FilterOp = "OR" + } + + // assign root filter conditions + for _, f := range config.FilterSet.Items { + + switch f.Value.(type) { + case string: + value := f.Value.(string) + policy.StringAttributeCfgs = append(policy.StringAttributeCfgs, tsp.StringAttributeCfg{Key: f.Key, Values: []string{value}}) + case []string: + values := f.Value.([]string) + policy.StringAttributeCfgs = append(policy.StringAttributeCfgs, tsp.StringAttributeCfg{Key: f.Key, Values: values}) + case int, int32, int64: + value := f.Value.(int) + policy.NumericAttributeCfgs = append(policy.NumericAttributeCfgs, tsp.NumericAttributeCfg{Key: f.Key, MinValue: int64(value), MaxValue: int64(value)}) + case float64: + // improvement: currrently only int64 is supported by tail based sampler, add float64 support + value := f.Value.(float64) + policy.NumericAttributeCfgs = append(policy.NumericAttributeCfgs, tsp.NumericAttributeCfg{Key: f.Key, MinValue: int64(value), MaxValue: int64(value)}) + case float32: + // improvement: currrently only int64 is supported by tail based sampler, add float32 support + value := f.Value.(float32) + policy.NumericAttributeCfgs = append(policy.NumericAttributeCfgs, tsp.NumericAttributeCfg{Key: f.Key, MinValue: int64(value), MaxValue: int64(value)}) + } + } + + policy.ProbabilisticCfg = tsp.ProbabilisticCfg{ + // HashSalt: // + SamplingPercentage: float64(config.SamplingPercent), + } + + return policy +} + +func isPolicyCfgOk(p tsp.PolicyCfg) bool { + // todo: added more validations for policy config + return true +} + +// PrepareTailSamplingParams transforms sampling rules into tail sampler processor config +func PrepareTailSamplingParams(rules []model.IngestionRule) (*tsp.Config, error) { + + if len(rules) == 0 { + // no rules setup, configure always sample policy + // which allows all records to pass-through and no dropouts + return &tsp.Config{}, nil + } + + // we want the order to be maintained, so a list + var finalPolicies []tsp.PolicyCfg + + sort.Sort(model.IngestionRulesByPriority(rules)) + + for _, root := range rules { + if root.RuleType != model.IngestionRuleTypeSampling || root.Config == nil { + continue + } + rootConfig := root.Config.SamplingConfig + conditions := rootConfig.Conditions + + rootPolicy := preparePolicycfg(rootConfig) + + if len(conditions) == 0 { + zap.S().Warnf("found a sampling rule with no default condtion, skipping the policy", rootConfig.Name) + continue + } + + // build sub-policies for conditions + + // first, sort conditions to order by priority and move default to the end + sort.Sort(model.SamplingConfigByPriority(conditions)) + + for _, condition := range conditions { + + if condition.Default { + rootPolicy.ProbabilisticCfg = tsp.ProbabilisticCfg{ + // HashSalt: // + SamplingPercentage: float64(condition.SamplingPercent), + } + continue + } + + // prepare subpolicy for this condition + rootPolicy.SubPolicies = append(rootPolicy.SubPolicies, preparePolicycfg(condition)) + + } // end for loop conditions + + if isPolicyCfgOk(rootPolicy) { + finalPolicies = append(finalPolicies, rootPolicy) + } + } + + zap.S().Debugf("sorted sampling policies:", finalPolicies) + + return &tsp.Config{ + DecisionWait: 30 * time.Second, + NumTraces: 50000, + PolicyCfgs: finalPolicies, + }, nil +} diff --git a/ee/query-service/ingestionRules/sqlite/init.go b/ee/query-service/ingestionRules/sqlite/init.go new file mode 100644 index 0000000000..37806dfa09 --- /dev/null +++ b/ee/query-service/ingestionRules/sqlite/init.go @@ -0,0 +1,38 @@ +package sqlite + +import ( + "fmt" + + "github.com/pkg/errors" + + "github.com/jmoiron/sqlx" +) + +func InitDB(db *sqlx.DB) error { + var err error + if db == nil { + return fmt.Errorf("invalid db connection") + } + + table_schema := `CREATE TABLE IF NOT EXISTS ingestion_rules( + id TEXT PRIMARY KEY, + created_by TEXT, + created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updated_by TEXT, + updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + source VARCHAR(100), + priority INTEGER, + rule_type VARCHAR(100), + rule_subtype VARCHAR(100), + name VARCHAR(400) NOT NULL, + config_json TEXT, + error_message TEXT + ); + ` + + _, err = db.Exec(table_schema) + if err != nil { + return errors.Wrap(err, "Error in creating ingestion rules table") + } + return nil +} diff --git a/ee/query-service/model/audit.go b/ee/query-service/model/audit.go new file mode 100644 index 0000000000..7143d8d499 --- /dev/null +++ b/ee/query-service/model/audit.go @@ -0,0 +1,20 @@ +package model + +import ( + "time" +) + +type Creator struct { + CreatedBy string `json:"created_by" db:"created_by"` + Created time.Time `json:"created_at" db:"created_at"` +} + +type Updater struct { + UpdatedBy string + Updated time.Time +} + +type AuditRecord struct { + Creator + Updater +} diff --git a/ee/query-service/model/ingestionRules.go b/ee/query-service/model/ingestionRules.go new file mode 100644 index 0000000000..644e6a06a9 --- /dev/null +++ b/ee/query-service/model/ingestionRules.go @@ -0,0 +1,219 @@ +package model + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" + basemodel "go.signoz.io/signoz/pkg/query-service/model" +) + +// this file contains all the model definitions +// required for cost optimizer + +const ( + MetricName = "metric_name" + ResourceAttribute = "resource_attribute" + Label = "label" +) + +// IngestionSource identifies the source data on which the ingestion rule will run on +type IngestionSource string + +const ( + IngestionSourceMetrics = "metrics" + IngestionSourceLogs = "logs" + IngestionSourceTraces = "traces" +) + +// IngestionRuleType identifies the type of rule: drop, sampling, inclusion +type IngestionRuleType string + +const ( + IngestionRuleTypeDrop = "drop_rule" + IngestionRuleTypeSampling = "sampling_rule" +) + +// IngestionRuleSubtype identifies sub type of rules and is +// dependent on IngestionRuleType +type IngestionRuleSubtype string + +const ( + IngestionRuleSubTypeAO IngestionRuleSubtype = "always_on" + IngestionRuleSubTypeCond IngestionRuleSubtype = "conditional" +) + +// IngestionRule is stored and also deployed finally to collector config +type IngestionRule struct { + Id string `json:"id,omitempty" db:"id"` + Name string `json:"name,omitempty" db:"name"` + Priority int `json:"priority,omitempty" db:"priority"` + Source IngestionSource `json:"source" db:"source"` + RuleType IngestionRuleType `json:"ruleType" db:"rule_type"` + RuleSubType IngestionRuleSubtype `json:"ruleSubType" db:"rule_subtype"` + + // configuration for drop and sampling rules + RawConfig string `db:"config_json"` + + Config *IngestionRuleConfig `json:"config"` + + Creator +} + +func (i *IngestionRule) ParseRawConfig() error { + c := IngestionRuleConfig{} + err := json.Unmarshal([]byte(i.RawConfig), &c) + if err != nil { + return errors.Wrap(err, "failed to parse ingestion rule config") + } + i.Config = &c + return nil +} + +// IngestionRuleConfig identifies a rule configuration that turns into +// an input for processor in collector config. An ingestion rule is +// more likely to have either drop or a sampling config but not both +// at the same time. however this structure does not discourage such a +// condition. +type IngestionRuleConfig struct { + DropConfig + SamplingConfig +} + +// DropConfig configuration for dropping metrics, logs or traces record +type DropConfig struct { + // identifies filters for this drop rule + DropFilter basemodel.FilterSet `json:"dropFilter"` +} + +// PrepareExpression prepares an OTTL expression that can be passed to +// filter processor. +func (d *DropConfig) PrepareExpression() (result string, fnerr error) { + + // holds a list of expressions + var exprs []string + + for _, i := range d.DropFilter.Items { + var expr string + switch i.KeyType { + case MetricName: + // todo(amol): may need to transform operator to OTTL + // https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/pkg/ottl/README.md#boolean-expressions + expr = fmt.Sprintf("name %s \"%s\"", i.Operator, i.Value) + case ResourceAttribute: + expr = fmt.Sprintf("resource.attributes[\"%s\"] %s \"%s\"", i.Key, i.Operator, i.Value) + case Label: + expr = fmt.Sprintf("attributes[\"%s\"] %s \"%s\"", i.Key, i.Operator, i.Value) + } + if expr != "" { + exprs = append(exprs, expr) + } + } + + if len(exprs) == 0 { + return + } + + result = exprs[0] + + for _, e := range exprs[1:] { + result += fmt.Sprintf(" %s %s", d.DropFilter.Operator, e) + } + return +} + +type SamplingMethod string + +const ( + // allows probabilistic dropping of ingested record to meet + // a storage objective + SamplingMethodProbabilistic = "probabilistic" + + // ensures the ingested records that match a the filter are always + // included in sampled data + SamplingMethodIncludeAll = "include_all" +) + +// SamplingConfig defines a sampling strategy rule +type SamplingConfig struct { + ID string `json:"id"` + Name string `json:"name"` + + // Set to true for sampling rule (root) and false for conditions + Root bool `json:"root"` + + Priority int `json:"priority"` + + // filter conditions + FilterSet basemodel.FilterSet `json:"filterSet"` + + SamplingMethod SamplingMethod `json:"samplingMethod"` + SamplingPercent int `json:"samplingPercent"` + + // each rule supports multiple condition filters + // with their own sampling method and percent. + // This is expected to be just one level deep structure. + // e.g This rule will retain all audit logs with threat=true + // and sample remaining records using prabilistic sampling (50%) + // root (source: audit) + // condition (threat: true) IncludeAll 100% + // condition (no filter set) Default Probabilistic 50% + Conditions []SamplingConfig `json:"conditions"` + + // Default would identify capture-all case. The conditions or rules + // with default flag (true) will always have least priority and + // no filters. + Default bool +} + +// Valid checks the validity of a sampling rule and its conditions +func (sc SamplingConfig) Valid() error { + if sc.Name == "" { + return fmt.Errorf("name is required for all sampling rules") + } + + if sc.Root { + if !sc.Default && len(sc.FilterSet.Items) != 1 { + // non-default rule has no filter set, raise an error + return fmt.Errorf(fmt.Sprintf("invalid filter for sampling rule (%s)", sc.Name)) + } + + // go through conditions and ensure only on empty filter exists. here we wont + // worry about priority of empty filter since at the builder side (samplingBuilder.go) + // we will prioritize empty filter at the end always. + for _, c := range sc.Conditions { + if !c.Default && len(c.FilterSet.Items) == 0 { + return fmt.Errorf(fmt.Sprintf("rule (%s) has empty filter condition ", sc.Name)) + } + } + + } + + return nil +} + +// SamplingConfigByPriority implements sort.Interface for []SamplingConfig based on +// the priority. +type SamplingConfigByPriority []SamplingConfig + +func (sp SamplingConfigByPriority) Len() int { return len(sp) } +func (sp SamplingConfigByPriority) Swap(i, j int) { sp[i], sp[j] = sp[j], sp[i] } +func (sp SamplingConfigByPriority) Less(i, j int) bool { + if sp[j].Default { + // default priority always ranks highest + return true + } + if sp[i].Default { + // default priority always ranks highest + return false + } + return sp[i].Priority < sp[j].Priority +} + +// IngestionRulesByPriority implements sort.Interface for []IngestionRule based on +// the priority. +type IngestionRulesByPriority []IngestionRule + +func (ip IngestionRulesByPriority) Len() int { return len(ip) } +func (ip IngestionRulesByPriority) Swap(i, j int) { ip[i], ip[j] = ip[j], ip[i] } +func (ip IngestionRulesByPriority) Less(i, j int) bool { return ip[i].Priority < ip[j].Priority } diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 5d556bdec5..d5440c72fe 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -135,8 +135,8 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st } }() - userPayload, err := auth.ExtractUserFromContext(ctx) - if err != nil || userPayload == nil { + userId, err := auth.UserIdFromContext(ctx) + if err != nil || userId == "" { zap.S().Error("failed to find user in the context", err) return fmt.Errorf("failed to identify user of the request") } @@ -158,7 +158,7 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st configQuery, c.ID, c.Version, - userPayload.User.Id, + userId, c.ElementType, false, false, diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go index b4f59a1d6f..af04abb723 100644 --- a/pkg/query-service/app/opamp/configure_ingestionRules.go +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -48,7 +48,7 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ for _, agent := range agents { - agenthash, err := addIngestionControlToAgent(agent, signal, processors, withLB) + agenthash, err := addIngestionControlToAgent(agent, signal, processors, false) if err != nil { zap.S().Error("failed to push ingestion rules config to agent", agent.ID, err) continue diff --git a/pkg/query-service/auth/jwt.go b/pkg/query-service/auth/jwt.go index 35a76838f2..705b7892ad 100644 --- a/pkg/query-service/auth/jwt.go +++ b/pkg/query-service/auth/jwt.go @@ -92,15 +92,3 @@ func ExtractJwtFromContext(ctx context.Context) (string, error) { func ExtractJwtFromRequest(r *http.Request) (string, error) { return jwtmiddleware.FromAuthHeader(r) } - -// ExtractUserFromContext extracts user from context.Context using accssJwt. here, -// we do not validate if user exists as it would be done so while adding -// user to the context -func ExtractUserFromContext(ctx context.Context) (*model.UserPayload, error) { - jwt, err := ExtractJwtFromContext(ctx) - if err != nil { - return nil, err - } - - return validateUser(jwt) -} diff --git a/pkg/query-service/auth/userContext.go b/pkg/query-service/auth/userContext.go new file mode 100644 index 0000000000..ce047fe38e --- /dev/null +++ b/pkg/query-service/auth/userContext.go @@ -0,0 +1,43 @@ +package auth + +import ( + "context" + "net/http" + + "github.com/pkg/errors" + "google.golang.org/grpc/metadata" +) + +// AttachUserToContext extracts user from request and adds it to +// context +func AttachUserToContext(ctx context.Context, r *http.Request) (context.Context, error) { + userPayload, err := GetUserFromRequest(r) + if err != nil { + return ctx, err + } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + md = metadata.New(nil) + } + + md.Append("userId", userPayload.User.Id) + ctx = metadata.NewIncomingContext(ctx, md) + + return ctx, nil +} + +// UserIdFromContext extracts user from context.Context using accssJwt. here, +// we do not validate if user exists as it would be done so while adding +// user to the context +func UserIdFromContext(ctx context.Context) (string, error) { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return "", errors.New("No user metadata found in context") + } + userId := md.Get("userId") + if len(userId) == 0 { + return "", errors.New("No user found in the context") + } + + return userId[0], nil +} diff --git a/pkg/query-service/model/queryParams.go b/pkg/query-service/model/queryParams.go index 4730b1fabf..95a3c011e2 100644 --- a/pkg/query-service/model/queryParams.go +++ b/pkg/query-service/model/queryParams.go @@ -464,6 +464,7 @@ type GetErrorParams struct { type FilterItem struct { Key string `json:"key"` + KeyType string `json:"keyType"` Value interface{} `json:"value"` Operator string `json:"op"` } From 42a53a2efe6e8317c108595375fc7129a26a89f6 Mon Sep 17 00:00:00 2001 From: mindhash Date: Mon, 13 Mar 2023 20:53:45 +0530 Subject: [PATCH 12/24] feat: added user context logic --- ee/query-service/app/api/api.go | 88 ++++++--- ee/query-service/app/api/ingestionRules.go | 3 +- ee/query-service/app/server.go | 44 ++--- pkg/query-service/app/http_handler.go | 218 +++++++++++---------- pkg/query-service/app/server.go | 21 +- pkg/query-service/auth/userContext.go | 19 -- 6 files changed, 201 insertions(+), 192 deletions(-) diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 303b8a74e0..3068aadfd6 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -1,6 +1,7 @@ package api import ( + "context" "net/http" "github.com/gorilla/mux" @@ -13,6 +14,7 @@ import ( basemodel "go.signoz.io/signoz/pkg/query-service/model" rules "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/version" + "google.golang.org/grpc/metadata" ) type APIHandlerOptions struct { @@ -22,6 +24,7 @@ type APIHandlerOptions struct { FeatureFlags baseint.FeatureLookup LicenseManager *license.Manager IngestionController *ingestionRules.IngestionController + Authenticator *baseapp.AuthMiddleware } type APIHandler struct { @@ -33,10 +36,11 @@ type APIHandler struct { func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { baseHandler, err := baseapp.NewAPIHandler(baseapp.APIHandlerOpts{ - Reader: opts.DataConnector, - AppDao: opts.AppDao, - RuleManager: opts.RulesManager, - FeatureFlags: opts.FeatureFlags}) + Reader: opts.DataConnector, + AppDao: opts.AppDao, + RuleManager: opts.RulesManager, + FeatureFlags: opts.FeatureFlags, + Authenticator: opts.Authenticator}) if err != nil { return nil, err @@ -49,6 +53,10 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { return ah, nil } +func (ah *APIHandler) Auth() *baseapp.AuthMiddleware { + return ah.opts.Authenticator +} + func (ah *APIHandler) FF() baseint.FeatureLookup { return ah.opts.FeatureFlags } @@ -71,89 +79,89 @@ func (ah *APIHandler) CheckFeature(f string) bool { } // RegisterRoutes registers routes for this handler on the given router -func (ah *APIHandler) RegisterRoutes(router *mux.Router, am *baseapp.AuthMiddleware) { +func (ah *APIHandler) RegisterRoutes(router *mux.Router) { // note: add ee override methods first // routes available only in ee version router.HandleFunc("/api/v1/licenses", - am.AdminAccess(ah.listLicenses)). + ah.Auth().AdminAccess(ah.listLicenses)). Methods(http.MethodGet) router.HandleFunc("/api/v1/licenses", - am.AdminAccess(ah.applyLicense)). + ah.Auth().AdminAccess(ah.applyLicense)). Methods(http.MethodPost) router.HandleFunc("/api/v1/featureFlags", - am.OpenAccess(ah.getFeatureFlags)). + ah.Auth().OpenAccess(ah.getFeatureFlags)). Methods(http.MethodGet) router.HandleFunc("/api/v1/loginPrecheck", - am.OpenAccess(ah.precheckLogin)). + ah.Auth().OpenAccess(ah.precheckLogin)). Methods(http.MethodGet) // paid plans specific routes router.HandleFunc("/api/v1/complete/saml", - am.OpenAccess(ah.receiveSAML)). + ah.Auth().OpenAccess(ah.receiveSAML)). Methods(http.MethodPost) router.HandleFunc("/api/v1/complete/google", - am.OpenAccess(ah.receiveGoogleAuth)). + ah.Auth().OpenAccess(ah.receiveGoogleAuth)). Methods(http.MethodGet) router.HandleFunc("/api/v1/orgs/{orgId}/domains", - am.AdminAccess(ah.listDomainsByOrg)). + ah.Auth().AdminAccess(ah.listDomainsByOrg)). Methods(http.MethodGet) router.HandleFunc("/api/v1/domains", - am.AdminAccess(ah.postDomain)). + ah.Auth().AdminAccess(ah.postDomain)). Methods(http.MethodPost) router.HandleFunc("/api/v1/domains/{id}", - am.AdminAccess(ah.putDomain)). + ah.Auth().AdminAccess(ah.putDomain)). Methods(http.MethodPut) router.HandleFunc("/api/v1/domains/{id}", - am.AdminAccess(ah.deleteDomain)). + ah.Auth().AdminAccess(ah.deleteDomain)). Methods(http.MethodDelete) router.HandleFunc("/api/v1/dropRules/{version}", - am.AdminAccess(ah.listDropRules)). + ah.Auth().AdminAccess(ah.listDropRules)). Methods(http.MethodGet) router.HandleFunc("/api/v1/dropRules/{version}/deploy", - am.AdminAccess(ah.deployDropRules)). + ah.Auth().AdminAccess(ah.deployDropRules)). Methods(http.MethodPost) router.HandleFunc("/api/v1/dropRules", - am.AdminAccess(ah.createDropRule)). + ah.Auth().AdminAccess(ah.createDropRule)). Methods(http.MethodPost) router.HandleFunc("/api/v1/samplingRules/{version}/deploy", - am.AdminAccess(ah.deploySamplingRules)). + ah.Auth().AdminAccess(ah.deploySamplingRules)). Methods(http.MethodPost) router.HandleFunc("/api/v1/samplingRules/{version}", - am.AdminAccess(ah.listSamplingRules)). + ah.Auth().AdminAccess(ah.listSamplingRules)). Methods(http.MethodGet) router.HandleFunc("/api/v1/samplingRules", - am.AdminAccess(ah.createSamplingRule)). + ah.Auth().AdminAccess(ah.createSamplingRule)). Methods(http.MethodPost) // base overrides - router.HandleFunc("/api/v1/version", am.OpenAccess(ah.getVersion)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/invite/{token}", am.OpenAccess(ah.getInvite)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/register", am.OpenAccess(ah.registerUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/login", am.OpenAccess(ah.loginUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/traces/{traceId}", am.ViewAccess(ah.searchTraces)).Methods(http.MethodGet) - router.HandleFunc("/api/v2/metrics/query_range", am.ViewAccess(ah.queryRangeMetricsV2)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/version", ah.Auth().OpenAccess(ah.getVersion)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/invite/{token}", ah.Auth().OpenAccess(ah.getInvite)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/register", ah.Auth().OpenAccess(ah.registerUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/login", ah.Auth().OpenAccess(ah.loginUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/traces/{traceId}", ah.Auth().ViewAccess(ah.searchTraces)).Methods(http.MethodGet) + router.HandleFunc("/api/v2/metrics/query_range", ah.Auth().ViewAccess(ah.queryRangeMetricsV2)).Methods(http.MethodPost) // PAT APIs - router.HandleFunc("/api/v1/pat", am.OpenAccess(ah.createPAT)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/pat", am.OpenAccess(ah.getPATs)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/pat/{id}", am.OpenAccess(ah.deletePAT)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/pat", ah.Auth().OpenAccess(ah.createPAT)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/pat", ah.Auth().OpenAccess(ah.getPATs)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/pat/{id}", ah.Auth().OpenAccess(ah.deletePAT)).Methods(http.MethodDelete) - ah.APIHandler.RegisterRoutes(router, am) + ah.APIHandler.RegisterRoutes(router) } @@ -167,3 +175,21 @@ func (ah *APIHandler) getVersion(w http.ResponseWriter, r *http.Request) { ah.WriteJSON(w, r, versionResponse) } + +// AttachUserToContext extracts user from request and adds it to +// context +func (api *APIHandler) AttachUserToContext(ctx context.Context, r *http.Request) (context.Context, error) { + userPayload, err := api.Auth().GetUserFromRequest(r) + if err != nil { + return ctx, err + } + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + md = metadata.New(nil) + } + + md.Append("userId", userPayload.User.Id) + ctx = metadata.NewIncomingContext(ctx, md) + + return ctx, nil +} diff --git a/ee/query-service/app/api/ingestionRules.go b/ee/query-service/app/api/ingestionRules.go index a18f2ba05a..301866db11 100644 --- a/ee/query-service/app/api/ingestionRules.go +++ b/ee/query-service/app/api/ingestionRules.go @@ -10,7 +10,6 @@ import ( "go.signoz.io/signoz/ee/query-service/ingestionRules" "go.signoz.io/signoz/ee/query-service/model" "go.signoz.io/signoz/pkg/query-service/agentConf" - baseauth "go.signoz.io/signoz/pkg/query-service/auth" "go.uber.org/zap" ) @@ -99,7 +98,7 @@ func (ah *APIHandler) listIngestionRulesByVersion(ctx context.Context, version i func (ah *APIHandler) createIngestionRule(w http.ResponseWriter, r *http.Request, elementType agentConf.ElementTypeDef) { - ctx, err := baseauth.AttachUserToContext(context.Background(), r) + ctx, err := ah.AttachUserToContext(context.Background(), r) if err != nil { RespondError(w, model.BadRequestStr("failed to find or attach user to the context"), nil) return diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index bd9685df9c..f47a2be813 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -161,7 +161,24 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } telemetry.GetInstance().SetReader(reader) + getUserFromRequest := func(r *http.Request) (*basemodel.UserPayload, error) { + patToken := r.Header.Get("SIGNOZ-API-KEY") + if len(patToken) > 0 { + zap.S().Debugf("Received a non-zero length PAT token") + ctx := context.Background() + user, err := modelDao.GetUserByPAT(ctx, patToken) + if err == nil && user != nil { + zap.S().Debugf("Found valid PAT user: %+v", user) + return user, nil + } + if err != nil { + zap.S().Debugf("Error while getting user for PAT: %+v", err) + } + } + return baseauth.GetUserFromRequest(r) + } + authMiddleware := baseapp.NewAuthMiddleware(getUserFromRequest) apiOpts := api.APIHandlerOptions{ DataConnector: reader, AppDao: modelDao, @@ -169,6 +186,7 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { FeatureFlags: lm, LicenseManager: lm, IngestionController: ingestionController, + Authenticator: authMiddleware, } apiHandler, err := api.NewAPIHandler(apiOpts) @@ -232,33 +250,13 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e r := mux.NewRouter() - getUserFromRequest := func(r *http.Request) (*basemodel.UserPayload, error) { - patToken := r.Header.Get("SIGNOZ-API-KEY") - if len(patToken) > 0 { - zap.S().Debugf("Received a non-zero length PAT token") - ctx := context.Background() - dao := apiHandler.AppDao() - - user, err := dao.GetUserByPAT(ctx, patToken) - if err == nil && user != nil { - zap.S().Debugf("Found valid PAT user: %+v", user) - return user, nil - } - if err != nil { - zap.S().Debugf("Error while getting user for PAT: %+v", err) - } - } - return baseauth.GetUserFromRequest(r) - } - - am := baseapp.NewAuthMiddleware(getUserFromRequest) r.Use(setTimeoutMiddleware) r.Use(s.analyticsMiddleware) r.Use(loggingMiddleware) - apiHandler.RegisterRoutes(r, am) - apiHandler.RegisterMetricsRoutes(r, am) - apiHandler.RegisterLogsRoutes(r, am) + apiHandler.RegisterRoutes(r) + apiHandler.RegisterMetricsRoutes(r) + apiHandler.RegisterLogsRoutes(r) c := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index eb8409d239..9dc77fef9d 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -55,14 +55,15 @@ func NewRouter() *mux.Router { type APIHandler struct { // queryService *querysvc.QueryService // queryParser queryParser - basePath string - apiPrefix string - reader interfaces.Reader - appDao dao.ModelDao - alertManager am.Manager - ruleManager *rules.Manager - featureFlags interfaces.FeatureLookup - ready func(http.HandlerFunc) http.HandlerFunc + basePath string + apiPrefix string + reader interfaces.Reader + appDao dao.ModelDao + alertManager am.Manager + ruleManager *rules.Manager + featureFlags interfaces.FeatureLookup + authenticator *AuthMiddleware + ready func(http.HandlerFunc) http.HandlerFunc // SetupCompleted indicates if SigNoz is ready for general use. // at the moment, we mark the app ready when the first user @@ -83,6 +84,8 @@ type APIHandlerOpts struct { // feature flags querier FeatureFlags interfaces.FeatureLookup + + Authenticator *AuthMiddleware } // NewAPIHandler returns an APIHandler @@ -94,11 +97,12 @@ func NewAPIHandler(opts APIHandlerOpts) (*APIHandler, error) { } aH := &APIHandler{ - reader: opts.Reader, - appDao: opts.AppDao, - alertManager: alertManager, - ruleManager: opts.RuleManager, - featureFlags: opts.FeatureFlags, + reader: opts.Reader, + appDao: opts.AppDao, + alertManager: alertManager, + ruleManager: opts.RuleManager, + featureFlags: opts.FeatureFlags, + authenticator: opts.Authenticator, } aH.ready = aH.testReady @@ -231,17 +235,17 @@ func writeHttpResponse(w http.ResponseWriter, data interface{}) { } } -func (aH *APIHandler) RegisterMetricsRoutes(router *mux.Router, am *AuthMiddleware) { +func (aH *APIHandler) RegisterMetricsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v2/metrics").Subrouter() - subRouter.HandleFunc("/query_range", am.ViewAccess(aH.QueryRangeMetricsV2)).Methods(http.MethodPost) - subRouter.HandleFunc("/autocomplete/list", am.ViewAccess(aH.metricAutocompleteMetricName)).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/tagKey", am.ViewAccess(aH.metricAutocompleteTagKey)).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/tagValue", am.ViewAccess(aH.metricAutocompleteTagValue)).Methods(http.MethodGet) + subRouter.HandleFunc("/query_range", aH.authenticator.ViewAccess(aH.QueryRangeMetricsV2)).Methods(http.MethodPost) + subRouter.HandleFunc("/autocomplete/list", aH.authenticator.ViewAccess(aH.metricAutocompleteMetricName)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/tagKey", aH.authenticator.ViewAccess(aH.metricAutocompleteTagKey)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/tagValue", aH.authenticator.ViewAccess(aH.metricAutocompleteTagValue)).Methods(http.MethodGet) } -func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router, am *AuthMiddleware) { +func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router) { subRouter := router.PathPrefix("/api/v3").Subrouter() - subRouter.HandleFunc("/autocomplete/aggregate_attributes", am.ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/aggregate_attributes", aH.authenticator.ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet) } func (aH *APIHandler) Respond(w http.ResponseWriter, data interface{}) { @@ -254,98 +258,98 @@ func (aH *APIHandler) RegisterPrivateRoutes(router *mux.Router) { } // RegisterRoutes registers routes for this handler on the given router -func (aH *APIHandler) RegisterRoutes(router *mux.Router, am *AuthMiddleware) { - router.HandleFunc("/api/v1/query_range", am.ViewAccess(aH.queryRangeMetrics)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/query", am.ViewAccess(aH.queryMetrics)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/channels", am.ViewAccess(aH.listChannels)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/channels/{id}", am.ViewAccess(aH.getChannel)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/channels/{id}", am.AdminAccess(aH.editChannel)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/channels/{id}", am.AdminAccess(aH.deleteChannel)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/channels", am.EditAccess(aH.createChannel)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/testChannel", am.EditAccess(aH.testChannel)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/rules", am.ViewAccess(aH.listRules)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/rules/{id}", am.ViewAccess(aH.getRule)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/rules", am.EditAccess(aH.createRule)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/rules/{id}", am.EditAccess(aH.editRule)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/rules/{id}", am.EditAccess(aH.deleteRule)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/rules/{id}", am.EditAccess(aH.patchRule)).Methods(http.MethodPatch) - router.HandleFunc("/api/v1/testRule", am.EditAccess(aH.testRule)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/dashboards", am.ViewAccess(aH.getDashboards)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/dashboards", am.EditAccess(aH.createDashboards)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/dashboards/grafana", am.EditAccess(aH.createDashboardsTransform)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/dashboards/{uuid}", am.ViewAccess(aH.getDashboard)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/dashboards/{uuid}", am.EditAccess(aH.updateDashboard)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/dashboards/{uuid}", am.EditAccess(aH.deleteDashboard)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/variables/query", am.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet) - router.HandleFunc("/api/v2/variables/query", am.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/explorer/queries", am.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/explorer/queries", am.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", am.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete) - - router.HandleFunc("/api/v1/feedback", am.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost) +func (aH *APIHandler) RegisterRoutes(router *mux.Router) { + router.HandleFunc("/api/v1/query_range", aH.authenticator.ViewAccess(aH.queryRangeMetrics)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/query", aH.authenticator.ViewAccess(aH.queryMetrics)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/channels", aH.authenticator.ViewAccess(aH.listChannels)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/channels/{id}", aH.authenticator.ViewAccess(aH.getChannel)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/channels/{id}", aH.authenticator.AdminAccess(aH.editChannel)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/channels/{id}", aH.authenticator.AdminAccess(aH.deleteChannel)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/channels", aH.authenticator.EditAccess(aH.createChannel)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/testChannel", aH.authenticator.EditAccess(aH.testChannel)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/rules", aH.authenticator.ViewAccess(aH.listRules)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.ViewAccess(aH.getRule)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rules", aH.authenticator.EditAccess(aH.createRule)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.EditAccess(aH.editRule)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.EditAccess(aH.deleteRule)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.EditAccess(aH.patchRule)).Methods(http.MethodPatch) + router.HandleFunc("/api/v1/testRule", aH.authenticator.EditAccess(aH.testRule)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/dashboards", aH.authenticator.ViewAccess(aH.getDashboards)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/dashboards", aH.authenticator.EditAccess(aH.createDashboards)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/dashboards/grafana", aH.authenticator.EditAccess(aH.createDashboardsTransform)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/dashboards/{uuid}", aH.authenticator.ViewAccess(aH.getDashboard)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/dashboards/{uuid}", aH.authenticator.EditAccess(aH.updateDashboard)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/dashboards/{uuid}", aH.authenticator.EditAccess(aH.deleteDashboard)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/variables/query", aH.authenticator.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet) + router.HandleFunc("/api/v2/variables/query", aH.authenticator.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/explorer/queries", aH.authenticator.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/queries", aH.authenticator.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.authenticator.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.authenticator.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.authenticator.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete) + + router.HandleFunc("/api/v1/feedback", aH.authenticator.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost) // router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet) - router.HandleFunc("/api/v1/services", am.ViewAccess(aH.getServices)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/services/list", am.ViewAccess(aH.getServicesList)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/service/overview", am.ViewAccess(aH.getServiceOverview)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/service/top_operations", am.ViewAccess(aH.getTopOperations)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/service/top_level_operations", am.ViewAccess(aH.getServicesTopLevelOps)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/traces/{traceId}", am.ViewAccess(aH.SearchTraces)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/usage", am.ViewAccess(aH.getUsage)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/dependency_graph", am.ViewAccess(aH.dependencyGraph)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/settings/ttl", am.AdminAccess(aH.setTTL)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/settings/ttl", am.ViewAccess(aH.getTTL)).Methods(http.MethodGet) - - router.HandleFunc("/api/v1/version", am.OpenAccess(aH.getVersion)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/featureFlags", am.OpenAccess(aH.getFeatureFlags)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/configs", am.OpenAccess(aH.getConfigs)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/health", am.OpenAccess(aH.getHealth)).Methods(http.MethodGet) - - router.HandleFunc("/api/v1/getSpanFilters", am.ViewAccess(aH.getSpanFilters)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getTagFilters", am.ViewAccess(aH.getTagFilters)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getFilteredSpans", am.ViewAccess(aH.getFilteredSpans)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getFilteredSpans/aggregates", am.ViewAccess(aH.getFilteredSpanAggregates)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getTagValues", am.ViewAccess(aH.getTagValues)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/listErrors", am.ViewAccess(aH.listErrors)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/countErrors", am.ViewAccess(aH.countErrors)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/errorFromErrorID", am.ViewAccess(aH.getErrorFromErrorID)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/errorFromGroupID", am.ViewAccess(aH.getErrorFromGroupID)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/nextPrevErrorIDs", am.ViewAccess(aH.getNextPrevErrorIDs)).Methods(http.MethodGet) - - router.HandleFunc("/api/v1/disks", am.ViewAccess(aH.getDisks)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/services", aH.authenticator.ViewAccess(aH.getServices)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/services/list", aH.authenticator.ViewAccess(aH.getServicesList)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/service/overview", aH.authenticator.ViewAccess(aH.getServiceOverview)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/service/top_operations", aH.authenticator.ViewAccess(aH.getTopOperations)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/service/top_level_operations", aH.authenticator.ViewAccess(aH.getServicesTopLevelOps)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/traces/{traceId}", aH.authenticator.ViewAccess(aH.SearchTraces)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/usage", aH.authenticator.ViewAccess(aH.getUsage)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/dependency_graph", aH.authenticator.ViewAccess(aH.dependencyGraph)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/settings/ttl", aH.authenticator.AdminAccess(aH.setTTL)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/settings/ttl", aH.authenticator.ViewAccess(aH.getTTL)).Methods(http.MethodGet) + + router.HandleFunc("/api/v1/version", aH.authenticator.OpenAccess(aH.getVersion)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/featureFlags", aH.authenticator.OpenAccess(aH.getFeatureFlags)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/configs", aH.authenticator.OpenAccess(aH.getConfigs)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/health", aH.authenticator.OpenAccess(aH.getHealth)).Methods(http.MethodGet) + + router.HandleFunc("/api/v1/getSpanFilters", aH.authenticator.ViewAccess(aH.getSpanFilters)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getTagFilters", aH.authenticator.ViewAccess(aH.getTagFilters)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getFilteredSpans", aH.authenticator.ViewAccess(aH.getFilteredSpans)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getFilteredSpans/aggregates", aH.authenticator.ViewAccess(aH.getFilteredSpanAggregates)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getTagValues", aH.authenticator.ViewAccess(aH.getTagValues)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/listErrors", aH.authenticator.ViewAccess(aH.listErrors)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/countErrors", aH.authenticator.ViewAccess(aH.countErrors)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/errorFromErrorID", aH.authenticator.ViewAccess(aH.getErrorFromErrorID)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/errorFromGroupID", aH.authenticator.ViewAccess(aH.getErrorFromGroupID)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/nextPrevErrorIDs", aH.authenticator.ViewAccess(aH.getNextPrevErrorIDs)).Methods(http.MethodGet) + + router.HandleFunc("/api/v1/disks", aH.authenticator.ViewAccess(aH.getDisks)).Methods(http.MethodGet) // === Authentication APIs === - router.HandleFunc("/api/v1/invite", am.AdminAccess(aH.inviteUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/invite/{token}", am.OpenAccess(aH.getInvite)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/invite/{email}", am.AdminAccess(aH.revokeInvite)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/invite", am.AdminAccess(aH.listPendingInvites)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/invite", aH.authenticator.AdminAccess(aH.inviteUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/invite/{token}", aH.authenticator.OpenAccess(aH.getInvite)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/invite/{email}", aH.authenticator.AdminAccess(aH.revokeInvite)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/invite", aH.authenticator.AdminAccess(aH.listPendingInvites)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/register", am.OpenAccess(aH.registerUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/login", am.OpenAccess(aH.loginUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/register", aH.authenticator.OpenAccess(aH.registerUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/login", aH.authenticator.OpenAccess(aH.loginUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/user", am.AdminAccess(aH.listUsers)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/user/{id}", am.SelfAccess(aH.getUser)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/user/{id}", am.SelfAccess(aH.editUser)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/user/{id}", am.AdminAccess(aH.deleteUser)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/user", aH.authenticator.AdminAccess(aH.listUsers)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/user/{id}", aH.authenticator.SelfAccess(aH.getUser)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/user/{id}", aH.authenticator.SelfAccess(aH.editUser)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/user/{id}", aH.authenticator.AdminAccess(aH.deleteUser)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/user/{id}/flags", am.SelfAccess(aH.patchUserFlag)).Methods(http.MethodPatch) + router.HandleFunc("/api/v1/user/{id}/flags", aH.authenticator.SelfAccess(aH.patchUserFlag)).Methods(http.MethodPatch) - router.HandleFunc("/api/v1/rbac/role/{id}", am.SelfAccess(aH.getRole)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/rbac/role/{id}", am.AdminAccess(aH.editRole)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/rbac/role/{id}", aH.authenticator.SelfAccess(aH.getRole)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rbac/role/{id}", aH.authenticator.AdminAccess(aH.editRole)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/org", am.AdminAccess(aH.getOrgs)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/org/{id}", am.AdminAccess(aH.getOrg)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/org/{id}", am.AdminAccess(aH.editOrg)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/orgUsers/{id}", am.AdminAccess(aH.getOrgUsers)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/org", aH.authenticator.AdminAccess(aH.getOrgs)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/org/{id}", aH.authenticator.AdminAccess(aH.getOrg)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/org/{id}", aH.authenticator.AdminAccess(aH.editOrg)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/orgUsers/{id}", aH.authenticator.AdminAccess(aH.getOrgUsers)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/getResetPasswordToken/{id}", am.AdminAccess(aH.getResetPasswordToken)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/resetPassword", am.OpenAccess(aH.resetPassword)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/changePassword/{id}", am.SelfAccess(aH.changePassword)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getResetPasswordToken/{id}", aH.authenticator.AdminAccess(aH.getResetPasswordToken)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/resetPassword", aH.authenticator.OpenAccess(aH.resetPassword)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/changePassword/{id}", aH.authenticator.SelfAccess(aH.changePassword)).Methods(http.MethodPost) } func Intersection(a, b []int) (c []int) { @@ -2144,11 +2148,11 @@ func (aH *APIHandler) WriteJSON(w http.ResponseWriter, r *http.Request, response // logs func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router, am *AuthMiddleware) { subRouter := router.PathPrefix("/api/v1/logs").Subrouter() - subRouter.HandleFunc("", am.ViewAccess(aH.getLogs)).Methods(http.MethodGet) - subRouter.HandleFunc("/tail", am.ViewAccess(aH.tailLogs)).Methods(http.MethodGet) - subRouter.HandleFunc("/fields", am.ViewAccess(aH.logFields)).Methods(http.MethodGet) + subRouter.HandleFunc("", aH.authenticator.ViewAccess(aH.getLogs)).Methods(http.MethodGet) + subRouter.HandleFunc("/tail", aH.authenticator.ViewAccess(aH.tailLogs)).Methods(http.MethodGet) + subRouter.HandleFunc("/fields", aH.authenticator.ViewAccess(aH.logFields)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", am.EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) - subRouter.HandleFunc("/aggregate", am.ViewAccess(aH.logAggregate)).Methods(http.MethodGet) + subRouter.HandleFunc("/aggregate", aH.authenticator.ViewAccess(aH.logAggregate)).Methods(http.MethodGet) } func (aH *APIHandler) logFields(w http.ResponseWriter, r *http.Request) { diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index f737b3925c..3eac5eabd3 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -111,12 +111,15 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { return nil, err } + authmw := NewAuthMiddleware(auth.GetUserFromRequest) + telemetry.GetInstance().SetReader(reader) apiHandler, err := NewAPIHandler(APIHandlerOpts{ - Reader: reader, - AppDao: dao.DB(), - RuleManager: rm, - FeatureFlags: fm, + Reader: reader, + AppDao: dao.DB(), + RuleManager: rm, + FeatureFlags: fm, + Authenticator: authmw, }) if err != nil { return nil, err @@ -187,12 +190,10 @@ func (s *Server) createPublicServer(api *APIHandler) (*http.Server, error) { r.Use(s.analyticsMiddleware) r.Use(loggingMiddleware) - am := NewAuthMiddleware(auth.GetUserFromRequest) - - api.RegisterRoutes(r, am) - api.RegisterMetricsRoutes(r, am) - api.RegisterLogsRoutes(r, am) - api.RegisterQueryRangeV3Routes(r, am) + api.RegisterRoutes(r) + api.RegisterMetricsRoutes(r) + api.RegisterLogsRoutes(r) + api.RegisterQueryRangeV3Routes(r) c := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, diff --git a/pkg/query-service/auth/userContext.go b/pkg/query-service/auth/userContext.go index ce047fe38e..764f600d44 100644 --- a/pkg/query-service/auth/userContext.go +++ b/pkg/query-service/auth/userContext.go @@ -2,30 +2,11 @@ package auth import ( "context" - "net/http" "github.com/pkg/errors" "google.golang.org/grpc/metadata" ) -// AttachUserToContext extracts user from request and adds it to -// context -func AttachUserToContext(ctx context.Context, r *http.Request) (context.Context, error) { - userPayload, err := GetUserFromRequest(r) - if err != nil { - return ctx, err - } - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - md = metadata.New(nil) - } - - md.Append("userId", userPayload.User.Id) - ctx = metadata.NewIncomingContext(ctx, md) - - return ctx, nil -} - // UserIdFromContext extracts user from context.Context using accssJwt. here, // we do not validate if user exists as it would be done so while adding // user to the context From 77f4da89fbc73f3dfeab50d78970b8ba6ae4dd3d Mon Sep 17 00:00:00 2001 From: mindhash Date: Mon, 13 Mar 2023 22:22:42 +0530 Subject: [PATCH 13/24] fix: added userid to ConfigNewVersion() --- pkg/query-service/agentConf/db.go | 11 ++--------- .../app/opamp/configure_ingestionRules.go | 2 +- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 5d556bdec5..2567985836 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -9,7 +9,6 @@ import ( "github.com/google/uuid" "github.com/jmoiron/sqlx" "go.signoz.io/signoz/pkg/query-service/agentConf/sqlite" - "go.signoz.io/signoz/pkg/query-service/auth" "go.signoz.io/signoz/pkg/query-service/model" "go.uber.org/zap" ) @@ -98,7 +97,7 @@ func (r *Repo) GetLatestVersion(ctx context.Context, typ ElementTypeDef) (*Confi return &c, err } -func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []string) (fnerr error) { +func (r *Repo) insertConfig(ctx context.Context, userId string, c *ConfigVersion, elements []string) (fnerr error) { if string(c.ElementType) == "" { return fmt.Errorf("element type is required for creating agent config version") @@ -135,12 +134,6 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st } }() - userPayload, err := auth.ExtractUserFromContext(ctx) - if err != nil || userPayload == nil { - zap.S().Error("failed to find user in the context", err) - return fmt.Errorf("failed to identify user of the request") - } - // insert config configQuery := `INSERT INTO agent_config_versions( id, @@ -158,7 +151,7 @@ func (r *Repo) insertConfig(ctx context.Context, c *ConfigVersion, elements []st configQuery, c.ID, c.Version, - userPayload.User.Id, + userId, c.ElementType, false, false, diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go index b4f59a1d6f..af04abb723 100644 --- a/pkg/query-service/app/opamp/configure_ingestionRules.go +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -48,7 +48,7 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ for _, agent := range agents { - agenthash, err := addIngestionControlToAgent(agent, signal, processors, withLB) + agenthash, err := addIngestionControlToAgent(agent, signal, processors, false) if err != nil { zap.S().Error("failed to push ingestion rules config to agent", agent.ID, err) continue From f37b836a42ebc750062f6d9d3f1fc7fd8d01de9d Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 14 Mar 2023 13:10:09 +0530 Subject: [PATCH 14/24] chore: removed user id from contxt and added config parser --- pkg/query-service/agentConf/db.go | 7 ++ pkg/query-service/agentConf/manager.go | 4 +- pkg/query-service/agentConf/version.go | 24 ++++-- .../app/opamp/otelconfig/config_parser.go | 85 +++++++++++-------- .../opamp/otelconfig/config_parser_test.go | 58 +++++++++++++ .../app/opamp/otelconfig/testdata/basic.yaml | 76 +++++++++++++++++ .../opamp/otelconfig/testdata/service.yaml | 11 +++ 7 files changed, 218 insertions(+), 47 deletions(-) create mode 100644 pkg/query-service/app/opamp/otelconfig/config_parser_test.go create mode 100644 pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml create mode 100644 pkg/query-service/app/opamp/otelconfig/testdata/service.yaml diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 2567985836..6392513f6a 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -38,6 +38,9 @@ func (r *Repo) GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]Conf version, element_type, COALESCE(created_by, -1) as created_by, + created_at, + COALESCE((SELECT NAME FROM users + WHERE id = v.created_by), "unknown") created_by_name, active, is_valid, disabled, @@ -56,6 +59,7 @@ func (r *Repo) GetConfigVersion(ctx context.Context, typ ElementTypeDef, v int) version, element_type, COALESCE(created_by, -1) as created_by, + created_at, COALESCE((SELECT NAME FROM users WHERE id = v.created_by), "unknown") created_by_name, active, @@ -80,6 +84,9 @@ func (r *Repo) GetLatestVersion(ctx context.Context, typ ElementTypeDef) (*Confi version, element_type, COALESCE(created_by, -1) as created_by, + created_at, + COALESCE((SELECT NAME FROM users + WHERE id = v.created_by), "unknown") created_by_name, active, is_valid, disabled, diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index 49717cf83f..3fb83df1af 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -56,7 +56,7 @@ func GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]ConfigVersion, } // StartNewVersion launches a new config version for given set of elements -func StartNewVersion(ctx context.Context, eleType ElementTypeDef, elementIds []string) (*ConfigVersion, error) { +func StartNewVersion(ctx context.Context, userId string, eleType ElementTypeDef, elementIds []string) (*ConfigVersion, error) { if !m.Ready() { // agent is already being updated, ask caller to wait and re-try after sometime @@ -67,7 +67,7 @@ func StartNewVersion(ctx context.Context, eleType ElementTypeDef, elementIds []s cfg := NewConfigversion(eleType) // insert new config and elements into database - err := m.insertConfig(ctx, cfg, elementIds) + err := m.insertConfig(ctx, userId, cfg, elementIds) if err != nil { return nil, err } diff --git a/pkg/query-service/agentConf/version.go b/pkg/query-service/agentConf/version.go index d568c01d06..c6b8c01ee4 100644 --- a/pkg/query-service/agentConf/version.go +++ b/pkg/query-service/agentConf/version.go @@ -1,6 +1,10 @@ package agentConf -import "github.com/google/uuid" +import ( + "time" + + "github.com/google/uuid" +) type ElementTypeDef string @@ -22,20 +26,22 @@ const ( ) type ConfigVersion struct { - ID string `json:"id" db:"id"` - Version int `json:"version" db:"version"` - ElementType ElementTypeDef `json:"elementType" db:"element_type"` - CreatedBy string `json:"createdBy" db:"created_by"` - CreatedByName string `json:"createdByName" db:"created_by_name"` - Active bool `json:"active" db:"active"` - IsValid bool `json:"is_valid" db:"is_valid"` - Disabled bool `json:"disabled" db:"disabled"` + ID string `json:"id" db:"id"` + Version int `json:"version" db:"version"` + ElementType ElementTypeDef `json:"elementType" db:"element_type"` + Active bool `json:"active" db:"active"` + IsValid bool `json:"is_valid" db:"is_valid"` + Disabled bool `json:"disabled" db:"disabled"` DeployStatus DeployStatus `json:"deployStatus" db:"deploy_status"` DeployResult string `json:"deployResult" db:"deploy_result"` LastHash string `json:"lastHash" db:"last_hash"` LastConf string `json:"lastConf" db:"last_config"` + + CreatedBy string `json:"createdBy" db:"created_by"` + CreatedByName string `json:"createdByName" db:"created_by_name"` + CreatedAt time.Time `json:"createdAt" db:"created_at"` } func NewConfigversion(typeDef ElementTypeDef) *ConfigVersion { diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser.go b/pkg/query-service/app/opamp/otelconfig/config_parser.go index b08dd0b27d..5d6c8adba4 100644 --- a/pkg/query-service/app/opamp/otelconfig/config_parser.go +++ b/pkg/query-service/app/opamp/otelconfig/config_parser.go @@ -6,10 +6,6 @@ import ( "go.opentelemetry.io/collector/confmap" ) -type ConfigPart map[string]interface{} - -type ConfigList []interface{} - type ConfigParser struct { lock sync.Mutex agentConf *confmap.Conf @@ -21,93 +17,110 @@ func NewConfigParser(agentConf *confmap.Conf) ConfigParser { } } -func (cp *ConfigParser) Service() ConfigPart { +func toMap(i interface{}) map[string]interface{} { + return i.(map[string]interface{}) +} + +func toList(i interface{}) []interface{} { + return i.([]interface{}) +} + +// sent when key is not found in config +func emptyMap() map[string]interface{} { + return map[string]interface{}{} +} + +func emptyList() []interface{} { + return []interface{}{} +} + +func (cp *ConfigParser) Service() map[string]interface{} { service := cp.agentConf.Get("service") if service == nil { - return ConfigPart{} + return emptyMap() } - return service.(ConfigPart) + return toMap(service) } // components gets the high level parts like receivers, exporters, processors etc -func (cp *ConfigParser) components(partName, nameOptional string) ConfigPart { +func (cp *ConfigParser) components(partName, nameOptional string) map[string]interface{} { parts := cp.agentConf.Get(partName) if parts == nil { - return ConfigPart{} + return emptyMap() } - parsedParts := parts.(ConfigPart) + parsedParts := toMap(parts) if nameOptional != "" { if p, ok := parsedParts[nameOptional]; ok { - return p.(ConfigPart) + return p.(map[string]interface{}) } else { - return ConfigPart{} + return emptyMap() } } return parsedParts } -func (cp *ConfigParser) Processors() ConfigPart { +func (cp *ConfigParser) Processors() map[string]interface{} { return cp.components("processors", "") } -func (cp *ConfigParser) Processor(name string) ConfigPart { +func (cp *ConfigParser) Processor(name string) map[string]interface{} { return cp.components("processors", name) } -func (cp *ConfigParser) Exporters() ConfigPart { +func (cp *ConfigParser) Exporters() map[string]interface{} { return cp.components("exporters", "") } -func (cp *ConfigParser) Exporter(name string) ConfigPart { +func (cp *ConfigParser) Exporter(name string) map[string]interface{} { return cp.components("exporters", name) } -func (cp *ConfigParser) Receivers() ConfigPart { +func (cp *ConfigParser) Receivers() map[string]interface{} { return cp.components("receivers", "") } -func (cp *ConfigParser) Receiver(name string) ConfigPart { +func (cp *ConfigParser) Receiver(name string) map[string]interface{} { return cp.components("receivers", name) } -func (cp *ConfigParser) Pipelines(nameOptional string) ConfigPart { +func (cp *ConfigParser) Pipelines(nameOptional string) map[string]interface{} { services := cp.Service() if p, ok := services["pipelines"]; ok { - pipelines := p.(ConfigPart) + pipelines := toMap(p) if nameOptional != "" { if namedPipeline, ok := pipelines[nameOptional]; ok { - return namedPipeline.(ConfigPart) + return toMap(namedPipeline) } else { - return ConfigPart{} + return emptyMap() } } return pipelines } - return ConfigPart{} + return emptyMap() } // component can be "recevers", "exporter" or "processors" -func (cp *ConfigParser) PipelineComponent(pipelineName, pipelineComponent string) ConfigList { +func (cp *ConfigParser) PipelineComponent(pipelineName, pipelineComponent string) []interface{} { pipeline := cp.Pipelines(pipelineName) if exporters, ok := pipeline[pipelineComponent]; ok { - exporters := exporters.(ConfigList) + exporters := toList(exporters) return exporters } - return ConfigList{} + return emptyList() } -func (cp *ConfigParser) PipelineExporters(pipelineName string) ConfigList { +func (cp *ConfigParser) PipelineExporters(pipelineName string) []interface{} { return cp.PipelineComponent(pipelineName, "exporters") } -func (cp *ConfigParser) PipelineReceivers(pipelineName string) ConfigList { +func (cp *ConfigParser) PipelineReceivers(pipelineName string) []interface{} { return cp.PipelineComponent(pipelineName, "receivers") } -func (cp *ConfigParser) PipelineProcessors(pipelineName string) ConfigList { +func (cp *ConfigParser) PipelineProcessors(pipelineName string) []interface{} { return cp.PipelineComponent(pipelineName, "processors") } @@ -150,14 +163,14 @@ func (cp *ConfigParser) Merge(c *confmap.Conf) { cp.agentConf.Merge(c) } -func (cp *ConfigParser) UpdateProcessors(processors ConfigPart) { +func (cp *ConfigParser) UpdateProcessors(processors map[string]interface{}) { updates := cp.Processors() for key, params := range processors { updates[key] = params } - updatedProcessors := ConfigPart{ + updatedProcessors := map[string]interface{}{ "processors": updates, } @@ -166,12 +179,12 @@ func (cp *ConfigParser) UpdateProcessors(processors ConfigPart) { cp.Merge(updatedProcessorConf) } -func (cp *ConfigParser) UpdateProcsInPipeline(pipelineName string, list ConfigList) { +func (cp *ConfigParser) UpdateProcsInPipeline(pipelineName string, list []interface{}) { - serviceConf := ConfigPart{ - "service": ConfigPart{ - "pipelines": ConfigPart{ - pipelineName: ConfigPart{ + serviceConf := map[string]interface{}{ + "service": map[string]interface{}{ + "pipelines": map[string]interface{}{ + pipelineName: map[string]interface{}{ "processors": list, }, }, diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser_test.go b/pkg/query-service/app/opamp/otelconfig/config_parser_test.go new file mode 100644 index 0000000000..61ceb1f613 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/config_parser_test.go @@ -0,0 +1,58 @@ +package otelconfig + +import ( + "fmt" + "io/ioutil" + "testing" + + "github.com/knadh/koanf/parsers/yaml" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap" +) + +func TestServiceConfig(t *testing.T) { + yamlFile, err := ioutil.ReadFile("./testdata/service.yaml") + if err != nil { + fmt.Printf("yamlFile.Get err #%v ", err) + t.Fail() + return + } + + c, err := yaml.Parser().Unmarshal([]byte(yamlFile)) + if err != nil { + fmt.Println("failed to parse config file as yaml", err) + t.Fail() + return + } + + agentConf := confmap.NewFromStringMap(c) + configParser := NewConfigParser(agentConf) + + expected := map[string]interface{}{ + "extensions": []interface{}{"zpages"}, + "pipelines": map[string]interface{}{ + "traces": map[string]interface{}{ + "receivers": []interface{}{"jaeger", "otlp"}, + "processors": []interface{}{ + "signozspanmetrics/prometheus", "batch", + }, + "exporters": []interface{}{ + "clickhousetraces", + }, + }, + "metrics": map[string]interface{}{ + "receivers": []interface{}{ + "otlp", "hostmetrics", + }, + "processors": []interface{}{ + "batch", + }, + "exporters": []interface{}{ + "clickhousemetricswrite", + }, + }, + }, + } + + require.Equal(t, expected, configParser.Service(), "expected same service config after parsing") +} diff --git a/pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml b/pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml new file mode 100644 index 0000000000..d5ef74e00f --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/testdata/basic.yaml @@ -0,0 +1,76 @@ +receivers: + otlp/spanmetrics: + protocols: + grpc: + endpoint: "localhost:12345" + otlp: + protocols: + grpc: + http: + jaeger: + protocols: + grpc: + thrift_http: + hostmetrics: + collection_interval: 30s + scrapers: + cpu: + load: + memory: + disk: + filesystem: + network: +processors: + batch: + send_batch_size: 1000 + timeout: 10s + signozspanmetrics/prometheus: + metrics_exporter: prometheus + latency_histogram_buckets: [100us, 1ms, 2ms, 6ms, 10ms, 50ms, 100ms, 250ms, 500ms, 1000ms, 1400ms, 2000ms, 5s, 10s, 20s, 40s, 60s ] + dimensions_cache_size: 10000 + dimensions: + - name: service.namespace + default: default + - name: deployment.environment + default: default + # memory_limiter: + # # 80% of maximum memory up to 2G + # limit_mib: 1500 + # # 25% of limit up to 2G + # spike_limit_mib: 512 + # check_interval: 5s + # + # # 50% of the maximum memory + # limit_percentage: 50 + # # 20% of max memory usage spike expected + # spike_limit_percentage: 20 + # queued_retry: + # num_workers: 4 + # queue_size: 100 + # retry_on_failure: true +extensions: + zpages: {} +exporters: + clickhousetraces: + datasource: tcp://localhost:9000/?database=signoz_traces + migrations: exporter/clickhousetracesexporter/migrations + clickhousemetricswrite: + endpoint: tcp://localhost:9000/?database=signoz_metrics + resource_to_telemetry_conversion: + enabled: true + prometheus: + endpoint: "0.0.0.0:8889" +service: + extensions: [zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhousetraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] + metrics/spanmetrics: + receivers: [otlp/spanmetrics] + exporters: [prometheus] \ No newline at end of file diff --git a/pkg/query-service/app/opamp/otelconfig/testdata/service.yaml b/pkg/query-service/app/opamp/otelconfig/testdata/service.yaml new file mode 100644 index 0000000000..dd562fba0d --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/testdata/service.yaml @@ -0,0 +1,11 @@ +service: + extensions: [zpages] + pipelines: + traces: + receivers: [jaeger, otlp] + processors: [signozspanmetrics/prometheus, batch] + exporters: [clickhousetraces] + metrics: + receivers: [otlp, hostmetrics] + processors: [batch] + exporters: [clickhousemetricswrite] From c098d59b075d3584175ad8708bfac0f2965c2a5c Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 14 Mar 2023 13:29:31 +0530 Subject: [PATCH 15/24] chore: resolved some changes to newConfigVersion() --- pkg/query-service/agentConf/manager.go | 37 +++++++------------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index 3fb83df1af..db7d477999 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -75,6 +75,9 @@ func StartNewVersion(ctx context.Context, userId string, eleType ElementTypeDef, return cfg, nil } +// Redeploy picks up a config for a version from config_agent_versions +// and re-sends it to collector for deployment. This method is useful +// in enabling previous configs from history func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { return fmt.Errorf("agent updater is busy") @@ -92,45 +95,24 @@ func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { zap.S().Debug("config version has no conf yaml", configVersion) return fmt.Errorf("the config version can not be redeployed") } + switch typ { case ElementTypeSamplingRules: var config *tsp.Config + if err := yaml.Unmarshal([]byte(configVersion.LastConf), &config); err != nil { - zap.S().Error("failed to read last conf correctly", err) + zap.S().Error("sampling rule re-deployment failures: failed to read last conf correctly", err) return fmt.Errorf("failed to read the stored config correctly") } - // merge current config with new filter params - processorConf := map[string]interface{}{ - "signoz_tail_sampling": config, - } - - opamp.AddToTracePipelineSpec("signoz_tail_sampling") - configHash, err := opamp.UpsertControlProcessors(ctx, "traces", processorConf, m.OnConfigUpdate) - if err != nil { - zap.S().Error("failed to call agent config update for trace processor:", err) - return fmt.Errorf("failed to deploy the config") - } - - m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf) + return UpsertSamplingProcessor(ctx, configVersion.Version, config) case ElementTypeDropRules: var filterConfig *filterprocessor.Config if err := yaml.Unmarshal([]byte(configVersion.LastConf), &filterConfig); err != nil { - zap.S().Error("failed to read last conf correctly", err) + zap.S().Error("drop rule re-deployment failures: failed to read last conf correctly", err) return fmt.Errorf("failed to read the stored config correctly") } - processorConf := map[string]interface{}{ - "filter": filterConfig, - } - - opamp.AddToMetricsPipelineSpec("filter") - configHash, err := opamp.UpsertControlProcessors(ctx, "metrics", processorConf, m.OnConfigUpdate) - if err != nil { - zap.S().Error("failed to call agent config update for trace processor:", err) - return err - } - - m.updateDeployStatus(ctx, ElementTypeSamplingRules, version, string(DeployInitiated), "Deployment started", configHash, configVersion.LastConf) + return UpsertFilterProcessor(ctx, configVersion.Version, filterConfig) } return nil @@ -143,7 +125,6 @@ func UpsertFilterProcessor(ctx context.Context, version int, config *filterproce } defer atomic.StoreUint32(&m.lock, 0) - // merge current config with new filter params // merge current config with new filter params processorConf := map[string]interface{}{ "filter": config, From f979385cad7370c3738a9b335d0489e3b3962ee9 Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 14 Mar 2023 13:29:39 +0530 Subject: [PATCH 16/24] chore: resolved some changes to newConfigVersion() --- ee/query-service/app/api/ingestionRules.go | 13 ++++++------- ee/query-service/ingestionRules/controller.go | 14 +++++++------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/ee/query-service/app/api/ingestionRules.go b/ee/query-service/app/api/ingestionRules.go index 301866db11..ad3a2fc4ff 100644 --- a/ee/query-service/app/api/ingestionRules.go +++ b/ee/query-service/app/api/ingestionRules.go @@ -97,11 +97,10 @@ func (ah *APIHandler) listIngestionRulesByVersion(ctx context.Context, version i } func (ah *APIHandler) createIngestionRule(w http.ResponseWriter, r *http.Request, elementType agentConf.ElementTypeDef) { - - ctx, err := ah.AttachUserToContext(context.Background(), r) + ctx := context.Background() + userPayload, err := ah.Auth().GetUserFromRequest(r) if err != nil { - RespondError(w, model.BadRequestStr("failed to find or attach user to the context"), nil) - return + RespondError(w, model.BadRequestStr("failed to identify user from the request")) } req := ingestionRules.PostableIngestionRules{} @@ -111,7 +110,7 @@ func (ah *APIHandler) createIngestionRule(w http.ResponseWriter, r *http.Request return } - createRule := func(ctx context.Context, postable []ingestionRules.PostableIngestionRule) (*ingestionRules.IngestionRulesResponse, *model.ApiError) { + createRule := func(ctx context.Context, userId string, postable []ingestionRules.PostableIngestionRule) (*ingestionRules.IngestionRulesResponse, *model.ApiError) { if len(postable) == 0 { zap.S().Warnf("found no rules in the http request, this will delete all the rules") } @@ -123,10 +122,10 @@ func (ah *APIHandler) createIngestionRule(w http.ResponseWriter, r *http.Request } } - return ah.opts.IngestionController.ApplyRules(ctx, elementType, postable) + return ah.opts.IngestionController.ApplyRules(ctx, userId, elementType, postable) } - ingestionRuleResponse, apierr := createRule(ctx, req.Rules) + ingestionRuleResponse, apierr := createRule(ctx, userPayload.User.Id, req.Rules) if apierr != nil { RespondError(w, apierr, nil) return diff --git a/ee/query-service/ingestionRules/controller.go b/ee/query-service/ingestionRules/controller.go index 31e2bf1463..0f13013c6f 100644 --- a/ee/query-service/ingestionRules/controller.go +++ b/ee/query-service/ingestionRules/controller.go @@ -30,18 +30,18 @@ type IngestionRulesResponse struct { } // ApplyRules conditionally calls applyDropRules or applySampling rules -func (ic *IngestionController) ApplyRules(ctx context.Context, elementType agentConf.ElementTypeDef, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { +func (ic *IngestionController) ApplyRules(ctx context.Context, userId string, elementType agentConf.ElementTypeDef, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { switch elementType { case agentConf.ElementTypeDropRules: - return ic.ApplyDropRules(ctx, postable) + return ic.ApplyDropRules(ctx, userId, postable) case agentConf.ElementTypeSamplingRules: - return ic.ApplySamplingRules(ctx, postable) + return ic.ApplySamplingRules(ctx, userId, postable) } return nil, model.BadRequestStr("unexpected element type") } // ApplyDropRules stores new or changed drop rules and initiates a new config update -func (ic *IngestionController) ApplyDropRules(ctx context.Context, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { +func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { var dropRules []model.IngestionRule // scan through postable rules, to select the existing rules or insert missing ones @@ -94,7 +94,7 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, postable []Po } // prepare config by calling gen func - cfg, err := agentConf.StartNewVersion(ctx, agentConf.ElementTypeDropRules, elements) + cfg, err := agentConf.StartNewVersion(ctx, userId, agentConf.ElementTypeDropRules, elements) if err != nil || cfg == nil { zap.S().Errorf("failed to start a new config version for drop rules", err) return nil, model.InternalError(err) @@ -141,7 +141,7 @@ func (ic *IngestionController) GetRulesByVersion(ctx context.Context, version in } // ApplySamplingRules stores new or changed sampling rules and initiates a new config update -func (ic *IngestionController) ApplySamplingRules(ctx context.Context, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { +func (ic *IngestionController) ApplySamplingRules(ctx context.Context, userId string, postable []PostableIngestionRule) (*IngestionRulesResponse, *model.ApiError) { var smplRules []model.IngestionRule // scan through postable rules, to select the existing rules or insert missing ones @@ -196,7 +196,7 @@ func (ic *IngestionController) ApplySamplingRules(ctx context.Context, postable } // prepare config by calling gen func - cfg, err := agentConf.StartNewVersion(ctx, agentConf.ElementTypeSamplingRules, elements) + cfg, err := agentConf.StartNewVersion(ctx, userId, agentConf.ElementTypeSamplingRules, elements) if err != nil || cfg == nil { return nil, model.InternalError(err) } From 7306f05066327181d296f7c82bc1d49529655267 Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 14 Mar 2023 13:34:35 +0530 Subject: [PATCH 17/24] chore: fixed minor compile issues --- ee/query-service/app/api/ingestionRules.go | 2 +- pkg/query-service/app/http_handler.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ee/query-service/app/api/ingestionRules.go b/ee/query-service/app/api/ingestionRules.go index ad3a2fc4ff..48c4a47b43 100644 --- a/ee/query-service/app/api/ingestionRules.go +++ b/ee/query-service/app/api/ingestionRules.go @@ -100,7 +100,7 @@ func (ah *APIHandler) createIngestionRule(w http.ResponseWriter, r *http.Request ctx := context.Background() userPayload, err := ah.Auth().GetUserFromRequest(r) if err != nil { - RespondError(w, model.BadRequestStr("failed to identify user from the request")) + RespondError(w, model.BadRequestStr("failed to identify user from the request"), nil) } req := ingestionRules.PostableIngestionRules{} diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 9dc77fef9d..66d0052ace 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -2146,12 +2146,12 @@ func (aH *APIHandler) WriteJSON(w http.ResponseWriter, r *http.Request, response } // logs -func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router, am *AuthMiddleware) { +func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v1/logs").Subrouter() subRouter.HandleFunc("", aH.authenticator.ViewAccess(aH.getLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/tail", aH.authenticator.ViewAccess(aH.tailLogs)).Methods(http.MethodGet) subRouter.HandleFunc("/fields", aH.authenticator.ViewAccess(aH.logFields)).Methods(http.MethodGet) - subRouter.HandleFunc("/fields", am.EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) + subRouter.HandleFunc("/fields", aH.authenticator.EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) subRouter.HandleFunc("/aggregate", aH.authenticator.ViewAccess(aH.logAggregate)).Methods(http.MethodGet) } From 2755551e7cf88718bbced3057218af6f1c96e5b5 Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 14 Mar 2023 13:36:49 +0530 Subject: [PATCH 18/24] chore: removed user context --- ee/query-service/app/api/api.go | 20 -------------------- pkg/query-service/auth/userContext.go | 24 ------------------------ 2 files changed, 44 deletions(-) delete mode 100644 pkg/query-service/auth/userContext.go diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 3068aadfd6..0cd73bc266 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -1,7 +1,6 @@ package api import ( - "context" "net/http" "github.com/gorilla/mux" @@ -14,7 +13,6 @@ import ( basemodel "go.signoz.io/signoz/pkg/query-service/model" rules "go.signoz.io/signoz/pkg/query-service/rules" "go.signoz.io/signoz/pkg/query-service/version" - "google.golang.org/grpc/metadata" ) type APIHandlerOptions struct { @@ -175,21 +173,3 @@ func (ah *APIHandler) getVersion(w http.ResponseWriter, r *http.Request) { ah.WriteJSON(w, r, versionResponse) } - -// AttachUserToContext extracts user from request and adds it to -// context -func (api *APIHandler) AttachUserToContext(ctx context.Context, r *http.Request) (context.Context, error) { - userPayload, err := api.Auth().GetUserFromRequest(r) - if err != nil { - return ctx, err - } - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - md = metadata.New(nil) - } - - md.Append("userId", userPayload.User.Id) - ctx = metadata.NewIncomingContext(ctx, md) - - return ctx, nil -} diff --git a/pkg/query-service/auth/userContext.go b/pkg/query-service/auth/userContext.go deleted file mode 100644 index 764f600d44..0000000000 --- a/pkg/query-service/auth/userContext.go +++ /dev/null @@ -1,24 +0,0 @@ -package auth - -import ( - "context" - - "github.com/pkg/errors" - "google.golang.org/grpc/metadata" -) - -// UserIdFromContext extracts user from context.Context using accssJwt. here, -// we do not validate if user exists as it would be done so while adding -// user to the context -func UserIdFromContext(ctx context.Context) (string, error) { - md, ok := metadata.FromIncomingContext(ctx) - if !ok { - return "", errors.New("No user metadata found in context") - } - userId := md.Get("userId") - if len(userId) == 0 { - return "", errors.New("No user found in the context") - } - - return userId[0], nil -} From 3d8cbaa4c3e94c5a4528d26e24ae7ea84f28c49a Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 14 Mar 2023 13:54:47 +0530 Subject: [PATCH 19/24] fix: issue with sql get latest config --- ee/query-service/ingestionRules/controller.go | 28 +++++++++---------- ee/query-service/ingestionRules/db.go | 13 +++------ pkg/query-service/agentConf/db.go | 4 +-- 3 files changed, 20 insertions(+), 25 deletions(-) diff --git a/ee/query-service/ingestionRules/controller.go b/ee/query-service/ingestionRules/controller.go index 0f13013c6f..fb46e9241e 100644 --- a/ee/query-service/ingestionRules/controller.go +++ b/ee/query-service/ingestionRules/controller.go @@ -53,10 +53,10 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string if r.Id == "" { // looks like a new or changed rule, store it first - inserted, err := ic.insertRule(ctx, &r) + inserted, err := ic.insertRule(ctx, userId, &r) if err != nil || inserted == nil { - zap.S().Errorf("failed to insert edited ingestion rule", err) - return nil, model.BadRequestStr("failed to insert edited rule") + zap.S().Error("failed to insert ingestion rule", err) + return nil, model.BadRequestStr("failed to insert rule") } else { dropRules = append(dropRules, *inserted) } @@ -67,7 +67,7 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string // instead of picking the update from client (browser) request selected, err := ic.GetRule(ctx, r.Id) if err != nil || selected == nil { - zap.S().Errorf("failed to find edited ingestion rule", err, r.Id) + zap.S().Error("failed to find edited ingestion rule", err, r.Id) return nil, model.BadRequestStr("failed to find edited rule, invalid request") } dropRules = append(dropRules, *selected) @@ -78,7 +78,7 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string // prepare filter config (processor) from the drop rules filterConfig, err := PrepareDropFilter(dropRules) if err != nil { - zap.S().Errorf("failed to generate processor config from ingestion rules for deployment", err) + zap.S().Error("failed to generate processor config from ingestion rules for deployment", err) return nil, model.BadRequest(err) } @@ -96,7 +96,7 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string // prepare config by calling gen func cfg, err := agentConf.StartNewVersion(ctx, userId, agentConf.ElementTypeDropRules, elements) if err != nil || cfg == nil { - zap.S().Errorf("failed to start a new config version for drop rules", err) + zap.S().Error("failed to start a new config version for drop rules", err) return nil, model.InternalError(err) } @@ -113,7 +113,7 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string } if err != nil { - zap.S().Errorf("failed to insert drop rules into agent config", filterConfig, err) + zap.S().Error("failed to insert drop rules into agent config", filterConfig, err) return response, model.InternalErrorStr(fmt.Sprintf("failed to apply drop rules: %s", err.Error())) } @@ -124,13 +124,13 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string func (ic *IngestionController) GetRulesByVersion(ctx context.Context, version int, typ agentConf.ElementTypeDef) (*IngestionRulesResponse, *model.ApiError) { rules, apierr := ic.getRulesByVersion(ctx, version) if apierr != nil { - zap.S().Errorf("failed to get ingestion rules for version", version, apierr) + zap.S().Error("failed to get ingestion rules for version", version, apierr) return nil, model.InternalErrorStr("failed to get drop rules for given version") } configVersion, err := agentConf.GetConfigVersion(ctx, typ, version) if err != nil || configVersion == nil { - zap.S().Errorf("failed to get version info", version, err) + zap.S().Error("failed to get version info", version, err) return nil, model.InternalErrorStr("failed to get info on the given version") } @@ -160,9 +160,9 @@ func (ic *IngestionController) ApplySamplingRules(ctx context.Context, userId st if r.Id == "" { // looks like a new or changed rule, store it first - inserted, err := ic.insertRule(ctx, &r) + inserted, err := ic.insertRule(ctx, userId, &r) if err != nil || inserted == nil { - zap.S().Errorf("failed to insert edited ingestion rule", err) + zap.S().Error("failed to insert edited ingestion rule", err) return nil, model.BadRequestStr("failed to insert edited rule") } else { smplRules = append(smplRules, *inserted) @@ -170,7 +170,7 @@ func (ic *IngestionController) ApplySamplingRules(ctx context.Context, userId st } else { selected, err := ic.GetRule(ctx, r.Id) if err != nil || selected == nil { - zap.S().Errorf("failed to find edited ingestion rule", err) + zap.S().Error("failed to find edited ingestion rule", err) return nil, model.BadRequestStr("failed to find edited rule, invalid request") } smplRules = append(smplRules, *selected) @@ -181,7 +181,7 @@ func (ic *IngestionController) ApplySamplingRules(ctx context.Context, userId st // prepare params for sampling processor from the rules params, err := PrepareTailSamplingParams(smplRules) if err != nil { - zap.S().Errorf("failed to generate processor config from ingestion rules for deployment", err) + zap.S().Error("failed to generate processor config from ingestion rules for deployment", err) return nil, model.BadRequest(err) } @@ -215,7 +215,7 @@ func (ic *IngestionController) ApplySamplingRules(ctx context.Context, userId st } if err != nil { - zap.S().Errorf("failed to insert sampling rules into agent config: ", params, err) + zap.S().Error("failed to insert sampling rules into agent config: ", params, err) return response, model.InternalErrorStr(fmt.Sprintf("failed to apply drop rules: %s", err.Error())) } return response, nil diff --git a/ee/query-service/ingestionRules/db.go b/ee/query-service/ingestionRules/db.go index 7fa83b9b78..3c7973de62 100644 --- a/ee/query-service/ingestionRules/db.go +++ b/ee/query-service/ingestionRules/db.go @@ -11,7 +11,6 @@ import ( "github.com/pkg/errors" "go.signoz.io/signoz/ee/query-service/ingestionRules/sqlite" "go.signoz.io/signoz/ee/query-service/model" - baseauth "go.signoz.io/signoz/pkg/query-service/auth" "go.uber.org/zap" ) @@ -37,11 +36,7 @@ func (r *Repo) InitDB(engine string) error { } // InsertRule stores a given postable rule to database -func (r *Repo) insertRule(ctx context.Context, postable *PostableIngestionRule) (*model.IngestionRule, error) { - userId, err := baseauth.UserIdFromContext(ctx) - if err != nil { - return nil, err - } +func (r *Repo) insertRule(ctx context.Context, userId string, postable *PostableIngestionRule) (*model.IngestionRule, error) { if err := postable.IsValid(); err != nil { return nil, errors.Wrap(err, "failed to validate postable ingestion rule") @@ -150,19 +145,19 @@ func (r *Repo) GetRule(ctx context.Context, id string) (*model.IngestionRule, *m err := r.db.SelectContext(ctx, &rules, ruleQuery, id) if err != nil { - zap.S().Errorf("failed to get ingestion rule from db", err) + zap.S().Error("failed to get ingestion rule from db", err) return nil, model.BadRequestStr("failed to get ingestion rule from db") } if len(rules) == 0 { - zap.S().Warnf("No row found for ingestion rule id", id) + zap.S().Warn("No row found for ingestion rule id", id) return nil, nil } if len(rules) == 1 { err := rules[0].ParseRawConfig() if err != nil { - zap.S().Errorf("invalid rule config found", id, err) + zap.S().Error("invalid rule config found", id, err) return &rules[0], model.InternalErrorStr("found an invalid rule config ") } return &rules[0], nil diff --git a/pkg/query-service/agentConf/db.go b/pkg/query-service/agentConf/db.go index 6392513f6a..591a7ee799 100644 --- a/pkg/query-service/agentConf/db.go +++ b/pkg/query-service/agentConf/db.go @@ -46,7 +46,7 @@ func (r *Repo) GetConfigHistory(ctx context.Context, typ ElementTypeDef) ([]Conf disabled, deploy_status, deploy_result - FROM agent_config_versions + FROM agent_config_versions AS v WHERE element_type = $1`, typ) return c, err @@ -92,7 +92,7 @@ func (r *Repo) GetLatestVersion(ctx context.Context, typ ElementTypeDef) (*Confi disabled, deploy_status, deploy_result - FROM agent_config_versions + FROM agent_config_versions AS v WHERE element_type = $1 AND version = ( SELECT MAX(version) From 703ba3ac04472409f3a60054620a600ce3db2c70 Mon Sep 17 00:00:00 2001 From: mindhash Date: Sat, 18 Mar 2023 16:42:39 +0530 Subject: [PATCH 20/24] feat: added lb exporter config --- go.mod | 18 ++- go.sum | 42 ++++++- .../app/opamp/configure_ingestionRules.go | 63 +++------- pkg/query-service/app/opamp/configure_lb.go | 54 ++++++++ .../app/opamp/configure_lb_test.go | 45 +++++++ pkg/query-service/app/opamp/model/agent.go | 88 +++++++------ pkg/query-service/app/opamp/opamp_server.go | 11 +- .../app/opamp/otelconfig/config_parser.go | 117 +++++++++++++----- .../opamp/otelconfig/config_parser_test.go | 47 +++++++ .../app/opamp/otelconfig/lbexporter/config.go | 68 ++++++++++ .../opamp/otelconfig/otlpreceiver/config.go | 6 - .../otelconfig/otlpreceiver/grpcSettings.go | 14 --- .../otelconfig/otlpreceiver/httpSettings.go | 9 -- .../app/opamp/otelconfig/otlpreceiver/tls.go | 38 ------ .../app/opamp/otelconfig/otlpworker/config.go | 19 +++ .../opamp/otelconfig/testdata/processors.yaml | 4 + .../otelconfig/testdata/tail_processor.yaml | 6 + .../app/opamp/otelconfig/utils.go | 10 ++ 18 files changed, 459 insertions(+), 200 deletions(-) create mode 100644 pkg/query-service/app/opamp/configure_lb.go create mode 100644 pkg/query-service/app/opamp/configure_lb_test.go create mode 100644 pkg/query-service/app/opamp/otelconfig/lbexporter/config.go delete mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go delete mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go delete mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go delete mode 100644 pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go create mode 100644 pkg/query-service/app/opamp/otelconfig/otlpworker/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/testdata/processors.yaml create mode 100644 pkg/query-service/app/opamp/otelconfig/testdata/tail_processor.yaml create mode 100644 pkg/query-service/app/opamp/otelconfig/utils.go diff --git a/go.mod b/go.mod index ac1ab45631..fe03c821b3 100644 --- a/go.mod +++ b/go.mod @@ -30,14 +30,15 @@ require ( github.com/sethvargo/go-password v0.2.0 github.com/smartystreets/goconvey v1.6.4 github.com/soheilhy/cmux v0.1.5 - go.opentelemetry.io/collector/confmap v0.73.0 + go.opentelemetry.io/collector v0.66.0 + go.opentelemetry.io/collector/exporter/otlpexporter v0.66.0 go.uber.org/zap v1.24.0 gopkg.in/segmentio/analytics-go.v3 v3.1.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/apimachinery v0.26.0 ) require ( - github.com/benbjohnson/clock v1.3.0 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/dennwc/varint v1.0.0 // indirect github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect @@ -66,8 +67,8 @@ require ( github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/andybalholm/brotli v1.0.4 // indirect github.com/armon/go-metrics v0.4.0 // indirect - github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df // indirect github.com/beevik/etree v1.1.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -79,12 +80,19 @@ require ( github.com/minio/sha256-simd v0.1.1 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect + github.com/mostynb/go-grpc-compression v1.1.17 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.14.0 // indirect - go.opentelemetry.io/collector/featuregate v0.73.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/collector/component v0.65.0 // indirect + go.opentelemetry.io/collector/consumer v0.65.0 // indirect + go.opentelemetry.io/collector/featuregate v0.66.0 // indirect + go.opentelemetry.io/collector/pdata v0.65.0 // indirect + go.opentelemetry.io/collector/receiver/otlpreceiver v0.66.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 // indirect + google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index a435711f2f..c4309e892e 100644 --- a/go.sum +++ b/go.sum @@ -28,6 +28,7 @@ cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Ud cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= +cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= @@ -40,6 +41,8 @@ cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6m cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= +cloud.google.com/go/compute v1.13.0 h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU= +cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= @@ -53,6 +56,7 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v65.0.0+incompatible h1:HzKLt3kIwMm4KeJYTdx9EbjRYTySD/t8i1Ee/W5EGXw= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -110,12 +114,9 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72H github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= -github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df h1:GSoSVRLoBaFpOOds6QyY1L8AX7uoY+Ln3BHc22W40X0= -github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df/go.mod h1:hiVxq5OP2bUGBRNS3Z/bt/reCLFNbdcST6gISi1fiOM= github.com/beevik/etree v1.1.0 h1:T0xke/WvNtMoCqgzPhkX2r4rjY3GDZFi+FjpRZY2Jbs= github.com/beevik/etree v1.1.0/go.mod h1:r8Aw8JqVegEf0w2fDnATrX9VpkMcyFeM0FhwO62wh+A= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= -github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -123,6 +124,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= +github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -187,6 +190,7 @@ github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.5+incompatible h1:/l4kBbb4/vGSsdtB5nUe8L7B9mImVMaBPw9L/0TBHU8= github.com/form3tech-oss/jwt-go v3.2.5+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -238,6 +242,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -420,6 +425,7 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.15.13 h1:NFn1Wr8cfnenSJSA46lLq4wHCcBzKTSjnBIexDMMOV0= github.com/klauspost/compress v1.15.13/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= @@ -494,6 +500,8 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mostynb/go-grpc-compression v1.1.17 h1:N9t6taOJN3mNTTi0wDf4e3lp/G/ON1TP67Pn0vTUA9I= +github.com/mostynb/go-grpc-compression v1.1.17/go.mod h1:FUSBr0QjKqQgoDG/e0yiqlR6aqyXC39+g/hFLDfSsEY= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= @@ -521,6 +529,7 @@ github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKf github.com/pelletier/go-toml v1.7.0 h1:7utD74fnzVc/cpcyy8sjrlFr5vYpypUixARcHIMIGuI= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.17 h1:kV4Ip+/hUBC+8T6+2EgburRtkE9ef4nbY3f4dFhGjMc= github.com/pierrec/lz4/v4 v4.1.17/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= @@ -564,6 +573,7 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -623,6 +633,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -648,16 +659,35 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= -go.opentelemetry.io/collector/confmap v0.73.0 h1:tC8x8sDk7JQ3QcbosqrxLe756sYcg4iUdTXsx7Ie4CM= -go.opentelemetry.io/collector/confmap v0.73.0/go.mod h1:zEYi9hTAYmCHvN3XWxlN8LvHBrhs2FyQ99k7Ox60L1A= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/collector v0.66.0 h1:5+0N2PCyqHoE3MYV8tgCsyeD86KbmIVDKmqKF9A7u9k= +go.opentelemetry.io/collector v0.66.0/go.mod h1:hE6jCs+0rfiufCrVPucKZTMwfHit1okfDPnwPT2eW1I= +go.opentelemetry.io/collector/component v0.65.0 h1:6r71RfuzXyqyVeuU2+YBBS5xWfMQJ1INSKCTtGjBQLc= +go.opentelemetry.io/collector/component v0.65.0/go.mod h1:0c84EqXUhvYe6KW7hJfh76tiI/5yjWCH2amwyQ06XLM= +go.opentelemetry.io/collector/consumer v0.65.0 h1:9pjl1zV8nQ2QtQLuCcu0X7i9k+qXlTsE3YS9IaIQqHY= +go.opentelemetry.io/collector/consumer v0.65.0/go.mod h1:WtoRZa5SnxQO1ZEQdVxYpFcXCmq62rakv0oUSlPO0NQ= +go.opentelemetry.io/collector/exporter/otlpexporter v0.66.0 h1:uXNkGlAXHyvUZL51qsQR8hS8oSt80m1ncbqwXRRkJMk= +go.opentelemetry.io/collector/exporter/otlpexporter v0.66.0/go.mod h1:IOoHR1lsV1r2TFOVctPogn9RUXT4jiU7kH+IhfUlP1w= +go.opentelemetry.io/collector/featuregate v0.66.0 h1:WW3IYWxOu9cfXa6fQwov0jswlf2Y/NEBHgiDkRPm4Uw= +go.opentelemetry.io/collector/featuregate v0.66.0/go.mod h1:tewuFKJYalWBU0bmNKg++MC1ipINXUr6szYzOw2p1GI= go.opentelemetry.io/collector/featuregate v0.73.0 h1:hpHKXmRiJqMLefIzXwIuqDo9df2HcI/66IAKLo+g7nc= go.opentelemetry.io/collector/featuregate v0.73.0/go.mod h1:oIrO4ysPfThwkYKoNlgpHjDU/TgS9kIv2OI74pDUkp4= +go.opentelemetry.io/collector/pdata v0.65.0 h1:9m/hYC98sSQFjGP77/DS+uJedjFwe8TPiMdWrE644Xo= +go.opentelemetry.io/collector/pdata v0.65.0/go.mod h1:pqyaznLzk21m+1KL6fwOsRryRELL+zNM0qiVSn0MbVc= +go.opentelemetry.io/collector/receiver/otlpreceiver v0.66.0 h1:ZHEOeGOIYIDTKhI9xs30/obFfhUbb9b2+e/eahp1fJA= +go.opentelemetry.io/collector/receiver/otlpreceiver v0.66.0/go.mod h1:pb+dcV+KJ44dP+CqV0N6L6dTeDDvCXB8StCqwlrrSkM= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4 h1:PRXhsszxTt5bbPriTjmaweWUsAnJYeWBhUMLRetUgBU= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.36.4/go.mod h1:05eWWy6ZWzmpeImD3UowLTB3VjDMU1yxQ+ENuVWDM3c= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0 h1:yt2NKzK7Vyo6h0+X8BA4FpreZQTlVEIarnsBP/H5mzs= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.37.0/go.mod h1:+ARmXlUlc51J7sZeCBkBJNdHGySrdOzgzxp6VWRWM1U= go.opentelemetry.io/otel v1.11.2 h1:YBZcQlsVekzFsFbjygXMOXSs6pialIZxcjfO/mBDmR0= go.opentelemetry.io/otel v1.11.2/go.mod h1:7p4EUV+AqgdlNV9gL97IgUZiVR3yrFXYo53f9BM3tRI= +go.opentelemetry.io/otel/exporters/prometheus v0.33.0 h1:xXhPj7SLKWU5/Zd4Hxmd+X1C4jdmvc0Xy+kvjFx2z60= go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= +go.opentelemetry.io/otel/sdk/metric v0.33.0 h1:oTqyWfksgKoJmbrs2q7O7ahkJzt+Ipekihf8vhpa9qo= go.opentelemetry.io/otel/trace v1.11.2 h1:Xf7hWSF2Glv0DE3MH7fBHvtpSBsjcBUe5MYAmZM/+y0= go.opentelemetry.io/otel/trace v1.11.2/go.mod h1:4N+yC7QEz7TTsG9BSRLNAa63eg5E06ObSbKPmxQ/pKA= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1113,6 +1143,7 @@ google.golang.org/genproto v0.0.0-20220523171625-347a074981d8/go.mod h1:RAyBrSAP google.golang.org/genproto v0.0.0-20220608133413-ed9918b62aac/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20220616135557-88e70c0c3a90/go.mod h1:KEWEmljWE5zPzLBa/oHl6DaEt9LmfH6WtH1OHIvleBA= google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70= +google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1141,6 +1172,7 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go index af04abb723..b8f8a1e9ec 100644 --- a/pkg/query-service/app/opamp/configure_ingestionRules.go +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -2,11 +2,8 @@ package opamp import ( "context" - "crypto/sha256" "fmt" - "github.com/knadh/koanf/parsers/yaml" - "github.com/open-telemetry/opamp-go/protobufs" "go.opentelemetry.io/collector/confmap" model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" @@ -40,15 +37,15 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ return } + withLB := false if len(agents) > 1 && signal == string(Traces) { - zap.S().Debug("found multiple agents. this feature is not supported for traces pipeline (sampling rules)") - fnerr = fmt.Errorf("multiple agents not supported in sampling rules") - return + // enable LB exporter config + withLB = true } for _, agent := range agents { - agenthash, err := addIngestionControlToAgent(agent, signal, processors, false) + agenthash, err := addIngestionControlToAgent(agent, signal, processors, withLB) if err != nil { zap.S().Error("failed to push ingestion rules config to agent", agent.ID, err) continue @@ -66,57 +63,25 @@ func UpsertControlProcessors(ctx context.Context, signal string, processors map[ } // addIngestionControlToAgent adds ingestion contorl rules to agent config -func addIngestionControlToAgent(agent *model.Agent, signal string, processors map[string]interface{}, withLB bool) (string, error) { - confHash := "" - config := agent.EffectiveConfig - c, err := yaml.Parser().Unmarshal([]byte(config)) +func addIngestionControlToAgent(agent *model.Agent, signal string, processors map[string]interface{}, withLB bool) (confhash string, err error) { + agentConf, err := agent.GetEffectiveConfMap() if err != nil { - return confHash, err + return confhash, err } - agentConf := confmap.NewFromStringMap(c) + if withLB { + // add LB exporter pipeline and OTLP worker pipeline + makeLbExporterSpec(agentConf) + } - // add ingestion control spec + // add ingestion control spec. err = makeIngestionControlSpec(agentConf, Signal(signal), processors) if err != nil { zap.S().Error("failed to prepare ingestion control processors for agent ", agent.ID, err) - return confHash, err - } - - // ------ complete adding processor - configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) - if err != nil { - return confHash, err - } - - zap.S().Debugf("sending new config", string(configR)) - hash := sha256.New() - _, err = hash.Write(configR) - if err != nil { - return confHash, err - } - confHash = string(hash.Sum(nil)) - agent.EffectiveConfig = string(configR) - err = agent.Upsert() - if err != nil { - return confHash, err + return confhash, err } - agent.SendToAgent(&protobufs.ServerToAgent{ - RemoteConfig: &protobufs.AgentRemoteConfig{ - Config: &protobufs.AgentConfigMap{ - ConfigMap: map[string]*protobufs.AgentConfigFile{ - "collector.yaml": { - Body: configR, - ContentType: "application/x-yaml", - }, - }, - }, - ConfigHash: []byte(confHash), - }, - }) - - return string(confHash), nil + return agent.SendConfMap(agentConf) } // prepare spec to introduce ingestion control in agent conf diff --git a/pkg/query-service/app/opamp/configure_lb.go b/pkg/query-service/app/opamp/configure_lb.go new file mode 100644 index 0000000000..5e6ae9eccd --- /dev/null +++ b/pkg/query-service/app/opamp/configure_lb.go @@ -0,0 +1,54 @@ +package opamp + +import ( + "go.opentelemetry.io/collector/confmap" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/lbexporter" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/otlpworker" +) + +const ( + LoadBalancerPipeline = "traces/lb" + LoadBalancerExporter = "loadbalancing" + TracesDefaultPipeline = "traces" + OTLPWorker = "otlp/worker" +) + +// +// makeLbExporterSpec adds a new pipeline for +// enabling LB exporter and adding OTLP worker on the same +// instance +// traces/lb: +// receivers: [otlp, jaeger] +// processors: [] +// exporters: [lbExporter] + +// update receiver in service > pipelines > traces +// traces: +// +// receivers: [otlp_internal] +// processors: [signoz_tail_sampling, batch] +// exporters: [clickhousetraceexporter] +func makeLbExporterSpec(agentConf *confmap.Conf) error { + + configParser := otelconfig.NewConfigParser(agentConf) + + if configParser.CheckPipelineExists(LoadBalancerPipeline) { + return nil + } + + receivers := configParser.PipelineReceivers(TracesDefaultPipeline) + processors := configParser.PipelineProcessors(TracesDefaultPipeline) + exporters := configParser.PipelineExporters(TracesDefaultPipeline) + + // alter pipelines + configParser.ReplacePipeline(LoadBalancerPipeline, receivers, otelconfig.StringsToIfaces([]string{}), otelconfig.StringsToIfaces([]string{LoadBalancerExporter})) + configParser.ReplacePipeline(TracesDefaultPipeline, otelconfig.StringsToIfaces([]string{OTLPWorker}), processors, exporters) + + // add otlp internal receiver + configParser.ReplaceReceiver(OTLPWorker, otlpworker.NewOtlpWorkerReceiver("0.0.0.0:4949")) + + // add lb exporter + configParser.ReplaceExporter(LoadBalancerExporter, lbexporter.NewDnsConfig("amol-signoz-otel-collector-worker", "4949")) + return nil +} diff --git a/pkg/query-service/app/opamp/configure_lb_test.go b/pkg/query-service/app/opamp/configure_lb_test.go new file mode 100644 index 0000000000..fa7bec6caf --- /dev/null +++ b/pkg/query-service/app/opamp/configure_lb_test.go @@ -0,0 +1,45 @@ +package opamp + +import ( + "fmt" + "io/ioutil" + "log" + "testing" + + "github.com/knadh/koanf/parsers/yaml" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/confmap" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" +) + +func TestMakeLBExporterSpec(t *testing.T) { + yamlFile, err := ioutil.ReadFile("./config.yaml") + if err != nil { + fmt.Printf("yamlFile.Get err #%v ", err) + t.Fail() + return + } + + c, err := yaml.Parser().Unmarshal([]byte(yamlFile)) + if err != nil { + log.Println("failed to parse config file as yaml", err) + t.Fail() + return + } + + agentConf := confmap.NewFromStringMap(c) + makeLbExporterSpec(agentConf) + parser := otelconfig.NewConfigParser(agentConf) + + require.Equal(t, true, parser.CheckExporterInPipeline(LoadBalancerPipeline, LoadBalancerExporter)) + require.Equal(t, true, parser.CheckRecevierInPipeline(TracesDefaultPipeline, OTLPWorker)) + + exporters := parser.Exporters() + if _, ok := exporters[LoadBalancerExporter]; !ok { + t.Fail() + } + receivers := parser.Receivers() + if _, ok := receivers[OTLPWorker]; !ok { + t.Fail() + } +} diff --git a/pkg/query-service/app/opamp/model/agent.go b/pkg/query-service/app/opamp/model/agent.go index ba2ecfcddc..59a023c536 100644 --- a/pkg/query-service/app/opamp/model/agent.go +++ b/pkg/query-service/app/opamp/model/agent.go @@ -7,8 +7,11 @@ import ( "sync" "time" + "go.opentelemetry.io/collector/confmap" + "go.uber.org/zap" "google.golang.org/protobuf/proto" + "github.com/knadh/koanf/parsers/yaml" "github.com/open-telemetry/opamp-go/protobufs" "github.com/open-telemetry/opamp-go/server/types" ) @@ -21,10 +24,6 @@ const ( AgentStatusDisconnected ) -// set in agent description when agent is capable of supporting -// lb exporter configuration. values: 1 (true) or 0 (false) -const lbExporterFlag = "capabilities.lbexporter" - type Agent struct { ID string `json:"agentId" yaml:"agentId" db:"agent_id"` StartedAt time.Time `json:"startedAt" yaml:"startedAt" db:"started_at"` @@ -34,12 +33,6 @@ type Agent struct { remoteConfig *protobufs.AgentRemoteConfig Status *protobufs.AgentToServer - // can this agent be load balancer - CanLB bool - - // is this agent setup as load balancer - IsLb bool - conn types.Connection connMutex sync.Mutex mux sync.RWMutex @@ -49,6 +42,16 @@ func New(ID string, conn types.Connection) *Agent { return &Agent{ID: ID, StartedAt: time.Now(), CurrentStatus: AgentStatusConnected, conn: conn} } +func (agent *Agent) GetEffectiveConfMap() (*confmap.Conf, error) { + config := agent.EffectiveConfig + c, err := yaml.Parser().Unmarshal([]byte(config)) + if err != nil { + return nil, nil + } + + return confmap.NewFromStringMap(c), nil +} + // Upsert inserts or updates the agent in the database. func (agent *Agent) Upsert() error { agent.mux.Lock() @@ -78,29 +81,6 @@ func (agent *Agent) UpdateStatus(statusMsg *protobufs.AgentToServer, response *p agent.processStatusUpdate(statusMsg, response) } -// extracts lb exporter support flag from agent description. the flag -// is used to decide if lb exporter can be enabled on the agent. -func ExtractLbFlag(agentDescr *protobufs.AgentDescription) bool { - - if agentDescr == nil { - return false - } - - if len(agentDescr.NonIdentifyingAttributes) > 0 { - for _, kv := range agentDescr.NonIdentifyingAttributes { - anyvalue, ok := kv.Value.Value.(*protobufs.AnyValue_StringValue) - if !ok { - continue - } - if kv.Key == lbExporterFlag && anyvalue.StringValue == "1" { - // agent can support load balancer config - return true - } - } - } - return false -} - func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) (agentDescrChanged bool) { prevStatus := agent.Status @@ -149,10 +129,6 @@ func (agent *Agent) updateAgentDescription(newStatus *protobufs.AgentToServer) ( } } - if agentDescrChanged { - agent.CanLB = ExtractLbFlag(newStatus.AgentDescription) - } - return agentDescrChanged } @@ -332,3 +308,41 @@ func (agent *Agent) SendToAgent(msg *protobufs.ServerToAgent) { agent.conn.Send(context.Background(), msg) } + +func (agent *Agent) SendConfMap(agentConf *confmap.Conf) (confhash string, sendErr error) { + + configR, err := yaml.Parser().Marshal(agentConf.ToStringMap()) + if err != nil { + return confhash, err + } + + zap.S().Debug("sending new config to agent", agent.ID, string(configR)) + + hash := sha256.New() + _, err = hash.Write(configR) + if err != nil { + return confhash, err + } + + confhash = string(hash.Sum(nil)) + agent.EffectiveConfig = string(configR) + err = agent.Upsert() + if err != nil { + return confhash, err + } + + agent.SendToAgent(&protobufs.ServerToAgent{ + RemoteConfig: &protobufs.AgentRemoteConfig{ + Config: &protobufs.AgentConfigMap{ + ConfigMap: map[string]*protobufs.AgentConfigFile{ + "collector.yaml": { + Body: configR, + ContentType: "application/x-yaml", + }, + }, + }, + ConfigHash: []byte(confhash), + }, + }) + return confhash, nil +} diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go index 237b07f121..ae7b290bce 100644 --- a/pkg/query-service/app/opamp/opamp_server.go +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -74,19 +74,16 @@ func (srv *Server) onDisconnect(conn types.Connection) { func (srv *Server) onMessage(conn types.Connection, msg *protobufs.AgentToServer) *protobufs.ServerToAgent { agentID := msg.InstanceUid - agent, created, err := srv.agents.FindOrCreateAgent(agentID, conn) + agent, _, err := srv.agents.FindOrCreateAgent(agentID, conn) if err != nil { zap.S().Error("Failed to find or create agent %q: %v", agentID, err) // TODO: handle error } - if created { - agent.CanLB = model.ExtractLbFlag(msg.AgentDescription) - zap.S().Debugf("New agent added:", zap.Bool("canLb", agent.CanLB), zap.String("ID", agent.ID), zap.Any("status", agent.CurrentStatus)) - } + // todo(): when agent count goes to 2 and sampling rules on traces is enabled + // add lb - var response *protobufs.ServerToAgent - response = &protobufs.ServerToAgent{ + response := &protobufs.ServerToAgent{ InstanceUid: agentID, Capabilities: uint64(capabilities), } diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser.go b/pkg/query-service/app/opamp/otelconfig/config_parser.go index 5d6c8adba4..8a4dc2018e 100644 --- a/pkg/query-service/app/opamp/otelconfig/config_parser.go +++ b/pkg/query-service/app/opamp/otelconfig/config_parser.go @@ -1,11 +1,20 @@ package otelconfig import ( + "fmt" "sync" "go.opentelemetry.io/collector/confmap" ) +const ( + receiversKey = "receivers" + processorsKey = "processors" + pipelinesKey = "pipelines" + serviceKey = "service" + exportersKey = "exporters" +) + type ConfigParser struct { lock sync.Mutex agentConf *confmap.Conf @@ -34,8 +43,12 @@ func emptyList() []interface{} { return []interface{}{} } +func (cp *ConfigParser) Current() *confmap.Conf { + return cp.agentConf +} + func (cp *ConfigParser) Service() map[string]interface{} { - service := cp.agentConf.Get("service") + service := cp.agentConf.Get(serviceKey) if service == nil { return emptyMap() } @@ -43,51 +56,53 @@ func (cp *ConfigParser) Service() map[string]interface{} { } // components gets the high level parts like receivers, exporters, processors etc -func (cp *ConfigParser) components(partName, nameOptional string) map[string]interface{} { +func (cp *ConfigParser) components(partName string) map[string]interface{} { parts := cp.agentConf.Get(partName) if parts == nil { return emptyMap() } parsedParts := toMap(parts) - if nameOptional != "" { - if p, ok := parsedParts[nameOptional]; ok { - return p.(map[string]interface{}) - } else { - return emptyMap() - } - } return parsedParts } -func (cp *ConfigParser) Processors() map[string]interface{} { - return cp.components("processors", "") +// components gets the high level parts like receivers, exporters, processors etc +func (cp *ConfigParser) component(partName, name string) (interface{}, error) { + components := cp.components(partName) + if p, ok := components[name]; ok { + return p, nil + } + + return nil, fmt.Errorf("component not found") } -func (cp *ConfigParser) Processor(name string) map[string]interface{} { - return cp.components("processors", name) +func (cp *ConfigParser) Processors() map[string]interface{} { + return cp.components(processorsKey) } func (cp *ConfigParser) Exporters() map[string]interface{} { - return cp.components("exporters", "") + return cp.components(exportersKey) } -func (cp *ConfigParser) Exporter(name string) map[string]interface{} { - return cp.components("exporters", name) +func (cp *ConfigParser) Receivers() map[string]interface{} { + return cp.components(receiversKey) } -func (cp *ConfigParser) Receivers() map[string]interface{} { - return cp.components("receivers", "") +func (cp *ConfigParser) Exporter(name string) (interface{}, error) { + return cp.component(exportersKey, name) +} +func (cp *ConfigParser) Receiver(name string) (interface{}, error) { + return cp.component(receiversKey, name) } -func (cp *ConfigParser) Receiver(name string) map[string]interface{} { - return cp.components("receivers", name) +func (cp *ConfigParser) Processor(name string) (interface{}, error) { + return cp.component(processorsKey, name) } func (cp *ConfigParser) Pipelines(nameOptional string) map[string]interface{} { services := cp.Service() - if p, ok := services["pipelines"]; ok { + if p, ok := services[pipelinesKey]; ok { pipelines := toMap(p) if nameOptional != "" { if namedPipeline, ok := pipelines[nameOptional]; ok { @@ -102,7 +117,7 @@ func (cp *ConfigParser) Pipelines(nameOptional string) map[string]interface{} { return emptyMap() } -// component can be "recevers", "exporter" or "processors" +// component can be "recevers", "exporter" or processors func (cp *ConfigParser) PipelineComponent(pipelineName, pipelineComponent string) []interface{} { pipeline := cp.Pipelines(pipelineName) if exporters, ok := pipeline[pipelineComponent]; ok { @@ -113,15 +128,15 @@ func (cp *ConfigParser) PipelineComponent(pipelineName, pipelineComponent string } func (cp *ConfigParser) PipelineExporters(pipelineName string) []interface{} { - return cp.PipelineComponent(pipelineName, "exporters") + return cp.PipelineComponent(pipelineName, exportersKey) } func (cp *ConfigParser) PipelineReceivers(pipelineName string) []interface{} { - return cp.PipelineComponent(pipelineName, "receivers") + return cp.PipelineComponent(pipelineName, receiversKey) } func (cp *ConfigParser) PipelineProcessors(pipelineName string) []interface{} { - return cp.PipelineComponent(pipelineName, "processors") + return cp.PipelineComponent(pipelineName, processorsKey) } func (cp *ConfigParser) CheckPipelineExists(name string) bool { @@ -150,11 +165,15 @@ func (cp *ConfigParser) CheckEntryInPipeline(pipelineName, pipelineComponent, na } func (cp *ConfigParser) CheckExporterInPipeline(pipelineName, name string) bool { - return cp.CheckEntryInPipeline(pipelineName, "exporters", name) + return cp.CheckEntryInPipeline(pipelineName, exportersKey, name) } func (cp *ConfigParser) CheckProcessorInPipeline(pipelineName, name string) bool { - return cp.CheckEntryInPipeline(pipelineName, "processors", name) + return cp.CheckEntryInPipeline(pipelineName, processorsKey, name) +} + +func (cp *ConfigParser) CheckRecevierInPipeline(pipelineName, name string) bool { + return cp.CheckEntryInPipeline(pipelineName, receiversKey, name) } func (cp *ConfigParser) Merge(c *confmap.Conf) { @@ -171,7 +190,7 @@ func (cp *ConfigParser) UpdateProcessors(processors map[string]interface{}) { } updatedProcessors := map[string]interface{}{ - "processors": updates, + processorsKey: updates, } updatedProcessorConf := confmap.NewFromStringMap(updatedProcessors) @@ -182,10 +201,10 @@ func (cp *ConfigParser) UpdateProcessors(processors map[string]interface{}) { func (cp *ConfigParser) UpdateProcsInPipeline(pipelineName string, list []interface{}) { serviceConf := map[string]interface{}{ - "service": map[string]interface{}{ + serviceKey: map[string]interface{}{ "pipelines": map[string]interface{}{ pipelineName: map[string]interface{}{ - "processors": list, + processorsKey: list, }, }, }, @@ -193,3 +212,41 @@ func (cp *ConfigParser) UpdateProcsInPipeline(pipelineName string, list []interf cp.Merge(confmap.NewFromStringMap(serviceConf)) } + +func (cp *ConfigParser) ReplacePipeline(name string, receivers []interface{}, processors []interface{}, exporters []interface{}) { + serviceConf := map[string]interface{}{ + serviceKey: map[string]interface{}{ + "pipelines": map[string]interface{}{ + name: map[string]interface{}{ + receiversKey: receivers, + processorsKey: processors, + exportersKey: exporters, + }, + }, + }, + } + + cp.Merge(confmap.NewFromStringMap(serviceConf)) +} + +func (cp *ConfigParser) replaceComponent(partName, name string, params interface{}) { + partConf := map[string]interface{}{ + partName: map[string]interface{}{ + name: params, + }, + } + + cp.Merge(confmap.NewFromStringMap(partConf)) +} + +func (cp *ConfigParser) ReplaceProcessor(name string, params interface{}) { + cp.replaceComponent(processorsKey, name, params) +} + +func (cp *ConfigParser) ReplaceExporter(name string, params interface{}) { + cp.replaceComponent(exportersKey, name, params) +} + +func (cp *ConfigParser) ReplaceReceiver(name string, params interface{}) { + cp.replaceComponent(receiversKey, name, params) +} diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser_test.go b/pkg/query-service/app/opamp/otelconfig/config_parser_test.go index 61ceb1f613..4949090b75 100644 --- a/pkg/query-service/app/opamp/otelconfig/config_parser_test.go +++ b/pkg/query-service/app/opamp/otelconfig/config_parser_test.go @@ -3,11 +3,14 @@ package otelconfig import ( "fmt" "io/ioutil" + "log" "testing" + "time" "github.com/knadh/koanf/parsers/yaml" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/confmap" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/tailsampler" ) func TestServiceConfig(t *testing.T) { @@ -56,3 +59,47 @@ func TestServiceConfig(t *testing.T) { require.Equal(t, expected, configParser.Service(), "expected same service config after parsing") } + +func TestReplaceProcessor(t *testing.T) { + + runtest := func(file string) { + log.Println("TestReplaceProcessor:", file) + yamlFile, err := ioutil.ReadFile(file) + if err != nil { + fmt.Printf("yamlFile.Get err #%v ", err) + t.Fail() + return + } + + c, err := yaml.Parser().Unmarshal([]byte(yamlFile)) + if err != nil { + fmt.Println("failed to parse config file as yaml", err) + t.Fail() + return + } + + agentConf := confmap.NewFromStringMap(c) + configParser := NewConfigParser(agentConf) + + tailparams := tailsampler.Config{ + DecisionWait: 2 * time.Second, + NumTraces: 10000, + PolicyCfgs: []tailsampler.PolicyCfg(nil), + Version: 0, + ExpectedNewTracesPerSec: 10000, + } + configParser.ReplaceProcessor("signoz_tail_sampling", tailparams) + + p, err := configParser.Processor("signoz_tail_sampling") + require.NoError(t, err, "failed to get processor signoz_tail_sampling") + + returned := p.(tailsampler.Config) + require.Equal(t, tailparams, returned, "expected same parameters for signoz_tail_sampling processor") + + b, err := configParser.Processor("batch") + require.NoError(t, err, "expected no error fetching batch processor as it is not changed") + require.NotEmpty(t, b, "expected non empty batch processor") + } + runtest("./testdata/processors.yaml") + runtest("./testdata/tail_processor.yaml") +} diff --git a/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go b/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go new file mode 100644 index 0000000000..37ef574604 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go @@ -0,0 +1,68 @@ +package lbexporter + +// borrowed from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.66.x/exporter/loadbalancingexporter/config.go + +import ( + "time" + + otelgrpc "go.opentelemetry.io/collector/config/configgrpc" + oteltls "go.opentelemetry.io/collector/config/configtls" + otelexporterhelper "go.opentelemetry.io/collector/exporter/exporterhelper" + "go.opentelemetry.io/collector/exporter/otlpexporter" +) + +// config for lb exporter +type Config struct { + Protocol Protocol `mapstructure:"protocol"` + Resolver ResolverSettings `mapstructure:"resolver"` + RoutingKey string `mapstructure:"routing_key"` +} + +func NewDnsConfig(hostname string, port string) Config { + // needs improvement(amol): need a better way to set parameters outside of binary (query-service) + return Config{ + Protocol: Protocol{ + OTLP: otlpexporter.Config{ + GRPCClientSettings: otelgrpc.GRPCClientSettings{ + TLSSetting: oteltls.TLSClientSetting{ + // needs improvement(amol): add tls + Insecure: true, + }, + }, + // default timeout is 5 seconds + TimeoutSettings: otelexporterhelper.NewDefaultTimeoutSettings(), + }, + }, + Resolver: ResolverSettings{ + DNS: &DNSResolver{ + Hostname: hostname, + Port: port, + }, + }, + } + +} + +// Protocol holds the individual protocol-specific settings. Only OTLP is supported at the moment. +type Protocol struct { + OTLP otlpexporter.Config `mapstructure:"otlp"` +} + +// ResolverSettings defines the configurations for the backend resolver +type ResolverSettings struct { + Static *StaticResolver `mapstructure:"static"` + DNS *DNSResolver `mapstructure:"dns"` +} + +// StaticResolver defines the configuration for the resolver providing a fixed list of backends +type StaticResolver struct { + Hostnames []string `mapstructure:"hostnames"` +} + +// DNSResolver defines the configuration for the DNS resolver +type DNSResolver struct { + Hostname string `mapstructure:"hostname"` + Port string `mapstructure:"port"` + Interval time.Duration `mapstructure:"interval"` + Timeout time.Duration `mapstructure:"timeout"` +} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go deleted file mode 100644 index 869afc9f43..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/config.go +++ /dev/null @@ -1,6 +0,0 @@ -package otlpreceiver - -type Protocols struct { - GRPC *GRPCServerSettings `mapstructure:"grpc"` - HTTP *HTTPServerSettings `mapstructure:"http"` -} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go deleted file mode 100644 index f0f55eda10..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/grpcSettings.go +++ /dev/null @@ -1,14 +0,0 @@ -package otlpreceiver - -type GRPCServerSettings struct { - // Endpoint configures the address for this network connection. - // For TCP and UDP networks, the address has the form "host:port". The host must be a literal IP address, - // or a host name that can be resolved to IP addresses. The port must be a literal port number or a service name. - // If the host is a literal IPv6 address it must be enclosed in square brackets, as in "[2001:db8::1]:80" or - // "[fe80::1%zone]:80". The zone specifies the scope of the literal IPv6 address as defined in RFC 4007. - Endpoint string `mapstructure:"endpoint"` - - // Transport to use. Known protocols are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only), "udp", "udp4" (IPv4-only), - // "udp6" (IPv6-only), "ip", "ip4" (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and "unixpacket". - Transport string `mapstructure:"transport"` -} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go deleted file mode 100644 index 64c9a16fe5..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/httpSettings.go +++ /dev/null @@ -1,9 +0,0 @@ -package otlpreceiver - -type HTTPServerSettings struct { - // Endpoint configures the listening address for the server. - Endpoint string `mapstructure:"endpoint" yaml:"endpoint"` - - // TLSSetting struct exposes TLS client configuration. - TLSSetting *TLSServerSetting `mapstructure:"tls" yaml:"tls"` -} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go b/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go deleted file mode 100644 index 39978fcb29..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/otlpreceiver/tls.go +++ /dev/null @@ -1,38 +0,0 @@ -package otlpreceiver - -import "time" - -type TLSSetting struct { - // Path to the CA cert. For a client this verifies the server certificate. - // For a server this verifies client certificates. If empty uses system root CA. - // (optional) - CAFile string `mapstructure:"ca_file"` - - // Path to the TLS cert to use for TLS required connections. (optional) - CertFile string `mapstructure:"cert_file"` - - // Path to the TLS key to use for TLS required connections. (optional) - KeyFile string `mapstructure:"key_file"` - - // MinVersion sets the minimum TLS version that is acceptable. - // If not set, TLS 1.2 will be used. (optional) - MinVersion string `mapstructure:"min_version"` - - // MaxVersion sets the maximum TLS version that is acceptable. - // If not set, refer to crypto/tls for defaults. (optional) - MaxVersion string `mapstructure:"max_version"` - - // ReloadInterval specifies the duration after which the certificate will be reloaded - // If not set, it will never be reloaded (optional) - ReloadInterval time.Duration `mapstructure:"reload_interval"` -} - -type TLSServerSetting struct { - // squash ensures fields are correctly decoded in embedded struct. - TLSSetting `mapstructure:",squash"` - - // Path to the TLS cert to use by the server to verify a client certificate. (optional) - // This sets the ClientCAs and ClientAuth to RequireAndVerifyClientCert in the TLSConfig. Please refer to - // https://godoc.org/crypto/tls#Config for more information. (optional) - ClientCAFile string `mapstructure:"client_ca_file"` -} diff --git a/pkg/query-service/app/opamp/otelconfig/otlpworker/config.go b/pkg/query-service/app/opamp/otelconfig/otlpworker/config.go new file mode 100644 index 0000000000..7364da1b24 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/otlpworker/config.go @@ -0,0 +1,19 @@ +package otlpworker + +import ( + otelgrpc "go.opentelemetry.io/collector/config/configgrpc" + otelnet "go.opentelemetry.io/collector/config/confignet" + "go.opentelemetry.io/collector/receiver/otlpreceiver" +) + +func NewOtlpWorkerReceiver(endpoint string) otlpreceiver.Config { + return otlpreceiver.Config{ + Protocols: otlpreceiver.Protocols{ + GRPC: &otelgrpc.GRPCServerSettings{ + NetAddr: otelnet.NetAddr{ + Endpoint: endpoint, + }, + }, + }, + } +} diff --git a/pkg/query-service/app/opamp/otelconfig/testdata/processors.yaml b/pkg/query-service/app/opamp/otelconfig/testdata/processors.yaml new file mode 100644 index 0000000000..9cb43070fc --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/testdata/processors.yaml @@ -0,0 +1,4 @@ +processors: + batch: + send_batch_size: 1000 + timeout: 10s \ No newline at end of file diff --git a/pkg/query-service/app/opamp/otelconfig/testdata/tail_processor.yaml b/pkg/query-service/app/opamp/otelconfig/testdata/tail_processor.yaml new file mode 100644 index 0000000000..beaa415fa6 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/testdata/tail_processor.yaml @@ -0,0 +1,6 @@ +processors: + batch: + send_batch_size: 1000 + timeout: 10s + signoz_tail_sampling: + decision_wait: 1s \ No newline at end of file diff --git a/pkg/query-service/app/opamp/otelconfig/utils.go b/pkg/query-service/app/opamp/otelconfig/utils.go new file mode 100644 index 0000000000..e4a1fe14b7 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/utils.go @@ -0,0 +1,10 @@ +package otelconfig + +// converts string array to interface array +func StringsToIfaces(s []string) []interface{} { + a := make([]interface{}, len(s)) + for i := range s { + a[i] = s[i] + } + return a +} From 1735844a11e2f4e478e56c325a1d5506672ac7ce Mon Sep 17 00:00:00 2001 From: mindhash Date: Sat, 18 Mar 2023 18:06:45 +0530 Subject: [PATCH 21/24] feat: added default-otel config for lb exporter --- ee/query-service/app/server.go | 7 ++-- ee/query-service/main.go | 17 ++++++---- pkg/query-service/app/opamp/config.yaml | 17 ++++++++++ .../app/opamp/configure_ingestionRules.go | 2 +- pkg/query-service/app/opamp/configure_lb.go | 30 +++++++++++++---- .../app/opamp/configure_lb_test.go | 11 ++++++- pkg/query-service/app/opamp/opamp_server.go | 14 ++++++-- .../app/opamp/otelconfig/config_parser.go | 17 ++++++++++ .../app/opamp/otelconfig/lbexporter/config.go | 32 +++---------------- .../app/opamp/otelconfig/otlpworker/config.go | 19 ----------- .../app/opamp/otelconfig/worker/config.go | 9 ++++++ pkg/query-service/app/server.go | 4 ++- pkg/query-service/constants/otel-default.yaml | 19 +++++++++++ pkg/query-service/main.go | 3 ++ 14 files changed, 133 insertions(+), 68 deletions(-) delete mode 100644 pkg/query-service/app/opamp/otelconfig/otlpworker/config.go create mode 100644 pkg/query-service/app/opamp/otelconfig/worker/config.go create mode 100644 pkg/query-service/constants/otel-default.yaml diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index f47a2be813..298ef371de 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -52,8 +52,9 @@ type ServerOptions struct { HTTPHostPort string PrivateHostPort string // alert specific params - DisableRules bool - RuleRepoURL string + DisableRules bool + RuleRepoURL string + DefaultOtelConfig string } // Server runs HTTP api service @@ -512,7 +513,7 @@ func (s *Server) Start() error { go func() { zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", baseconst.OpAmpWsEndpoint)) - err := opamp.InitalizeServer(baseconst.OpAmpWsEndpoint, &opAmpModel.AllAgents) + err := opamp.InitalizeServer(baseconst.OpAmpWsEndpoint, &opAmpModel.AllAgents, s.serverOptions.DefaultOtelConfig) if err != nil { zap.S().Info("opamp ws server failed to start", err) s.unavailableChannel <- healthcheck.Unavailable diff --git a/ee/query-service/main.go b/ee/query-service/main.go index e29b86797a..90bb5bd830 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -34,9 +34,13 @@ func main() { // the url used to build link in the alert messages in slack and other systems var ruleRepoURL string - flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") + // path of default otel config used by opamp + var defaultOtelConfig string + + flag.StringVar(&promConfigPath, "config", "../../pkg/query-service/config/prometheus.yml", "(prometheus config to read metrics)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") flag.StringVar(&ruleRepoURL, "rules.repo-url", baseconst.AlertHelpPage, "(host address used to build rule link in alert messages)") + flag.StringVar(&defaultOtelConfig, "opamp.defaultOtelConfig", "../../pkg/query-service/config/otel-default.yaml", "(default otel config used by opamp)") flag.Parse() loggerMgr := initZapLog() @@ -47,11 +51,12 @@ func main() { version.PrintVersion() serverOptions := &app.ServerOptions{ - HTTPHostPort: baseconst.HTTPHostPort, - PromConfigPath: promConfigPath, - PrivateHostPort: baseconst.PrivateHostPort, - DisableRules: disableRules, - RuleRepoURL: ruleRepoURL, + HTTPHostPort: baseconst.HTTPHostPort, + PromConfigPath: promConfigPath, + PrivateHostPort: baseconst.PrivateHostPort, + DisableRules: disableRules, + RuleRepoURL: ruleRepoURL, + DefaultOtelConfig: defaultOtelConfig, } // Read the jwt secret key diff --git a/pkg/query-service/app/opamp/config.yaml b/pkg/query-service/app/opamp/config.yaml index d5ef74e00f..c4914d2948 100644 --- a/pkg/query-service/app/opamp/config.yaml +++ b/pkg/query-service/app/opamp/config.yaml @@ -1,4 +1,11 @@ receivers: + otlp/worker: + protocols: + grpc: + endpoint: 0.0.0.0:4949 + max_recv_msg_size_mib: 16 + http: + endpoint: 0.0.0.0:4950 otlp/spanmetrics: protocols: grpc: @@ -51,6 +58,16 @@ processors: extensions: zpages: {} exporters: + loadbalancing: + protocol: + otlp: + tls: + insecure: true + timeout: 1s + resolver: + dns: + hostname: amol-signoz-otel-collector-worker + port: 4949 clickhousetraces: datasource: tcp://localhost:9000/?database=signoz_traces migrations: exporter/clickhousetracesexporter/migrations diff --git a/pkg/query-service/app/opamp/configure_ingestionRules.go b/pkg/query-service/app/opamp/configure_ingestionRules.go index b8f8a1e9ec..4ca76caa76 100644 --- a/pkg/query-service/app/opamp/configure_ingestionRules.go +++ b/pkg/query-service/app/opamp/configure_ingestionRules.go @@ -71,7 +71,7 @@ func addIngestionControlToAgent(agent *model.Agent, signal string, processors ma if withLB { // add LB exporter pipeline and OTLP worker pipeline - makeLbExporterSpec(agentConf) + makeLbExporterSpec(agentConf, opAmpServer.defaultConfig) } // add ingestion control spec. diff --git a/pkg/query-service/app/opamp/configure_lb.go b/pkg/query-service/app/opamp/configure_lb.go index 5e6ae9eccd..591209ed7d 100644 --- a/pkg/query-service/app/opamp/configure_lb.go +++ b/pkg/query-service/app/opamp/configure_lb.go @@ -1,10 +1,10 @@ package opamp import ( + "fmt" + "go.opentelemetry.io/collector/confmap" "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" - "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/lbexporter" - "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig/otlpworker" ) const ( @@ -29,7 +29,7 @@ const ( // receivers: [otlp_internal] // processors: [signoz_tail_sampling, batch] // exporters: [clickhousetraceexporter] -func makeLbExporterSpec(agentConf *confmap.Conf) error { +func makeLbExporterSpec(agentConf *confmap.Conf, defaultConf *otelconfig.ConfigParser) error { configParser := otelconfig.NewConfigParser(agentConf) @@ -37,6 +37,10 @@ func makeLbExporterSpec(agentConf *confmap.Conf) error { return nil } + if defaultConf == nil { + return fmt.Errorf("opamp requires a default config to setup LB Exporter") + } + receivers := configParser.PipelineReceivers(TracesDefaultPipeline) processors := configParser.PipelineProcessors(TracesDefaultPipeline) exporters := configParser.PipelineExporters(TracesDefaultPipeline) @@ -45,10 +49,22 @@ func makeLbExporterSpec(agentConf *confmap.Conf) error { configParser.ReplacePipeline(LoadBalancerPipeline, receivers, otelconfig.StringsToIfaces([]string{}), otelconfig.StringsToIfaces([]string{LoadBalancerExporter})) configParser.ReplacePipeline(TracesDefaultPipeline, otelconfig.StringsToIfaces([]string{OTLPWorker}), processors, exporters) - // add otlp internal receiver - configParser.ReplaceReceiver(OTLPWorker, otlpworker.NewOtlpWorkerReceiver("0.0.0.0:4949")) + // fetch otlp worker params from default otel config in opamp server + recevierConfig, err := defaultConf.Receiver(OTLPWorker) + if err != nil || recevierConfig == nil { + return fmt.Errorf("invalid default otel config found in opamp server: %v", err) + } + + // add otlp/worker to receivers + configParser.ReplaceReceiver(OTLPWorker, recevierConfig) + + // fetch lb exporter config from default otel file in opamp server + exporterConfig, err := defaultConf.Exporter(LoadBalancerExporter) + if err != nil || exporterConfig == nil { + return fmt.Errorf("invalid default otel config found in opamp server: %v", err) + } - // add lb exporter - configParser.ReplaceExporter(LoadBalancerExporter, lbexporter.NewDnsConfig("amol-signoz-otel-collector-worker", "4949")) + // add loadbalancing to exporters + configParser.ReplaceExporter(LoadBalancerExporter, exporterConfig) return nil } diff --git a/pkg/query-service/app/opamp/configure_lb_test.go b/pkg/query-service/app/opamp/configure_lb_test.go index fa7bec6caf..f73445d7e4 100644 --- a/pkg/query-service/app/opamp/configure_lb_test.go +++ b/pkg/query-service/app/opamp/configure_lb_test.go @@ -28,7 +28,15 @@ func TestMakeLBExporterSpec(t *testing.T) { } agentConf := confmap.NewFromStringMap(c) - makeLbExporterSpec(agentConf) + + defaultConfig, err := otelconfig.LoadConfigParserFromFile("./config.yaml") + if err != nil { + log.Println("default config is needed for settting up load balancer") + t.Fail() + return + } + + makeLbExporterSpec(agentConf, defaultConfig) parser := otelconfig.NewConfigParser(agentConf) require.Equal(t, true, parser.CheckExporterInPipeline(LoadBalancerPipeline, LoadBalancerExporter)) @@ -38,6 +46,7 @@ func TestMakeLBExporterSpec(t *testing.T) { if _, ok := exporters[LoadBalancerExporter]; !ok { t.Fail() } + receivers := parser.Receivers() if _, ok := receivers[OTLPWorker]; !ok { t.Fail() diff --git a/pkg/query-service/app/opamp/opamp_server.go b/pkg/query-service/app/opamp/opamp_server.go index ae7b290bce..8260829fb2 100644 --- a/pkg/query-service/app/opamp/opamp_server.go +++ b/pkg/query-service/app/opamp/opamp_server.go @@ -12,6 +12,7 @@ import ( "github.com/open-telemetry/opamp-go/server/types" "go.opentelemetry.io/collector/confmap" model "go.signoz.io/signoz/pkg/query-service/app/opamp/model" + "go.signoz.io/signoz/pkg/query-service/app/opamp/otelconfig" "go.uber.org/zap" ) @@ -23,20 +24,29 @@ type Server struct { agents *model.Agents logger *zap.Logger capabilities int32 + + // default config used to initiate parts of otel config like otlp/worker receiver or lbexporter + defaultConfig *otelconfig.ConfigParser } const capabilities = protobufs.ServerCapabilities_ServerCapabilities_AcceptsEffectiveConfig | protobufs.ServerCapabilities_ServerCapabilities_OffersRemoteConfig | protobufs.ServerCapabilities_ServerCapabilities_AcceptsStatus -func InitalizeServer(listener string, agents *model.Agents) error { +func InitalizeServer(listener string, agents *model.Agents, defaultOtelConfig string) error { if agents == nil { agents = &model.AllAgents } + defaultConfig, err := otelconfig.LoadConfigParserFromFile(defaultOtelConfig) + if err != nil { + return err + } + opAmpServer = &Server{ - agents: agents, + agents: agents, + defaultConfig: defaultConfig, } opAmpServer.server = server.New(zap.S()) diff --git a/pkg/query-service/app/opamp/otelconfig/config_parser.go b/pkg/query-service/app/opamp/otelconfig/config_parser.go index 8a4dc2018e..974ae6eefa 100644 --- a/pkg/query-service/app/opamp/otelconfig/config_parser.go +++ b/pkg/query-service/app/opamp/otelconfig/config_parser.go @@ -2,8 +2,10 @@ package otelconfig import ( "fmt" + "io/ioutil" "sync" + "github.com/knadh/koanf/parsers/yaml" "go.opentelemetry.io/collector/confmap" ) @@ -26,6 +28,21 @@ func NewConfigParser(agentConf *confmap.Conf) ConfigParser { } } +func LoadConfigParserFromFile(filename string) (*ConfigParser, error) { + yamlFile, err := ioutil.ReadFile(filename) + if err != nil { + return nil, fmt.Errorf("failed to load default collector config in opamp server: %v", err) + } + + stringMap, err := yaml.Parser().Unmarshal([]byte(yamlFile)) + if err != nil { + return nil, fmt.Errorf("failed to parse default collector config in opamp server: %v", err) + } + agentConf := confmap.NewFromStringMap(stringMap) + parser := NewConfigParser(agentConf) + return &parser, err +} + func toMap(i interface{}) map[string]interface{} { return i.(map[string]interface{}) } diff --git a/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go b/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go index 37ef574604..673cf1fcf2 100644 --- a/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go +++ b/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go @@ -5,12 +5,13 @@ package lbexporter import ( "time" - otelgrpc "go.opentelemetry.io/collector/config/configgrpc" - oteltls "go.opentelemetry.io/collector/config/configtls" - otelexporterhelper "go.opentelemetry.io/collector/exporter/exporterhelper" "go.opentelemetry.io/collector/exporter/otlpexporter" ) +func MakeConfig(i interface{}) Config { + return i.(Config) +} + // config for lb exporter type Config struct { Protocol Protocol `mapstructure:"protocol"` @@ -18,31 +19,6 @@ type Config struct { RoutingKey string `mapstructure:"routing_key"` } -func NewDnsConfig(hostname string, port string) Config { - // needs improvement(amol): need a better way to set parameters outside of binary (query-service) - return Config{ - Protocol: Protocol{ - OTLP: otlpexporter.Config{ - GRPCClientSettings: otelgrpc.GRPCClientSettings{ - TLSSetting: oteltls.TLSClientSetting{ - // needs improvement(amol): add tls - Insecure: true, - }, - }, - // default timeout is 5 seconds - TimeoutSettings: otelexporterhelper.NewDefaultTimeoutSettings(), - }, - }, - Resolver: ResolverSettings{ - DNS: &DNSResolver{ - Hostname: hostname, - Port: port, - }, - }, - } - -} - // Protocol holds the individual protocol-specific settings. Only OTLP is supported at the moment. type Protocol struct { OTLP otlpexporter.Config `mapstructure:"otlp"` diff --git a/pkg/query-service/app/opamp/otelconfig/otlpworker/config.go b/pkg/query-service/app/opamp/otelconfig/otlpworker/config.go deleted file mode 100644 index 7364da1b24..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/otlpworker/config.go +++ /dev/null @@ -1,19 +0,0 @@ -package otlpworker - -import ( - otelgrpc "go.opentelemetry.io/collector/config/configgrpc" - otelnet "go.opentelemetry.io/collector/config/confignet" - "go.opentelemetry.io/collector/receiver/otlpreceiver" -) - -func NewOtlpWorkerReceiver(endpoint string) otlpreceiver.Config { - return otlpreceiver.Config{ - Protocols: otlpreceiver.Protocols{ - GRPC: &otelgrpc.GRPCServerSettings{ - NetAddr: otelnet.NetAddr{ - Endpoint: endpoint, - }, - }, - }, - } -} diff --git a/pkg/query-service/app/opamp/otelconfig/worker/config.go b/pkg/query-service/app/opamp/otelconfig/worker/config.go new file mode 100644 index 0000000000..33de5e3e72 --- /dev/null +++ b/pkg/query-service/app/opamp/otelconfig/worker/config.go @@ -0,0 +1,9 @@ +package worker + +import ( + "go.opentelemetry.io/collector/receiver/otlpreceiver" +) + +func MakeConfig(i interface{}) otlpreceiver.Config { + return i.(otlpreceiver.Config) +} diff --git a/pkg/query-service/app/server.go b/pkg/query-service/app/server.go index 3eac5eabd3..b9c5b09d98 100644 --- a/pkg/query-service/app/server.go +++ b/pkg/query-service/app/server.go @@ -46,6 +46,8 @@ type ServerOptions struct { // alert specific params DisableRules bool RuleRepoURL string + // opamp related config + defaultOtelConfig string } // Server runs HTTP, Mux and a grpc server @@ -450,7 +452,7 @@ func (s *Server) Start() error { go func() { zap.S().Info("Starting OpAmp Websocket server", zap.String("addr", constants.OpAmpWsEndpoint)) - err := opamp.InitalizeServer(constants.OpAmpWsEndpoint, &opAmpModel.AllAgents) + err := opamp.InitalizeServer(constants.OpAmpWsEndpoint, &opAmpModel.AllAgents, s.serverOptions.defaultOtelConfig) if err != nil { zap.S().Info("opamp ws server failed to start", err) s.unavailableChannel <- healthcheck.Unavailable diff --git a/pkg/query-service/constants/otel-default.yaml b/pkg/query-service/constants/otel-default.yaml new file mode 100644 index 0000000000..5b095169e7 --- /dev/null +++ b/pkg/query-service/constants/otel-default.yaml @@ -0,0 +1,19 @@ +receivers: + otlp/worker: + protocols: + grpc: + endpoint: 0.0.0.0:4949 + max_recv_msg_size_mib: 16 + http: + endpoint: 0.0.0.0:4950 +exporters: + loadbalancing: + protocol: + otlp: + tls: + insecure: true + timeout: 1s + resolver: + dns: + hostname: amol-signoz-otel-collector-worker + port: 4949 \ No newline at end of file diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index 10bfe67306..ff21363432 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -34,9 +34,12 @@ func main() { // the url used to build link in the alert messages in slack and other systems var ruleRepoURL string + var defaultOtelConfig string + flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") flag.StringVar(&ruleRepoURL, "rules.repo-url", constants.AlertHelpPage, "(host address used to build rule link in alert messages)") + flag.StringVar(&defaultOtelConfig, "opamp.defaultOtelConfig", "./config/otel-default.yaml", "(default otel config used by opamp)") flag.Parse() loggerMgr := initZapLog() From a3c153e85ac0b6a8f5f8d9d3d6de04497d09b197 Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 21 Mar 2023 15:10:48 +0530 Subject: [PATCH 22/24] feat: added default otel config for opamp --- ee/query-service/main.go | 2 +- pkg/query-service/agentConf/manager.go | 5 --- .../app/opamp/otelconfig/lbexporter/config.go | 44 ------------------- .../app/opamp/otelconfig/worker/config.go | 9 ---- pkg/query-service/config/otel-default.yml | 23 ++++++++++ pkg/query-service/main.go | 2 +- 6 files changed, 25 insertions(+), 60 deletions(-) delete mode 100644 pkg/query-service/app/opamp/otelconfig/lbexporter/config.go delete mode 100644 pkg/query-service/app/opamp/otelconfig/worker/config.go create mode 100644 pkg/query-service/config/otel-default.yml diff --git a/ee/query-service/main.go b/ee/query-service/main.go index 90bb5bd830..028263822c 100644 --- a/ee/query-service/main.go +++ b/ee/query-service/main.go @@ -40,7 +40,7 @@ func main() { flag.StringVar(&promConfigPath, "config", "../../pkg/query-service/config/prometheus.yml", "(prometheus config to read metrics)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") flag.StringVar(&ruleRepoURL, "rules.repo-url", baseconst.AlertHelpPage, "(host address used to build rule link in alert messages)") - flag.StringVar(&defaultOtelConfig, "opamp.defaultOtelConfig", "../../pkg/query-service/config/otel-default.yaml", "(default otel config used by opamp)") + flag.StringVar(&defaultOtelConfig, "opamp.defaultOtelConfig", "../../pkg/query-service/config/otel-default.yml", "(default otel config used by opamp)") flag.Parse() loggerMgr := initZapLog() diff --git a/pkg/query-service/agentConf/manager.go b/pkg/query-service/agentConf/manager.go index db7d477999..ffc22b8064 100644 --- a/pkg/query-service/agentConf/manager.go +++ b/pkg/query-service/agentConf/manager.go @@ -79,11 +79,6 @@ func StartNewVersion(ctx context.Context, userId string, eleType ElementTypeDef, // and re-sends it to collector for deployment. This method is useful // in enabling previous configs from history func Redeploy(ctx context.Context, typ ElementTypeDef, version int) error { - if !atomic.CompareAndSwapUint32(&m.lock, 0, 1) { - return fmt.Errorf("agent updater is busy") - } - - defer atomic.StoreUint32(&m.lock, 0) configVersion, err := GetConfigVersion(ctx, typ, version) if err != nil { diff --git a/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go b/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go deleted file mode 100644 index 673cf1fcf2..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/lbexporter/config.go +++ /dev/null @@ -1,44 +0,0 @@ -package lbexporter - -// borrowed from https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/release/v0.66.x/exporter/loadbalancingexporter/config.go - -import ( - "time" - - "go.opentelemetry.io/collector/exporter/otlpexporter" -) - -func MakeConfig(i interface{}) Config { - return i.(Config) -} - -// config for lb exporter -type Config struct { - Protocol Protocol `mapstructure:"protocol"` - Resolver ResolverSettings `mapstructure:"resolver"` - RoutingKey string `mapstructure:"routing_key"` -} - -// Protocol holds the individual protocol-specific settings. Only OTLP is supported at the moment. -type Protocol struct { - OTLP otlpexporter.Config `mapstructure:"otlp"` -} - -// ResolverSettings defines the configurations for the backend resolver -type ResolverSettings struct { - Static *StaticResolver `mapstructure:"static"` - DNS *DNSResolver `mapstructure:"dns"` -} - -// StaticResolver defines the configuration for the resolver providing a fixed list of backends -type StaticResolver struct { - Hostnames []string `mapstructure:"hostnames"` -} - -// DNSResolver defines the configuration for the DNS resolver -type DNSResolver struct { - Hostname string `mapstructure:"hostname"` - Port string `mapstructure:"port"` - Interval time.Duration `mapstructure:"interval"` - Timeout time.Duration `mapstructure:"timeout"` -} diff --git a/pkg/query-service/app/opamp/otelconfig/worker/config.go b/pkg/query-service/app/opamp/otelconfig/worker/config.go deleted file mode 100644 index 33de5e3e72..0000000000 --- a/pkg/query-service/app/opamp/otelconfig/worker/config.go +++ /dev/null @@ -1,9 +0,0 @@ -package worker - -import ( - "go.opentelemetry.io/collector/receiver/otlpreceiver" -) - -func MakeConfig(i interface{}) otlpreceiver.Config { - return i.(otlpreceiver.Config) -} diff --git a/pkg/query-service/config/otel-default.yml b/pkg/query-service/config/otel-default.yml new file mode 100644 index 0000000000..d780e75739 --- /dev/null +++ b/pkg/query-service/config/otel-default.yml @@ -0,0 +1,23 @@ +# otel configuration file used for defaulting component +# parameters (e.g. exporters, receviers or processors) +receivers: + otlp/worker: + protocols: + grpc: + endpoint: 0.0.0.0:4949 + max_recv_msg_size_mib: 16 + http: + endpoint: 0.0.0.0:4950 +# processors: +# signoz_tail_sampling: +exporters: + loadbalancing: + protocol: + otlp: + tls: + insecure: true + timeout: 1s + resolver: + dns: + hostname: amol-signoz-otel-collector-worker + port: 4949 diff --git a/pkg/query-service/main.go b/pkg/query-service/main.go index ff21363432..4b8e36c921 100644 --- a/pkg/query-service/main.go +++ b/pkg/query-service/main.go @@ -39,7 +39,7 @@ func main() { flag.StringVar(&promConfigPath, "config", "./config/prometheus.yml", "(prometheus config to read metrics)") flag.BoolVar(&disableRules, "rules.disable", false, "(disable rule evaluation)") flag.StringVar(&ruleRepoURL, "rules.repo-url", constants.AlertHelpPage, "(host address used to build rule link in alert messages)") - flag.StringVar(&defaultOtelConfig, "opamp.defaultOtelConfig", "./config/otel-default.yaml", "(default otel config used by opamp)") + flag.StringVar(&defaultOtelConfig, "opamp.defaultOtelConfig", "./config/otel-default.yml", "(default otel config used by opamp)") flag.Parse() loggerMgr := initZapLog() From 77c73d914749063897bdc6c85b7b76797e7652c6 Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 21 Mar 2023 15:47:43 +0530 Subject: [PATCH 23/24] chore: added some comments and merged auth middleware --- ee/query-service/app/api/api.go | 11 +- ee/query-service/app/api/ingestionRules.go | 6 + ee/query-service/app/api/pat.go | 16 ++ ee/query-service/app/server.go | 18 +- ee/query-service/model/plans.go | 5 + pkg/query-service/app/http_handler.go | 250 +++++++++++++-------- 6 files changed, 199 insertions(+), 107 deletions(-) diff --git a/ee/query-service/app/api/api.go b/ee/query-service/app/api/api.go index 0cd73bc266..c2934966f5 100644 --- a/ee/query-service/app/api/api.go +++ b/ee/query-service/app/api/api.go @@ -21,8 +21,8 @@ type APIHandlerOptions struct { RulesManager *rules.Manager FeatureFlags baseint.FeatureLookup LicenseManager *license.Manager - IngestionController *ingestionRules.IngestionController Authenticator *baseapp.AuthMiddleware + IngestionController *ingestionRules.IngestionController } type APIHandler struct { @@ -51,10 +51,6 @@ func NewAPIHandler(opts APIHandlerOptions) (*APIHandler, error) { return ah, nil } -func (ah *APIHandler) Auth() *baseapp.AuthMiddleware { - return ah.opts.Authenticator -} - func (ah *APIHandler) FF() baseint.FeatureLookup { return ah.opts.FeatureFlags } @@ -67,6 +63,10 @@ func (ah *APIHandler) LM() *license.Manager { return ah.opts.LicenseManager } +func (ah *APIHandler) Auth() *baseapp.AuthMiddleware { + return ah.opts.Authenticator +} + func (ah *APIHandler) AppDao() dao.ModelDao { return ah.opts.AppDao } @@ -160,7 +160,6 @@ func (ah *APIHandler) RegisterRoutes(router *mux.Router) { router.HandleFunc("/api/v1/pat/{id}", ah.Auth().OpenAccess(ah.deletePAT)).Methods(http.MethodDelete) ah.APIHandler.RegisterRoutes(router) - } func (ah *APIHandler) getVersion(w http.ResponseWriter, r *http.Request) { diff --git a/ee/query-service/app/api/ingestionRules.go b/ee/query-service/app/api/ingestionRules.go index 48c4a47b43..793e6e385a 100644 --- a/ee/query-service/app/api/ingestionRules.go +++ b/ee/query-service/app/api/ingestionRules.go @@ -15,7 +15,13 @@ import ( // ingestion rules handler - combines common methods for drop and sampling rules +// parseAgentConfigVersion parses the version string from URL path. func parseAgentConfigVersion(r *http.Request) (int, *model.ApiError) { + // parse version from path and respond with + // - (-1) to indicate no version specified in the url path but the user is looking for the latest version + // - (0) indicates a failure in parsing + // - non-zero (>0) to indicate user is looking for a specific version + versionString := mux.Vars(r)["version"] if versionString == "latest" { diff --git a/ee/query-service/app/api/pat.go b/ee/query-service/app/api/pat.go index 619c875c8f..cec3b6845b 100644 --- a/ee/query-service/app/api/pat.go +++ b/ee/query-service/app/api/pat.go @@ -25,6 +25,12 @@ func generatePATToken() string { } func (ah *APIHandler) createPAT(w http.ResponseWriter, r *http.Request) { + + if !ah.CheckFeature(model.Pat) { + RespondError(w, model.BadRequestStr("feature unavailable, please upgrade to EE"), nil) + return + } + ctx := context.Background() req := model.PAT{} @@ -57,6 +63,11 @@ func (ah *APIHandler) createPAT(w http.ResponseWriter, r *http.Request) { } func (ah *APIHandler) getPATs(w http.ResponseWriter, r *http.Request) { + if !ah.CheckFeature(model.Pat) { + RespondError(w, model.BadRequestStr("feature unavailable, please upgrade to EE"), nil) + return + } + ctx := context.Background() user, err := auth.GetUserFromRequest(r) if err != nil { @@ -76,6 +87,11 @@ func (ah *APIHandler) getPATs(w http.ResponseWriter, r *http.Request) { } func (ah *APIHandler) deletePAT(w http.ResponseWriter, r *http.Request) { + if !ah.CheckFeature(model.Pat) { + RespondError(w, model.BadRequestStr("feature unavailable, please upgrade to EE"), nil) + return + } + ctx := context.Background() id := mux.Vars(r)["id"] user, err := auth.GetUserFromRequest(r) diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index f47a2be813..eb5fe611b9 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -161,6 +161,8 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } telemetry.GetInstance().SetReader(reader) + + // auth middleware for EE getUserFromRequest := func(r *http.Request) (*basemodel.UserPayload, error) { patToken := r.Header.Get("SIGNOZ-API-KEY") if len(patToken) > 0 { @@ -177,16 +179,15 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { } return baseauth.GetUserFromRequest(r) } + authm := baseapp.NewAuthMiddleware(getUserFromRequest) - authMiddleware := baseapp.NewAuthMiddleware(getUserFromRequest) apiOpts := api.APIHandlerOptions{ - DataConnector: reader, - AppDao: modelDao, - RulesManager: rm, - FeatureFlags: lm, - LicenseManager: lm, - IngestionController: ingestionController, - Authenticator: authMiddleware, + DataConnector: reader, + AppDao: modelDao, + RulesManager: rm, + FeatureFlags: lm, + LicenseManager: lm, + Authenticator: authm, } apiHandler, err := api.NewAPIHandler(apiOpts) @@ -257,6 +258,7 @@ func (s *Server) createPublicServer(apiHandler *api.APIHandler) (*http.Server, e apiHandler.RegisterRoutes(r) apiHandler.RegisterMetricsRoutes(r) apiHandler.RegisterLogsRoutes(r) + apiHandler.RegisterQueryRangeV3Routes(r) c := cors.New(cors.Options{ AllowedOrigins: []string{"*"}, diff --git a/ee/query-service/model/plans.go b/ee/query-service/model/plans.go index c42712f693..1f16f5000a 100644 --- a/ee/query-service/model/plans.go +++ b/ee/query-service/model/plans.go @@ -10,6 +10,9 @@ const Pro = "PRO_PLAN" const Enterprise = "ENTERPRISE_PLAN" const DisableUpsell = "DISABLE_UPSELL" +// personal access tokens feature +const Pat = "PAT" + var BasicPlan = basemodel.FeatureSet{ Basic: true, SSO: false, @@ -21,6 +24,7 @@ var ProPlan = basemodel.FeatureSet{ SSO: true, basemodel.SmartTraceDetail: true, basemodel.CustomMetricsFunction: true, + Pat: true, } var EnterprisePlan = basemodel.FeatureSet{ @@ -28,4 +32,5 @@ var EnterprisePlan = basemodel.FeatureSet{ SSO: true, basemodel.SmartTraceDetail: true, basemodel.CustomMetricsFunction: true, + Pat: true, } diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index 66d0052ace..b18db20d00 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -61,8 +61,8 @@ type APIHandler struct { appDao dao.ModelDao alertManager am.Manager ruleManager *rules.Manager - featureFlags interfaces.FeatureLookup authenticator *AuthMiddleware + featureFlags interfaces.FeatureLookup ready func(http.HandlerFunc) http.HandlerFunc // SetupCompleted indicates if SigNoz is ready for general use. @@ -235,17 +235,23 @@ func writeHttpResponse(w http.ResponseWriter, data interface{}) { } } +func (ah *APIHandler) Auth() *AuthMiddleware { + return ah.authenticator +} + func (aH *APIHandler) RegisterMetricsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v2/metrics").Subrouter() - subRouter.HandleFunc("/query_range", aH.authenticator.ViewAccess(aH.QueryRangeMetricsV2)).Methods(http.MethodPost) - subRouter.HandleFunc("/autocomplete/list", aH.authenticator.ViewAccess(aH.metricAutocompleteMetricName)).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/tagKey", aH.authenticator.ViewAccess(aH.metricAutocompleteTagKey)).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/tagValue", aH.authenticator.ViewAccess(aH.metricAutocompleteTagValue)).Methods(http.MethodGet) + subRouter.HandleFunc("/query_range", aH.Auth().ViewAccess(aH.QueryRangeMetricsV2)).Methods(http.MethodPost) + subRouter.HandleFunc("/autocomplete/list", aH.Auth().ViewAccess(aH.metricAutocompleteMetricName)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/tagKey", aH.Auth().ViewAccess(aH.metricAutocompleteTagKey)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/tagValue", aH.Auth().ViewAccess(aH.metricAutocompleteTagValue)).Methods(http.MethodGet) } func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router) { subRouter := router.PathPrefix("/api/v3").Subrouter() - subRouter.HandleFunc("/autocomplete/aggregate_attributes", aH.authenticator.ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/aggregate_attributes", aH.Auth().ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/attribute_keys", aH.Auth().ViewAccess(aH.autoCompleteAttributeKeys)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/attribute_values", aH.Auth().ViewAccess(aH.autoCompleteAttributeValues)).Methods(http.MethodGet) } func (aH *APIHandler) Respond(w http.ResponseWriter, data interface{}) { @@ -259,97 +265,97 @@ func (aH *APIHandler) RegisterPrivateRoutes(router *mux.Router) { // RegisterRoutes registers routes for this handler on the given router func (aH *APIHandler) RegisterRoutes(router *mux.Router) { - router.HandleFunc("/api/v1/query_range", aH.authenticator.ViewAccess(aH.queryRangeMetrics)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/query", aH.authenticator.ViewAccess(aH.queryMetrics)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/channels", aH.authenticator.ViewAccess(aH.listChannels)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/channels/{id}", aH.authenticator.ViewAccess(aH.getChannel)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/channels/{id}", aH.authenticator.AdminAccess(aH.editChannel)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/channels/{id}", aH.authenticator.AdminAccess(aH.deleteChannel)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/channels", aH.authenticator.EditAccess(aH.createChannel)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/testChannel", aH.authenticator.EditAccess(aH.testChannel)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/rules", aH.authenticator.ViewAccess(aH.listRules)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.ViewAccess(aH.getRule)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/rules", aH.authenticator.EditAccess(aH.createRule)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.EditAccess(aH.editRule)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.EditAccess(aH.deleteRule)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/rules/{id}", aH.authenticator.EditAccess(aH.patchRule)).Methods(http.MethodPatch) - router.HandleFunc("/api/v1/testRule", aH.authenticator.EditAccess(aH.testRule)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/dashboards", aH.authenticator.ViewAccess(aH.getDashboards)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/dashboards", aH.authenticator.EditAccess(aH.createDashboards)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/dashboards/grafana", aH.authenticator.EditAccess(aH.createDashboardsTransform)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/dashboards/{uuid}", aH.authenticator.ViewAccess(aH.getDashboard)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/dashboards/{uuid}", aH.authenticator.EditAccess(aH.updateDashboard)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/dashboards/{uuid}", aH.authenticator.EditAccess(aH.deleteDashboard)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/variables/query", aH.authenticator.ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet) - router.HandleFunc("/api/v2/variables/query", aH.authenticator.ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/explorer/queries", aH.authenticator.ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/explorer/queries", aH.authenticator.EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.authenticator.ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.authenticator.EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.authenticator.EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete) - - router.HandleFunc("/api/v1/feedback", aH.authenticator.OpenAccess(aH.submitFeedback)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/query_range", aH.Auth().ViewAccess(aH.queryRangeMetrics)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/query", aH.Auth().ViewAccess(aH.queryMetrics)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/channels", aH.Auth().ViewAccess(aH.listChannels)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/channels/{id}", aH.Auth().ViewAccess(aH.getChannel)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/channels/{id}", aH.Auth().AdminAccess(aH.editChannel)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/channels/{id}", aH.Auth().AdminAccess(aH.deleteChannel)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/channels", aH.Auth().EditAccess(aH.createChannel)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/testChannel", aH.Auth().EditAccess(aH.testChannel)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/rules", aH.Auth().ViewAccess(aH.listRules)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rules/{id}", aH.Auth().ViewAccess(aH.getRule)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rules", aH.Auth().EditAccess(aH.createRule)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/rules/{id}", aH.Auth().EditAccess(aH.editRule)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/rules/{id}", aH.Auth().EditAccess(aH.deleteRule)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/rules/{id}", aH.Auth().EditAccess(aH.patchRule)).Methods(http.MethodPatch) + router.HandleFunc("/api/v1/testRule", aH.Auth().EditAccess(aH.testRule)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/dashboards", aH.Auth().ViewAccess(aH.getDashboards)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/dashboards", aH.Auth().EditAccess(aH.createDashboards)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/dashboards/grafana", aH.Auth().EditAccess(aH.createDashboardsTransform)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/dashboards/{uuid}", aH.Auth().ViewAccess(aH.getDashboard)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/dashboards/{uuid}", aH.Auth().EditAccess(aH.updateDashboard)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/dashboards/{uuid}", aH.Auth().EditAccess(aH.deleteDashboard)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/variables/query", aH.Auth().ViewAccess(aH.queryDashboardVars)).Methods(http.MethodGet) + router.HandleFunc("/api/v2/variables/query", aH.Auth().ViewAccess(aH.queryDashboardVarsV2)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/explorer/queries", aH.Auth().ViewAccess(aH.getExplorerQueries)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/queries", aH.Auth().EditAccess(aH.createExplorerQueries)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.Auth().ViewAccess(aH.getExplorerQuery)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.Auth().EditAccess(aH.updateExplorerQuery)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/explorer/queries/{queryId}", aH.Auth().EditAccess(aH.deleteExplorerQuery)).Methods(http.MethodDelete) + + router.HandleFunc("/api/v1/feedback", aH.Auth().OpenAccess(aH.submitFeedback)).Methods(http.MethodPost) // router.HandleFunc("/api/v1/get_percentiles", aH.getApplicationPercentiles).Methods(http.MethodGet) - router.HandleFunc("/api/v1/services", aH.authenticator.ViewAccess(aH.getServices)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/services/list", aH.authenticator.ViewAccess(aH.getServicesList)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/service/overview", aH.authenticator.ViewAccess(aH.getServiceOverview)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/service/top_operations", aH.authenticator.ViewAccess(aH.getTopOperations)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/service/top_level_operations", aH.authenticator.ViewAccess(aH.getServicesTopLevelOps)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/traces/{traceId}", aH.authenticator.ViewAccess(aH.SearchTraces)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/usage", aH.authenticator.ViewAccess(aH.getUsage)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/dependency_graph", aH.authenticator.ViewAccess(aH.dependencyGraph)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/settings/ttl", aH.authenticator.AdminAccess(aH.setTTL)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/settings/ttl", aH.authenticator.ViewAccess(aH.getTTL)).Methods(http.MethodGet) - - router.HandleFunc("/api/v1/version", aH.authenticator.OpenAccess(aH.getVersion)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/featureFlags", aH.authenticator.OpenAccess(aH.getFeatureFlags)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/configs", aH.authenticator.OpenAccess(aH.getConfigs)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/health", aH.authenticator.OpenAccess(aH.getHealth)).Methods(http.MethodGet) - - router.HandleFunc("/api/v1/getSpanFilters", aH.authenticator.ViewAccess(aH.getSpanFilters)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getTagFilters", aH.authenticator.ViewAccess(aH.getTagFilters)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getFilteredSpans", aH.authenticator.ViewAccess(aH.getFilteredSpans)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getFilteredSpans/aggregates", aH.authenticator.ViewAccess(aH.getFilteredSpanAggregates)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/getTagValues", aH.authenticator.ViewAccess(aH.getTagValues)).Methods(http.MethodPost) - - router.HandleFunc("/api/v1/listErrors", aH.authenticator.ViewAccess(aH.listErrors)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/countErrors", aH.authenticator.ViewAccess(aH.countErrors)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/errorFromErrorID", aH.authenticator.ViewAccess(aH.getErrorFromErrorID)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/errorFromGroupID", aH.authenticator.ViewAccess(aH.getErrorFromGroupID)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/nextPrevErrorIDs", aH.authenticator.ViewAccess(aH.getNextPrevErrorIDs)).Methods(http.MethodGet) - - router.HandleFunc("/api/v1/disks", aH.authenticator.ViewAccess(aH.getDisks)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/services", aH.Auth().ViewAccess(aH.getServices)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/services/list", aH.Auth().ViewAccess(aH.getServicesList)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/service/overview", aH.Auth().ViewAccess(aH.getServiceOverview)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/service/top_operations", aH.Auth().ViewAccess(aH.getTopOperations)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/service/top_level_operations", aH.Auth().ViewAccess(aH.getServicesTopLevelOps)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/traces/{traceId}", aH.Auth().ViewAccess(aH.SearchTraces)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/usage", aH.Auth().ViewAccess(aH.getUsage)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/dependency_graph", aH.Auth().ViewAccess(aH.dependencyGraph)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/settings/ttl", aH.Auth().AdminAccess(aH.setTTL)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/settings/ttl", aH.Auth().ViewAccess(aH.getTTL)).Methods(http.MethodGet) + + router.HandleFunc("/api/v1/version", aH.Auth().OpenAccess(aH.getVersion)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/featureFlags", aH.Auth().OpenAccess(aH.getFeatureFlags)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/configs", aH.Auth().OpenAccess(aH.getConfigs)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/health", aH.Auth().OpenAccess(aH.getHealth)).Methods(http.MethodGet) + + router.HandleFunc("/api/v1/getSpanFilters", aH.Auth().ViewAccess(aH.getSpanFilters)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getTagFilters", aH.Auth().ViewAccess(aH.getTagFilters)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getFilteredSpans", aH.Auth().ViewAccess(aH.getFilteredSpans)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getFilteredSpans/aggregates", aH.Auth().ViewAccess(aH.getFilteredSpanAggregates)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getTagValues", aH.Auth().ViewAccess(aH.getTagValues)).Methods(http.MethodPost) + + router.HandleFunc("/api/v1/listErrors", aH.Auth().ViewAccess(aH.listErrors)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/countErrors", aH.Auth().ViewAccess(aH.countErrors)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/errorFromErrorID", aH.Auth().ViewAccess(aH.getErrorFromErrorID)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/errorFromGroupID", aH.Auth().ViewAccess(aH.getErrorFromGroupID)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/nextPrevErrorIDs", aH.Auth().ViewAccess(aH.getNextPrevErrorIDs)).Methods(http.MethodGet) + + router.HandleFunc("/api/v1/disks", aH.Auth().ViewAccess(aH.getDisks)).Methods(http.MethodGet) // === Authentication APIs === - router.HandleFunc("/api/v1/invite", aH.authenticator.AdminAccess(aH.inviteUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/invite/{token}", aH.authenticator.OpenAccess(aH.getInvite)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/invite/{email}", aH.authenticator.AdminAccess(aH.revokeInvite)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/invite", aH.authenticator.AdminAccess(aH.listPendingInvites)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/invite", aH.Auth().AdminAccess(aH.inviteUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/invite/{token}", aH.Auth().OpenAccess(aH.getInvite)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/invite/{email}", aH.Auth().AdminAccess(aH.revokeInvite)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/invite", aH.Auth().AdminAccess(aH.listPendingInvites)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/register", aH.authenticator.OpenAccess(aH.registerUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/login", aH.authenticator.OpenAccess(aH.loginUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/register", aH.Auth().OpenAccess(aH.registerUser)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/login", aH.Auth().OpenAccess(aH.loginUser)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/user", aH.authenticator.AdminAccess(aH.listUsers)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/user/{id}", aH.authenticator.SelfAccess(aH.getUser)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/user/{id}", aH.authenticator.SelfAccess(aH.editUser)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/user/{id}", aH.authenticator.AdminAccess(aH.deleteUser)).Methods(http.MethodDelete) + router.HandleFunc("/api/v1/user", aH.Auth().AdminAccess(aH.listUsers)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/user/{id}", aH.Auth().SelfAccess(aH.getUser)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/user/{id}", aH.Auth().SelfAccess(aH.editUser)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/user/{id}", aH.Auth().AdminAccess(aH.deleteUser)).Methods(http.MethodDelete) - router.HandleFunc("/api/v1/user/{id}/flags", aH.authenticator.SelfAccess(aH.patchUserFlag)).Methods(http.MethodPatch) + router.HandleFunc("/api/v1/user/{id}/flags", aH.Auth().SelfAccess(aH.patchUserFlag)).Methods(http.MethodPatch) - router.HandleFunc("/api/v1/rbac/role/{id}", aH.authenticator.SelfAccess(aH.getRole)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/rbac/role/{id}", aH.authenticator.AdminAccess(aH.editRole)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/rbac/role/{id}", aH.Auth().SelfAccess(aH.getRole)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/rbac/role/{id}", aH.Auth().AdminAccess(aH.editRole)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/org", aH.authenticator.AdminAccess(aH.getOrgs)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/org/{id}", aH.authenticator.AdminAccess(aH.getOrg)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/org/{id}", aH.authenticator.AdminAccess(aH.editOrg)).Methods(http.MethodPut) - router.HandleFunc("/api/v1/orgUsers/{id}", aH.authenticator.AdminAccess(aH.getOrgUsers)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/org", aH.Auth().AdminAccess(aH.getOrgs)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/org/{id}", aH.Auth().AdminAccess(aH.getOrg)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/org/{id}", aH.Auth().AdminAccess(aH.editOrg)).Methods(http.MethodPut) + router.HandleFunc("/api/v1/orgUsers/{id}", aH.Auth().AdminAccess(aH.getOrgUsers)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/getResetPasswordToken/{id}", aH.authenticator.AdminAccess(aH.getResetPasswordToken)).Methods(http.MethodGet) - router.HandleFunc("/api/v1/resetPassword", aH.authenticator.OpenAccess(aH.resetPassword)).Methods(http.MethodPost) - router.HandleFunc("/api/v1/changePassword/{id}", aH.authenticator.SelfAccess(aH.changePassword)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/getResetPasswordToken/{id}", aH.Auth().AdminAccess(aH.getResetPasswordToken)).Methods(http.MethodGet) + router.HandleFunc("/api/v1/resetPassword", aH.Auth().OpenAccess(aH.resetPassword)).Methods(http.MethodPost) + router.HandleFunc("/api/v1/changePassword/{id}", aH.Auth().SelfAccess(aH.changePassword)).Methods(http.MethodPost) } func Intersection(a, b []int) (c []int) { @@ -2148,11 +2154,11 @@ func (aH *APIHandler) WriteJSON(w http.ResponseWriter, r *http.Request, response // logs func (aH *APIHandler) RegisterLogsRoutes(router *mux.Router) { subRouter := router.PathPrefix("/api/v1/logs").Subrouter() - subRouter.HandleFunc("", aH.authenticator.ViewAccess(aH.getLogs)).Methods(http.MethodGet) - subRouter.HandleFunc("/tail", aH.authenticator.ViewAccess(aH.tailLogs)).Methods(http.MethodGet) - subRouter.HandleFunc("/fields", aH.authenticator.ViewAccess(aH.logFields)).Methods(http.MethodGet) - subRouter.HandleFunc("/fields", aH.authenticator.EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) - subRouter.HandleFunc("/aggregate", aH.authenticator.ViewAccess(aH.logAggregate)).Methods(http.MethodGet) + subRouter.HandleFunc("", aH.Auth().ViewAccess(aH.getLogs)).Methods(http.MethodGet) + subRouter.HandleFunc("/tail", aH.Auth().ViewAccess(aH.tailLogs)).Methods(http.MethodGet) + subRouter.HandleFunc("/fields", aH.Auth().ViewAccess(aH.logFields)).Methods(http.MethodGet) + subRouter.HandleFunc("/fields", aH.Auth().EditAccess(aH.logFieldUpdate)).Methods(http.MethodPost) + subRouter.HandleFunc("/aggregate", aH.Auth().ViewAccess(aH.logAggregate)).Methods(http.MethodGet) } func (aH *APIHandler) logFields(w http.ResponseWriter, r *http.Request) { @@ -2368,3 +2374,61 @@ func (aH *APIHandler) autocompleteAggregateAttributes(w http.ResponseWriter, r * aH.Respond(w, response) } + +func (aH *APIHandler) autoCompleteAttributeKeys(w http.ResponseWriter, r *http.Request) { + var response *v3.FilterAttributeKeyResponse + req, err := parseFilterAttributeKeyRequest(r) + + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + switch req.DataSource { + case v3.DataSourceMetrics: + response, err = aH.reader.GetMetricAttributeKeys(r.Context(), req) + case v3.DataSourceLogs: + // TODO: implement + case v3.DataSourceTraces: + // TODO: implement + default: + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("invalid data source")}, nil) + return + } + + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + aH.Respond(w, response) +} + +func (aH *APIHandler) autoCompleteAttributeValues(w http.ResponseWriter, r *http.Request) { + var response *v3.FilterAttributeValueResponse + req, err := parseFilterAttributeValueRequest(r) + + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + switch req.DataSource { + case v3.DataSourceMetrics: + response, err = aH.reader.GetMetricAttributeValues(r.Context(), req) + case v3.DataSourceLogs: + // TODO: implement + case v3.DataSourceTraces: + // TODO: implement + default: + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: fmt.Errorf("invalid data source")}, nil) + return + } + + if err != nil { + RespondError(w, &model.ApiError{Typ: model.ErrorBadData, Err: err}, nil) + return + } + + aH.Respond(w, response) +} From c36271910249637a887e6b8f9e6cc279174ca731 Mon Sep 17 00:00:00 2001 From: mindhash Date: Tue, 21 Mar 2023 16:05:03 +0530 Subject: [PATCH 24/24] fix: solved some merge issues --- ee/query-service/app/api/ingestionRules.go | 4 ++-- ee/query-service/app/server.go | 13 +++++++------ ee/query-service/ingestionRules/controller.go | 4 ++-- pkg/query-service/app/http_handler.go | 6 +++--- 4 files changed, 14 insertions(+), 13 deletions(-) diff --git a/ee/query-service/app/api/ingestionRules.go b/ee/query-service/app/api/ingestionRules.go index 793e6e385a..25e8a3889e 100644 --- a/ee/query-service/app/api/ingestionRules.go +++ b/ee/query-service/app/api/ingestionRules.go @@ -76,7 +76,7 @@ func (ah *APIHandler) listIngestionRules(ctx context.Context, elementType agentC return payload, apierr } - history, err := agentConf.GetConfigHistory(ctx, elementType) + history, err := agentConf.GetConfigHistory(ctx, elementType, 10) if apierr != nil { return payload, apierr } @@ -92,7 +92,7 @@ func (ah *APIHandler) listIngestionRulesByVersion(ctx context.Context, version i return payload, apierr } - history, err := agentConf.GetConfigHistory(ctx, elementType) + history, err := agentConf.GetConfigHistory(ctx, elementType, 10) if err != nil { zap.S().Errorf("failed to retreive config history for element type", elementType, err) return payload, model.InternalErrorStr("failed to retrieve agent config history") diff --git a/ee/query-service/app/server.go b/ee/query-service/app/server.go index eb5fe611b9..5c0b724ed0 100644 --- a/ee/query-service/app/server.go +++ b/ee/query-service/app/server.go @@ -182,12 +182,13 @@ func NewServer(serverOptions *ServerOptions) (*Server, error) { authm := baseapp.NewAuthMiddleware(getUserFromRequest) apiOpts := api.APIHandlerOptions{ - DataConnector: reader, - AppDao: modelDao, - RulesManager: rm, - FeatureFlags: lm, - LicenseManager: lm, - Authenticator: authm, + DataConnector: reader, + AppDao: modelDao, + RulesManager: rm, + FeatureFlags: lm, + LicenseManager: lm, + Authenticator: authm, + IngestionController: ingestionController, } apiHandler, err := api.NewAPIHandler(apiOpts) diff --git a/ee/query-service/ingestionRules/controller.go b/ee/query-service/ingestionRules/controller.go index 4f7010e4c1..7a54a32872 100644 --- a/ee/query-service/ingestionRules/controller.go +++ b/ee/query-service/ingestionRules/controller.go @@ -103,7 +103,7 @@ func (ic *IngestionController) ApplyDropRules(ctx context.Context, userId string // queue up the config to push to opamp err = agentConf.UpsertFilterProcessor(ctx, cfg.Version, filterConfig) - history, _ := agentConf.GetConfigHistory(ctx, agentConf.ElementTypeDropRules) + history, _ := agentConf.GetConfigHistory(ctx, agentConf.ElementTypeDropRules, 10) response := &IngestionRulesResponse{ ConfigVersion: cfg, @@ -205,7 +205,7 @@ func (ic *IngestionController) ApplySamplingRules(ctx context.Context, userId st // queue up the config to push to opamp err = agentConf.UpsertSamplingProcessor(ctx, cfg.Version, params) - history, _ := agentConf.GetConfigHistory(ctx, agentConf.ElementTypeSamplingRules) + history, _ := agentConf.GetConfigHistory(ctx, agentConf.ElementTypeSamplingRules, 10) response := &IngestionRulesResponse{ ConfigVersion: cfg, diff --git a/pkg/query-service/app/http_handler.go b/pkg/query-service/app/http_handler.go index c51a666e81..b18db20d00 100644 --- a/pkg/query-service/app/http_handler.go +++ b/pkg/query-service/app/http_handler.go @@ -249,9 +249,9 @@ func (aH *APIHandler) RegisterMetricsRoutes(router *mux.Router) { func (aH *APIHandler) RegisterQueryRangeV3Routes(router *mux.Router) { subRouter := router.PathPrefix("/api/v3").Subrouter() - subRouter.HandleFunc("/autocomplete/aggregate_attributes", am.ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/attribute_keys", am.ViewAccess(aH.autoCompleteAttributeKeys)).Methods(http.MethodGet) - subRouter.HandleFunc("/autocomplete/attribute_values", am.ViewAccess(aH.autoCompleteAttributeValues)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/aggregate_attributes", aH.Auth().ViewAccess(aH.autocompleteAggregateAttributes)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/attribute_keys", aH.Auth().ViewAccess(aH.autoCompleteAttributeKeys)).Methods(http.MethodGet) + subRouter.HandleFunc("/autocomplete/attribute_values", aH.Auth().ViewAccess(aH.autoCompleteAttributeValues)).Methods(http.MethodGet) } func (aH *APIHandler) Respond(w http.ResponseWriter, data interface{}) {