From e85ab996e2df308afbe15394fee85525775e84f4 Mon Sep 17 00:00:00 2001 From: stone1100 Date: Thu, 16 May 2024 21:48:38 +0800 Subject: [PATCH] [feat]: add hash for metric name --- .github/workflows/lindb-common.yml | 4 +- .golangci.yml | 47 +- Makefile | 2 +- go.mod | 2 +- models/result_set.go | 16 +- proto/gen/v1/flatMetricsV1/CompoundField.go | 68 +-- proto/gen/v1/flatMetricsV1/Exemplar.go | 71 +-- proto/gen/v1/flatMetricsV1/Metric.go | 71 ++- proto/gen/v1/flatMetricsV1/SimpleField.go | 28 +- proto/gen/v1/linmetrics/linmetrics.pb.go | 500 ++++++++------------ proto/v1/linmetrics.proto | 44 +- proto/v1/metrics.fbs | 59 ++- series/checker_test.go | 8 +- series/row_builder.go | 115 ++++- series/row_builder_test.go | 4 +- 15 files changed, 518 insertions(+), 521 deletions(-) diff --git a/.github/workflows/lindb-common.yml b/.github/workflows/lindb-common.yml index 53c2684..72ed0b6 100644 --- a/.github/workflows/lindb-common.yml +++ b/.github/workflows/lindb-common.yml @@ -12,8 +12,8 @@ jobs: - name: Setup Go uses: actions/setup-go@v3 with: - go-version: 1.19 - cache: true + go-version: 1.22 + cache: false id: go - name: Test run: make test diff --git a/.golangci.yml b/.golangci.yml index f6bf489..ff0dbb1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,13 +4,42 @@ linters-settings: # Default: 30 (but we recommend 10-20) min-complexity: 50 depguard: - list-type: blacklist - packages: - # logging is allowed only by logutils.Log, logrus - # is allowed to use only in logutils package - - github.com/sirupsen/logrus - packages-with-error-message: - - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + # Rules to apply. + # + # Variables: + # - File Variables + # you can still use and exclamation mark ! in front of a variable to say not to use it. + # Example !$test will match any file that is not a go test file. + # + # `$all` - matches all go files + # `$test` - matches all go test files + # + # - Package Variables + # + # `$gostd` - matches all of go's standard library (Pulled from `GOROOT`) + # + # Default: Only allow $gostd in all files. + rules: + # Name of a rule. + main: + # Used to determine the package matching priority. + # There are three different modes: `original`, `strict`, and `lax`. + # Default: "original" + list-mode: lax + # List of file globs that will match this list of settings to compare against. + # Default: $all + files: + - "!**/*_a _file.go" + # List of allowed packages. + allow: + - $gostd + # Packages that are not allowed where the value is a suggestion. + deny: + - pkg: "github.com/sirupsen/logrus" + desc: not allowed + - pkg: "github.com/pkg/errors" + desc: Should be replaced by standard lib errors package + dupl: threshold: 100 funlen: @@ -36,7 +65,7 @@ linters-settings: local-prefixes: github.com/lindb/lindb govet: - check-shadowing: true + shadow: true settings: printf: funcs: @@ -172,8 +201,6 @@ issues: run: timeout: 5m - skip-dirs: - - test/testdata_etc # timeout for analysis, e.g. 30s, 5m, default is 1m deadline: 10m # list of build tags, all linters use it. Default is empty list. diff --git a/Makefile b/Makefile index daf25a6..fef4c0d 100644 --- a/Makefile +++ b/Makefile @@ -17,7 +17,7 @@ header: ## check and add license header. lint: ## run lint ifeq (, $(shell which golangci-lint)) # binary will be $(go env GOPATH)/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.51.2 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.57.2 else echo "Found golangci-lint" endif diff --git a/go.mod b/go.mod index 3b95d32..ce16669 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/lindb/common -go 1.19 +go 1.22 require ( github.com/BurntSushi/toml v1.2.1 diff --git a/models/result_set.go b/models/result_set.go index f28b65b..4b9b03b 100644 --- a/models/result_set.go +++ b/models/result_set.go @@ -29,14 +29,15 @@ import ( // ResultSet represents the query result set type ResultSet struct { + Stats *NodeStats `json:"stats,omitempty"` + Namespace string `json:"namespace,omitempty"` MetricName string `json:"metricName,omitempty"` GroupBy []string `json:"groupBy,omitempty"` Fields []string `json:"fields,omitempty"` + Series []*Series `json:"series,omitempty"` StartTime int64 `json:"startTime,omitempty"` EndTime int64 `json:"endTime,omitempty"` Interval int64 `json:"interval,omitempty"` - Series []*Series `json:"series,omitempty"` - Stats *NodeStats `json:"stats,omitempty"` } // NewResultSet creates a new result set @@ -49,10 +50,17 @@ func (rs *ResultSet) AddSeries(series *Series) { rs.Series = append(rs.Series, series) } +type Exemplar struct { + TraceID string `json:"traceId"` + SpanID string `json:"spanId"` + Duration int64 `json:"duration"` +} + // Series represents one time series for metric. type Series struct { - Tags map[string]string `json:"tags,omitempty"` - Fields map[string]map[int64]float64 `json:"fields,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + Fields map[string]map[int64]float64 `json:"fields,omitempty"` + Exemplars map[string]map[int64][]*Exemplar `json:"exemplars,omitempty"` TagValues string `json:"-"` // return series in order by tag values } diff --git a/proto/gen/v1/flatMetricsV1/CompoundField.go b/proto/gen/v1/flatMetricsV1/CompoundField.go index d541548..eff6189 100644 --- a/proto/gen/v1/flatMetricsV1/CompoundField.go +++ b/proto/gen/v1/flatMetricsV1/CompoundField.go @@ -50,28 +50,8 @@ func (rcv *CompoundField) Table() flatbuffers.Table { return rcv._tab } -func (rcv *CompoundField) Exemplars(obj *Exemplar, j int) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - x := rcv._tab.Vector(o) - x += flatbuffers.UOffsetT(j) * 4 - x = rcv._tab.Indirect(x) - obj.Init(rcv._tab.Bytes, x) - return true - } - return false -} - -func (rcv *CompoundField) ExemplarsLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - func (rcv *CompoundField) Min() float64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { return rcv._tab.GetFloat64(o + rcv._tab.Pos) } @@ -79,11 +59,11 @@ func (rcv *CompoundField) Min() float64 { } func (rcv *CompoundField) MutateMin(n float64) bool { - return rcv._tab.MutateFloat64Slot(6, n) + return rcv._tab.MutateFloat64Slot(4, n) } func (rcv *CompoundField) Max() float64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { return rcv._tab.GetFloat64(o + rcv._tab.Pos) } @@ -91,11 +71,11 @@ func (rcv *CompoundField) Max() float64 { } func (rcv *CompoundField) MutateMax(n float64) bool { - return rcv._tab.MutateFloat64Slot(8, n) + return rcv._tab.MutateFloat64Slot(6, n) } func (rcv *CompoundField) Sum() float64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { return rcv._tab.GetFloat64(o + rcv._tab.Pos) } @@ -103,11 +83,11 @@ func (rcv *CompoundField) Sum() float64 { } func (rcv *CompoundField) MutateSum(n float64) bool { - return rcv._tab.MutateFloat64Slot(10, n) + return rcv._tab.MutateFloat64Slot(8, n) } func (rcv *CompoundField) Count() float64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { return rcv._tab.GetFloat64(o + rcv._tab.Pos) } @@ -115,11 +95,11 @@ func (rcv *CompoundField) Count() float64 { } func (rcv *CompoundField) MutateCount(n float64) bool { - return rcv._tab.MutateFloat64Slot(12, n) + return rcv._tab.MutateFloat64Slot(10, n) } func (rcv *CompoundField) ExplicitBounds(j int) float64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { a := rcv._tab.Vector(o) return rcv._tab.GetFloat64(a + flatbuffers.UOffsetT(j*8)) @@ -128,7 +108,7 @@ func (rcv *CompoundField) ExplicitBounds(j int) float64 { } func (rcv *CompoundField) ExplicitBoundsLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { return rcv._tab.VectorLen(o) } @@ -136,7 +116,7 @@ func (rcv *CompoundField) ExplicitBoundsLength() int { } func (rcv *CompoundField) MutateExplicitBounds(j int, n float64) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { a := rcv._tab.Vector(o) return rcv._tab.MutateFloat64(a+flatbuffers.UOffsetT(j*8), n) @@ -145,7 +125,7 @@ func (rcv *CompoundField) MutateExplicitBounds(j int, n float64) bool { } func (rcv *CompoundField) Values(j int) float64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { a := rcv._tab.Vector(o) return rcv._tab.GetFloat64(a + flatbuffers.UOffsetT(j*8)) @@ -154,7 +134,7 @@ func (rcv *CompoundField) Values(j int) float64 { } func (rcv *CompoundField) ValuesLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { return rcv._tab.VectorLen(o) } @@ -162,7 +142,7 @@ func (rcv *CompoundField) ValuesLength() int { } func (rcv *CompoundField) MutateValues(j int, n float64) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { a := rcv._tab.Vector(o) return rcv._tab.MutateFloat64(a+flatbuffers.UOffsetT(j*8), n) @@ -171,34 +151,28 @@ func (rcv *CompoundField) MutateValues(j int, n float64) bool { } func CompoundFieldStart(builder *flatbuffers.Builder) { - builder.StartObject(7) -} -func CompoundFieldAddExemplars(builder *flatbuffers.Builder, exemplars flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(exemplars), 0) -} -func CompoundFieldStartExemplarsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(4, numElems, 4) + builder.StartObject(6) } func CompoundFieldAddMin(builder *flatbuffers.Builder, min float64) { - builder.PrependFloat64Slot(1, min, 0.0) + builder.PrependFloat64Slot(0, min, 0.0) } func CompoundFieldAddMax(builder *flatbuffers.Builder, max float64) { - builder.PrependFloat64Slot(2, max, 0.0) + builder.PrependFloat64Slot(1, max, 0.0) } func CompoundFieldAddSum(builder *flatbuffers.Builder, sum float64) { - builder.PrependFloat64Slot(3, sum, 0.0) + builder.PrependFloat64Slot(2, sum, 0.0) } func CompoundFieldAddCount(builder *flatbuffers.Builder, count float64) { - builder.PrependFloat64Slot(4, count, 0.0) + builder.PrependFloat64Slot(3, count, 0.0) } func CompoundFieldAddExplicitBounds(builder *flatbuffers.Builder, explicitBounds flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(explicitBounds), 0) + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(explicitBounds), 0) } func CompoundFieldStartExplicitBoundsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(8, numElems, 8) } func CompoundFieldAddValues(builder *flatbuffers.Builder, values flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(values), 0) + builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(values), 0) } func CompoundFieldStartValuesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(8, numElems, 8) diff --git a/proto/gen/v1/flatMetricsV1/Exemplar.go b/proto/gen/v1/flatMetricsV1/Exemplar.go index 51acc1d..7e36a8c 100644 --- a/proto/gen/v1/flatMetricsV1/Exemplar.go +++ b/proto/gen/v1/flatMetricsV1/Exemplar.go @@ -50,60 +50,32 @@ func (rcv *Exemplar) Table() flatbuffers.Table { return rcv._tab } -func (rcv *Exemplar) SpanID(j int) int8 { +func (rcv *Exemplar) Name() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1)) + return rcv._tab.ByteVector(o + rcv._tab.Pos) } - return 0 -} - -func (rcv *Exemplar) SpanIDLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func (rcv *Exemplar) MutateSpanID(j int, n int8) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(4)) - if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.MutateInt8(a+flatbuffers.UOffsetT(j*1), n) - } - return false + return nil } -func (rcv *Exemplar) TraceID(j int) int8 { +func (rcv *Exemplar) SpanId() []byte { o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.GetInt8(a + flatbuffers.UOffsetT(j*1)) + return rcv._tab.ByteVector(o + rcv._tab.Pos) } - return 0 + return nil } -func (rcv *Exemplar) TraceIDLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - -func (rcv *Exemplar) MutateTraceID(j int, n int8) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(6)) +func (rcv *Exemplar) TraceId() []byte { + o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) if o != 0 { - a := rcv._tab.Vector(o) - return rcv._tab.MutateInt8(a+flatbuffers.UOffsetT(j*1), n) + return rcv._tab.ByteVector(o + rcv._tab.Pos) } - return false + return nil } func (rcv *Exemplar) Duration() int64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(8)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) if o != 0 { return rcv._tab.GetInt64(o + rcv._tab.Pos) } @@ -111,26 +83,23 @@ func (rcv *Exemplar) Duration() int64 { } func (rcv *Exemplar) MutateDuration(n int64) bool { - return rcv._tab.MutateInt64Slot(8, n) + return rcv._tab.MutateInt64Slot(10, n) } func ExemplarStart(builder *flatbuffers.Builder) { - builder.StartObject(3) -} -func ExemplarAddSpanID(builder *flatbuffers.Builder, spanID flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(spanID), 0) + builder.StartObject(4) } -func ExemplarStartSpanIDVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(1, numElems, 1) +func ExemplarAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0) } -func ExemplarAddTraceID(builder *flatbuffers.Builder, traceID flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(traceID), 0) +func ExemplarAddSpanId(builder *flatbuffers.Builder, spanId flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(1, flatbuffers.UOffsetT(spanId), 0) } -func ExemplarStartTraceIDVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(1, numElems, 1) +func ExemplarAddTraceId(builder *flatbuffers.Builder, traceId flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(2, flatbuffers.UOffsetT(traceId), 0) } func ExemplarAddDuration(builder *flatbuffers.Builder, duration int64) { - builder.PrependInt64Slot(2, duration, 0) + builder.PrependInt64Slot(3, duration, 0) } func ExemplarEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() diff --git a/proto/gen/v1/flatMetricsV1/Metric.go b/proto/gen/v1/flatMetricsV1/Metric.go index 0b49b1d..7841477 100644 --- a/proto/gen/v1/flatMetricsV1/Metric.go +++ b/proto/gen/v1/flatMetricsV1/Metric.go @@ -78,8 +78,20 @@ func (rcv *Metric) MutateTimestamp(n int64) bool { return rcv._tab.MutateInt64Slot(8, n) } -func (rcv *Metric) KeyValues(obj *KeyValue, j int) bool { +func (rcv *Metric) NameHash() uint64 { o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + if o != 0 { + return rcv._tab.GetUint64(o + rcv._tab.Pos) + } + return 0 +} + +func (rcv *Metric) MutateNameHash(n uint64) bool { + return rcv._tab.MutateUint64Slot(10, n) +} + +func (rcv *Metric) KeyValues(obj *KeyValue, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { x := rcv._tab.Vector(o) x += flatbuffers.UOffsetT(j) * 4 @@ -91,27 +103,27 @@ func (rcv *Metric) KeyValues(obj *KeyValue, j int) bool { } func (rcv *Metric) KeyValuesLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) if o != 0 { return rcv._tab.VectorLen(o) } return 0 } -func (rcv *Metric) Hash() uint64 { - o := flatbuffers.UOffsetT(rcv._tab.Offset(12)) +func (rcv *Metric) KvsHash() uint64 { + o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) if o != 0 { return rcv._tab.GetUint64(o + rcv._tab.Pos) } return 0 } -func (rcv *Metric) MutateHash(n uint64) bool { - return rcv._tab.MutateUint64Slot(12, n) +func (rcv *Metric) MutateKvsHash(n uint64) bool { + return rcv._tab.MutateUint64Slot(14, n) } func (rcv *Metric) SimpleFields(obj *SimpleField, j int) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) if o != 0 { x := rcv._tab.Vector(o) x += flatbuffers.UOffsetT(j) * 4 @@ -123,7 +135,7 @@ func (rcv *Metric) SimpleFields(obj *SimpleField, j int) bool { } func (rcv *Metric) SimpleFieldsLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(14)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) if o != 0 { return rcv._tab.VectorLen(o) } @@ -131,7 +143,7 @@ func (rcv *Metric) SimpleFieldsLength() int { } func (rcv *Metric) CompoundField(obj *CompoundField) *CompoundField { - o := flatbuffers.UOffsetT(rcv._tab.Offset(16)) + o := flatbuffers.UOffsetT(rcv._tab.Offset(18)) if o != 0 { x := rcv._tab.Indirect(o + rcv._tab.Pos) if obj == nil { @@ -143,8 +155,28 @@ func (rcv *Metric) CompoundField(obj *CompoundField) *CompoundField { return nil } +func (rcv *Metric) Exemplars(obj *Exemplar, j int) bool { + o := flatbuffers.UOffsetT(rcv._tab.Offset(20)) + if o != 0 { + x := rcv._tab.Vector(o) + x += flatbuffers.UOffsetT(j) * 4 + x = rcv._tab.Indirect(x) + obj.Init(rcv._tab.Bytes, x) + return true + } + return false +} + +func (rcv *Metric) ExemplarsLength() int { + o := flatbuffers.UOffsetT(rcv._tab.Offset(20)) + if o != 0 { + return rcv._tab.VectorLen(o) + } + return 0 +} + func MetricStart(builder *flatbuffers.Builder) { - builder.StartObject(7) + builder.StartObject(9) } func MetricAddNamespace(builder *flatbuffers.Builder, namespace flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(namespace), 0) @@ -155,23 +187,32 @@ func MetricAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { func MetricAddTimestamp(builder *flatbuffers.Builder, timestamp int64) { builder.PrependInt64Slot(2, timestamp, 0) } +func MetricAddNameHash(builder *flatbuffers.Builder, nameHash uint64) { + builder.PrependUint64Slot(3, nameHash, 0) +} func MetricAddKeyValues(builder *flatbuffers.Builder, keyValues flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(keyValues), 0) + builder.PrependUOffsetTSlot(4, flatbuffers.UOffsetT(keyValues), 0) } func MetricStartKeyValuesVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems, 4) } -func MetricAddHash(builder *flatbuffers.Builder, hash uint64) { - builder.PrependUint64Slot(4, hash, 0) +func MetricAddKvsHash(builder *flatbuffers.Builder, kvsHash uint64) { + builder.PrependUint64Slot(5, kvsHash, 0) } func MetricAddSimpleFields(builder *flatbuffers.Builder, simpleFields flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(5, flatbuffers.UOffsetT(simpleFields), 0) + builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(simpleFields), 0) } func MetricStartSimpleFieldsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { return builder.StartVector(4, numElems, 4) } func MetricAddCompoundField(builder *flatbuffers.Builder, compoundField flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(6, flatbuffers.UOffsetT(compoundField), 0) + builder.PrependUOffsetTSlot(7, flatbuffers.UOffsetT(compoundField), 0) +} +func MetricAddExemplars(builder *flatbuffers.Builder, exemplars flatbuffers.UOffsetT) { + builder.PrependUOffsetTSlot(8, flatbuffers.UOffsetT(exemplars), 0) +} +func MetricStartExemplarsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { + return builder.StartVector(4, numElems, 4) } func MetricEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() diff --git a/proto/gen/v1/flatMetricsV1/SimpleField.go b/proto/gen/v1/flatMetricsV1/SimpleField.go index 6bc9590..0a5c647 100644 --- a/proto/gen/v1/flatMetricsV1/SimpleField.go +++ b/proto/gen/v1/flatMetricsV1/SimpleField.go @@ -82,28 +82,8 @@ func (rcv *SimpleField) MutateValue(n float64) bool { return rcv._tab.MutateFloat64Slot(8, n) } -func (rcv *SimpleField) Exemplars(obj *Exemplar, j int) bool { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) - if o != 0 { - x := rcv._tab.Vector(o) - x += flatbuffers.UOffsetT(j) * 4 - x = rcv._tab.Indirect(x) - obj.Init(rcv._tab.Bytes, x) - return true - } - return false -} - -func (rcv *SimpleField) ExemplarsLength() int { - o := flatbuffers.UOffsetT(rcv._tab.Offset(10)) - if o != 0 { - return rcv._tab.VectorLen(o) - } - return 0 -} - func SimpleFieldStart(builder *flatbuffers.Builder) { - builder.StartObject(4) + builder.StartObject(3) } func SimpleFieldAddName(builder *flatbuffers.Builder, name flatbuffers.UOffsetT) { builder.PrependUOffsetTSlot(0, flatbuffers.UOffsetT(name), 0) @@ -114,12 +94,6 @@ func SimpleFieldAddType(builder *flatbuffers.Builder, type_ SimpleFieldType) { func SimpleFieldAddValue(builder *flatbuffers.Builder, value float64) { builder.PrependFloat64Slot(2, value, 0.0) } -func SimpleFieldAddExemplars(builder *flatbuffers.Builder, exemplars flatbuffers.UOffsetT) { - builder.PrependUOffsetTSlot(3, flatbuffers.UOffsetT(exemplars), 0) -} -func SimpleFieldStartExemplarsVector(builder *flatbuffers.Builder, numElems int) flatbuffers.UOffsetT { - return builder.StartVector(4, numElems, 4) -} func SimpleFieldEnd(builder *flatbuffers.Builder) flatbuffers.UOffsetT { return builder.EndObject() } diff --git a/proto/gen/v1/linmetrics/linmetrics.pb.go b/proto/gen/v1/linmetrics/linmetrics.pb.go index a0e947b..d40a626 100644 --- a/proto/gen/v1/linmetrics/linmetrics.pb.go +++ b/proto/gen/v1/linmetrics/linmetrics.pb.go @@ -38,7 +38,7 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package type SimpleFieldType int32 @@ -124,45 +124,6 @@ func (m *MetricList) GetMetrics() []*Metric { return nil } -// Defines a Metric which has one or more time series. The following is a -// brief summary of the Metric data model. For more details, see: -// -// https://lindb.io/zh/docs/concept/datamodel.html -// -// Here, "Field" is the term used to refer any specific field with exemplars. -// and "Data" is the term used to refer to the specific underlying value for the field. -// -// - Metric is composed of metadata态timestamp and fields. -// - Metadata part contains a namespace, name, tags and its sorted-concat-string hash. -// - Fields is the array of the possible fields (Sum, Last, Histogram). -// - Fields contains exemplars, names, and the underlying value or value list -// -// Metric -// +---------------+ -// |namespace | -// |name | -// |tags | -// |tags-hash | -// |timestamp | +------------------------------------+ -// |simple-fields |---> |Last, Sum, ... | -// |compound-field |---> |Histogram | -// +---------------+ +------------------------------------+ -// -// SimpleField [One of Last, DeltaSum, Min, Max ...] -// +-----------+ -// |name | // field-name -// |type | // field-type -// |exemplars | // exemplars of series -// +-----------+ -// |value | -// +-----------+ -// -// CompoundField [DeltaHistogram ...] -// +-----------+ -// |exemplars | // exemplars of series -// +-----+-----+-----+-----+-----+-----+ -// |min |max |sum |value|value|.....| -// +-----+-----+-----+-----+-----+-----+ type Metric struct { Namespace string `protobuf:"bytes,1,opt,name=namespace,proto3" json:"namespace,omitempty"` Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` @@ -172,6 +133,7 @@ type Metric struct { TagsHash uint64 `protobuf:"varint,5,opt,name=tags_hash,json=tagsHash,proto3" json:"tags_hash,omitempty"` SimpleFields []*SimpleField `protobuf:"bytes,6,rep,name=simple_fields,json=simpleFields,proto3" json:"simple_fields,omitempty"` CompoundField *CompoundField `protobuf:"bytes,7,opt,name=compound_field,json=compoundField,proto3" json:"compound_field,omitempty"` + Exemplars []*Exemplar `protobuf:"bytes,8,rep,name=exemplars,proto3" json:"exemplars,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -259,10 +221,16 @@ func (m *Metric) GetCompoundField() *CompoundField { return nil } +func (m *Metric) GetExemplars() []*Exemplar { + if m != nil { + return m.Exemplars + } + return nil +} + type SimpleField struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Type SimpleFieldType `protobuf:"varint,2,opt,name=type,proto3,enum=protoMetricsV1.SimpleFieldType" json:"type,omitempty"` - Exemplars []*Exemplar `protobuf:"bytes,3,rep,name=exemplars,proto3" json:"exemplars,omitempty"` Value float64 `protobuf:"fixed64,4,opt,name=value,proto3" json:"value,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -316,13 +284,6 @@ func (m *SimpleField) GetType() SimpleFieldType { return SimpleFieldType_SIMPLE_UNSPECIFIED } -func (m *SimpleField) GetExemplars() []*Exemplar { - if m != nil { - return m.Exemplars - } - return nil -} - func (m *SimpleField) GetValue() float64 { if m != nil { return m.Value @@ -332,11 +293,10 @@ func (m *SimpleField) GetValue() float64 { // CompoundData is compound data used for histogram field. type CompoundField struct { - Exemplars []*Exemplar `protobuf:"bytes,1,rep,name=exemplars,proto3" json:"exemplars,omitempty"` - Min float64 `protobuf:"fixed64,2,opt,name=min,proto3" json:"min,omitempty"` - Max float64 `protobuf:"fixed64,3,opt,name=max,proto3" json:"max,omitempty"` - Sum float64 `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"` - Count float64 `protobuf:"fixed64,5,opt,name=count,proto3" json:"count,omitempty"` + Min float64 `protobuf:"fixed64,2,opt,name=min,proto3" json:"min,omitempty"` + Max float64 `protobuf:"fixed64,3,opt,name=max,proto3" json:"max,omitempty"` + Sum float64 `protobuf:"fixed64,4,opt,name=sum,proto3" json:"sum,omitempty"` + Count float64 `protobuf:"fixed64,5,opt,name=count,proto3" json:"count,omitempty"` // same as open-telemetry metrics definition // explicit_bounds specifies buckets with explicitly defined bounds for values. // @@ -391,13 +351,6 @@ func (m *CompoundField) XXX_DiscardUnknown() { var xxx_messageInfo_CompoundField proto.InternalMessageInfo -func (m *CompoundField) GetExemplars() []*Exemplar { - if m != nil { - return m.Exemplars - } - return nil -} - func (m *CompoundField) GetMin() float64 { if m != nil { return m.Min @@ -500,12 +453,14 @@ func (m *KeyValue) GetValue() string { // Exemplars in LindDB wont' hold any information about the environment // it is used to record span and trace ID for a specify series. type Exemplar struct { + // Exemplar Name + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Span ID of the exemplar trace. - SpanId []byte `protobuf:"bytes,1,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` + SpanId string `protobuf:"bytes,2,opt,name=span_id,json=spanId,proto3" json:"span_id,omitempty"` // Trace ID of the exemplar trace. - TraceId []byte `protobuf:"bytes,2,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` + TraceId string `protobuf:"bytes,3,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` // Duration of the exemplar span. - Duration int64 `protobuf:"varint,3,opt,name=duration,proto3" json:"duration,omitempty"` + Duration int64 `protobuf:"varint,4,opt,name=duration,proto3" json:"duration,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -544,18 +499,25 @@ func (m *Exemplar) XXX_DiscardUnknown() { var xxx_messageInfo_Exemplar proto.InternalMessageInfo -func (m *Exemplar) GetSpanId() []byte { +func (m *Exemplar) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *Exemplar) GetSpanId() string { if m != nil { return m.SpanId } - return nil + return "" } -func (m *Exemplar) GetTraceId() []byte { +func (m *Exemplar) GetTraceId() string { if m != nil { return m.TraceId } - return nil + return "" } func (m *Exemplar) GetDuration() int64 { @@ -578,44 +540,43 @@ func init() { func init() { proto.RegisterFile("linmetrics.proto", fileDescriptor_e09a430c1694f179) } var fileDescriptor_e09a430c1694f179 = []byte{ - // 578 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0xcd, 0x6e, 0xd3, 0x4c, - 0x14, 0xed, 0xc4, 0x4e, 0x62, 0xdf, 0x36, 0xa9, 0x35, 0xfa, 0xd4, 0x6f, 0xa0, 0x10, 0xac, 0x6c, - 0xb0, 0x10, 0xaa, 0x20, 0x95, 0x58, 0x22, 0xfa, 0x93, 0x0a, 0x8b, 0x04, 0x55, 0xe3, 0xb4, 0x0b, - 0x36, 0xd6, 0xd4, 0x1e, 0xe8, 0x08, 0xff, 0x29, 0xe3, 0xa0, 0xe4, 0x4d, 0x78, 0x00, 0x76, 0xbc, - 0x08, 0x4b, 0x16, 0x3c, 0x00, 0x2a, 0x2f, 0x82, 0x66, 0xec, 0x34, 0x4d, 0x84, 0x10, 0xab, 0xb9, - 0xe7, 0xdc, 0x33, 0x77, 0xee, 0x3d, 0x77, 0xc0, 0x49, 0x44, 0x96, 0xf2, 0x72, 0x2a, 0x22, 0x79, - 0x50, 0x4c, 0xf3, 0x32, 0xc7, 0x5d, 0x7d, 0x8c, 0x2b, 0xee, 0xf2, 0x79, 0xff, 0x25, 0x40, 0x05, - 0x46, 0x42, 0x96, 0xf8, 0x19, 0xb4, 0x6b, 0x39, 0x69, 0xb8, 0x86, 0xb7, 0x3d, 0xd8, 0x3b, 0x58, - 0xd7, 0x1f, 0x54, 0x11, 0x5d, 0xca, 0xfa, 0x5f, 0x1b, 0xd0, 0xaa, 0x38, 0xfc, 0x00, 0xec, 0x8c, - 0xa5, 0x5c, 0x16, 0x2c, 0xe2, 0x04, 0xb9, 0xc8, 0xb3, 0xe9, 0x8a, 0xc0, 0x18, 0x4c, 0x05, 0x48, - 0x43, 0x27, 0x74, 0xac, 0x6e, 0x94, 0x22, 0xe5, 0xb2, 0x64, 0x69, 0x41, 0x0c, 0x17, 0x79, 0x06, - 0x5d, 0x11, 0xf8, 0x29, 0x98, 0x25, 0xfb, 0x20, 0x89, 0xa9, 0x3b, 0x21, 0x9b, 0x9d, 0xbc, 0xe1, - 0x8b, 0x4b, 0x96, 0xcc, 0x38, 0xd5, 0x2a, 0xbc, 0x0f, 0xb6, 0x3a, 0xc3, 0x6b, 0x26, 0xaf, 0x49, - 0xd3, 0x45, 0x9e, 0x49, 0x2d, 0x45, 0xbc, 0x66, 0xf2, 0x1a, 0xbf, 0x82, 0x8e, 0x14, 0x69, 0x91, - 0xf0, 0xf0, 0xbd, 0xe0, 0x49, 0x2c, 0x49, 0x4b, 0xd7, 0xdc, 0xdf, 0xac, 0x19, 0x68, 0xd1, 0x99, - 0xd2, 0xd0, 0x1d, 0xb9, 0x02, 0x12, 0x9f, 0x42, 0x37, 0xca, 0xd3, 0x22, 0x9f, 0x65, 0x71, 0x55, - 0x83, 0xb4, 0x5d, 0xe4, 0x6d, 0x0f, 0x1e, 0x6e, 0x96, 0x38, 0xa9, 0x55, 0x55, 0x91, 0x4e, 0x74, - 0x17, 0xf6, 0xbf, 0x20, 0xd8, 0xbe, 0xf3, 0xc6, 0xad, 0x29, 0xe8, 0x8e, 0x29, 0x87, 0x60, 0x96, - 0x8b, 0xa2, 0x32, 0xaa, 0x3b, 0x78, 0xf4, 0x97, 0x16, 0x27, 0x8b, 0x42, 0x4d, 0xbf, 0x28, 0x38, - 0x7e, 0x01, 0x36, 0x9f, 0xf3, 0xb4, 0x48, 0xd8, 0x54, 0x12, 0xe3, 0xcf, 0x86, 0x0d, 0x6b, 0x01, - 0x5d, 0x49, 0xf1, 0x7f, 0xd0, 0xfc, 0xa4, 0x4c, 0x24, 0xa6, 0x8b, 0x3c, 0x44, 0x2b, 0xd0, 0xff, - 0x81, 0xa0, 0xb3, 0x36, 0xc7, 0x7a, 0x7d, 0xf4, 0xef, 0xf5, 0x1d, 0x30, 0x52, 0x91, 0xe9, 0x59, - 0x10, 0x55, 0xa1, 0x66, 0xd8, 0x5c, 0x6f, 0x5b, 0x31, 0x6c, 0xae, 0x18, 0x39, 0x4b, 0xeb, 0x0e, - 0x54, 0xa8, 0xba, 0x8a, 0xf2, 0x59, 0x56, 0xea, 0x3d, 0x22, 0x5a, 0x01, 0xfc, 0x18, 0x76, 0xf9, - 0xbc, 0x48, 0x44, 0x24, 0xca, 0xf0, 0x4a, 0xb5, 0x56, 0xad, 0x11, 0xd1, 0xee, 0x92, 0x3e, 0xd6, - 0x2c, 0xde, 0x83, 0x96, 0x9e, 0x43, 0x92, 0xb6, 0xce, 0xd7, 0xa8, 0x3f, 0x00, 0x6b, 0xf9, 0x69, - 0xd4, 0xa3, 0x1f, 0xf9, 0xa2, 0x36, 0x5e, 0x85, 0x2b, 0x2b, 0xaa, 0x1f, 0x5a, 0x5b, 0xf1, 0x0e, - 0xac, 0xe5, 0x5c, 0xf8, 0x7f, 0x68, 0xcb, 0x82, 0x65, 0xa1, 0x88, 0xf5, 0xbd, 0x1d, 0xda, 0x52, - 0xd0, 0x8f, 0xf1, 0x3d, 0xb0, 0xca, 0x29, 0x8b, 0xb8, 0xca, 0x34, 0x74, 0xa6, 0xad, 0xb1, 0x1f, - 0xe3, 0xfb, 0x60, 0xc5, 0xb3, 0x29, 0x2b, 0x45, 0x9e, 0xd5, 0x3f, 0xfc, 0x16, 0x3f, 0x09, 0x61, - 0x77, 0x63, 0x9b, 0x78, 0x0f, 0x70, 0xe0, 0x8f, 0xcf, 0x47, 0xc3, 0xf0, 0xe2, 0x6d, 0x70, 0x3e, - 0x3c, 0xf1, 0xcf, 0xfc, 0xe1, 0xa9, 0xb3, 0x85, 0x2d, 0x30, 0x47, 0x47, 0xc1, 0xc4, 0x41, 0xb8, - 0x03, 0xf6, 0xe9, 0x70, 0x34, 0x39, 0x0a, 0x83, 0x8b, 0xb1, 0xd3, 0xc0, 0x6d, 0x30, 0xc6, 0x22, - 0x73, 0x0c, 0x1d, 0xb0, 0xb9, 0x63, 0x62, 0x1b, 0x9a, 0x67, 0x3e, 0x0d, 0x26, 0x4e, 0xf3, 0xd8, - 0xf9, 0x76, 0xd3, 0x43, 0xdf, 0x6f, 0x7a, 0xe8, 0xe7, 0x4d, 0x0f, 0x7d, 0xfe, 0xd5, 0xdb, 0xba, - 0x6a, 0xe9, 0x9d, 0x1d, 0xfe, 0x0e, 0x00, 0x00, 0xff, 0xff, 0x4b, 0xb5, 0x3d, 0x31, 0x19, 0x04, - 0x00, 0x00, + // 565 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x52, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xed, 0xc6, 0x4e, 0x6c, 0x4f, 0x49, 0x6a, 0xad, 0x50, 0x59, 0x28, 0x04, 0x2b, 0x17, 0x2c, + 0x84, 0x2a, 0x48, 0x25, 0x8e, 0x88, 0x7e, 0xa4, 0xc2, 0x22, 0x41, 0xd5, 0x3a, 0xed, 0xd5, 0xda, + 0xda, 0x0b, 0x5d, 0xe1, 0x2f, 0x65, 0x1d, 0x94, 0xfc, 0x13, 0x0e, 0xfc, 0x20, 0x8e, 0xfc, 0x04, + 0xd4, 0xfe, 0x11, 0xb4, 0x6b, 0xa7, 0x69, 0xa3, 0x8a, 0x93, 0xe7, 0xbd, 0x79, 0x7e, 0x9e, 0x79, + 0x63, 0x70, 0x53, 0x91, 0x67, 0xbc, 0x9a, 0x89, 0x58, 0xee, 0x97, 0xb3, 0xa2, 0x2a, 0x70, 0x4f, + 0x3f, 0x26, 0x35, 0x77, 0xf1, 0x6e, 0xf0, 0x01, 0xa0, 0x06, 0x63, 0x21, 0x2b, 0xfc, 0x16, 0xac, + 0x46, 0x4e, 0x5a, 0x9e, 0xe1, 0x6f, 0x0f, 0x77, 0xf7, 0xef, 0xeb, 0xf7, 0xeb, 0x8a, 0xae, 0x64, + 0x83, 0x9b, 0x16, 0x74, 0x6a, 0x0e, 0x3f, 0x07, 0x27, 0x67, 0x19, 0x97, 0x25, 0x8b, 0x39, 0x41, + 0x1e, 0xf2, 0x1d, 0xba, 0x26, 0x30, 0x06, 0x53, 0x01, 0xd2, 0xd2, 0x0d, 0x5d, 0xab, 0x37, 0x2a, + 0x91, 0x71, 0x59, 0xb1, 0xac, 0x24, 0x86, 0x87, 0x7c, 0x83, 0xae, 0x09, 0xfc, 0x06, 0xcc, 0x8a, + 0x7d, 0x93, 0xc4, 0xd4, 0x93, 0x90, 0xcd, 0x49, 0x3e, 0xf3, 0xe5, 0x05, 0x4b, 0xe7, 0x9c, 0x6a, + 0x15, 0xde, 0x03, 0x47, 0x3d, 0xa3, 0x2b, 0x26, 0xaf, 0x48, 0xdb, 0x43, 0xbe, 0x49, 0x6d, 0x45, + 0x7c, 0x62, 0xf2, 0x0a, 0x7f, 0x84, 0xae, 0x14, 0x59, 0x99, 0xf2, 0xe8, 0xab, 0xe0, 0x69, 0x22, + 0x49, 0x47, 0x7b, 0xee, 0x6d, 0x7a, 0x86, 0x5a, 0x74, 0xaa, 0x34, 0xf4, 0x91, 0x5c, 0x03, 0x89, + 0x4f, 0xa0, 0x17, 0x17, 0x59, 0x59, 0xcc, 0xf3, 0xa4, 0xf6, 0x20, 0x96, 0x87, 0xfc, 0xed, 0xe1, + 0x8b, 0x4d, 0x8b, 0xe3, 0x46, 0x55, 0x9b, 0x74, 0xe3, 0xbb, 0x10, 0xbf, 0x07, 0x87, 0x2f, 0x78, + 0x56, 0xa6, 0x6c, 0x26, 0x89, 0xfd, 0xf0, 0x5e, 0xa3, 0x46, 0x40, 0xd7, 0xd2, 0x41, 0x0a, 0xdb, + 0x77, 0x46, 0xbb, 0xcd, 0x12, 0xdd, 0xc9, 0xf2, 0x00, 0xcc, 0x6a, 0x59, 0xd6, 0xf9, 0xf6, 0x86, + 0x2f, 0xff, 0xb3, 0xd9, 0x74, 0x59, 0xaa, 0xd0, 0x96, 0x25, 0xc7, 0x8f, 0xa1, 0xfd, 0x43, 0x65, + 0x48, 0x4c, 0x0f, 0xf9, 0x88, 0xd6, 0x60, 0xf0, 0x0b, 0x41, 0xf7, 0xde, 0x1a, 0xd8, 0x05, 0x23, + 0x13, 0xb9, 0xf6, 0x46, 0x54, 0x95, 0x9a, 0x61, 0x0b, 0x7d, 0x34, 0xc5, 0xb0, 0x85, 0x62, 0xe4, + 0x3c, 0x6b, 0x9c, 0x54, 0xa9, 0xdc, 0xe3, 0x62, 0x9e, 0x57, 0xfa, 0x1c, 0x88, 0xd6, 0x00, 0xbf, + 0x82, 0x1d, 0xbe, 0x28, 0x53, 0x11, 0x8b, 0x2a, 0xba, 0x54, 0x9f, 0xa8, 0xaf, 0x81, 0x68, 0x6f, + 0x45, 0x1f, 0x69, 0x16, 0xef, 0x42, 0x47, 0xcf, 0x23, 0x89, 0xa5, 0xfb, 0x0d, 0x1a, 0x0c, 0xc1, + 0x5e, 0xdd, 0x5e, 0x7d, 0xf4, 0x3b, 0x5f, 0x36, 0x41, 0xa8, 0x72, 0xbd, 0x52, 0xfd, 0xa3, 0x35, + 0x2b, 0xe5, 0x60, 0xaf, 0x72, 0x7d, 0x30, 0xbd, 0x27, 0x60, 0xc9, 0x92, 0xe5, 0x91, 0x48, 0x9a, + 0xf7, 0x3a, 0x0a, 0x06, 0x09, 0x7e, 0x0a, 0x76, 0x35, 0x63, 0x31, 0x57, 0x1d, 0x43, 0x77, 0x2c, + 0x8d, 0x83, 0x04, 0x3f, 0x03, 0x3b, 0x99, 0xcf, 0x58, 0x25, 0x8a, 0x5c, 0x6f, 0x6d, 0xd0, 0x5b, + 0xfc, 0x3a, 0x82, 0x9d, 0x8d, 0xc4, 0xf1, 0x2e, 0xe0, 0x30, 0x98, 0x9c, 0x8d, 0x47, 0xd1, 0xf9, + 0x97, 0xf0, 0x6c, 0x74, 0x1c, 0x9c, 0x06, 0xa3, 0x13, 0x77, 0x0b, 0xdb, 0x60, 0x8e, 0x0f, 0xc3, + 0xa9, 0x8b, 0x70, 0x17, 0x9c, 0x93, 0xd1, 0x78, 0x7a, 0x18, 0x85, 0xe7, 0x13, 0xb7, 0x85, 0x2d, + 0x30, 0x26, 0x22, 0x77, 0x0d, 0x5d, 0xb0, 0x85, 0x6b, 0x62, 0x07, 0xda, 0xa7, 0x01, 0x0d, 0xa7, + 0x6e, 0xfb, 0xc8, 0xfd, 0x7d, 0xdd, 0x47, 0x7f, 0xae, 0xfb, 0xe8, 0xef, 0x75, 0x1f, 0xfd, 0xbc, + 0xe9, 0x6f, 0x5d, 0x76, 0xf4, 0xc5, 0x0f, 0xfe, 0x05, 0x00, 0x00, 0xff, 0xff, 0x85, 0x41, 0xfc, + 0xd4, 0xf4, 0x03, 0x00, 0x00, } func (m *MetricList) Marshal() (dAtA []byte, err error) { @@ -683,6 +644,20 @@ func (m *Metric) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.Exemplars) > 0 { + for iNdEx := len(m.Exemplars) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Exemplars[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLinmetrics(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 + } + } if m.CompoundField != nil { { size, err := m.CompoundField.MarshalToSizedBuffer(dAtA[:i]) @@ -780,20 +755,6 @@ func (m *SimpleField) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x21 } - if len(m.Exemplars) > 0 { - for iNdEx := len(m.Exemplars) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Exemplars[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintLinmetrics(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x1a - } - } if m.Type != 0 { i = encodeVarintLinmetrics(dAtA, i, uint64(m.Type)) i-- @@ -877,20 +838,6 @@ func (m *CompoundField) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x11 } - if len(m.Exemplars) > 0 { - for iNdEx := len(m.Exemplars) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Exemplars[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintLinmetrics(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0xa - } - } return len(dAtA) - i, nil } @@ -962,20 +909,27 @@ func (m *Exemplar) MarshalToSizedBuffer(dAtA []byte) (int, error) { if m.Duration != 0 { i = encodeVarintLinmetrics(dAtA, i, uint64(m.Duration)) i-- - dAtA[i] = 0x18 + dAtA[i] = 0x20 } if len(m.TraceId) > 0 { i -= len(m.TraceId) copy(dAtA[i:], m.TraceId) i = encodeVarintLinmetrics(dAtA, i, uint64(len(m.TraceId))) i-- - dAtA[i] = 0x12 + dAtA[i] = 0x1a } if len(m.SpanId) > 0 { i -= len(m.SpanId) copy(dAtA[i:], m.SpanId) i = encodeVarintLinmetrics(dAtA, i, uint64(len(m.SpanId))) i-- + dAtA[i] = 0x12 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintLinmetrics(dAtA, i, uint64(len(m.Name))) + i-- dAtA[i] = 0xa } return len(dAtA) - i, nil @@ -1046,6 +1000,12 @@ func (m *Metric) Size() (n int) { l = m.CompoundField.Size() n += 1 + l + sovLinmetrics(uint64(l)) } + if len(m.Exemplars) > 0 { + for _, e := range m.Exemplars { + l = e.Size() + n += 1 + l + sovLinmetrics(uint64(l)) + } + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -1065,12 +1025,6 @@ func (m *SimpleField) Size() (n int) { if m.Type != 0 { n += 1 + sovLinmetrics(uint64(m.Type)) } - if len(m.Exemplars) > 0 { - for _, e := range m.Exemplars { - l = e.Size() - n += 1 + l + sovLinmetrics(uint64(l)) - } - } if m.Value != 0 { n += 9 } @@ -1086,12 +1040,6 @@ func (m *CompoundField) Size() (n int) { } var l int _ = l - if len(m.Exemplars) > 0 { - for _, e := range m.Exemplars { - l = e.Size() - n += 1 + l + sovLinmetrics(uint64(l)) - } - } if m.Min != 0 { n += 9 } @@ -1142,6 +1090,10 @@ func (m *Exemplar) Size() (n int) { } var l int _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovLinmetrics(uint64(l)) + } l = len(m.SpanId) if l > 0 { n += 1 + l + sovLinmetrics(uint64(l)) @@ -1234,10 +1186,7 @@ func (m *MetricList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthLinmetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLinmetrics } if (iNdEx + skippy) > l { @@ -1488,16 +1437,47 @@ func (m *Metric) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Exemplars", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLinmetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLinmetrics + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLinmetrics + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Exemplars = append(m.Exemplars, &Exemplar{}) + if err := m.Exemplars[len(m.Exemplars)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLinmetrics(dAtA[iNdEx:]) if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthLinmetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLinmetrics } if (iNdEx + skippy) > l { @@ -1593,40 +1573,6 @@ func (m *SimpleField) Unmarshal(dAtA []byte) error { break } } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Exemplars", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLinmetrics - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthLinmetrics - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthLinmetrics - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Exemplars = append(m.Exemplars, &Exemplar{}) - if err := m.Exemplars[len(m.Exemplars)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 4: if wireType != 1 { return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) @@ -1644,10 +1590,7 @@ func (m *SimpleField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthLinmetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLinmetrics } if (iNdEx + skippy) > l { @@ -1692,40 +1635,6 @@ func (m *CompoundField) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: CompoundField: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Exemplars", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowLinmetrics - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthLinmetrics - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthLinmetrics - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Exemplars = append(m.Exemplars, &Exemplar{}) - if err := m.Exemplars[len(m.Exemplars)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex case 2: if wireType != 1 { return fmt.Errorf("proto: wrong wireType = %d for field Min", wireType) @@ -1884,10 +1793,7 @@ func (m *CompoundField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthLinmetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLinmetrics } if (iNdEx + skippy) > l { @@ -2002,10 +1908,7 @@ func (m *KeyValue) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthLinmetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLinmetrics } if (iNdEx + skippy) > l { @@ -2052,9 +1955,9 @@ func (m *Exemplar) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SpanId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) } - var byteLen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLinmetrics @@ -2064,31 +1967,29 @@ func (m *Exemplar) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthLinmetrics } - postIndex := iNdEx + byteLen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthLinmetrics } if postIndex > l { return io.ErrUnexpectedEOF } - m.SpanId = append(m.SpanId[:0], dAtA[iNdEx:postIndex]...) - if m.SpanId == nil { - m.SpanId = []byte{} - } + m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field TraceId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field SpanId", wireType) } - var byteLen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowLinmetrics @@ -2098,27 +1999,57 @@ func (m *Exemplar) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - byteLen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if byteLen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthLinmetrics } - postIndex := iNdEx + byteLen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthLinmetrics } if postIndex > l { return io.ErrUnexpectedEOF } - m.TraceId = append(m.TraceId[:0], dAtA[iNdEx:postIndex]...) - if m.TraceId == nil { - m.TraceId = []byte{} - } + m.SpanId = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TraceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLinmetrics + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLinmetrics + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLinmetrics + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TraceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Duration", wireType) } @@ -2143,10 +2074,7 @@ func (m *Exemplar) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthLinmetrics - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthLinmetrics } if (iNdEx + skippy) > l { @@ -2165,6 +2093,7 @@ func (m *Exemplar) Unmarshal(dAtA []byte) error { func skipLinmetrics(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 + depth := 0 for iNdEx < l { var wire uint64 for shift := uint(0); ; shift += 7 { @@ -2196,10 +2125,8 @@ func skipLinmetrics(dAtA []byte) (n int, err error) { break } } - return iNdEx, nil case 1: iNdEx += 8 - return iNdEx, nil case 2: var length int for shift := uint(0); ; shift += 7 { @@ -2220,55 +2147,30 @@ func skipLinmetrics(dAtA []byte) (n int, err error) { return 0, ErrInvalidLengthLinmetrics } iNdEx += length - if iNdEx < 0 { - return 0, ErrInvalidLengthLinmetrics - } - return iNdEx, nil case 3: - for { - var innerWire uint64 - var start int = iNdEx - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return 0, ErrIntOverflowLinmetrics - } - if iNdEx >= l { - return 0, io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - innerWire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - innerWireType := int(innerWire & 0x7) - if innerWireType == 4 { - break - } - next, err := skipLinmetrics(dAtA[start:]) - if err != nil { - return 0, err - } - iNdEx = start + next - if iNdEx < 0 { - return 0, ErrInvalidLengthLinmetrics - } - } - return iNdEx, nil + depth++ case 4: - return iNdEx, nil + if depth == 0 { + return 0, ErrUnexpectedEndOfGroupLinmetrics + } + depth-- case 5: iNdEx += 4 - return iNdEx, nil default: return 0, fmt.Errorf("proto: illegal wireType %d", wireType) } + if iNdEx < 0 { + return 0, ErrInvalidLengthLinmetrics + } + if depth == 0 { + return iNdEx, nil + } } - panic("unreachable") + return 0, io.ErrUnexpectedEOF } var ( - ErrInvalidLengthLinmetrics = fmt.Errorf("proto: negative length found during unmarshaling") - ErrIntOverflowLinmetrics = fmt.Errorf("proto: integer overflow") + ErrInvalidLengthLinmetrics = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowLinmetrics = fmt.Errorf("proto: integer overflow") + ErrUnexpectedEndOfGroupLinmetrics = fmt.Errorf("proto: unexpected end of group") ) diff --git a/proto/v1/linmetrics.proto b/proto/v1/linmetrics.proto index 3da21fd..99007c3 100644 --- a/proto/v1/linmetrics.proto +++ b/proto/v1/linmetrics.proto @@ -20,31 +20,35 @@ message MetricList { // - Fields contains exemplars, names, and the underlying value or value list // // Metric -// +---------------+ -// |namespace | -// |name | -// |tags | -// |tags-hash | -// |timestamp | +------------------------------------+ -// |simple-fields |---> |Last, Sum, ... | -// |compound-field |---> |Histogram | -// +---------------+ +------------------------------------+ +// +----------------+ +// |namespace | +// |name | +// |tags | +// |tags-hash | +// |timestamp | +------------------------------------+ +// |simple-fields |---> |Last, Sum, ... | +// |compound-field |---> |Histogram | +// |exemplar-fields |---> |Exemplar | +// +----------------+ +------------------------------------+ // // SimpleField [One of Last, DeltaSum, Min, Max ...] // +-----------+ // |name | // field-name // |type | // field-type -// |exemplars | // exemplars of series // +-----------+ // |value | // +-----------+ // // CompoundField [DeltaHistogram ...] -// +-----------+ -// |exemplars | // exemplars of series // +-----+-----+-----+-----+-----+-----+ // |min |max |sum |value|value|.....| // +-----+-----+-----+-----+-----+-----+ +// +// ExemplarField [Exemplar ...] +// +-----+----------+---------+----- ----+ +// |name | trace id | span id | duration | +// +-----+----------+---------+----------+ + message Metric { string namespace = 1; string name = 2; @@ -54,7 +58,9 @@ message Metric { uint64 tags_hash = 5; repeated SimpleField simple_fields = 6; CompoundField compound_field = 7; + repeated Exemplar exemplars = 8; } + enum SimpleFieldType { SIMPLE_UNSPECIFIED = 0; LAST = 1; @@ -64,17 +70,14 @@ enum SimpleFieldType { FIRST = 5; } - message SimpleField { string name = 1; SimpleFieldType type = 2; - repeated Exemplar exemplars = 3; double value = 4; } // CompoundData is compound data used for histogram field. message CompoundField { - repeated Exemplar exemplars = 1; double min = 2; double max = 3; double sum = 4; @@ -107,12 +110,15 @@ message KeyValue { // Exemplars in LindDB wont' hold any information about the environment // it is used to record span and trace ID for a specify series. message Exemplar { + // Exemplar Name + string name = 1; + // Span ID of the exemplar trace. - bytes span_id = 1; + string span_id = 2; // Trace ID of the exemplar trace. - bytes trace_id = 2; + string trace_id = 3; // Duration of the exemplar span. - int64 duration = 3; -} \ No newline at end of file + int64 duration = 4; +} diff --git a/proto/v1/metrics.fbs b/proto/v1/metrics.fbs index 169192f..667063d 100644 --- a/proto/v1/metrics.fbs +++ b/proto/v1/metrics.fbs @@ -13,12 +13,10 @@ table SimpleField { name: string; type: SimpleFieldType; value: double; - exemplars: [Exemplar]; } // CompoundField holds compound data used for histogram field. table CompoundField{ - exemplars: [Exemplar]; min: double; // lindb specific field max: double; // lindb specific field sum: double; @@ -37,7 +35,7 @@ table CompoundField{ // Histogram buckets are inclusive of their upper boundary, except the last // bucket where the boundary is at infinity. This format is intentionally // compatible with the OpenMetrics histogram definition. - explicitBounds: [double]; + explicit_bounds: [double]; values: [double]; } @@ -51,12 +49,14 @@ table KeyValue { // Exemplars in LindDB wont' hold any information about the environment // it is used to record span and trace ID for a specify series. table Exemplar { - // Span ID of the exemplar trace. - spanID: [byte]; - // Trace ID of the exemplar trace. - traceID: [byte]; - // Duration of the exemplar span. - duration: int64; + // Exemplar name + name: string; + // Span ID of the exemplar trace. + span_id: string; + // Trace ID of the exemplar trace. + trace_id: string; + // Duration of the exemplar span.(optional) + duration: int64; } // Defines a Metric which has one or more timeseries. The following is a @@ -73,40 +73,47 @@ table Exemplar { // - Fields contains exemplars, names, and the underlying value or value list // // Metric -// +---------------+ -// |namespace | -// |name | -// |tags | -// |tags-hash | -// |timestamp | +------------------------------------+ -// |simple-fields |---> |Last, Sum, ... | -// |compound-field |---> |Histogram | -// +---------------+ +------------------------------------+ +// +----------------+ +// |namespace | +// |name | +// |tags | +// |tags-hash | +// |timestamp | +------------------------------------+ +// |simple-fields |---> |Last, Sum, ... | +// |compound-field |---> |Histogram | +// |exemplar-fields |---> |Exemplar | +// +----------------+ +------------------------------------+ // // SimpleField [One of Last, DeltaSum, Min, Max ...] // +-----------+ // |name | // field-name // |type | // field-type -// |exemplars | // exemplars of series // +-----------+ // |value | // +-----------+ // // CompoundField [DeltaHistogram ...] -// +-----------+ -// |exemplars | // exemplars of series // +-----+-----+-----+-----+-----+-----+ // |min |max |sum |value|value|.....| // +-----+-----+-----+-----+-----+-----+ +// +// ExemplarField [Exemplar ...] +// +-----+----------+---------+----- ----+ +// |name | trace id | span id | duration | +// +-----+----------+---------+----------+ + table Metric { namespace: string; name:string; // metric-name timestamp: int64; // in milliseconds - keyValues: [KeyValue]; + // xxhash.Sum64String(namespace name), broker side generate before write wal + name_hash: uint64 ; + key_values: [KeyValue]; // xxhash.Sum64String(sorted tags), broker side generate before write wal - hash: uint64 ; - simpleFields: [SimpleField]; - compoundField: CompoundField; + kvs_hash: uint64 ; + simple_fields: [SimpleField]; + compound_field: CompoundField; + exemplars: [Exemplar]; } -root_type Metric; \ No newline at end of file +root_type Metric; diff --git a/series/checker_test.go b/series/checker_test.go index 2605777..69a7e09 100644 --- a/series/checker_test.go +++ b/series/checker_test.go @@ -101,7 +101,8 @@ func Benchmark_Marshal_Proto(b *testing.B) { for i := 0; i < 10; i++ { m.SimpleFields = append(m.SimpleFields, &protoMetricsV1.SimpleField{ - Name: "counter" + strconv.Itoa(i), Type: protoMetricsV1.SimpleFieldType_LAST, Value: float64(i)}) + Name: "counter" + strconv.Itoa(i), Type: protoMetricsV1.SimpleFieldType_LAST, Value: float64(i), + }) m.Tags = append(m.Tags, &protoMetricsV1.KeyValue{Key: "key" + strconv.Itoa(i), Value: "value" + strconv.Itoa(i)}) } b.ReportAllocs() @@ -117,7 +118,8 @@ func Benchmark_Unmarshal_Proto_10Fields(b *testing.B) { for i := 0; i < 10; i++ { m.SimpleFields = append(m.SimpleFields, &protoMetricsV1.SimpleField{ - Name: "counter" + strconv.Itoa(i), Type: protoMetricsV1.SimpleFieldType_LAST, Value: float64(i)}) + Name: "counter" + strconv.Itoa(i), Type: protoMetricsV1.SimpleFieldType_LAST, Value: float64(i), + }) m.Tags = append(m.Tags, &protoMetricsV1.KeyValue{Key: "key" + strconv.Itoa(i), Value: "value" + strconv.Itoa(i)}) } data, _ := m.Marshal() @@ -153,7 +155,7 @@ func Test_FlatMetric(t *testing.T) { assert.Equal(t, "hello", string(m.Name())) assert.Equal(t, "default-ns", string(m.Namespace())) assert.Zero(t, m.Timestamp()) - assert.Zero(t, m.Hash()) + assert.Zero(t, m.KvsHash()) } func Test_FlatMetric_Size(t *testing.T) { diff --git a/series/row_builder.go b/series/row_builder.go index 22733e8..df15bf2 100644 --- a/series/row_builder.go +++ b/series/row_builder.go @@ -27,7 +27,6 @@ import ( "github.com/cespare/xxhash/v2" flatbuffers "github.com/google/flatbuffers/go" - "github.com/lindb/common/constants" "github.com/lindb/common/pkg/fasttime" "github.com/lindb/common/proto/gen/v1/flatMetricsV1" ) @@ -51,6 +50,13 @@ func (items rowKVs) Less(i, j int) bool { return bytes.Compare(items.kvs[i].key, items.kvs[j].key) < 0 } +type rowExemplar struct { + name []byte + traceID []byte + spanID []byte + duration int64 +} + type rowSimpleField struct { name []byte fType flatMetricsV1.SimpleFieldType @@ -70,6 +76,9 @@ type RowBuilder struct { simpleFields []rowSimpleField simpleFieldCount int + exemplarFields []rowExemplar + exemplarFieldCount int + compoundFieldValues []float64 compoundFieldExplicitValues []float64 compoundFieldMin float64 @@ -78,12 +87,16 @@ type RowBuilder struct { compoundFieldCount float64 // context for building flat metrics - flatBuilder *flatbuffers.Builder - keys []flatbuffers.UOffsetT - values []flatbuffers.UOffsetT - kvs []flatbuffers.UOffsetT - fieldNames []flatbuffers.UOffsetT - fields []flatbuffers.UOffsetT + flatBuilder *flatbuffers.Builder + keys []flatbuffers.UOffsetT + values []flatbuffers.UOffsetT + kvs []flatbuffers.UOffsetT + fieldNames []flatbuffers.UOffsetT + fields []flatbuffers.UOffsetT + exemplarNames []flatbuffers.UOffsetT + exemplarTraces []flatbuffers.UOffsetT + exemplarSpans []flatbuffers.UOffsetT + exemplars []flatbuffers.UOffsetT } var rowBuilderPool sync.Pool @@ -162,6 +175,34 @@ func (rb *RowBuilder) AddSimpleField(fieldName []byte, fieldType flatMetricsV1.S return nil } +// AddExemplar appends a exemplar +// Return false if exemplar is invalid +func (rb *RowBuilder) AddExemplar(name, traceID, spanID []byte, duration int64) error { + if len(name) == 0 { + return fmt.Errorf("exemplar name is empty") + } + if len(traceID) == 0 { + return fmt.Errorf("trace id is empty") + } + if len(spanID) == 0 { + return fmt.Errorf("span id is empty") + } + + rb.exemplarFieldCount++ + // add exemplar(cache) + if rb.exemplarFieldCount > len(rb.exemplarFields) { + rb.exemplarFields = append(rb.exemplarFields, rowExemplar{}) + } + exemplarIdx := rb.exemplarFieldCount - 1 + // copy exemplar name/trace/span + rb.exemplarFields[exemplarIdx].name = append(rb.exemplarFields[exemplarIdx].name[:0], name...) + rb.exemplarFields[exemplarIdx].traceID = append(rb.exemplarFields[exemplarIdx].traceID[:0], traceID...) + rb.exemplarFields[exemplarIdx].spanID = append(rb.exemplarFields[exemplarIdx].spanID[:0], spanID...) + // copy duration + rb.exemplarFields[exemplarIdx].duration = duration + return nil +} + func (rb *RowBuilder) AddTimestamp(ts int64) { rb.timestamp = ts } func (rb *RowBuilder) AddCompoundFieldData(values, bounds []float64) error { @@ -241,6 +282,8 @@ func (rb *RowBuilder) Reset() { // reset simple fields context rb.simpleFieldCount = 0 + // reset exemplars context + rb.exemplarFieldCount = 0 // reset compound context rb.compoundFieldValues = rb.compoundFieldValues[:0] @@ -255,10 +298,15 @@ func (rb *RowBuilder) Reset() { rb.kvs = rb.kvs[:0] rb.fieldNames = rb.fieldNames[:0] rb.fields = rb.fields[:0] + rb.exemplarNames = rb.exemplarNames[:0] + rb.exemplarTraces = rb.exemplarTraces[:0] + rb.exemplarSpans = rb.exemplarSpans[:0] + rb.exemplars = rb.exemplars[:0] } var ( emptyStringHash = xxhash.Sum64String("") + emptyNamespace = []byte{} ) func (rb *RowBuilder) _xxHashOfKVs() uint64 { @@ -277,6 +325,16 @@ func (rb *RowBuilder) _xxHashOfKVs() uint64 { return xxhash.Sum64(rb.hashBuf.Bytes()) } +func (rb *RowBuilder) _xxHashOfName() uint64 { + rb.hashBuf.Reset() + + if len(rb.nameSpace) > 0 { + _, _ = rb.hashBuf.Write(rb.nameSpace) + } + _, _ = rb.hashBuf.Write(rb.metricName) + return xxhash.Sum64(rb.hashBuf.Bytes()) +} + // dedupTags removes duplicated tags func (rb *RowBuilder) dedupTagsThenXXHash() uint64 { if rb.rowKVs.kvCount < 2 { @@ -300,7 +358,7 @@ func (rb *RowBuilder) dedupTagsThenXXHash() uint64 { // tags with same key will keep order as they are appended after sorting // high index key has higher priority // use 2-pointer algorithm - var slow = 0 + slow := 0 for high := 1; high < rb.rowKVs.kvCount; high++ { if !bytes.Equal(rb.rowKVs.kvs[slow].key, rb.rowKVs.kvs[high].key) { slow++ @@ -333,16 +391,33 @@ func (rb *RowBuilder) Build() ([]byte, error) { } // building field names for i := 0; i < rb.simpleFieldCount; i++ { + // field name offsets rb.fieldNames = append(rb.fieldNames, rb.flatBuilder.CreateByteString(rb.simpleFields[i].name)) } - for i := 0; i < rb.simpleFieldCount; i++ { flatMetricsV1.SimpleFieldStart(rb.flatBuilder) - flatMetricsV1.SimpleFieldAddName(rb.flatBuilder, rb.fieldNames[i]) + flatMetricsV1.SimpleFieldAddName(rb.flatBuilder, rb.fieldNames[i]) // write field name offset flatMetricsV1.SimpleFieldAddType(rb.flatBuilder, rb.simpleFields[i].fType) flatMetricsV1.SimpleFieldAddValue(rb.flatBuilder, rb.simpleFields[i].value) rb.fields = append(rb.fields, flatMetricsV1.SimpleFieldEnd(rb.flatBuilder)) } + + // building exemplars + for i := 0; i < rb.exemplarFieldCount; i++ { + // exemplar name/trace/span offset + rb.exemplarNames = append(rb.exemplarNames, rb.flatBuilder.CreateByteString(rb.exemplarFields[i].name)) + rb.exemplarTraces = append(rb.exemplarTraces, rb.flatBuilder.CreateByteString(rb.exemplarFields[i].traceID)) + rb.exemplarSpans = append(rb.exemplarSpans, rb.flatBuilder.CreateByteString(rb.exemplarFields[i].spanID)) + } + for i := 0; i < rb.exemplarFieldCount; i++ { + flatMetricsV1.ExemplarStart(rb.flatBuilder) + flatMetricsV1.ExemplarAddName(rb.flatBuilder, rb.exemplarNames[i]) // write exemplar name offset + flatMetricsV1.ExemplarAddTraceId(rb.flatBuilder, rb.exemplarTraces[i]) // write exemplar trace offset + flatMetricsV1.ExemplarAddSpanId(rb.flatBuilder, rb.exemplarSpans[i]) // write exemplar span offset + flatMetricsV1.ExemplarAddDuration(rb.flatBuilder, rb.exemplarFields[i].duration) + rb.exemplars = append(rb.exemplars, flatMetricsV1.ExemplarEnd(rb.flatBuilder)) + } + flatMetricsV1.MetricStartKeyValuesVector(rb.flatBuilder, rb.rowKVs.kvCount) for i := rb.rowKVs.kvCount - 1; i >= 0; i-- { rb.flatBuilder.PrependUOffsetT(rb.kvs[i]) @@ -355,6 +430,13 @@ func (rb *RowBuilder) Build() ([]byte, error) { } fields := rb.flatBuilder.EndVector(rb.simpleFieldCount) + // serialize exemplars + flatMetricsV1.MetricStartExemplarsVector(rb.flatBuilder, rb.exemplarFieldCount) + for i := rb.exemplarFieldCount - 1; i >= 0; i-- { + rb.flatBuilder.PrependUOffsetT(rb.exemplars[i]) + } + exemplars := rb.flatBuilder.EndVector(rb.exemplarFieldCount) + var ( compoundFieldBounds flatbuffers.UOffsetT compoundFieldValues flatbuffers.UOffsetT @@ -388,13 +470,14 @@ func (rb *RowBuilder) Build() ([]byte, error) { Serialize: metricName := rb.flatBuilder.CreateByteString(rb.metricName) + var namespace flatbuffers.UOffsetT if len(rb.nameSpace) == 0 { - // be careful, rb.namespace is reused, need create new default value - rb.nameSpace = []byte(constants.DefaultNamespace) + namespace = rb.flatBuilder.CreateByteString(emptyNamespace) + } else { + namespace = rb.flatBuilder.CreateByteString(rb.nameSpace) } - namespace := rb.flatBuilder.CreateByteString(rb.nameSpace) flatMetricsV1.MetricStart(rb.flatBuilder) flatMetricsV1.MetricAddNamespace(rb.flatBuilder, namespace) flatMetricsV1.MetricAddName(rb.flatBuilder, metricName) @@ -402,9 +485,11 @@ Serialize: rb.timestamp = fasttime.UnixMilliseconds() } flatMetricsV1.MetricAddTimestamp(rb.flatBuilder, rb.timestamp) + flatMetricsV1.MetricAddNameHash(rb.flatBuilder, rb._xxHashOfName()) flatMetricsV1.MetricAddKeyValues(rb.flatBuilder, kvs) - flatMetricsV1.MetricAddHash(rb.flatBuilder, hash) + flatMetricsV1.MetricAddKvsHash(rb.flatBuilder, hash) flatMetricsV1.MetricAddSimpleFields(rb.flatBuilder, fields) + flatMetricsV1.MetricAddExemplars(rb.flatBuilder, exemplars) if compoundField != 0 { flatMetricsV1.MetricAddCompoundField(rb.flatBuilder, compoundField) } @@ -416,3 +501,5 @@ Serialize: } func (rb *RowBuilder) SimpleFieldsLen() int { return rb.simpleFieldCount } + +func (rb *RowBuilder) ExemplarsLen() int { return rb.exemplarFieldCount } diff --git a/series/row_builder_test.go b/series/row_builder_test.go index ebcba84..51ea28e 100644 --- a/series/row_builder_test.go +++ b/series/row_builder_test.go @@ -65,7 +65,7 @@ func Test_RowBuilder(t *testing.T) { if i%2 == 0 { assert.Equal(t, "test", string(rb.nameSpace)) } else { - assert.Equal(t, "default-ns", string(rb.nameSpace)) + assert.Equal(t, "", string(rb.nameSpace)) } rb.Reset() } @@ -293,7 +293,7 @@ func buildFlatMetric(builder *flatbuffers.Builder) { flatMetricsV1.MetricAddName(builder, metricName) flatMetricsV1.MetricAddTimestamp(builder, fasttime.UnixMilliseconds()) flatMetricsV1.MetricAddKeyValues(builder, kvsAt) - flatMetricsV1.MetricAddHash(builder, xxhash.Sum64String("hello")) + flatMetricsV1.MetricAddKvsHash(builder, xxhash.Sum64String("hello")) flatMetricsV1.MetricAddSimpleFields(builder, fieldsAt) flatMetricsV1.MetricAddCompoundField(builder, compoundField)