forked from opensearch-project/observability
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
adding serviceMap dashboards related Integration (opensearch-project#…
…1494) Signed-off-by: YANGDB <yang.db.dev@gmail.com>
- Loading branch information
Showing
12 changed files
with
954 additions
and
0 deletions.
There are no files selected for viewing
266 changes: 266 additions & 0 deletions
266
integrations/service-map/assets/display/services-dashboard.ndjson
Large diffs are not rendered by default.
Oops, something went wrong.
62 changes: 62 additions & 0 deletions
62
integrations/service-map/assets/mapping/data-prepper-services.mapping
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,62 @@ | ||
| { | ||
| "index_patterns": [ | ||
| "otel-v1-apm-service-map" | ||
| ], | ||
| "mappings": { | ||
| "dynamic_templates": [ | ||
| { | ||
| "strings_as_keyword": { | ||
| "match_mapping_type": "string", | ||
| "mapping": { | ||
| "ignore_above": 1024, | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| } | ||
| ], | ||
| "date_detection": false, | ||
| "properties": { | ||
| "destination": { | ||
| "properties": { | ||
| "domain": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "resource": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| } | ||
| } | ||
| }, | ||
| "hashId": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "kind": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "serviceName": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "target": { | ||
| "properties": { | ||
| "domain": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "resource": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| } | ||
| } | ||
| }, | ||
| "traceGroupName": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| } | ||
| } | ||
| }, | ||
| "version": 1 | ||
| } |
157 changes: 157 additions & 0 deletions
157
integrations/service-map/assets/mapping/data-prepper-traces.mapping
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,157 @@ | ||
| { | ||
| "index_patterns": [ | ||
| "otel-v1-apm-span-*" | ||
| ], | ||
| "data_stream": {}, | ||
| "template": { | ||
| "mappings": { | ||
| "dynamic_templates": [ | ||
| { | ||
| "resource_attributes_map": { | ||
| "path_match": "resource.attributes.*", | ||
| "mapping": { | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| }, | ||
| { | ||
| "span_attributes_map": { | ||
| "path_match": "span.attributes.*", | ||
| "mapping": { | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| } | ||
| ], | ||
| "date_detection": false, | ||
| "properties": { | ||
| "droppedAttributesCount": { | ||
| "type": "long" | ||
| }, | ||
| "droppedEventsCount": { | ||
| "type": "long" | ||
| }, | ||
| "droppedLinksCount": { | ||
| "type": "long" | ||
| }, | ||
| "durationInNanos": { | ||
| "type": "long" | ||
| }, | ||
| "endTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "events": { | ||
| "type": "nested", | ||
| "properties": { | ||
| "attributes": { | ||
| "properties": { | ||
| "time": { | ||
| "type": "date_nanos" | ||
| } | ||
| } | ||
| }, | ||
| "droppedAttributesCount": { | ||
| "type": "long" | ||
| }, | ||
| "name": { | ||
| "type": "text", | ||
| "fields": { | ||
| "keyword": { | ||
| "type": "keyword", | ||
| "ignore_above": 256 | ||
| } | ||
| } | ||
| }, | ||
| "time": { | ||
| "type": "date_nanos" | ||
| } | ||
| } | ||
| }, | ||
| "kind": { | ||
| "type": "keyword", | ||
| "ignore_above": 128 | ||
| }, | ||
| "links": { | ||
| "type": "nested" | ||
| }, | ||
| "name": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "parentSpanId": { | ||
| "type": "keyword", | ||
| "ignore_above": 256 | ||
| }, | ||
| "resource": { | ||
| "properties": { | ||
| "attributes": { | ||
| "properties": { | ||
| "service@name": { | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "serviceName": { | ||
| "type": "keyword" | ||
| }, | ||
| "span": { | ||
| "properties": { | ||
| "attributes": { | ||
| "properties": {} | ||
| } | ||
| } | ||
| }, | ||
| "spanId": { | ||
| "type": "keyword", | ||
| "ignore_above": 256 | ||
| }, | ||
| "startTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "status": { | ||
| "properties": { | ||
| "code": { | ||
| "type": "integer" | ||
| }, | ||
| "message": { | ||
| "type": "keyword" | ||
| } | ||
| } | ||
| }, | ||
| "traceGroup": { | ||
| "type": "keyword", | ||
| "ignore_above": 1024 | ||
| }, | ||
| "traceGroupFields": { | ||
| "properties": { | ||
| "durationInNanos": { | ||
| "type": "long" | ||
| }, | ||
| "endTime": { | ||
| "type": "date_nanos" | ||
| }, | ||
| "statusCode": { | ||
| "type": "integer" | ||
| } | ||
| } | ||
| }, | ||
| "traceId": { | ||
| "type": "keyword", | ||
| "ignore_above": 256 | ||
| }, | ||
| "traceState": { | ||
| "type": "text", | ||
| "fields": { | ||
| "keyword": { | ||
| "type": "keyword", | ||
| "ignore_above": 256 | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| }, | ||
| "version": 1 | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,47 @@ | ||
| { | ||
| "name": "data-prepper", | ||
| "version": { | ||
| "integ": "0.1.0", | ||
| "schema": "1.0.0", | ||
| "resource": "^2.6.0" | ||
| }, | ||
| "description": "Data-Prepper Service Map and Tracing correlations Integration", | ||
| "catalog": "observability", | ||
| "components": [ | ||
| "traces,traceGroups,metrics" | ||
| ], | ||
| "collection":[ | ||
| { | ||
| "traces": [{ | ||
| "info": "traces signals", | ||
| "input_type":"traces", | ||
| "dataset":"spans", | ||
| "labels" :["traceGroups","traces"], | ||
| "fields-mapping" : [ | ||
| {"alias":"attributes.serviceName","field":"serviceName"} , | ||
| {"alias":"events.@timestamp","field":"events.time"} | ||
| ] | ||
| }] | ||
| }, | ||
| { | ||
| "services": [{ | ||
| "info": "service map", | ||
| "input_type": "services", | ||
| "dataset": "service", | ||
| "labels": ["services"] | ||
| }] | ||
| }, | ||
| { | ||
| "metrics": [{ | ||
| "info": "status metrics", | ||
| "input_type":"metrics", | ||
| "dataset":"metrics", | ||
| "labels" :["metrics"] | ||
| }] | ||
| } | ||
| ], | ||
| "repo": { | ||
| "github": "https://github.com/opensearch-project/observability/tree/main/integrarions/service-map" | ||
| } | ||
| } | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,107 @@ | ||
|  | ||
|
|
||
| # What is Data Prepper | ||
|
|
||
| [Data Prepper](https://github.com/opensearch-project/data-prepper/blob/main/docs/overview.md) is an open source utility service. Data Prepper is a server side data collector with abilities to filter, enrich, transform, normalize and aggregate data for downstream analytics and visualization. The broader vision for Data Prepper is to enable an end-to-end data analysis life cycle from gathering raw logs to facilitating sophisticated and actionable interactive ad-hoc analyses on the data. | ||
|
|
||
| # What is Data Prepper Integration | ||
|
|
||
| Data Prepper integration is concerned with the following aspects | ||
|
|
||
| - Allow simple and automatic generation of all schematic structured | ||
| - traces ( including specific fields mapping to map to SS4O schema) | ||
| - services ( adding support for specific service mapping category) | ||
| - metrics (using the standard SS4O schema) | ||
|
|
||
| - Add Dashboard Assets for correlation between traces-services-metrics | ||
|
|
||
| - Add correlation queries to investigate traces based metrics | ||
|
|
||
| # Data - Prepper Trace Fields | ||
| Data Prepper uses the following [Traces](https://github.com/opensearch-project/data-prepper/blob/main/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md) mapping file | ||
| The next fields are used: | ||
| ```text | ||
| - traceId - A unique identifier for a trace. All spans from the same trace share the same traceId. | ||
| - spanId - A unique identifier for a span within a trace, assigned when the span is created. | ||
| - traceState - Conveys information about request position in multiple distributed tracing graphs. | ||
| - parentSpanId - The spanId of this span's parent span. If this is a root span, then this field must be empty. | ||
| - name - A description of the span's operation. | ||
| - kind - The type of span. See OpenTelemetry - SpanKind. | ||
| - startTime - The start time of the span. | ||
| - endTime - The end time of the span. | ||
| - durationInNanos - Difference in nanoseconds between startTime and endTime. | ||
| - serviceName - Currently derived from the opentelemetry.proto.resource.v1.Resource associated with the span, the resource from the span originates. | ||
| - events - A list of events. See OpenTelemetry - Events. | ||
| - links - A list of linked spans. See OpenTelemetry - Links. | ||
| - droppedAttributesCount - The number of attributes that were discarded. | ||
| - droppedEventsCount - The number of events that were discarded. | ||
| - droppedLinksCount - The number of links that were dropped. | ||
| - span.attributes.* - All span attributes are split into a list of keywords. | ||
| - resource.attributes.* - All resource attributes are split into a list of keywords. | ||
| - status.code - The status of the span. See OpenTelemetry - Status. | ||
| ``` | ||
| These fields have a high overlap with the [`sso_traces`](../../../schema/observability/traces/README.md) fields from the observability catalog | ||
|
|
||
| There are some additional `trace.group` related fields which are not part of the [OTEL spec](https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md) for traces | ||
| ```text | ||
| - traceGroup - A derived field, the name of the trace's root span. | ||
| - traceGroupFields.endTime - A derived field, the endTime of the trace's root span. | ||
| - traceGroupFields.statusCode - A derived field, the status.code of the trace's root span. | ||
| - traceGroupFields.durationInNanos - A derived field, the durationInNanos of the trace's root span. | ||
| ``` | ||
|
|
||
| # Data - Prepper Trace Fields Mapping to SSO Trace Schema | ||
| To compensate for the minor differences in fields naming between the [Data-Prepper-Trace mapping](https://github.com/opensearch-project/data-prepper/blob/main/docs/schemas/trace-analytics/otel-v1-apm-span-index-template.md) and [SSO Traces mapping](../../../schema/observability/traces/traces.mapping) | ||
| We will define the next mapping alias that will be part of the [config.json](../config.json) for this integration | ||
|
|
||
| ```json5 | ||
| ... | ||
| "fields-mapping" : [ | ||
| {"alias":"attributes.serviceName","field":"serviceName"} , | ||
| {"alias":"events.@timestamp","field":"events.time"} | ||
| ] | ||
| ... | ||
| ``` | ||
|
|
||
|
|
||
| To address this difference, the `trace` signal can be augmented with additional trace-component - in this case it will be the [traceGroup](../../../schema/observability/traces/traceGroup). | ||
|
|
||
| ## Service dashboard | ||
| Service will have a dashboard comprised of the following visualizations: | ||
| - Latency per service with traces / metrics links | ||
| - Latency per trace-group with traces / metrics links | ||
| - Error rate per service with traces / metrics links | ||
| - Error rate per trace-group with traces / metrics links | ||
| - Throughput rate per service with traces / metrics links | ||
| - Throughput rate per trace-group with traces / metrics links | ||
|
|
||
| - Services view including error rate, latency, throughput, connected-services, traces, metrics | ||
|
|
||
| ```text | ||
| +--------------------------------+--------------------------------+ | ||
| | | | | ||
| | Latency per service | Latency per trace-group | | ||
| | with traces/metrics links | with traces/metrics links | | ||
| | | | | ||
| +--------------------------------+--------------------------------+ | ||
| | | | | ||
| | Error rate per service | Error rate per trace-group | | ||
| | with traces/metrics links | with traces/metrics links | | ||
| | | | | ||
| +--------------------------------+--------------------------------+ | ||
| | | | | ||
| | Throughput rate per service | Throughput rate per trace-group| | ||
| | with traces/metrics links | with traces/metrics links | | ||
| | | | | ||
| +--------------------------------+--------------------------------+ | ||
| | | | ||
| | Services view with error rate, latency, | | ||
| | throughput, connected-services, traces, | | ||
| | and metrics | | ||
| | | | ||
| +--------------------------------------------------------+ | ||
| ``` |
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| # Samples | ||
| The sample folder contains any type of sampled data that explains and demonstrates the expected input signals. | ||
|
|
||
| Specifically this folder contains two inner folder | ||
| - **preloaded** containing a ready-made signals with detailed instructions on how to load them into the appropriate opensearch data-streams. | ||
|
|
||
| Any other internal folder can be added that represents additional aspects of this integration expected ingesting content. |
Oops, something went wrong.