add internal component metrics

.chloggen/ck-internal-metrics.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: clickhouseexporter
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Add internal metrics to track exporter performance
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [17559]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

exporter/clickhouseexporter/documentation.md

@@ -0,0 +1,63 @@
+[comment]: <> (Code generated by mdatagen. DO NOT EDIT.)
+
+# clickhouse
+
+## Internal Telemetry
+
+The following telemetry is emitted by this component.
+
+### otelcol_exporter_clickhouse_sent_batch_size
+
+How many count of a batch sent to Clickhouse. Clickhouse recommend inserting data in packets of at least 1000 rows, or no more than a single request per second.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| int | Histogram | Int |
+
+### otelcol_exporter_clickhouse_sent_data_points
+
+Number of data points the exporter sent.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| {data_points} | Sum | Int | true |
+
+### otelcol_exporter_clickhouse_sent_data_points_latency
+
+Latency (in milliseconds) of each sent data points batch.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| ms | Histogram | Int |
+
+### otelcol_exporter_clickhouse_sent_logs
+
+Number of logs the exporter sent.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| {logs} | Sum | Int | true |
+
+### otelcol_exporter_clickhouse_sent_logs_latency
+
+Latency (in milliseconds) of each sent log batch.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| ms | Histogram | Int |
+
+### otelcol_exporter_clickhouse_sent_spans
+
+Number of spans the exporter sent.
+
+| Unit | Metric Type | Value Type | Monotonic |
+| ---- | ----------- | ---------- | --------- |
+| {spans} | Sum | Int | true |
+
+### otelcol_exporter_clickhouse_sent_spans_latency
+
+Latency (in milliseconds) of each sent span batch.
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| ms | Histogram | Int |

exporter/clickhouseexporter/exporter_logs.go

@@ -11,23 +11,33 @@ import (
 
 	_ "github.com/ClickHouse/clickhouse-go/v2" // For register database driver.
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/plog"
 	conventions "go.opentelemetry.io/collector/semconv/v1.18.0"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal/metadata"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil"
 )
 
 type logsExporter struct {
 	client    *sql.DB
 	insertSQL string
 
-	logger *zap.Logger
-	cfg    *Config
+	logger    *zap.Logger
+	cfg       *Config
+	telemetry *metadata.TelemetryBuilder
 }
 
-func newLogsExporter(logger *zap.Logger, cfg *Config) (*logsExporter, error) {
+func newLogsExporter(settings exporter.Settings, cfg *Config) (*logsExporter, error) {
+	telemetry, err := metadata.NewTelemetryBuilder(settings.TelemetrySettings,
+		metadata.WithLevel(settings.TelemetrySettings.MetricsLevel))
+	if err != nil {
+		return nil, err
+	}
 	client, err := newClickhouseClient(cfg)
 	if err != nil {
 		return nil, err
@@ -36,8 +46,9 @@ func newLogsExporter(logger *zap.Logger, cfg *Config) (*logsExporter, error) {
 	return &logsExporter{
 		client:    client,
 		insertSQL: renderInsertLogsSQL(cfg),
-		logger:    logger,
+		logger:    settings.Logger,
 		cfg:       cfg,
+		telemetry: telemetry,
 	}, nil
 }
 
@@ -116,6 +127,8 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error {
 						logAttr,
 					)
 					if err != nil {
+						duration := time.Since(start)
+						recordLogsInternalMetrics(ctx, e.telemetry, int64(ld.LogRecordCount()), duration, true)
 						return fmt.Errorf("ExecContext:%w", err)
 					}
 				}
@@ -124,6 +137,7 @@ func (e *logsExporter) pushLogsData(ctx context.Context, ld plog.Logs) error {
 		return nil
 	})
 	duration := time.Since(start)
+	recordLogsInternalMetrics(ctx, e.telemetry, int64(ld.LogRecordCount()), duration, false)
 	e.logger.Debug("insert logs", zap.Int("records", ld.LogRecordCount()),
 		zap.String("cost", duration.String()))
 	return err
@@ -271,3 +285,11 @@ func doWithTx(_ context.Context, db *sql.DB, fn func(tx *sql.Tx) error) error {
 	}
 	return tx.Commit()
 }
+
+func recordLogsInternalMetrics(ctx context.Context, telemetry *metadata.TelemetryBuilder, count int64, duration time.Duration, hasError bool) {
+	if !hasError {
+		telemetry.ExporterClickhouseSentBatchSize.Record(ctx, count, metric.WithAttributes(attribute.KeyValue{Key: "telemetry_type", Value: attribute.StringValue("logs")}))
+	}
+	telemetry.ExporterClickhouseSentLogs.Add(ctx, count, metric.WithAttributes(attribute.KeyValue{Key: "error", Value: attribute.BoolValue(hasError)}))
+	telemetry.ExporterClickhouseSentLogsLatency.Record(ctx, duration.Milliseconds(), metric.WithAttributes(attribute.KeyValue{Key: "error", Value: attribute.BoolValue(hasError)}))
+}

exporter/clickhouseexporter/exporter_logs_test.go

@@ -13,11 +13,10 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/exporter/exportertest"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/plog"
 	conventions "go.opentelemetry.io/collector/semconv/v1.18.0"
-	"go.uber.org/zap"
-	"go.uber.org/zap/zaptest"
 )
 
 func TestLogsExporter_New(t *testing.T) {
@@ -59,7 +58,7 @@ func TestLogsExporter_New(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 
 			var err error
-			exporter, err := newLogsExporter(zap.NewNop(), test.config)
+			exporter, err := newLogsExporter(exportertest.NewNopSettings(), test.config)
 			err = errors.Join(err, err)
 
 			if exporter != nil {
@@ -147,7 +146,7 @@ func TestLogsTableEngineConfig(t *testing.T) {
 }
 
 func newTestLogsExporter(t *testing.T, dsn string, fns ...func(*Config)) *logsExporter {
-	exporter, err := newLogsExporter(zaptest.NewLogger(t), withTestExporterConfig(fns...)(dsn))
+	exporter, err := newLogsExporter(exportertest.NewNopSettings(), withTestExporterConfig(fns...)(dsn))
 	require.NoError(t, err)
 	require.NoError(t, exporter.start(context.TODO(), nil))
 

exporter/clickhouseexporter/exporter_metrics.go

@@ -10,29 +10,38 @@ import (
 	"fmt"
 
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.uber.org/zap"
 
 	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal/metadata"
 )
 
 type metricsExporter struct {
 	client *sql.DB
 
-	logger *zap.Logger
-	cfg    *Config
+	logger    *zap.Logger
+	cfg       *Config
+	telemetry *metadata.TelemetryBuilder
 }
 
-func newMetricsExporter(logger *zap.Logger, cfg *Config) (*metricsExporter, error) {
+func newMetricsExporter(settings exporter.Settings, cfg *Config) (*metricsExporter, error) {
+	telemetry, err := metadata.NewTelemetryBuilder(settings.TelemetrySettings,
+		metadata.WithLevel(settings.TelemetrySettings.MetricsLevel))
+	if err != nil {
+		return nil, err
+	}
 	client, err := newClickhouseClient(cfg)
 	if err != nil {
 		return nil, err
 	}
 
 	return &metricsExporter{
-		client: client,
-		logger: logger,
-		cfg:    cfg,
+		client:    client,
+		logger:    settings.Logger,
+		cfg:       cfg,
+		telemetry: telemetry,
 	}, nil
 }
 
@@ -95,5 +104,5 @@ func (e *metricsExporter) pushMetricsData(ctx context.Context, md pmetric.Metric
 		}
 	}
 	// batch insert https://clickhouse.com/docs/en/about-us/performance/#performance-when-inserting-data
-	return internal.InsertMetrics(ctx, e.client, metricsMap)
+	return internal.InsertMetrics(ctx, e.client, metricsMap, e.telemetry)
 }

exporter/clickhouseexporter/exporter_metrics_test.go

@@ -14,9 +14,9 @@ import (
 
 	"github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/exporter/exportertest"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
-	"go.uber.org/zap/zaptest"
 )
 
 func TestMetricsClusterConfig(t *testing.T) {
@@ -498,7 +498,7 @@ func mustPushMetricsData(t *testing.T, exporter *metricsExporter, md pmetric.Met
 
 // nolint:unparam // not need to check this func
 func newTestMetricsExporter(t *testing.T, dsn string, fns ...func(*Config)) *metricsExporter {
-	exporter, err := newMetricsExporter(zaptest.NewLogger(t), withTestExporterConfig(fns...)(dsn))
+	exporter, err := newMetricsExporter(exportertest.NewNopSettings(), withTestExporterConfig(fns...)(dsn))
 	require.NoError(t, err)
 	require.NoError(t, exporter.start(context.TODO(), nil))
 

exporter/clickhouseexporter/exporter_traces.go

@@ -12,22 +12,32 @@ import (
 
 	_ "github.com/ClickHouse/clickhouse-go/v2" // For register database driver.
 	"go.opentelemetry.io/collector/component"
+	"go.opentelemetry.io/collector/exporter"
 	"go.opentelemetry.io/collector/pdata/ptrace"
 	conventions "go.opentelemetry.io/collector/semconv/v1.18.0"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
 	"go.uber.org/zap"
 
+	"github.com/open-telemetry/opentelemetry-collector-contrib/exporter/clickhouseexporter/internal/metadata"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/traceutil"
 )
 
 type tracesExporter struct {
 	client    *sql.DB
 	insertSQL string
 
-	logger *zap.Logger
-	cfg    *Config
+	logger    *zap.Logger
+	cfg       *Config
+	telemetry *metadata.TelemetryBuilder
 }
 
-func newTracesExporter(logger *zap.Logger, cfg *Config) (*tracesExporter, error) {
+func newTracesExporter(settings exporter.Settings, cfg *Config) (*tracesExporter, error) {
+	telemetry, err := metadata.NewTelemetryBuilder(settings.TelemetrySettings,
+		metadata.WithLevel(settings.TelemetrySettings.MetricsLevel))
+	if err != nil {
+		return nil, err
+	}
 	client, err := newClickhouseClient(cfg)
 	if err != nil {
 		return nil, err
@@ -36,8 +46,9 @@ func newTracesExporter(logger *zap.Logger, cfg *Config) (*tracesExporter, error)
 	return &tracesExporter{
 		client:    client,
 		insertSQL: renderInsertTracesSQL(cfg),
-		logger:    logger,
+		logger:    settings.Logger,
 		cfg:       cfg,
+		telemetry: telemetry,
 	}, nil
 }
 
@@ -114,6 +125,8 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er
 						linksAttrs,
 					)
 					if err != nil {
+						duration := time.Since(start)
+						recordSpansInternalMetrics(ctx, e.telemetry, int64(td.SpanCount()), duration, false)
 						return fmt.Errorf("ExecContext:%w", err)
 					}
 				}
@@ -122,6 +135,7 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er
 		return nil
 	})
 	duration := time.Since(start)
+	recordSpansInternalMetrics(ctx, e.telemetry, int64(td.SpanCount()), duration, true)
 	e.logger.Debug("insert traces", zap.Int("records", td.SpanCount()),
 		zap.String("cost", duration.String()))
 	return err
@@ -308,3 +322,11 @@ func renderTraceIDTsMaterializedViewSQL(cfg *Config) string {
 	return fmt.Sprintf(createTraceIDTsMaterializedViewSQL, cfg.TracesTableName,
 		cfg.clusterString(), cfg.Database, cfg.TracesTableName, cfg.Database, cfg.TracesTableName)
 }
+
+func recordSpansInternalMetrics(ctx context.Context, telemetry *metadata.TelemetryBuilder, count int64, duration time.Duration, hasError bool) {
+	if !hasError {
+		telemetry.ExporterClickhouseSentBatchSize.Record(ctx, count, metric.WithAttributes(attribute.KeyValue{Key: "telemetry_type", Value: attribute.StringValue("Traces")}))
+	}
+	telemetry.ExporterClickhouseSentSpans.Add(ctx, count, metric.WithAttributes(attribute.KeyValue{Key: "error", Value: attribute.BoolValue(hasError)}))
+	telemetry.ExporterClickhouseSentSpansLatency.Record(ctx, duration.Milliseconds(), metric.WithAttributes(attribute.KeyValue{Key: "error", Value: attribute.BoolValue(hasError)}))
+}

exporter/clickhouseexporter/exporter_traces_test.go

@@ -11,10 +11,10 @@ import (
 	"time"
 
 	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/collector/exporter/exportertest"
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/ptrace"
 	conventions "go.opentelemetry.io/collector/semconv/v1.18.0"
-	"go.uber.org/zap/zaptest"
 )
 
 func TestExporter_pushTracesData(t *testing.T) {
@@ -49,7 +49,7 @@ func TestExporter_pushTracesData(t *testing.T) {
 }
 
 func newTestTracesExporter(t *testing.T, dsn string, fns ...func(*Config)) *tracesExporter {
-	exporter, err := newTracesExporter(zaptest.NewLogger(t), withTestExporterConfig(fns...)(dsn))
+	exporter, err := newTracesExporter(exportertest.NewNopSettings(), withTestExporterConfig(fns...)(dsn))
 	require.NoError(t, err)
 	require.NoError(t, exporter.start(context.TODO(), nil))
 

exporter/clickhouseexporter/factory.go

@@ -53,7 +53,7 @@ func createLogsExporter(
 	cfg component.Config,
 ) (exporter.Logs, error) {
 	c := cfg.(*Config)
-	exporter, err := newLogsExporter(set.Logger, c)
+	exporter, err := newLogsExporter(set, c)
 	if err != nil {
 		return nil, fmt.Errorf("cannot configure clickhouse logs exporter: %w", err)
 	}
@@ -79,7 +79,7 @@ func createTracesExporter(
 	cfg component.Config,
 ) (exporter.Traces, error) {
 	c := cfg.(*Config)
-	exporter, err := newTracesExporter(set.Logger, c)
+	exporter, err := newTracesExporter(set, c)
 	if err != nil {
 		return nil, fmt.Errorf("cannot configure clickhouse traces exporter: %w", err)
 	}
@@ -103,7 +103,7 @@ func createMetricExporter(
 	cfg component.Config,
 ) (exporter.Metrics, error) {
 	c := cfg.(*Config)
-	exporter, err := newMetricsExporter(set.Logger, c)
+	exporter, err := newMetricsExporter(set, c)
 	if err != nil {
 		return nil, fmt.Errorf("cannot configure clickhouse metrics exporter: %w", err)
 	}