From 44b578f445c97b629ede9311a63f0289d08153d7 Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Thu, 22 Aug 2024 21:53:00 +0200 Subject: [PATCH 1/3] statsd: get `host` and `port` from config --- karapace/config.py | 4 ++++ karapace/statsd.py | 11 ++--------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/karapace/config.py b/karapace/config.py index 9212f6348..894164586 100644 --- a/karapace/config.py +++ b/karapace/config.py @@ -78,6 +78,8 @@ class Config(TypedDict): name_strategy_validation: bool master_election_strategy: str protobuf_runtime_directory: str + statsd_host: str + statsd_port: int sentry: NotRequired[Mapping[str, object]] tags: NotRequired[Mapping[str, object]] @@ -150,6 +152,8 @@ class ConfigDefaults(Config, total=False): "name_strategy_validation": True, "master_election_strategy": "lowest", "protobuf_runtime_directory": "runtime", + "statsd_host": "127.0.0.1", + "statsd_port": 8125, } SECRET_CONFIG_OPTIONS = [SASL_PLAIN_PASSWORD] diff --git a/karapace/statsd.py b/karapace/statsd.py index be37d92ef..3c32e09d8 100644 --- a/karapace/statsd.py +++ b/karapace/statsd.py @@ -20,19 +20,12 @@ import socket import time -STATSD_HOST: Final = "127.0.0.1" -STATSD_PORT: Final = 8125 LOG = logging.getLogger(__name__) class StatsClient: - def __init__( - self, - config: Config, - host: str = STATSD_HOST, - port: int = STATSD_PORT, - ) -> None: - self._dest_addr: Final = (host, port) + def __init__(self, config: Config) -> None: + self._dest_addr: Final = (config["statsd_host"], config["statsd_port"]) self._socket: Final = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self._tags: Final = config.get("tags", {}) self.sentry_client: Final = get_sentry_client(sentry_config=config.get("sentry", None)) From 7f5a4b70640cc4f5ee1900fde88394f39004bcfd Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Fri, 23 Aug 2024 10:49:34 +0200 Subject: [PATCH 2/3] metrics: added `statsd-exporter` and simple prometheus rules --- container/compose.yml | 26 ++++++++++++++++++++++- container/{ => prometheus}/prometheus.yml | 13 +++++++++++- container/prometheus/rules.yml | 20 +++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) rename container/{ => prometheus}/prometheus.yml (60%) create mode 100644 container/prometheus/rules.yml diff --git a/container/compose.yml b/container/compose.yml index 6e17cee5b..511d924e8 100644 --- a/container/compose.yml +++ b/container/compose.yml @@ -78,6 +78,8 @@ services: KARAPACE_TOPIC_NAME: _schemas KARAPACE_LOG_LEVEL: WARNING KARAPACE_COMPATIBILITY: FULL + KARAPACE_STATSD_HOST: statsd-exporter + KARAPACE_STATSD_PORT: 8125 karapace-rest: image: ghcr.io/aiven-open/karapace:develop @@ -102,10 +104,32 @@ services: KARAPACE_REGISTRY_PORT: 8081 KARAPACE_ADMIN_METADATA_MAX_AGE: 0 KARAPACE_LOG_LEVEL: WARNING + KARAPACE_STATSD_HOST: statsd-exporter + KARAPACE_STATSD_PORT: 8125 prometheus: image: prom/prometheus volumes: - - ./prometheus.yml:/etc/prometheus/prometheus.yml + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + - ./prometheus/rules.yml:/etc/prometheus/rules.yml ports: - 9090:9090 + + grafana: + image: grafana/grafana + environment: + GF_SECURITY_ADMIN_USER: karapace + GF_SECURITY_ADMIN_PASSWORD: karapace + GF_PATHS_PROVISIONING: /grafana/provisioning + ports: + - 3000:3000 + volumes: + - ./grafana/dashboards:/grafana/dashboards + - ./grafana/provisioning:/grafana/provisioning + + statsd-exporter: + image: prom/statsd-exporter + command: "--statsd.listen-udp=:8125 --web.listen-address=:9102" + ports: + - 9102:9102 + - 8125:8125/udp diff --git a/container/prometheus.yml b/container/prometheus/prometheus.yml similarity index 60% rename from container/prometheus.yml rename to container/prometheus/prometheus.yml index 20731bb4b..f62e8082a 100644 --- a/container/prometheus.yml +++ b/container/prometheus/prometheus.yml @@ -1,7 +1,10 @@ global: scrape_interval: 10s # How frequently to scrape targets by default. scrape_timeout: 5s # How long until a scrape request times out. - evaluation_interval: 60s # How frequently to evaluate rules. + evaluation_interval: 10s # How frequently to evaluate rules. + +rule_files: + - /etc/prometheus/rules.yml # A scrape configuration scrape_configs: @@ -16,3 +19,11 @@ scrape_configs: static_configs: - targets: - karapace-rest:8082 + + - job_name: statsd-exporter + metrics_path: /metrics + tls_config: + insecure_skip_verify: true + static_configs: + - targets: + - statsd-exporter:9102 diff --git a/container/prometheus/rules.yml b/container/prometheus/rules.yml new file mode 100644 index 000000000..9bdc83f0a --- /dev/null +++ b/container/prometheus/rules.yml @@ -0,0 +1,20 @@ +groups: + - name: karapace + rules: + - record: karapace_exceptions_sum_by_exception + expr: sum by (exception) (exception) + - alert: HighHTTPRequests + expr: karapace_http_requests_total > 10 + for: 5m + labels: + severity: warning + annotations: + summary: High HTTP requests for (instance={{ $labels.instance }}) + description: "Service received\n HTTP Requests = {{ $value }}\n" + - alert: FireImmidiately + expr: karapace_schema_reader_schemas > 1 + labels: + severity: page + annotations: + summary: Lots of schems on (instance={{ $labels.instance }}) + description: "\n Schema count = {{ $value }}\n" From 83241550297865252fadef77f788aa8d6666acfd Mon Sep 17 00:00:00 2001 From: Emmanuel Evbuomwan Date: Fri, 23 Aug 2024 10:50:44 +0200 Subject: [PATCH 3/3] visualisation: added simple `grafana` dashboard for karapace - we use prometheus as source - we also make use of simple [grafana provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) --- container/grafana/dashboards/karapace.json | 845 ++++++++++++++++++ .../provisioning/dashboards/default.yaml | 11 + .../provisioning/datasources/default.yaml | 6 + 3 files changed, 862 insertions(+) create mode 100644 container/grafana/dashboards/karapace.json create mode 100644 container/grafana/provisioning/dashboards/default.yaml create mode 100644 container/grafana/provisioning/datasources/default.yaml diff --git a/container/grafana/dashboards/karapace.json b/container/grafana/dashboards/karapace.json new file mode 100644 index 000000000..66db1d797 --- /dev/null +++ b/container/grafana/dashboards/karapace.json @@ -0,0 +1,845 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 9, + "panels": [], + "title": "Karapace Exceptions", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "exception", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{where}} - {{exception}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Karapace Exceptions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_exceptions_sum_by_exception", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "exception={{exception}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Count by Exception", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 8, + "panels": [], + "title": "Karapace HTTP API", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_http_requests_total", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "instance={{instance}} method={{method}} path={{path}} status={{status}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "HTTP Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "right", + "showLegend": true + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_http_requests_duration_seconds_bucket", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "instance={{instance}} method={{method}} path={{path}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "HTTP Requests Duration", + "type": "histogram" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 7, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_schema_reader_subjects", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Subjects Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 42 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_schema_reader_schemas", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Schemas Count", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 50 + }, + "id": 4, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_schema_reader_records_processed", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{keymode}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Records Processed by Keymode", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 50 + }, + "id": 2, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_schema_reader_records_per_keymode", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{keymode}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Records by Keymode", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 5, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "11.1.4", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "karapace_schema_reader_subject_data_schema_versions", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{state}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Schema reader schema versions by state", + "type": "gauge" + } + ], + "title": "Karapace Schema Reader", + "type": "row" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "kubernetes" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Karapace", + "uid": "bdvmm8aqj5tkwd", + "version": 1, + "weekStart": "" +} diff --git a/container/grafana/provisioning/dashboards/default.yaml b/container/grafana/provisioning/dashboards/default.yaml new file mode 100644 index 000000000..2db6363e9 --- /dev/null +++ b/container/grafana/provisioning/dashboards/default.yaml @@ -0,0 +1,11 @@ +apiVersion: 1 +providers: + - name: dashboards + folder: General + type: file + editable: true + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /grafana/dashboards + foldersFromFilesStructure: true diff --git a/container/grafana/provisioning/datasources/default.yaml b/container/grafana/provisioning/datasources/default.yaml new file mode 100644 index 000000000..52e30ce1c --- /dev/null +++ b/container/grafana/provisioning/datasources/default.yaml @@ -0,0 +1,6 @@ +apiVersion: 1 +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090