diff --git a/diagrams/api-detection-screenshot.png b/diagrams/api-detection-screenshot.png index 106b29e..1fae719 100644 Binary files a/diagrams/api-detection-screenshot.png and b/diagrams/api-detection-screenshot.png differ diff --git a/services/clickhouse/docker-entrypoint-initdb.d/init-api-detection-db.sh b/services/clickhouse/docker-entrypoint-initdb.d/init-api-detection-db.sh index 93161ee..580c745 100644 --- a/services/clickhouse/docker-entrypoint-initdb.d/init-api-detection-db.sh +++ b/services/clickhouse/docker-entrypoint-initdb.d/init-api-detection-db.sh @@ -13,6 +13,12 @@ clickhouse client -nm <<-EOSQL accesses UInt64, auth_headers_seen UInt64, auth_errors UInt64, + sensitive_headers_seen UInt64, + sensitive_payload_seen UInt64, + ssn_seen UInt64, + dob_seen UInt64, + creditcard_seen UInt64, + email_seen UInt64, ) ENGINE = MergeTree ORDER BY (accesses, Timestamp) diff --git a/services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh b/services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh index 9341170..95ac2ca 100644 --- a/services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh +++ b/services/clickhouse/docker-entrypoint-initdb.d/init-otel-db.sh @@ -35,12 +35,15 @@ clickhouse client <<-EOSQL Timestamp DateTime, path String CODEC(ZSTD(1)), hasAuthorization Bool, + hasSensitiveHeaders Bool, + hasSensitivePayload Bool, httpv LowCardinality(String) CODEC(ZSTD(1)), reqCType LowCardinality(String) CODEC(ZSTD(1)), resCType LowCardinality(String) CODEC(ZSTD(1)), host LowCardinality(String) CODEC(ZSTD(1)), hostname LowCardinality(String) CODEC(ZSTD(1)), method LowCardinality(String) CODEC(ZSTD(1)), + sensitiveDataTypes LowCardinality(String) CODEC(ZSTD(1)), statusCode UInt16, ) ENGINE = MergeTree @@ -57,6 +60,9 @@ clickhouse client <<-EOSQL path(LogAttributes['uri']) AS path, LogAttributes['host'] AS host, LogAttributes['hasAuthorization'] AS hasAuthorization, + LogAttributes['sensitiveInHeaders'] AS hasSensitiveHeaders, + LogAttributes['sensitiveInPayload'] AS hasSensitivePayload, + LogAttributes['sensitiveDataTypes'] AS sensitiveDataTypes, LogAttributes['method'] AS method, LogAttributes['statusCode'] AS statusCode, LogAttributes['reqCType'] AS reqCType, diff --git a/services/clickhouse/schema/access_log.schema b/services/clickhouse/schema/access_log.schema deleted file mode 100644 index b8b7161..0000000 --- a/services/clickhouse/schema/access_log.schema +++ /dev/null @@ -1,44 +0,0 @@ -# These files do nothing, actual table creation in ../docker-entrypoint-initdb.d -CREATE TABLE otel.cbip_access_logs -( -`Timestamp` DateTime, -`TraceId` String CODEC(ZSTD(1)), -`SpanId` String CODEC(ZSTD(1)), -`TraceFlags` UInt8, -`SeverityText` LowCardinality(String) CODEC(ZSTD(1)), -`SeverityNumber` UInt8, -`ServiceName` LowCardinality(String) CODEC(ZSTD(1)), -`Body` String CODEC(ZSTD(1)), -`ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), -`ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), -`ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), -`ScopeName` String CODEC(ZSTD(1)), -`ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), -`ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), -`LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), -`cIP` String CODEC(ZSTD(1)), -`cPort` UInt16, -`uri` String CODEC(ZSTD(1)), -`path` String CODEC(ZSTD(1)), -`queryString` String CODEC(ZSTD(1)), -`host` LowCardinality(String) CODEC(ZSTD(1)), -`method` LowCardinality(String) CODEC(ZSTD(1)), -`reqLength` UInt32, -`statusCode`UInt16, -`resLength` UInt32, -`vs` LowCardinality(String) CODEC(ZSTD(1)), -`pool` LowCardinality(String) CODEC(ZSTD(1)), -`referrer` String CODEC(ZSTD(1)), -`cType` LowCardinality(String) CODEC(ZSTD(1)), -`hostname` LowCardinality(String) CODEC(ZSTD(1)), -`userAgent` LowCardinality(String) CODEC(ZSTD(1)), -`httpv` LowCardinality(String) CODEC(ZSTD(1)), -`vip` LowCardinality(String) CODEC(ZSTD(1)), -`clientsslprofile` LowCardinality(String) CODEC(ZSTD(1)), -`responseHeaders` String CODEC(ZSTD(1)), -`requestHeaders` String CODEC(ZSTD(1)), -`payload` String CODEC(ZSTD(1)) -) -ENGINE = MergeTree -ORDER BY (ServiceName, Timestamp) -TTL Timestamp + INTERVAL 6 HOUR \ No newline at end of file diff --git a/services/clickhouse/schema/api_endpoints.schema b/services/clickhouse/schema/api_endpoints.schema deleted file mode 100644 index eedf41e..0000000 --- a/services/clickhouse/schema/api_endpoints.schema +++ /dev/null @@ -1,15 +0,0 @@ -# These files do nothing, actual table creation in ../docker-entrypoint-initdb.d -CREATE TABLE api_detection.detected_api_endpoints -( -`Timestamp` DateTime, -`path` String CODEC(ZSTD(1)), -`host` LowCardinality(String) CODEC(ZSTD(1)), -`method` LowCardinality(String) CODEC(ZSTD(1)), -`isAuthenticated` LowCardinality(String) CODEC(ZSTD(1)), -`accesses` UInt64, -`auth_headers_seen` UInt64, -`auth_errors` UInt64, -) -ENGINE = MergeTree -ORDER BY (accesses, Timestamp) -TTL Timestamp + INTERVAL 6 HOUR diff --git a/services/clickhouse/schema/otel_null_log.schema b/services/clickhouse/schema/otel_null_log.schema deleted file mode 100644 index 2def3d8..0000000 --- a/services/clickhouse/schema/otel_null_log.schema +++ /dev/null @@ -1,20 +0,0 @@ -# These files do nothing, actual table creation in ../docker-entrypoint-initdb.d -CREATE TABLE otel.otel_logs_null -( - `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), - `TraceId` String CODEC(ZSTD(1)), - `SpanId` String CODEC(ZSTD(1)), - `TraceFlags` UInt8, - `SeverityText` LowCardinality(String) CODEC(ZSTD(1)), - `SeverityNumber` UInt8, - `ServiceName` LowCardinality(String) CODEC(ZSTD(1)), - `Body` String CODEC(ZSTD(1)), - `ResourceSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ResourceAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `ScopeSchemaUrl` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeName` String CODEC(ZSTD(1)), - `ScopeVersion` LowCardinality(String) CODEC(ZSTD(1)), - `ScopeAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), - `LogAttributes` Map(LowCardinality(String), String) CODEC(ZSTD(1)), -) -ENGINE = Null \ No newline at end of file diff --git a/services/clickhouse/schema/view.schema b/services/clickhouse/schema/view.schema deleted file mode 100644 index df3ec74..0000000 --- a/services/clickhouse/schema/view.schema +++ /dev/null @@ -1,40 +0,0 @@ -# These files do nothing, actual table creation in ../docker-entrypoint-initdb.d -CREATE MATERIALIZED VIEW otel.cbip_access_log_mv TO otel.cbip_access_logs AS -SELECT Timestamp::DateTime AS Timestamp, -TraceId, -SpanId, -TraceFlags, -SeverityText, -SeverityNumber, -ServiceName, -Body, -ResourceSchemaUrl, -ResourceAttributes, -ScopeSchemaUrl, -ScopeName, -ScopeVersion, -ScopeAttributes, -LogAttributes, -LogAttributes['hostname'] AS `hostname`, -LogAttributes['cIP'] AS `cIP`, -LogAttributes['cPort'] AS `cPort`, -LogAttributes['uri'] AS `uri`, -path(LogAttributes['uri']) AS `path`, -queryString(LogAttributes['uri']) AS `queryString`, -LogAttributes['host'] AS `host`, -LogAttributes['method'] AS `method`, -LogAttributes['reqLength'] AS `reqLength`, -LogAttributes['statusCode'] AS `statusCode`, -LogAttributes['resLength'] AS `resLength`, -LogAttributes['vs'] AS `vs`, -LogAttributes['pool'] AS `pool`, -LogAttributes['referrer'] AS `referrer`, -LogAttributes['cType'] AS `cType`, -LogAttributes['userAgent'] AS `userAgent`, -LogAttributes['httpv'] AS `httpv`, -LogAttributes['vip'] AS `vip`, -LogAttributes['clientsslprofile'] AS `clientsslprofile`, -LogAttributes['requestHeaders'] AS `requestHeaders`, -LogAttributes['responseHeaders'] AS `responseHeaders`, -LogAttributes['payload'] AS `payload` -FROM otel.otel_logs_null; \ No newline at end of file diff --git a/services/grafana/provisioning/dashboards/api-discovery/api-detection.json b/services/grafana/provisioning/dashboards/api-discovery/api-detection.json index ed47565..eb1b40f 100644 --- a/services/grafana/provisioning/dashboards/api-discovery/api-detection.json +++ b/services/grafana/provisioning/dashboards/api-discovery/api-detection.json @@ -1,67 +1,4 @@ { - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "description": "", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - }, - { - "name": "DS_CLICKHOUSE", - "label": "ClickHouse", - "description": "", - "type": "datasource", - "pluginId": "grafana-clickhouse-datasource", - "pluginName": "ClickHouse" - } - ], - "__elements": {}, - "__requires": [ - { - "type": "grafana", - "id": "grafana", - "name": "Grafana", - "version": "11.2.0" - }, - { - "type": "datasource", - "id": "grafana-clickhouse-datasource", - "name": "ClickHouse", - "version": "4.4.0" - }, - { - "type": "panel", - "id": "piechart", - "name": "Pie chart", - "version": "" - }, - { - "type": "datasource", - "id": "prometheus", - "name": "Prometheus", - "version": "1.0.0" - }, - { - "type": "panel", - "id": "stat", - "name": "Stat", - "version": "" - }, - { - "type": "panel", - "id": "table", - "name": "Table", - "version": "" - }, - { - "type": "panel", - "id": "timeseries", - "name": "Time series", - "version": "" - } - ], "annotations": { "list": [ { @@ -81,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": null, + "id": 3, "links": [], "panels": [ { @@ -689,8 +626,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -915,8 +851,7 @@ "mode": "absolute", "steps": [ { - "color": "blue", - "value": null + "color": "blue" } ] }, @@ -1014,8 +949,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1117,8 +1051,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1221,8 +1154,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1355,8 +1287,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -1461,7 +1392,7 @@ }, { "datasource": { - "name": "${DS_CLICKHOUSE}", + "default": false, "type": "grafana-clickhouse-datasource", "uid": "${DS_CLICKHOUSE}" }, @@ -1550,6 +1481,30 @@ } } ] + }, + { + "matcher": { + "id": "byName", + "options": "Path" + }, + "properties": [ + { + "id": "custom.width", + "value": 520 + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.* Present/" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + } + ] } ] }, @@ -1708,9 +1663,9 @@ "table": "detected_api_endpoints" } }, - "pluginVersion": "4.4.0", + "pluginVersion": "4.5.0", "queryType": "table", - "rawSql": "SELECT path, host, method, isAuthenticated, accesses FROM \"api_detection\".\"detected_api_endpoints\" t1 WHERE Timestamp = (SELECT MAX(Timestamp) from \"api_detection\".\"detected_api_endpoints\" t2) AND host LIKE '${host}' AND startsWith(path, '$path_prefix' ) ORDER BY accesses DESC LIMIT 1000", + "rawSql": "SELECT\n path,\n host,\n method,\n isAuthenticated,\n sensitive_headers_seen / accesses as sensitive_headers_seen,\n sensitive_payload_seen / accesses as sensitive_payload_seen,\n -- ssn_seen / accesses as ssn_seen,\n -- creditcard_seen / accesses as creditcard_seen,\n -- dob_seen / accesses as dob_seen,\n -- email_seen / accesses as email_seen,\n accesses,\n arrayStringConcat(\n arrayFilter(\n\n x -> x IS NOT NULL,\n [\n (if(ssn_seen > 0, 'SSN', null)),\n (if(dob_seen > 0, 'DOB', null)),\n (if(creditcard_seen > 0, 'Credit Card', null)),\n (if(email_seen > 0, 'Email', null))\n ]\n ),\n ', '\n ) AS seen_data_types\nFROM \"api_detection\".\"detected_api_endpoints\" t1 WHERE Timestamp = (SELECT MAX(Timestamp) from \"api_detection\".\"detected_api_endpoints\" t2) AND host LIKE '${host}' AND startsWith(path, '$path_prefix' ) AND accesses > 0 ORDER BY accesses DESC LIMIT 1000", "refId": "A" } ], @@ -1722,18 +1677,29 @@ "excludeByName": {}, "includeByName": {}, "indexByName": { - "accesses": 4, + "accesses": 7, "host": 0, "isAuthenticated": 3, "method": 1, - "path": 2 + "path": 2, + "seen_data_types": 6, + "sensitive_headers_seen": 4, + "sensitive_payload_seen": 5 }, "renameByName": { "accesses": "Requests", + "creditcard_seen": "Creditcard Present", + "dob_seen": "DOB Present", + "email_seen": "Email Present", "host": "Host", "isAuthenticated": "Authentication Detected", "method": "Method", - "path": "Path" + "path": "Path", + "seen_data_types": "Sensitive Data Types", + "sensitive_headers_seen": "Sensitive Headers Present", + "sensitive_payload_seen": "Sensitive Body Present", + "sensitive_seen": "Sensitive Data Present", + "ssn_seen": "SSN Present" } } } @@ -1747,7 +1713,11 @@ "templating": { "list": [ { - "current": {}, + "current": { + "selected": false, + "text": "ClickHouse", + "value": "PDEE91DDB90597936" + }, "hide": 0, "includeAll": false, "label": "Clickhouse", @@ -1761,7 +1731,11 @@ "type": "datasource" }, { - "current": {}, + "current": { + "selected": false, + "text": "Prometheus", + "value": "PBFA97CFB590B2093" + }, "hide": 0, "includeAll": false, "label": "Prometheus", @@ -1775,7 +1749,11 @@ "type": "datasource" }, { - "current": {}, + "current": { + "selected": false, + "text": "nginx.f5kc.com", + "value": "nginx.f5kc.com" + }, "datasource": { "type": "grafana-clickhouse-datasource", "uid": "${DS_CLICKHOUSE}" @@ -1824,6 +1802,6 @@ "timezone": "browser", "title": "API Detection", "uid": "bdy1jayi74npce", - "version": 40, + "version": 5, "weekStart": "" } \ No newline at end of file