chore: rip out tinybird from the API

unkeyed · Nov 12, 2024 · 92dbb53 · 92dbb53
1 parent 0e08b26
commit 92dbb53
Show file tree

Hide file tree

Showing 188 changed files with 221 additions and 4,104 deletions.
diff --git a/apps/api/package.json b/apps/api/package.json
@@ -22,7 +22,6 @@
   },
   "dependencies": {
     "@axiomhq/js": "1.0.0-rc.2",
-    "@chronark/zod-bird": "0.3.9",
     "@hono/zod-openapi": "^0.11.0",
     "@hono/zod-validator": "^0.2.1",
     "@planetscale/database": "^1.16.0",

diff --git a/apps/api/src/pkg/analytics.ts b/apps/api/src/pkg/analytics.ts
@@ -1,44 +1,13 @@
-import { NoopTinybird, Tinybird } from "@chronark/zod-bird";
 import { ClickHouse } from "@unkey/clickhouse";
-import { newId } from "@unkey/id";
-import { auditLogSchemaV1, unkeyAuditLogEvents } from "@unkey/schema/src/auditlog";
-import { ratelimitSchemaV1 } from "@unkey/schema/src/ratelimit-tinybird";
 import { z } from "zod";
-import type { MaybeArray } from "./types/maybe";
-// const datetimeToUnixMilli = z.string().transform((t) => new Date(t).getTime());
-
-/**
- * `t` has the format `2021-01-01 00:00:00`
- *
- * If we transform it as is, we get `1609459200000` which is `2021-01-01 01:00:00` due to fun timezone stuff.
- * So we split the string at the space and take the date part, and then parse that.
- */
-const dateToUnixMilli = z.string().transform((t) => new Date(t.split(" ").at(0) ?? t).getTime());
 
 export class Analytics {
-  public readonly readClient: Tinybird | NoopTinybird;
-  public readonly writeClient: Tinybird | NoopTinybird;
-  private clickhouse: ClickHouse;
+  private readonly clickhouse: ClickHouse;
 
   constructor(opts: {
-    tinybirdToken?: string;
-    tinybirdProxy?: {
-      url: string;
-      token: string;
-    };
-    clickhouse?: {
-      url: string;
-    };
+    clickhouseUrl: string;
   }) {
-    this.readClient = opts.tinybirdToken
-      ? new Tinybird({ token: opts.tinybirdToken })
-      : new NoopTinybird();
-
-    this.writeClient = opts.tinybirdProxy
-      ? new Tinybird({ token: opts.tinybirdProxy.token, baseUrl: opts.tinybirdProxy.url })
-      : this.readClient;
-
-    this.clickhouse = new ClickHouse({ url: opts.clickhouse?.url });
+    this.clickhouse = new ClickHouse({ url: opts.clickhouseUrl });
   }
 
   public get insertSdkTelemetry() {
@@ -53,218 +22,20 @@ export class Analytics {
       }),
     });
   }
-  //tinybird, to be removed
-  public get ingestSdkTelemetry() {
-    return this.writeClient.buildIngestEndpoint({
-      datasource: "sdk_telemetry__v1",
-      event: z.object({
-        runtime: z.string(),
-        platform: z.string(),
-        versions: z.array(z.string()),
-        requestId: z.string(),
-        time: z.number(),
-      }),
-    });
-  }
-
-  //tinybird
-  public ingestUnkeyAuditLogsTinybird(logs: MaybeArray<UnkeyAuditLog>) {
-    return this.writeClient.buildIngestEndpoint({
-      datasource: "audit_logs__v2",
-      event: auditLogSchemaV1
-        .merge(
-          z.object({
-            event: unkeyAuditLogEvents,
-            auditLogId: z.string().default(newId("auditLog")),
-            bucket: z.string().default("unkey_mutations"),
-            time: z.number().default(Date.now()),
-          }),
-        )
-        .transform((l) => ({
-          ...l,
-          meta: l.meta ? JSON.stringify(l.meta) : undefined,
-          actor: {
-            ...l.actor,
-            meta: l.actor.meta ? JSON.stringify(l.actor.meta) : undefined,
-          },
-          resources: JSON.stringify(l.resources),
-        })),
-    })(logs);
-  }
 
-  //tinybird
-  public get ingestGenericAuditLogsTinybird() {
-    return this.writeClient.buildIngestEndpoint({
-      datasource: "audit_logs__v2",
-      event: auditLogSchemaV1.transform((l) => ({
-        ...l,
-        meta: l.meta ? JSON.stringify(l.meta) : undefined,
-        actor: {
-          ...l.actor,
-          meta: l.actor.meta ? JSON.stringify(l.actor.meta) : undefined,
-        },
-        resources: JSON.stringify(l.resources),
-      })),
-    });
-  }
   public get insertRatelimit() {
-    return this.clickhouse.client.insert({
-      table: "ratelimits.raw_ratelimits_v1",
-      schema: z.object({
-        request_id: z.string(),
-        time: z.number().int(),
-        workspace_id: z.string(),
-        namespace_id: z.string(),
-        identifier: z.string(),
-        passed: z.boolean(),
-      }),
-    });
-  }
-
-  //tinybird
-  public get ingestRatelimit() {
-    return this.writeClient.buildIngestEndpoint({
-      datasource: "ratelimits__v2",
-      event: ratelimitSchemaV1,
-    });
+    return this.clickhouse.ratelimits.insert;
   }
 
   public get insertKeyVerification() {
-    return this.clickhouse.client.insert({
-      table: "verifications.raw_key_verifications_v1",
-      schema: z.object({
-        request_id: z.string(),
-        time: z.number().int(),
-        workspace_id: z.string(),
-        key_space_id: z.string(),
-        key_id: z.string(),
-        region: z.string(),
-        outcome: z.enum([
-          "VALID",
-          "RATE_LIMITED",
-          "EXPIRED",
-          "DISABLED",
-          "FORBIDDEN",
-          "USAGE_EXCEEDED",
-          "INSUFFICIENT_PERMISSIONS",
-        ]),
-        identity_id: z.string().optional().default(""),
-      }),
-    });
+    return this.clickhouse.verifications.insert;
   }
 
   public get insertApiRequest() {
-    return this.clickhouse.client.insert({
-      table: "metrics.raw_api_requests_v1",
-      schema: z.object({
-        request_id: z.string(),
-        time: z.number().int(),
-        workspace_id: z.string(),
-        host: z.string(),
-        method: z.string(),
-        path: z.string(),
-        request_headers: z.array(z.string()),
-        request_body: z.string(),
-        response_status: z.number().int(),
-        response_headers: z.array(z.string()),
-        response_body: z.string(),
-        error: z.string().optional().default(""),
-        service_latency: z.number().int(),
-        user_agent: z.string(),
-        ip_address: z.string(),
-        continent: z.string().nullable().default(""),
-        city: z.string().nullable().default(""),
-        country: z.string().nullable().default(""),
-        colo: z.string().nullable().default(""),
-      }),
-    });
-  }
-  // replaced by insertKeyVerification
-  public get ingestKeyVerification() {
-    return this.writeClient.buildIngestEndpoint({
-      datasource: "key_verifications__v2",
-      event: z.object({
-        workspaceId: z.string(),
-        apiId: z.string(),
-        keyId: z.string(),
-        deniedReason: z
-          .enum([
-            "RATE_LIMITED",
-            "USAGE_EXCEEDED",
-            "FORBIDDEN",
-            "UNAUTHORIZED",
-            "DISABLED",
-            "INSUFFICIENT_PERMISSIONS",
-            "EXPIRED",
-          ])
-          .optional(),
-        time: z.number(),
-        ipAddress: z.string().default(""),
-        userAgent: z.string().default(""),
-        requestedResource: z.string().default(""),
-        edgeRegion: z.string().default(""),
-        region: z.string(),
-        // deprecated, use deniedReason
-        ratelimited: z.boolean().default(false),
-        // deprecated, use deniedReason
-        usageExceeded: z.boolean().default(false),
-        ownerId: z.string().optional(),
-        keySpaceId: z.string(),
-        requestId: z.string().optional(),
-        requestBody: z.string().optional(),
-        responseBody: z.string().optional(),
-      }),
-    });
+    return this.clickhouse.api.insert;
   }
 
   public get getVerificationsDaily() {
-    return this.readClient.buildPipe({
-      pipe: "get_verifications_daily__v2",
-      parameters: z.object({
-        workspaceId: z.string(),
-        apiId: z.string(),
-        keyId: z.string().optional(),
-        start: z.number().optional(),
-        end: z.number().optional(),
-      }),
-      data: z.object({
-        time: dateToUnixMilli,
-        success: z.number(),
-        rateLimited: z.number(),
-        usageExceeded: z.number(),
-      }),
-    });
+    return this.clickhouse.verifications.perDay;
   }
 }
-
-export type UnkeyAuditLog = {
-  workspaceId: string;
-  event: z.infer<typeof unkeyAuditLogEvents>;
-  description: string;
-  actor: {
-    type: "user" | "key";
-    name?: string;
-    id: string;
-  };
-  resources: Array<{
-    type:
-      | "key"
-      | "api"
-      | "workspace"
-      | "role"
-      | "permission"
-      | "keyAuth"
-      | "vercelBinding"
-      | "vercelIntegration"
-      | "ratelimitIdentifier"
-      | "ratelimitNamespace"
-      | "identity"
-      | "ratelimit";
-    id: string;
-    meta?: Record<string, string | number | boolean | null | undefined>;
-  }>;
-  context: {
-    userAgent?: string;
-    location: string;
-  };
-};
diff --git a/apps/api/src/pkg/audit.ts b/apps/api/src/pkg/audit.ts
@@ -4,7 +4,16 @@ import { type Transaction, schema } from "./db";
 
 import { newId } from "@unkey/id";
 import { z } from "zod";
-import type { UnkeyAuditLog } from "./analytics";
+
+const unkeyAuditLogSchema = auditLogSchemaV1.merge(
+  z.object({
+    event: unkeyAuditLogEvents,
+    auditLogId: z.string().default(newId("auditLog")),
+    bucket: z.string().default("unkey_mutations"),
+    time: z.number().default(Date.now()),
+  }),
+);
+export type UnkeyAuditLog = z.input<typeof unkeyAuditLogSchema>;
 
 export async function insertUnkeyAuditLog(
   c: Context,

diff --git a/apps/api/src/pkg/auth/root_key.ts b/apps/api/src/pkg/auth/root_key.ts
@@ -53,22 +53,15 @@ export async function rootKeyAuth(c: Context<HonoEnv>, permissionQuery?: Permiss
   // otherwise, they likely sent garbage to us and we can't associate it with anything
 
   c.executionCtx.waitUntil(
-    analytics.ingestKeyVerification({
-      workspaceId: rootKey.key.workspaceId,
-      apiId: rootKey.api.id,
-      keyId: rootKey.key.id,
+    analytics.insertKeyVerification({
+      workspace_id: rootKey.key.workspaceId,
+      key_id: rootKey.key.id,
       time: Date.now(),
-      deniedReason: rootKey.code,
-      ipAddress: c.req.header("True-Client-IP") ?? c.req.header("CF-Connecting-IP"),
-      userAgent: c.req.header("User-Agent"),
-      requestedResource: "",
+      outcome: rootKey.code ?? "VALID",
+      key_space_id: rootKey.key.keyAuthId,
       // @ts-expect-error - the cf object will be there on cloudflare
-      region: c.req.raw?.cf?.country ?? "",
-      ownerId: rootKey.key.ownerId ?? undefined,
-      // @ts-expect-error - the cf object will be there on cloudflare
-      edgeRegion: c.req.raw?.cf?.colo ?? "",
-      keySpaceId: rootKey.key.keyAuthId,
-      requestId: c.get("requestId"),
+      region: c.req.cf?.region,
+      request_id: c.get("requestId"),
     }),
   );
 

diff --git a/apps/api/src/pkg/cache/index.ts b/apps/api/src/pkg/cache/index.ts
@@ -35,7 +35,7 @@ export function initCache(c: Context<HonoEnv>, metrics: Metrics): C<CacheNamespa
           cloudflareApiKey: c.env.CLOUDFLARE_API_KEY,
           zoneId: c.env.CLOUDFLARE_ZONE_ID,
           domain: "cache.unkey.dev",
-          cacheBuster: "v6",
+          cacheBuster: "v7",
         })
       : undefined;
 

diff --git a/apps/api/src/pkg/cache/namespaces.ts b/apps/api/src/pkg/cache/namespaces.ts
@@ -44,9 +44,8 @@ export type CacheNamespaces = {
   }[];
   verificationsByKeyId: {
     time: number;
-    success: number;
-    rateLimited: number;
-    usageExceeded: number;
+    count: number;
+    outcome: string;
   }[];
   ratelimitByIdentifier: {
     namespace: Pick<RatelimitNamespace, "id" | "workspaceId">;

diff --git a/apps/api/src/pkg/env.ts b/apps/api/src/pkg/env.ts
@@ -33,7 +33,7 @@ export const zEnv = z.object({
   AGENT_URL: z.string().url(),
   AGENT_TOKEN: z.string(),
 
-  CLICKHOUSE_URL: z.string().optional(),
+  CLICKHOUSE_URL: z.string(),
 
   SYNC_RATELIMIT_ON_NO_DATA: z
     .string()