From 3180f07635dec798bfbff6cfb2fe092ee468bb09 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Thu, 16 Nov 2023 18:56:41 +0100 Subject: [PATCH] a lot of pedestrian code --- front/lib/core_api.ts | 20 +- .../[name]/databases/[id]/query.ts | 135 +++++++++++ .../[name]/databases/[id]/schema.ts | 108 +++++++++ .../[name]/databases/[id]/tables/[id].ts | 119 ++++++++++ .../[id]/{tables.ts => tables/index.ts} | 19 +- .../[name]/databases/[id]/tables/rows/[id].ts | 135 +++++++++++ .../databases/[id]/tables/rows/index.ts | 212 ++++++++++++++++++ 7 files changed, 731 insertions(+), 17 deletions(-) create mode 100644 front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/query.ts create mode 100644 front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/schema.ts create mode 100644 front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/[id].ts rename front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/{tables.ts => tables/index.ts} (91%) create mode 100644 front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/[id].ts create mode 100644 front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/index.ts diff --git a/front/lib/core_api.ts b/front/lib/core_api.ts index 701d13131eb2..6d19c14b750a 100644 --- a/front/lib/core_api.ts +++ b/front/lib/core_api.ts @@ -134,6 +134,14 @@ export type CoreAPIDatabaseRow = { content: Record; }; +export type CoreAPIDatabaseSchema = Record< + string, + { + table: CoreAPIDatabaseTable; + schema: Record; + } +>; + export const CoreAPI = { async createProject(): Promise> { const response = await fetch(`${CORE_API}/projects`, { @@ -1061,13 +1069,7 @@ export const CoreAPI = { databaseId: string; }): Promise< CoreAPIResponse<{ - schema: Record< - string, - { - table: CoreAPIDatabaseTable; - schema: Record; - } - >; + schema: CoreAPIDatabaseSchema; }> > { const response = await fetch( @@ -1092,8 +1094,8 @@ export const CoreAPI = { query: string; }): Promise< CoreAPIResponse<{ - schema: Record; - rows: Record[]; + schema: CoreAPIDatabaseSchema; + rows: CoreAPIDatabaseRow[]; }> > { const response = await fetch( diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/query.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/query.ts new file mode 100644 index 000000000000..336c02ccbb3f --- /dev/null +++ b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/query.ts @@ -0,0 +1,135 @@ +import { isLeft } from "fp-ts/lib/Either"; +import * as t from "io-ts"; +import * as reporter from "io-ts-reporters"; +import { NextApiRequest, NextApiResponse } from "next"; + +import { getDataSource } from "@app/lib/api/data_sources"; +import { Authenticator, getAPIKey } from "@app/lib/auth"; +import { + CoreAPI, + CoreAPIDatabaseRow, + CoreAPIDatabaseSchema, +} from "@app/lib/core_api"; +import { isDevelopmentOrDustWorkspace } from "@app/lib/development"; +import logger from "@app/logger/logger"; +import { apiError, withLogging } from "@app/logger/withlogging"; + +const GetDatabaseSchemaReqBodySchema = t.type({ + query: t.string, +}); + +type QueryDatabaseSchemaResponseBody = { + schema: CoreAPIDatabaseSchema; + rows: CoreAPIDatabaseRow[]; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse +): Promise { + const keyRes = await getAPIKey(req); + if (keyRes.isErr()) { + return apiError(req, res, keyRes.error); + } + + const { auth } = await Authenticator.fromKey( + keyRes.value, + req.query.wId as string + ); + + const owner = auth.workspace(); + const plan = auth.plan(); + if (!owner || !plan) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you requested was not found.", + }, + }); + } + + if (!isDevelopmentOrDustWorkspace(owner)) { + res.status(404).end(); + return; + } + + const dataSource = await getDataSource(auth, req.query.name as string); + if (!dataSource) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "data_source_not_found", + message: "The data source you requested was not found.", + }, + }); + } + + const databaseId = req.query.id; + if (!databaseId || typeof databaseId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "Invalid request query: id is required.", + }, + }); + } + + switch (req.method) { + case "POST": + const bodyValidation = GetDatabaseSchemaReqBodySchema.decode(req.body); + if (isLeft(bodyValidation)) { + const pathError = reporter.formatValidationErrors(bodyValidation.left); + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: `Invalid request body: ${pathError}`, + }, + }); + } + + const { query } = bodyValidation.right; + + const queryRes = await CoreAPI.queryDatabase({ + projectId: dataSource.dustAPIProjectId, + dataSourceName: dataSource.name, + databaseId, + query, + }); + if (queryRes.isErr()) { + logger.error( + { + dataSourceName: dataSource.name, + workspaceId: owner.id, + databaseId, + error: queryRes.error, + }, + "Failed to query database." + ); + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: "Failed to query database.", + }, + }); + } + + const { schema, rows } = queryRes.value; + + return res.status(200).json({ schema, rows }); + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withLogging(handler); diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/schema.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/schema.ts new file mode 100644 index 000000000000..dbbda793bee4 --- /dev/null +++ b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/schema.ts @@ -0,0 +1,108 @@ +import { NextApiRequest, NextApiResponse } from "next"; + +import { getDataSource } from "@app/lib/api/data_sources"; +import { Authenticator, getAPIKey } from "@app/lib/auth"; +import { CoreAPI, CoreAPIDatabaseSchema } from "@app/lib/core_api"; +import { isDevelopmentOrDustWorkspace } from "@app/lib/development"; +import logger from "@app/logger/logger"; +import { apiError, withLogging } from "@app/logger/withlogging"; + +type GetDatabaseSchemaResponseBody = { + schema: CoreAPIDatabaseSchema; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse +): Promise { + const keyRes = await getAPIKey(req); + if (keyRes.isErr()) { + return apiError(req, res, keyRes.error); + } + + const { auth } = await Authenticator.fromKey( + keyRes.value, + req.query.wId as string + ); + + const owner = auth.workspace(); + const plan = auth.plan(); + if (!owner || !plan) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you requested was not found.", + }, + }); + } + + if (!isDevelopmentOrDustWorkspace(owner)) { + res.status(404).end(); + return; + } + + const dataSource = await getDataSource(auth, req.query.name as string); + if (!dataSource) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "data_source_not_found", + message: "The data source you requested was not found.", + }, + }); + } + + const databaseId = req.query.id; + if (!databaseId || typeof databaseId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "Invalid request query: id is required.", + }, + }); + } + + switch (req.method) { + case "GET": + const schemaRes = await CoreAPI.getDatabaseSchema({ + projectId: dataSource.dustAPIProjectId, + dataSourceName: dataSource.name, + databaseId, + }); + if (schemaRes.isErr()) { + logger.error( + { + dataSourceName: dataSource.name, + workspaceId: owner.id, + databaseId, + error: schemaRes.error, + }, + "Failed to get database schema." + ); + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: "Failed to get database schema.", + }, + }); + } + + const { schema } = schemaRes.value; + + return res.status(200).json({ schema }); + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withLogging(handler); diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/[id].ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/[id].ts new file mode 100644 index 000000000000..4e42eee4a91b --- /dev/null +++ b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/[id].ts @@ -0,0 +1,119 @@ +import { NextApiRequest, NextApiResponse } from "next"; + +import { getDataSource } from "@app/lib/api/data_sources"; +import { Authenticator, getAPIKey } from "@app/lib/auth"; +import { CoreAPI, CoreAPIDatabaseTable } from "@app/lib/core_api"; +import { isDevelopmentOrDustWorkspace } from "@app/lib/development"; +import logger from "@app/logger/logger"; +import { apiError, withLogging } from "@app/logger/withlogging"; + +type GetDatabaseTableResponseBody = { + table: CoreAPIDatabaseTable; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse +): Promise { + const keyRes = await getAPIKey(req); + if (keyRes.isErr()) { + return apiError(req, res, keyRes.error); + } + + const { auth } = await Authenticator.fromKey( + keyRes.value, + req.query.wId as string + ); + + const owner = auth.workspace(); + const plan = auth.plan(); + if (!owner || !plan) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you requested was not found.", + }, + }); + } + + if (!isDevelopmentOrDustWorkspace(owner)) { + res.status(404).end(); + return; + } + + const dataSource = await getDataSource(auth, req.query.name as string); + if (!dataSource) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "data_source_not_found", + message: "The data source you requested was not found.", + }, + }); + } + + const databaseId = req.query.id; + if (!databaseId || typeof databaseId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The database id is missing.", + }, + }); + } + + const tableId = req.query.tableId; + if (!tableId || typeof tableId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The table id is missing.", + }, + }); + } + + switch (req.method) { + case "GET": + const tableRes = await CoreAPI.getDatabaseTable({ + projectId: dataSource.dustAPIProjectId, + dataSourceName: dataSource.name, + databaseId, + tableId, + }); + if (tableRes.isErr()) { + logger.error( + { + dataSourcename: dataSource.name, + workspaceId: owner.id, + error: tableRes.error, + }, + "Failed to get database table." + ); + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: "Failed to get database.", + }, + }); + } + + const { table } = tableRes.value; + + return res.status(200).json({ table }); + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withLogging(handler); diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/index.ts similarity index 91% rename from front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables.ts rename to front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/index.ts index 054ff79706f8..11193c845510 100644 --- a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables.ts +++ b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/index.ts @@ -11,7 +11,7 @@ import { generateModelSId } from "@app/lib/utils"; import logger from "@app/logger/logger"; import { apiError, withLogging } from "@app/logger/withlogging"; -type GetDatabaseTablesResponseBody = { +type ListDatabaseTablesResponseBody = { tables: CoreAPIDatabaseTable[]; }; @@ -27,7 +27,7 @@ type UpsertDatabaseTableResponseBody = { async function handler( req: NextApiRequest, res: NextApiResponse< - GetDatabaseTablesResponseBody | UpsertDatabaseTableResponseBody + ListDatabaseTablesResponseBody | UpsertDatabaseTableResponseBody > ): Promise { const keyRes = await getAPIKey(req); @@ -87,16 +87,19 @@ async function handler( databaseId, }); if (tablesRes.isErr()) { - logger.error({ - dataSourcename: dataSource.name, - workspaceId: owner.id, - error: tablesRes.error, - }); + logger.error( + { + dataSourcename: dataSource.name, + workspaceId: owner.id, + error: tablesRes.error, + }, + "Failed to get database tables." + ); return apiError(req, res, { status_code: 500, api_error: { type: "internal_server_error", - message: "Failed to get database.", + message: "Failed to get database tables.", }, }); } diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/[id].ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/[id].ts new file mode 100644 index 000000000000..70430c8cc0f6 --- /dev/null +++ b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/[id].ts @@ -0,0 +1,135 @@ +import { NextApiRequest, NextApiResponse } from "next"; + +import { getDataSource } from "@app/lib/api/data_sources"; +import { Authenticator, getAPIKey } from "@app/lib/auth"; +import { CoreAPI, CoreAPIDatabaseRow } from "@app/lib/core_api"; +import { isDevelopmentOrDustWorkspace } from "@app/lib/development"; +import logger from "@app/logger/logger"; +import { apiError, withLogging } from "@app/logger/withlogging"; + +type GetDatabaseRowsResponseBody = { + row: CoreAPIDatabaseRow; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse +): Promise { + const keyRes = await getAPIKey(req); + if (keyRes.isErr()) { + return apiError(req, res, keyRes.error); + } + + const { auth } = await Authenticator.fromKey( + keyRes.value, + req.query.wId as string + ); + + const owner = auth.workspace(); + const plan = auth.plan(); + if (!owner || !plan) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you requested was not found.", + }, + }); + } + + if (!isDevelopmentOrDustWorkspace(owner)) { + res.status(404).end(); + return; + } + + const dataSource = await getDataSource(auth, req.query.name as string); + if (!dataSource) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "data_source_not_found", + message: "The data source you requested was not found.", + }, + }); + } + + const databaseId = req.query.id; + if (!databaseId || typeof databaseId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The database id is missing.", + }, + }); + } + + const tableId = req.query.tableId; + if (!tableId || typeof tableId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The table id is missing.", + }, + }); + } + + const rowId = req.query.rowId; + if (!rowId || typeof rowId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The row id is missing.", + }, + }); + } + + switch (req.method) { + case "GET": + const rowRes = await CoreAPI.getDatabaseRow({ + projectId: dataSource.dustAPIProjectId, + dataSourceName: dataSource.name, + databaseId, + tableId, + rowId, + }); + + if (rowRes.isErr()) { + logger.error( + { + dataSourceName: dataSource.name, + workspaceId: owner.id, + databaseName: name, + databaseId: databaseId, + tableId: tableId, + error: rowRes.error, + }, + "Failed to get database row." + ); + + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: "Failed to get database row.", + }, + }); + } + + const { row } = rowRes.value; + return res.status(200).json({ row }); + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withLogging(handler); diff --git a/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/index.ts b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/index.ts new file mode 100644 index 000000000000..831019886c13 --- /dev/null +++ b/front/pages/api/v1/w/[wId]/data_sources/[name]/databases/[id]/tables/rows/index.ts @@ -0,0 +1,212 @@ +import { isLeft } from "fp-ts/lib/Either"; +import * as t from "io-ts"; +import * as reporter from "io-ts-reporters"; +import { NextApiRequest, NextApiResponse } from "next"; + +import { getDataSource } from "@app/lib/api/data_sources"; +import { Authenticator, getAPIKey } from "@app/lib/auth"; +import { CoreAPI, CoreAPIDatabaseRow } from "@app/lib/core_api"; +import { isDevelopmentOrDustWorkspace } from "@app/lib/development"; +import logger from "@app/logger/logger"; +import { apiError, withLogging } from "@app/logger/withlogging"; + +const UpsertDatabaseRowsRequestBodySchema = t.type({ + contents: t.record( + t.string, + t.record(t.string, t.union([t.string, t.null, t.number, t.boolean])) + ), + truncate: t.union([t.boolean, t.undefined]), +}); + +type UpsertDatabaseRowsResponseBody = { + success: true; +}; + +const ListDatabaseRowsReqQuerySchema = t.type({ + offset: t.number, + limit: t.number, +}); + +type ListDatabaseRowsResponseBody = { + rows: CoreAPIDatabaseRow[]; + offset: number; + limit: number; + total: number; +}; + +async function handler( + req: NextApiRequest, + res: NextApiResponse< + UpsertDatabaseRowsResponseBody | ListDatabaseRowsResponseBody + > +): Promise { + const keyRes = await getAPIKey(req); + if (keyRes.isErr()) { + return apiError(req, res, keyRes.error); + } + + const { auth } = await Authenticator.fromKey( + keyRes.value, + req.query.wId as string + ); + + const owner = auth.workspace(); + const plan = auth.plan(); + if (!owner || !plan) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you requested was not found.", + }, + }); + } + + if (!isDevelopmentOrDustWorkspace(owner)) { + res.status(404).end(); + return; + } + + const dataSource = await getDataSource(auth, req.query.name as string); + if (!dataSource) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "data_source_not_found", + message: "The data source you requested was not found.", + }, + }); + } + + const databaseId = req.query.id; + if (!databaseId || typeof databaseId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The database id is missing.", + }, + }); + } + + const tableId = req.query.tableId; + if (!tableId || typeof tableId !== "string") { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "The table id is missing.", + }, + }); + } + + switch (req.method) { + case "GET": + const queryValidation = ListDatabaseRowsReqQuerySchema.decode(req.query); + if (isLeft(queryValidation)) { + const pathError = reporter.formatValidationErrors(queryValidation.left); + return apiError(req, res, { + api_error: { + type: "invalid_request_error", + message: `Invalid request query: ${pathError}`, + }, + status_code: 400, + }); + } + const { offset, limit } = queryValidation.right; + + const listRes = await CoreAPI.getDatabaseRows({ + projectId: dataSource.dustAPIProjectId, + dataSourceName: dataSource.name, + databaseId, + tableId, + offset, + limit, + }); + + if (listRes.isErr()) { + logger.error( + { + dataSourceName: dataSource.name, + workspaceId: owner.id, + databaseName: name, + databaseId: databaseId, + tableId: tableId, + error: listRes.error, + }, + "Failed to list database rows." + ); + + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: "Failed to list database rows.", + }, + }); + } + + const { rows, total } = listRes.value; + return res.status(200).json({ rows, offset, limit, total }); + + case "POST": + const bodyValidation = UpsertDatabaseRowsRequestBodySchema.decode( + req.body + ); + if (isLeft(bodyValidation)) { + const pathError = reporter.formatValidationErrors(bodyValidation.left); + return apiError(req, res, { + api_error: { + type: "invalid_request_error", + message: `Invalid request body: ${pathError}`, + }, + status_code: 400, + }); + } + const { contents, truncate } = bodyValidation.right; + + const upsertRes = await CoreAPI.upsertDatabaseRows({ + projectId: dataSource.dustAPIProjectId, + dataSourceName: dataSource.name, + databaseId, + tableId: tableId, + contents, + truncate, + }); + + if (upsertRes.isErr()) { + logger.error( + { + dataSourceName: dataSource.name, + workspaceId: owner.id, + databaseName: name, + databaseId: databaseId, + tableId: tableId, + error: upsertRes.error, + }, + "Failed to upsert database rows." + ); + + return apiError(req, res, { + status_code: 500, + api_error: { + type: "internal_server_error", + message: "Failed to upsert database rows.", + }, + }); + } + + return res.status(200).json({ success: true }); + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET, POST is expected.", + }, + }); + } +} + +export default withLogging(handler);