From ba630d898544ba1ed654328d42915923c35a1bcf Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Wed, 27 Nov 2024 15:25:37 +0100 Subject: [PATCH] feat(kw-search): allow passible title and mimetype on front endpoints when upserting tables --- front/lib/api/data_sources.ts | 12 +++++++ front/lib/api/files/upsert.ts | 2 ++ front/lib/api/tables.ts | 6 ++++ front/lib/upsert_queue.ts | 34 +++++++++++-------- .../data_sources/[dsId]/tables/index.ts | 15 ++++++++ front/temporal/upsert_tables/activities.ts | 2 ++ sdks/js/src/types.ts | 6 ++++ .../front/api_handlers/public/data_sources.ts | 32 +++++++++++------ types/src/front/lib/core_api.ts | 6 ++++ 9 files changed, 90 insertions(+), 25 deletions(-) diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts index 1094aaa365d2..eed39de91033 100644 --- a/front/lib/api/data_sources.ts +++ b/front/lib/api/data_sources.ts @@ -416,6 +416,8 @@ export async function upsertTable({ dataSource, auth, useAppForHeaderDetection, + title, + mimeType, }: { tableId?: string | null; name: string; @@ -429,6 +431,8 @@ export async function upsertTable({ dataSource: DataSourceResource; auth: Authenticator; useAppForHeaderDetection?: boolean; + title?: string; + mimeType?: string; }) { const nonNullTableId = tableId ?? generateRandomModelSId(); const tableParents: string[] = parents ?? []; @@ -476,6 +480,8 @@ export async function upsertTable({ truncate, useAppForHeaderDetection: useApp, detectedHeaders, + title, + mimeType, }, }); if (enqueueRes.isErr()) { @@ -497,6 +503,8 @@ export async function upsertTable({ csv: csv ?? null, truncate, useAppForHeaderDetection: useApp, + title, + mimeType, }); return tableRes; @@ -644,6 +652,8 @@ export async function handleDataSourceTableCSVUpsert({ truncate, useAppForHeaderDetection, detectedHeaders, + title: params.title, + mimeType: params.mimeType, }, }); if (enqueueRes.isErr()) { @@ -673,6 +683,8 @@ export async function handleDataSourceTableCSVUpsert({ csv: csv ?? null, truncate, useAppForHeaderDetection, + title: params.title, + mimeType: params.mimeType, }); if (tableRes.isErr()) { diff --git a/front/lib/api/files/upsert.ts b/front/lib/api/files/upsert.ts index e0661374213e..4679fc21482f 100644 --- a/front/lib/api/files/upsert.ts +++ b/front/lib/api/files/upsert.ts @@ -239,6 +239,8 @@ const upsertTableToDatasource: ProcessingFunction = async ({ dataSource, auth, useAppForHeaderDetection: true, + title: file.fileName, + mimeType: file.contentType, }); if (upsertTableRes.isErr()) { diff --git a/front/lib/api/tables.ts b/front/lib/api/tables.ts index 0f4c58b3b3a8..d28395c00f37 100644 --- a/front/lib/api/tables.ts +++ b/front/lib/api/tables.ts @@ -133,6 +133,8 @@ export async function upsertTableFromCsv({ truncate, useAppForHeaderDetection, detectedHeaders, + title, + mimeType, }: { auth: Authenticator; dataSource: DataSourceResource; @@ -146,6 +148,8 @@ export async function upsertTableFromCsv({ truncate: boolean; useAppForHeaderDetection: boolean; detectedHeaders?: DetectedHeadersType; + title?: string; + mimeType?: string; }): Promise> { const csvRowsRes = csv ? await rowsFromCsv({ @@ -232,6 +236,8 @@ export async function upsertTableFromCsv({ timestamp: tableTimestamp, tags: tableTags, parents: tableParents, + title, + mimeType, }); if (tableRes.isErr()) { diff --git a/front/lib/upsert_queue.ts b/front/lib/upsert_queue.ts index 090280283d2e..06cc6ada34de 100644 --- a/front/lib/upsert_queue.ts +++ b/front/lib/upsert_queue.ts @@ -44,20 +44,26 @@ const DetectedHeaders = t.type({ rowIndex: t.number, }); -export const EnqueueUpsertTable = t.type({ - workspaceId: t.string, - dataSourceId: t.string, - tableId: t.string, - tableName: t.string, - tableDescription: t.string, - tableTimestamp: t.union([t.number, t.undefined, t.null]), - tableTags: t.union([t.array(t.string), t.undefined, t.null]), - tableParents: t.union([t.array(t.string), t.undefined, t.null]), - csv: t.union([t.string, t.null]), - truncate: t.boolean, - useAppForHeaderDetection: t.union([t.boolean, t.undefined, t.null]), - detectedHeaders: t.union([DetectedHeaders, t.undefined]), -}); +export const EnqueueUpsertTable = t.intersection([ + t.type({ + workspaceId: t.string, + dataSourceId: t.string, + tableId: t.string, + tableName: t.string, + tableDescription: t.string, + tableTimestamp: t.union([t.number, t.undefined, t.null]), + tableTags: t.union([t.array(t.string), t.undefined, t.null]), + tableParents: t.union([t.array(t.string), t.undefined, t.null]), + csv: t.union([t.string, t.null]), + truncate: t.boolean, + useAppForHeaderDetection: t.union([t.boolean, t.undefined, t.null]), + detectedHeaders: t.union([DetectedHeaders, t.undefined]), + }), + t.partial({ + title: t.string, + mimeType: t.string, + }), +]); type EnqueueUpsertDocumentType = t.TypeOf; diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts index 0d3c4c88e612..9f1a288a4ce2 100644 --- a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts +++ b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts @@ -252,8 +252,21 @@ async function handler( parents, remote_database_table_id: remoteDatabaseTableId, remote_database_secret_id: remoteDatabaseSecretId, + title, + mimeType, } = r.data; + // If the request is not from a system key, we enforce that mimeType is not provided. + if (!auth.isSystemKey() && mimeType) { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "Invalid request body: mimeType must not be provided.", + }, + }); + } + const tableId = maybeTableId || generateRandomModelSId(); const tRes = await coreAPI.getTables({ @@ -302,6 +315,8 @@ async function handler( parents: parents || [], remoteDatabaseTableId: remoteDatabaseTableId ?? null, remoteDatabaseSecretId: remoteDatabaseSecretId ?? null, + title, + mimeType, }); if (upsertRes.isErr()) { diff --git a/front/temporal/upsert_tables/activities.ts b/front/temporal/upsert_tables/activities.ts index 876888141872..72d8d1fb229f 100644 --- a/front/temporal/upsert_tables/activities.ts +++ b/front/temporal/upsert_tables/activities.ts @@ -89,6 +89,8 @@ export async function upsertTableActivity( truncate: upsertQueueItem.truncate, useAppForHeaderDetection: upsertQueueItem.useAppForHeaderDetection ?? false, detectedHeaders: upsertQueueItem.detectedHeaders, + title: upsertQueueItem.title, + mimeType: upsertQueueItem.mimeType, }); if (tableRes.isErr()) { diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 8e624928a883..62e7e3e3d257 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -1981,6 +1981,8 @@ export const UpsertTableFromCsvRequestSchema = z.intersection( truncate: z.boolean(), useAppForHeaderDetection: z.boolean().nullable().optional(), async: z.boolean().optional(), + title: z.string().optional(), + mimeType: z.string().optional(), }) .transform((o) => ({ name: o.name, @@ -1991,6 +1993,8 @@ export const UpsertTableFromCsvRequestSchema = z.intersection( truncate: o.truncate, useAppForHeaderDetection: o.useAppForHeaderDetection, async: o.async, + title: o.title, + mimeType: o.mimeType, })), z.union([ z.object({ csv: z.string(), tableId: z.undefined() }).transform((o) => ({ @@ -2039,6 +2043,8 @@ export const UpsertDatabaseTableRequestSchema = z.object({ parents: z.array(z.string()).nullable().optional(), remote_database_table_id: z.string().nullable().optional(), remote_database_secret_id: z.string().nullable().optional(), + title: z.string().optional(), + mimeType: z.string().optional(), }); const UpsertTableResponseSchema = z.object({ diff --git a/types/src/front/api_handlers/public/data_sources.ts b/types/src/front/api_handlers/public/data_sources.ts index 53c6bebbcd9f..5667f9d0cb35 100644 --- a/types/src/front/api_handlers/public/data_sources.ts +++ b/types/src/front/api_handlers/public/data_sources.ts @@ -59,17 +59,23 @@ export type PatchDataSourceWithNameDocumentRequestBody = t.TypeOf< typeof PostDataSourceWithNameDocumentRequestBodySchema >; -export const PatchDataSourceTableRequestBodySchema = t.type({ - name: t.string, - description: t.string, - timestamp: t.union([t.number, t.undefined, t.null]), - tags: t.union([t.array(t.string), t.undefined, t.null]), - parents: t.union([t.array(t.string), t.undefined, t.null]), - truncate: t.boolean, - async: t.union([t.boolean, t.undefined]), - csv: t.union([t.string, t.undefined]), - useAppForHeaderDetection: t.union([t.boolean, t.undefined]), -}); +export const PatchDataSourceTableRequestBodySchema = t.intersection([ + t.type({ + name: t.string, + description: t.string, + timestamp: t.union([t.number, t.undefined, t.null]), + tags: t.union([t.array(t.string), t.undefined, t.null]), + parents: t.union([t.array(t.string), t.undefined, t.null]), + truncate: t.boolean, + async: t.union([t.boolean, t.undefined]), + csv: t.union([t.string, t.undefined]), + useAppForHeaderDetection: t.union([t.boolean, t.undefined]), + }), + t.partial({ + title: t.string, + mimeType: t.string, + }), +]); export const PostDataSourceTableRequestBodySchema = t.intersection([ PatchDataSourceTableRequestBodySchema, @@ -97,6 +103,10 @@ export const UpsertTableFromCsvRequestSchema = t.intersection([ tableId: t.string, }), ]), + t.partial({ + title: t.string, + mimeType: t.string, + }), ]); export type UpsertTableFromCsvRequestType = t.TypeOf< diff --git a/types/src/front/lib/core_api.ts b/types/src/front/lib/core_api.ts index 06b031d74001..33c6099aa9d7 100644 --- a/types/src/front/lib/core_api.ts +++ b/types/src/front/lib/core_api.ts @@ -1087,6 +1087,8 @@ export class CoreAPI { parents, remoteDatabaseTableId, remoteDatabaseSecretId, + title, + mimeType, }: { projectId: string; dataSourceId: string; @@ -1098,6 +1100,8 @@ export class CoreAPI { parents: string[]; remoteDatabaseTableId?: string | null; remoteDatabaseSecretId?: string | null; + title?: string; + mimeType?: string; }): Promise> { const response = await this._fetchWithError( `${this._url}/projects/${encodeURIComponent( @@ -1117,6 +1121,8 @@ export class CoreAPI { parents, remote_database_table_id: remoteDatabaseTableId ?? null, remote_database_secret_id: remoteDatabaseSecretId ?? null, + title, + mime_type: mimeType, }), } );