diff --git a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts index 41ac9919eacd..6cf4b0e07c95 100644 --- a/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts +++ b/connectors/migrations/20241219_backfill_intercom_data_source_folders.ts @@ -1,4 +1,4 @@ -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { makeScript } from "scripts/helpers"; import { @@ -36,7 +36,7 @@ async function createFolderNodes(execute: boolean) { parents: [getTeamsInternalId(connector.id)], parentId: null, title: "Conversations", - mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS, + mimeType: MIME_TYPES.INTERCOM.TEAMS_FOLDER, }); } @@ -60,7 +60,7 @@ async function createFolderNodes(execute: boolean) { parents: [teamInternalId, getTeamsInternalId(connector.id)], parentId: getTeamsInternalId(connector.id), title: team.name, - mimeType: INTERCOM_MIME_TYPES.TEAM, + mimeType: MIME_TYPES.INTERCOM.TEAM, }); } }, @@ -99,7 +99,7 @@ async function createFolderNodes(execute: boolean) { parents: [helpCenterInternalId], parentId: null, title: helpCenter.name, - mimeType: INTERCOM_MIME_TYPES.HELP_CENTER, + mimeType: MIME_TYPES.INTERCOM.HELP_CENTER, }); } @@ -133,7 +133,7 @@ async function createFolderNodes(execute: boolean) { parents: collectionParents, parentId: collectionParents[1] || null, title: collection.name, - mimeType: INTERCOM_MIME_TYPES.COLLECTION, + mimeType: MIME_TYPES.INTERCOM.COLLECTION, }); } }, diff --git a/connectors/src/api/get_content_node_parents.ts b/connectors/src/api/get_content_node_parents.ts deleted file mode 100644 index 26a12ebc4062..000000000000 --- a/connectors/src/api/get_content_node_parents.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { WithConnectorsAPIErrorReponse } from "@dust-tt/types"; -import type { Request, Response } from "express"; -import { isLeft } from "fp-ts/lib/Either"; -import * as t from "io-ts"; -import * as reporter from "io-ts-reporters"; - -import type { ContentNodeParentIdsBlob } from "@connectors/lib/api/content_nodes"; -import { getParentIdsForContentNodes } from "@connectors/lib/api/content_nodes"; -import logger from "@connectors/logger/logger"; -import { apiError, withLogging } from "@connectors/logger/withlogging"; -import { ConnectorResource } from "@connectors/resources/connector_resource"; - -const GetContentNodesParentsRequestBodySchema = t.type({ - internalIds: t.array(t.string), -}); - -export type GetContentNodesParentsRequestBody = t.TypeOf< - typeof GetContentNodesParentsRequestBodySchema ->; - -type GetContentNodesResponseBody = WithConnectorsAPIErrorReponse<{ - nodes: ContentNodeParentIdsBlob[]; -}>; - -const _getContentNodesParents = async ( - req: Request< - { connector_id: string }, - GetContentNodesResponseBody, - GetContentNodesParentsRequestBody - >, - res: Response -) => { - const connector = await ConnectorResource.fetchById(req.params.connector_id); - if (!connector) { - return apiError(req, res, { - status_code: 404, - api_error: { - type: "connector_not_found", - message: "Connector not found", - }, - }); - } - - const bodyValidation = GetContentNodesParentsRequestBodySchema.decode( - req.body - ); - if (isLeft(bodyValidation)) { - const pathError = reporter.formatValidationErrors(bodyValidation.left); - return apiError(req, res, { - status_code: 400, - api_error: { - type: "invalid_request_error", - message: `Invalid request body: ${pathError}`, - }, - }); - } - - const { internalIds } = bodyValidation.right; - - const parentsRes = await getParentIdsForContentNodes(connector, internalIds); - if (parentsRes.isErr()) { - logger.error(parentsRes.error, "Failed to get content node parents."); - return apiError(req, res, { - status_code: 500, - api_error: { - type: "internal_server_error", - message: parentsRes.error.message, - }, - }); - } - - return res.status(200).json({ nodes: parentsRes.value }); -}; - -export const getContentNodesParentsAPIHandler = withLogging( - _getContentNodesParents -); diff --git a/connectors/src/api_server.ts b/connectors/src/api_server.ts index 11490c655e99..ff0b3ebb9efd 100644 --- a/connectors/src/api_server.ts +++ b/connectors/src/api_server.ts @@ -13,7 +13,6 @@ import { getConnectorsAPIHandler, } from "@connectors/api/get_connector"; import { getConnectorPermissionsAPIHandler } from "@connectors/api/get_connector_permissions"; -import { getContentNodesParentsAPIHandler } from "@connectors/api/get_content_node_parents"; import { getContentNodesAPIHandler } from "@connectors/api/get_content_nodes"; import { pauseConnectorAPIHandler } from "@connectors/api/pause_connector"; import { resumeConnectorAPIHandler } from "@connectors/api/resume_connector"; @@ -112,11 +111,6 @@ export function startServer(port: number) { "/connectors/:connector_id/permissions", getConnectorPermissionsAPIHandler ); - app.post( - // must be POST because of body - "/connectors/:connector_id/content_nodes/parents", - getContentNodesParentsAPIHandler - ); app.post( // must be POST because of body "/connectors/:connector_id/content_nodes", diff --git a/connectors/src/connectors/confluence/lib/permissions.ts b/connectors/src/connectors/confluence/lib/permissions.ts index 4efe4b8a8c15..48ac475a6873 100644 --- a/connectors/src/connectors/confluence/lib/permissions.ts +++ b/connectors/src/connectors/confluence/lib/permissions.ts @@ -53,7 +53,6 @@ export function createContentNodeFromSpace( sourceUrl: `${baseUrl}/wiki${urlSuffix}`, expandable: isExpandable, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -75,7 +74,6 @@ export function createContentNodeFromPage( sourceUrl: `${baseUrl}/wiki${page.externalUrl}`, expandable: isExpandable, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }; } diff --git a/connectors/src/connectors/confluence/temporal/activities.ts b/connectors/src/connectors/confluence/temporal/activities.ts index c42bda84ed5d..56dffa86281d 100644 --- a/connectors/src/connectors/confluence/temporal/activities.ts +++ b/connectors/src/connectors/confluence/temporal/activities.ts @@ -1,8 +1,8 @@ import type { ModelId } from "@dust-tt/types"; import { - CONFLUENCE_MIME_TYPES, ConfluenceClientError, isConfluenceNotFoundError, + MIME_TYPES, } from "@dust-tt/types"; import { Op } from "sequelize"; import TurndownService from "turndown"; @@ -222,7 +222,7 @@ export async function confluenceUpsertSpaceFolderActivity({ parents: [makeSpaceInternalId(spaceId)], parentId: null, title: spaceName, - mimeType: CONFLUENCE_MIME_TYPES.SPACE, + mimeType: MIME_TYPES.CONFLUENCE.SPACE, }); } @@ -329,7 +329,7 @@ async function upsertConfluencePageToDataSource({ timestampMs: lastPageVersionCreatedAt.getTime(), upsertContext: { sync_type: syncType }, title: page.title, - mimeType: CONFLUENCE_MIME_TYPES.PAGE, + mimeType: MIME_TYPES.CONFLUENCE.PAGE, async: true, }); } diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts index ca4a9305e614..19ffd791ab8f 100644 --- a/connectors/src/connectors/github/index.ts +++ b/connectors/src/connectors/github/index.ts @@ -293,7 +293,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, })) ); @@ -364,7 +363,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/issues", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: latestIssue.updatedAt.getTime(), }); } @@ -378,7 +376,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: latestDiscussion.updatedAt.getTime(), }); } @@ -392,7 +389,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(), }); } @@ -434,7 +430,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: directory.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: directory.codeUpdatedAt.getTime(), }); }); @@ -448,7 +443,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: file.sourceUrl, expandable: false, permission: "read", - dustDocumentId: file.documentId, lastUpdatedAt: file.codeUpdatedAt.getTime(), }); }); @@ -610,7 +604,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -629,7 +622,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/issues", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -646,7 +638,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -665,7 +656,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + `/issues/${issueNumber}`, expandable: false, permission: "read", - dustDocumentId: getIssueInternalId(repoId, issueNumber), lastUpdatedAt: issue.updatedAt.getTime(), }); }); @@ -684,7 +674,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + `/discussions/${discussionNumber}`, expandable: false, permission: "read", - dustDocumentId: getDiscussionInternalId(repoId, discussionNumber), lastUpdatedAt: discussion.updatedAt.getTime(), }); }); @@ -699,7 +688,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: codeRepo.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(), }); }); @@ -714,7 +702,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: directory.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: directory.codeUpdatedAt.getTime(), }); }); @@ -729,7 +716,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: file.sourceUrl, expandable: false, permission: "read", - dustDocumentId: file.documentId, lastUpdatedAt: file.codeUpdatedAt.getTime(), }); }); diff --git a/connectors/src/connectors/github/temporal/activities.ts b/connectors/src/connectors/github/temporal/activities.ts index 57910729485c..11920a79999f 100644 --- a/connectors/src/connectors/github/temporal/activities.ts +++ b/connectors/src/connectors/github/temporal/activities.ts @@ -1,5 +1,5 @@ import type { CoreAPIDataSourceDocumentSection, ModelId } from "@dust-tt/types"; -import { assertNever, GITHUB_MIME_TYPES } from "@dust-tt/types"; +import { assertNever, MIME_TYPES } from "@dust-tt/types"; import { Context } from "@temporalio/activity"; import { hash as blake3 } from "blake3"; import { promises as fs } from "fs"; @@ -308,7 +308,7 @@ export async function githubUpsertIssueActivity( sync_type: isBatchSync ? "batch" : "incremental", }, title: issue.title, - mimeType: GITHUB_MIME_TYPES.ISSUE, + mimeType: MIME_TYPES.GITHUB.ISSUE, async: true, }); @@ -494,7 +494,7 @@ export async function githubUpsertDiscussionActivity( sync_type: isBatchSync ? "batch" : "incremental", }, title: discussion.title, - mimeType: GITHUB_MIME_TYPES.DISCUSSION, + mimeType: MIME_TYPES.GITHUB.DISCUSSION, async: true, }); @@ -960,7 +960,7 @@ export async function githubCodeSyncActivity({ title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.CODE_ROOT, + mimeType: MIME_TYPES.GITHUB.CODE_ROOT, }); let githubCodeRepository = await GithubCodeRepository.findOne({ @@ -1172,7 +1172,7 @@ export async function githubCodeSyncActivity({ sync_type: isBatchSync ? "batch" : "incremental", }, title: f.fileName, - mimeType: GITHUB_MIME_TYPES.CODE_FILE, + mimeType: MIME_TYPES.GITHUB.CODE_FILE, async: true, }); @@ -1216,7 +1216,7 @@ export async function githubCodeSyncActivity({ parents, parentId: parents[1], title: d.dirName, - mimeType: GITHUB_MIME_TYPES.CODE_DIRECTORY, + mimeType: MIME_TYPES.GITHUB.CODE_DIRECTORY, }); // Find directory or create it. @@ -1356,7 +1356,7 @@ export async function githubUpsertRepositoryFolderActivity({ title: repoName, parents: [getRepositoryInternalId(repoId)], parentId: null, - mimeType: GITHUB_MIME_TYPES.REPOSITORY, + mimeType: MIME_TYPES.GITHUB.REPOSITORY, }); } @@ -1377,7 +1377,7 @@ export async function githubUpsertIssuesFolderActivity({ title: "Issues", parents: [getIssuesInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.ISSUES, + mimeType: MIME_TYPES.GITHUB.ISSUES, }); } @@ -1401,7 +1401,7 @@ export async function githubUpsertDiscussionsFolderActivity({ getRepositoryInternalId(repoId), ], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.DISCUSSIONS, + mimeType: MIME_TYPES.GITHUB.DISCUSSIONS, }); } @@ -1422,6 +1422,6 @@ export async function githubUpsertCodeRootFolderActivity({ title: "Code", parents: [getCodeRootInternalId(repoId), getRepositoryInternalId(repoId)], parentId: getRepositoryInternalId(repoId), - mimeType: GITHUB_MIME_TYPES.CODE_ROOT, + mimeType: MIME_TYPES.GITHUB.CODE_ROOT, }); } diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts index d4341b8905f3..4987259ccb79 100644 --- a/connectors/src/connectors/google_drive/index.ts +++ b/connectors/src/connectors/google_drive/index.ts @@ -24,10 +24,7 @@ import { } from "@connectors/connectors/google_drive/lib"; import { GOOGLE_DRIVE_SHARED_WITH_ME_VIRTUAL_ID } from "@connectors/connectors/google_drive/lib/consts"; import { getGoogleDriveObject } from "@connectors/connectors/google_drive/lib/google_drive_api"; -import { - getGoogleDriveEntityDocumentId, - getPermissionViewType, -} from "@connectors/connectors/google_drive/lib/permissions"; +import { getPermissionViewType } from "@connectors/connectors/google_drive/lib/permissions"; import { folderHasChildren, getDrives, @@ -324,7 +321,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: null, type, title: f.name || "", - dustDocumentId: getGoogleDriveEntityDocumentId(f), lastUpdatedAt: f.lastUpsertedTs?.getTime() || null, sourceUrl: getSourceUrlForGoogleDriveFiles(f), expandable: await isDriveObjectExpandable({ @@ -350,7 +346,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: getInternalId(s.driveFileId), type: "database" as const, title: s.name || "", - dustDocumentId: null, lastUpdatedAt: s.updatedAt.getTime() || null, sourceUrl: null, expandable: false, @@ -394,7 +389,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { type: "folder" as const, title: driveObject.name, sourceUrl: driveObject.webViewLink || null, - dustDocumentId: null, lastUpdatedAt: driveObject.updatedAtMs || null, expandable: await folderHasChildren( this.connectorId, @@ -420,7 +414,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { preventSelection: true, title: "Shared with me", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -492,7 +485,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { this.connectorId, driveObject.id ), - dustDocumentId: null, lastUpdatedAt: driveObject.updatedAtMs || null, permission: (await GoogleDriveFolders.findOne({ where: { @@ -672,7 +664,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: null, type, title: f.name || "", - dustDocumentId: getGoogleDriveEntityDocumentId(f), lastUpdatedAt: f.lastUpsertedTs?.getTime() || null, sourceUrl, expandable: await isDriveObjectExpandable({ @@ -714,7 +705,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: getInternalId(s.driveFileId), type: "database", title: s.name || "", - dustDocumentId: null, lastUpdatedAt: s.updatedAt.getTime() || null, sourceUrl: `https://docs.google.com/spreadsheets/d/${s.driveFileId}/edit#gid=${s.driveSheetId}`, expandable: false, @@ -972,7 +962,6 @@ async function getFoldersAsContentNodes({ type: "folder", title: fd.name || "", sourceUrl, - dustDocumentId: null, lastUpdatedAt: fd.updatedAtMs || null, expandable: await isDriveObjectExpandable({ objectId: f.folderId, diff --git a/connectors/src/connectors/google_drive/lib/permissions.ts b/connectors/src/connectors/google_drive/lib/permissions.ts index 8983e8b3f722..226cd96dcea0 100644 --- a/connectors/src/connectors/google_drive/lib/permissions.ts +++ b/connectors/src/connectors/google_drive/lib/permissions.ts @@ -1,8 +1,4 @@ -import { - isGoogleDriveFolder, - isGoogleDriveSpreadSheetFile, -} from "@connectors/connectors/google_drive/temporal/mime_types"; -import { getInternalId } from "@connectors/connectors/google_drive/temporal/utils"; +import { isGoogleDriveFolder } from "@connectors/connectors/google_drive/temporal/mime_types"; import type { GoogleDriveFiles } from "@connectors/lib/models/google_drive"; export function getPermissionViewType(file: GoogleDriveFiles) { @@ -12,11 +8,3 @@ export function getPermissionViewType(file: GoogleDriveFiles) { return "file"; } - -export function getGoogleDriveEntityDocumentId(file: GoogleDriveFiles) { - if (isGoogleDriveSpreadSheetFile(file) || isGoogleDriveFolder(file)) { - return null; - } - - return getInternalId(file.driveFileId); -} diff --git a/connectors/src/connectors/google_drive/temporal/activities.ts b/connectors/src/connectors/google_drive/temporal/activities.ts index d9ee917aa548..ab15cd7c5729 100644 --- a/connectors/src/connectors/google_drive/temporal/activities.ts +++ b/connectors/src/connectors/google_drive/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { GOOGLE_DRIVE_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { uuid4 } from "@temporalio/workflow"; import type { drive_v3 } from "googleapis"; import type { GaxiosResponse, OAuth2Client } from "googleapis-common"; @@ -74,7 +74,7 @@ export async function upsertSharedWithMeFolder(connectorId: ModelId) { parents: [folderId], parentId: null, title: "Shared with me", - mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER, }); } @@ -515,7 +515,7 @@ export async function incrementalSync( parents, parentId: parents[1] || null, title: driveFile.name ?? "", - mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER, }); await GoogleDriveFiles.upsert({ @@ -860,7 +860,7 @@ export async function markFolderAsVisited( parents, parentId: parents[1] || null, title: file.name ?? "", - mimeType: GOOGLE_DRIVE_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.GOOGLE_DRIVE.FOLDER, }); await GoogleDriveFiles.upsert({ diff --git a/connectors/src/connectors/intercom/index.ts b/connectors/src/connectors/intercom/index.ts index 6a61b6e7655b..ff13edc25a57 100644 --- a/connectors/src/connectors/intercom/index.ts +++ b/connectors/src/connectors/intercom/index.ts @@ -629,7 +629,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: helpCenter.permission, - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -650,7 +649,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.lastUpsertedTs?.getTime() || null, }); } @@ -671,7 +669,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: article.url, expandable: false, permission: article.permission, - dustDocumentId: null, lastUpdatedAt: article.lastUpsertedTs?.getTime() || null, }); } @@ -687,7 +684,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { intercomWorkspace.syncAllConversations === "activated" ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -700,7 +696,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: null, }); } diff --git a/connectors/src/connectors/intercom/lib/conversation_permissions.ts b/connectors/src/connectors/intercom/lib/conversation_permissions.ts index 8e985bf589da..622e70ae7992 100644 --- a/connectors/src/connectors/intercom/lib/conversation_permissions.ts +++ b/connectors/src/connectors/intercom/lib/conversation_permissions.ts @@ -142,7 +142,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: false, preventSelection: false, permission: isAllConversationsSynced ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } else if (isRootLevel && hasTeamsWithReadPermission) { @@ -155,7 +154,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: true, preventSelection: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -170,7 +168,6 @@ export async function retrieveIntercomConversationsPermissions({ sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -188,7 +185,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: true, preventSelection: false, permission: isAllConversationsSynced ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -205,7 +201,6 @@ export async function retrieveIntercomConversationsPermissions({ sourceUrl: null, expandable: false, permission: isTeamInDb ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); }); diff --git a/connectors/src/connectors/intercom/lib/help_center_permissions.ts b/connectors/src/connectors/intercom/lib/help_center_permissions.ts index e494f19cf92b..995ca834e38e 100644 --- a/connectors/src/connectors/intercom/lib/help_center_permissions.ts +++ b/connectors/src/connectors/intercom/lib/help_center_permissions.ts @@ -374,7 +374,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: null, expandable: true, permission: helpCenter.permission, - dustDocumentId: null, lastUpdatedAt: helpCenter.updatedAt.getTime(), })); } else { @@ -388,7 +387,6 @@ export async function retrieveIntercomHelpCentersPermissions({ expandable: true, preventSelection: true, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, })); } @@ -435,7 +433,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.updatedAt.getTime() || null, })); } else { @@ -464,7 +461,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: false, // WE DO NOT LET EXPAND BELOW LEVEL 1 WHEN SELECTING NODES permission: matchingCollectionInDb ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: matchingCollectionInDb?.updatedAt.getTime() || null, }; }); @@ -504,7 +500,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.lastUpsertedTs?.getTime() || null, }) ); @@ -529,7 +524,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: article.url, expandable: false, permission: article.permission, - dustDocumentId: null, lastUpdatedAt: article.updatedAt.getTime(), })); diff --git a/connectors/src/connectors/intercom/lib/permissions.ts b/connectors/src/connectors/intercom/lib/permissions.ts index 961dd9fbf2a0..c4a5734c58ca 100644 --- a/connectors/src/connectors/intercom/lib/permissions.ts +++ b/connectors/src/connectors/intercom/lib/permissions.ts @@ -63,7 +63,6 @@ export async function retrieveSelectedNodes({ sourceUrl: collection.url, expandable, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.updatedAt.getTime() || null, }); }); @@ -85,7 +84,6 @@ export async function retrieveSelectedNodes({ sourceUrl: null, expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -105,7 +103,6 @@ export async function retrieveSelectedNodes({ sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: team.updatedAt.getTime() || null, }); }); diff --git a/connectors/src/connectors/intercom/temporal/activities.ts b/connectors/src/connectors/intercom/temporal/activities.ts index fd7196176193..3ee734db84e4 100644 --- a/connectors/src/connectors/intercom/temporal/activities.ts +++ b/connectors/src/connectors/intercom/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import { Op } from "sequelize"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -176,7 +176,7 @@ export async function syncHelpCenterOnlyActivity({ title: helpCenterOnIntercom.display_name || "Help Center", parents: [helpCenterInternalId], parentId: null, - mimeType: INTERCOM_MIME_TYPES.HELP_CENTER, + mimeType: MIME_TYPES.INTERCOM.HELP_CENTER, }); // If all children collections are not allowed anymore we delete the Help Center data @@ -509,7 +509,7 @@ export async function syncTeamOnlyActivity({ title: teamOnIntercom.name, parents: [teamInternalId, getTeamsInternalId(connectorId)], parentId: getTeamsInternalId(connectorId), - mimeType: INTERCOM_MIME_TYPES.TEAM, + mimeType: MIME_TYPES.INTERCOM.TEAM, }); return true; @@ -744,6 +744,6 @@ export async function upsertIntercomTeamsFolderActivity({ title: "Conversations", parents: [getTeamsInternalId(connectorId)], parentId: null, - mimeType: INTERCOM_MIME_TYPES.CONVERSATIONS, + mimeType: MIME_TYPES.INTERCOM.TEAMS_FOLDER, }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_conversation.ts b/connectors/src/connectors/intercom/temporal/sync_conversation.ts index 93c245ed4ae9..48622f7b4910 100644 --- a/connectors/src/connectors/intercom/temporal/sync_conversation.ts +++ b/connectors/src/connectors/intercom/temporal/sync_conversation.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -332,7 +332,7 @@ export async function syncConversation({ sync_type: syncType, }, title: convoTitle, - mimeType: INTERCOM_MIME_TYPES.CONVERSATION, + mimeType: MIME_TYPES.INTERCOM.CONVERSATION, async: true, }); } diff --git a/connectors/src/connectors/intercom/temporal/sync_help_center.ts b/connectors/src/connectors/intercom/temporal/sync_help_center.ts index 8868f7163376..9655b5fee61b 100644 --- a/connectors/src/connectors/intercom/temporal/sync_help_center.ts +++ b/connectors/src/connectors/intercom/temporal/sync_help_center.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { INTERCOM_MIME_TYPES } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; import TurndownService from "turndown"; import { getIntercomAccessToken } from "@connectors/connectors/intercom/lib/intercom_access_token"; @@ -229,7 +229,7 @@ export async function upsertCollectionWithChildren({ title: collection.name, parents: collectionParents, parentId: collectionParents[1], - mimeType: INTERCOM_MIME_TYPES.COLLECTION, + mimeType: MIME_TYPES.INTERCOM.COLLECTION, }); // Then we call ourself recursively on the children collections @@ -429,7 +429,7 @@ export async function upsertArticle({ sync_type: "batch", }, title: article.title, - mimeType: INTERCOM_MIME_TYPES.ARTICLE, + mimeType: MIME_TYPES.INTERCOM.ARTICLE, async: true, }); await articleOnDb.update({ diff --git a/connectors/src/connectors/microsoft/lib/content_nodes.ts b/connectors/src/connectors/microsoft/lib/content_nodes.ts index d71efb4db289..fe665918991a 100644 --- a/connectors/src/connectors/microsoft/lib/content_nodes.ts +++ b/connectors/src/connectors/microsoft/lib/content_nodes.ts @@ -25,7 +25,6 @@ export function getSitesRootAsContentNode(): ContentNode { type: "folder", title: "Sites", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -43,7 +42,6 @@ export function getTeamsRootAsContentNode(): ContentNode { type: "folder", title: "Teams", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -60,7 +58,6 @@ export function getTeamAsContentNode(team: microsoftgraph.Team): ContentNode { type: "folder", title: team.displayName || "unnamed", sourceUrl: team.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -85,7 +82,6 @@ export function getSiteAsContentNode( type: "folder", title: site.displayName || site.name || "unnamed", sourceUrl: site.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -115,7 +111,6 @@ export function getChannelAsContentNode( type: "channel", title: channel.displayName || "unnamed", sourceUrl: channel.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: false, permission: "none", @@ -136,7 +131,6 @@ export function getDriveAsContentNode( type: "folder", title: drive.name || "unnamed", sourceUrl: drive.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -152,7 +146,6 @@ export function getFolderAsContentNode( type: "folder", title: folder.name || "unnamed", sourceUrl: folder.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -169,7 +162,6 @@ export function getFileAsContentNode( type: "file", title: file.name || "unnamed", sourceUrl: file.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: false, permission: "none", @@ -204,7 +196,6 @@ export function getMicrosoftNodeAsContentNode( type, title: node.name || "unnamed", sourceUrl: node.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: isExpandable, permission: "none", diff --git a/connectors/src/connectors/microsoft/temporal/activities.ts b/connectors/src/connectors/microsoft/temporal/activities.ts index a64f72b3ac59..66fe7a5030c8 100644 --- a/connectors/src/connectors/microsoft/temporal/activities.ts +++ b/connectors/src/connectors/microsoft/temporal/activities.ts @@ -1,9 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { - cacheWithRedis, - MICROSOFT_MIME_TYPES, - removeNulls, -} from "@dust-tt/types"; +import { cacheWithRedis, MIME_TYPES, removeNulls } from "@dust-tt/types"; import type { Client } from "@microsoft/microsoft-graph-client"; import { GraphError } from "@microsoft/microsoft-graph-client"; import type { DriveItem } from "@microsoft/microsoft-graph-types"; @@ -213,7 +209,7 @@ export async function getRootNodesToSyncFromResources( parents: [createdOrUpdatedResource.internalId], parentId: null, title: createdOrUpdatedResource.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }), { concurrency: 5 } ); @@ -485,7 +481,7 @@ export async function syncFiles({ parents: [createdOrUpdatedResource.internalId, ...parentsOfParent], parentId: parentsOfParent[0], title: createdOrUpdatedResource.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }), { concurrency: 5 } ); @@ -659,7 +655,7 @@ export async function syncDeltaForRootNodesInDrive({ parents: [blob.internalId], parentId: null, title: blob.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }); // add parent information to new node resource. for the toplevel folder, @@ -875,7 +871,7 @@ async function updateDescendantsParentsInCore({ parents, parentId: parents[1] || null, title: folder.name ?? "", - mimeType: MICROSOFT_MIME_TYPES.FOLDER, + mimeType: MIME_TYPES.MICROSOFT.FOLDER, }); await concurrentExecutor( diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts index 741b41772564..e21c77ce5520 100644 --- a/connectors/src/connectors/notion/index.ts +++ b/connectors/src/connectors/notion/index.ts @@ -2,7 +2,7 @@ import type { ContentNode, ContentNodesViewType, Result } from "@dust-tt/types"; import { Err, getOAuthConnectionAccessToken, - NOTION_MIME_TYPES, + MIME_TYPES, Ok, } from "@dust-tt/types"; import _ from "lodash"; @@ -104,7 +104,7 @@ export class NotionConnectorManager extends BaseConnectorManager { parents: [folderId], parentId: null, title: "Orphaned Resources", - mimeType: NOTION_MIME_TYPES.UNKNOWN_FOLDER, + mimeType: MIME_TYPES.NOTION.UNKNOWN_FOLDER, }); try { @@ -464,7 +464,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: page.notionUrl || null, expandable, permission: "read", - dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, }; }; @@ -487,7 +486,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: db.notionUrl || null, expandable: true, permission: "read", - dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: db.structuredDataUpsertedTs?.getTime() ?? null, }; }; @@ -509,7 +507,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -559,7 +556,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: page.notionUrl || null, expandable: Boolean(hasChildrenByPageId[page.notionPageId]), permission: "read", - dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, })) ); @@ -575,7 +571,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: db.notionUrl || null, expandable: true, permission: "read", - dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: null, })); @@ -593,7 +588,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } diff --git a/connectors/src/connectors/notion/temporal/activities.ts b/connectors/src/connectors/notion/temporal/activities.ts index 864fcd25842f..719f3dd0829f 100644 --- a/connectors/src/connectors/notion/temporal/activities.ts +++ b/connectors/src/connectors/notion/temporal/activities.ts @@ -7,7 +7,7 @@ import type { import { assertNever, getNotionDatabaseTableId, - NOTION_MIME_TYPES, + MIME_TYPES, slugify, } from "@dust-tt/types"; import { isFullBlock, isFullPage, isNotionClientError } from "@notionhq/client"; @@ -1829,7 +1829,7 @@ export async function renderAndUpsertPageFromCache({ parents: parents, parentId: parents[1] || null, title: parentDb.title ?? "Untitled Notion Database", - mimeType: NOTION_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.NOTION.DATABASE, }), localLogger ); @@ -2053,7 +2053,7 @@ export async function renderAndUpsertPageFromCache({ sync_type: isFullSync ? "batch" : "incremental", }, title: title ?? "", - mimeType: NOTION_MIME_TYPES.PAGE, + mimeType: MIME_TYPES.NOTION.PAGE, async: true, }); } @@ -2550,7 +2550,7 @@ export async function upsertDatabaseStructuredDataFromCache({ parents: parentIds, parentId: parentIds[1] || null, title: dbModel.title ?? "Untitled Notion Database", - mimeType: NOTION_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.NOTION.DATABASE, }), localLogger ); @@ -2611,7 +2611,7 @@ export async function upsertDatabaseStructuredDataFromCache({ sync_type: "batch", }, title: databaseName, - mimeType: NOTION_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.NOTION.DATABASE, async: true, }); } else { diff --git a/connectors/src/connectors/slack/index.ts b/connectors/src/connectors/slack/index.ts index 4f6dc5a6f28b..f839596ad1b9 100644 --- a/connectors/src/connectors/slack/index.ts +++ b/connectors/src/connectors/slack/index.ts @@ -406,7 +406,6 @@ export class SlackConnectorManager extends BaseConnectorManager t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: SLACK_MIME_TYPES.MESSAGES, + mimeType: MIME_TYPES.SLACK.MESSAGES, async: true, }); } @@ -850,7 +846,7 @@ export async function syncThread( sync_type: isBatchSync ? "batch" : "incremental", }, title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: SLACK_MIME_TYPES.THREAD, + mimeType: MIME_TYPES.SLACK.THREAD, async: true, }); } diff --git a/connectors/src/connectors/snowflake/lib/content_nodes.ts b/connectors/src/connectors/snowflake/lib/content_nodes.ts index 919dcf0fceb7..7f3101ff3fe6 100644 --- a/connectors/src/connectors/snowflake/lib/content_nodes.ts +++ b/connectors/src/connectors/snowflake/lib/content_nodes.ts @@ -42,7 +42,6 @@ export const getContentNodeFromInternalId = ( expandable: true, preventSelection: false, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -56,7 +55,6 @@ export const getContentNodeFromInternalId = ( expandable: true, preventSelection: false, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -70,7 +68,6 @@ export const getContentNodeFromInternalId = ( expandable: false, preventSelection: false, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } diff --git a/connectors/src/connectors/snowflake/temporal/activities.ts b/connectors/src/connectors/snowflake/temporal/activities.ts index d7947bc4527e..2a6e8a3a4a51 100644 --- a/connectors/src/connectors/snowflake/temporal/activities.ts +++ b/connectors/src/connectors/snowflake/temporal/activities.ts @@ -1,5 +1,5 @@ import type { ModelId } from "@dust-tt/types"; -import { isSnowflakeCredentials, SNOWFLAKE_MIME_TYPES } from "@dust-tt/types"; +import { isSnowflakeCredentials, MIME_TYPES } from "@dust-tt/types"; import { connectToSnowflake, @@ -168,7 +168,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { title: table.databaseName, parents: [table.databaseName], parentId: null, - mimeType: SNOWFLAKE_MIME_TYPES.DATABASE, + mimeType: MIME_TYPES.SNOWFLAKE.DATABASE, }); // upsert a folder for the schema (child of the database) @@ -179,7 +179,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { title: table.schemaName, parents: [schemaId, table.databaseName], parentId: table.databaseName, - mimeType: SNOWFLAKE_MIME_TYPES.SCHEMA, + mimeType: MIME_TYPES.SNOWFLAKE.SCHEMA, }); await upsertDataSourceRemoteTable({ @@ -192,7 +192,7 @@ export async function syncSnowflakeConnection(connectorId: ModelId) { parents: [table.internalId, schemaId, table.databaseName], parentId: schemaId, title: table.name, - mimeType: SNOWFLAKE_MIME_TYPES.TABLE, + mimeType: MIME_TYPES.SNOWFLAKE.TABLE, }); await table.update({ lastUpsertedAt: new Date(), diff --git a/connectors/src/connectors/webcrawler/index.ts b/connectors/src/connectors/webcrawler/index.ts index d5d492c4f9bb..f53373063ecb 100644 --- a/connectors/src/connectors/webcrawler/index.ts +++ b/connectors/src/connectors/webcrawler/index.ts @@ -212,7 +212,6 @@ export class WebcrawlerConnectorManager extends BaseConnectorManager { this.ticketsPermission === "read" ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -359,7 +358,6 @@ export class ZendeskBrandResource extends BaseResource { sourceUrl: null, expandable: true, permission: this.helpCenterPermission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -380,7 +378,6 @@ export class ZendeskBrandResource extends BaseResource { sourceUrl: null, expandable: expandable, permission: this.ticketsPermission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -645,7 +642,6 @@ export class ZendeskCategoryResource extends BaseResource { sourceUrl: this.url, expandable: expandable, permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -730,7 +726,6 @@ export class ZendeskTicketResource extends BaseResource { sourceUrl: this.url, expandable: false, permission: this.permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -945,7 +940,6 @@ export class ZendeskArticleResource extends BaseResource { sourceUrl: this.url, expandable: false, permission: this.permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } diff --git a/front/admin/cli.ts b/front/admin/cli.ts index 7fd4b2c23933..0f290d8ddbc1 100644 --- a/front/admin/cli.ts +++ b/front/admin/cli.ts @@ -1,18 +1,7 @@ -import { - assertNever, - ConnectorsAPI, - removeNulls, - SUPPORTED_MODEL_CONFIGS, -} from "@dust-tt/types"; -import parseArgs from "minimist"; - import { getConversation } from "@app/lib/api/assistant/conversation"; import { renderConversationForModel } from "@app/lib/api/assistant/generation"; -import { - getTextContentFromMessage, - getTextRepresentationFromMessages, -} from "@app/lib/api/assistant/utils"; -import config from "@app/lib/api/config"; +import { getTextRepresentationFromMessages } from "@app/lib/api/assistant/utils"; +import { default as config } from "@app/lib/api/config"; import { getDataSources } from "@app/lib/api/data_sources"; import { garbageCollectGoogleDriveDocument } from "@app/lib/api/poke/plugins/data_sources/garbage_collect_google_drive_document"; import { Authenticator } from "@app/lib/auth"; @@ -38,6 +27,14 @@ import { stopRetrieveTranscriptsWorkflow, } from "@app/temporal/labs/client"; import { REGISTERED_CHECKS } from "@app/temporal/production_checks/activities"; +import { DustAPI } from "@dust-tt/client"; +import { + assertNever, + ConnectorsAPI, + removeNulls, + SUPPORTED_MODEL_CONFIGS, +} from "@dust-tt/types"; +import parseArgs from "minimist"; // `cli` takes an object type and a command as first two arguments and then a list of arguments. const workspace = async (command: string, args: parseArgs.ParsedArgs) => { @@ -493,6 +490,45 @@ const productionCheck = async (command: string, args: parseArgs.ParsedArgs) => { ); return; } + case "check-apps": { + if (!args.url) { + throw new Error("Missing --url argument"); + } + if (!args.wId) { + throw new Error("Missing --wId argument"); + } + if (!args.spaceId) { + throw new Error("Missing --spaceId argument"); + } + const api = new DustAPI( + config.getDustAPIConfig(), + { apiKey: args.apiKey, workspaceId: args.wId }, + logger, + args.url + ); + + const actions = Object.values(DustProdActionRegistry); + + const res = await api.checkApps( + { + apps: actions.map((action) => ({ + appId: action.app.appId, + appHash: action.app.appHash, + })), + }, + args.spaceId + ); + if (res.isErr()) { + throw new Error(res.error.message); + } + const notDeployedApps = res.value.filter((a) => !a.deployed); + if (notDeployedApps.length > 0) { + throw new Error( + "Missing apps: " + notDeployedApps.map((a) => a.appId).join(", ") + ); + } + console.log("All apps are deployed"); + } } }; diff --git a/front/components/ContentNodeTree.tsx b/front/components/ContentNodeTree.tsx index d41b07f481c2..e595d5e6f19c 100644 --- a/front/components/ContentNodeTree.tsx +++ b/front/components/ContentNodeTree.tsx @@ -249,15 +249,15 @@ function ContentNodeTreeChildren({ size="xs" icon={BracesIcon} onClick={() => { - if (n.dustDocumentId) { - onDocumentViewClick(n.dustDocumentId); + if (n.type === "file") { + onDocumentViewClick(n.internalId); } }} className={classNames( - n.dustDocumentId ? "" : "pointer-events-none opacity-0" + n.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!n.dustDocumentId} - variant="ghost" + disabled={n.type !== "file"} + variant="outline" /> )} diff --git a/front/components/app/blocks/Database.tsx b/front/components/app/blocks/Database.tsx index c77a23483116..c23613eb4d93 100644 --- a/front/components/app/blocks/Database.tsx +++ b/front/components/app/blocks/Database.tsx @@ -141,7 +141,7 @@ export function TablesManager({ currentTableId={table.table_id} onTableUpdate={(selectedTable) => { updateTableConfig(index, { - table_id: selectedTable.dustDocumentId!, + table_id: selectedTable.internalId, }); }} excludeTables={getSelectedTables()} diff --git a/front/components/assistant/conversation/AgentMessage.tsx b/front/components/assistant/conversation/AgentMessage.tsx index a5c925fcf6fd..2ef07dedcfb5 100644 --- a/front/components/assistant/conversation/AgentMessage.tsx +++ b/front/components/assistant/conversation/AgentMessage.tsx @@ -13,7 +13,6 @@ import { ConversationMessage, DocumentDuplicateIcon, EyeIcon, - FeedbackSelector, Markdown, Page, Popover, @@ -56,6 +55,7 @@ import type { PluggableList } from "react-markdown/lib/react-markdown"; import { makeDocumentCitation } from "@app/components/actions/retrieval/utils"; import { makeWebsearchResultsCitation } from "@app/components/actions/websearch/utils"; import { AgentMessageActions } from "@app/components/assistant/conversation/actions/AgentMessageActions"; +import { FeedbackSelector } from "@app/components/assistant/conversation/FeedbackSelector"; import { GenerationContext } from "@app/components/assistant/conversation/GenerationContextProvider"; import { CitationsContext, diff --git a/front/components/assistant/conversation/ConversationContainer.tsx b/front/components/assistant/conversation/ConversationContainer.tsx index 1b6e932b4075..c23d51325fe5 100644 --- a/front/components/assistant/conversation/ConversationContainer.tsx +++ b/front/components/assistant/conversation/ConversationContainer.tsx @@ -1,15 +1,14 @@ -import { Page } from "@dust-tt/sparkle"; -import { useSendNotification } from "@dust-tt/sparkle"; +import { Page, useSendNotification } from "@dust-tt/sparkle"; import type { AgentMention, LightAgentConfigurationType, MentionType, Result, SubscriptionType, + UploadedContentFragment, UserType, WorkspaceType, } from "@dust-tt/types"; -import type { UploadedContentFragment } from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; import { useRouter } from "next/router"; import { useCallback, useContext, useEffect, useRef, useState } from "react"; @@ -35,26 +34,22 @@ import { } from "@app/lib/swr/conversations"; interface ConversationContainerProps { - conversationId: string | null; owner: WorkspaceType; subscription: SubscriptionType; user: UserType; isBuilder: boolean; agentIdToMention: string | null; - messageRankToScrollTo: number | undefined; } export function ConversationContainer({ - conversationId, owner, subscription, user, isBuilder, agentIdToMention, - messageRankToScrollTo, }: ConversationContainerProps) { - const [activeConversationId, setActiveConversationId] = - useState(conversationId); + const { activeConversationId } = useConversationsNavigation(); + const [planLimitReached, setPlanLimitReached] = useState(false); const [stickyMentions, setStickyMentions] = useState([]); @@ -76,7 +71,6 @@ export function ConversationContainer({ conversationId: activeConversationId, workspaceId: owner.sId, limit: 50, - startAtRank: messageRankToScrollTo, }); const setInputbarMention = useCallback( @@ -242,7 +236,6 @@ export function ConversationContainer({ undefined, { shallow: true } ); - setActiveConversationId(conversationRes.value.sId); await mutateConversations(); await scrollConversationsToTop(); @@ -310,7 +303,6 @@ export function ConversationContainer({ conversationId={activeConversationId} // TODO(2024-06-20 flav): Fix extra-rendering loop with sticky mentions. onStickyMentionsChange={onStickyMentionsChange} - messageRankToScrollTo={messageRankToScrollTo} /> ) : (
diff --git a/front/components/assistant/conversation/ConversationLayout.tsx b/front/components/assistant/conversation/ConversationLayout.tsx index c42908d1c2ee..7b5332fcb43b 100644 --- a/front/components/assistant/conversation/ConversationLayout.tsx +++ b/front/components/assistant/conversation/ConversationLayout.tsx @@ -1,11 +1,14 @@ import type { SubscriptionType, WorkspaceType } from "@dust-tt/types"; import { useRouter } from "next/router"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { useMemo } from "react"; import RootLayout from "@app/components/app/RootLayout"; import { AssistantDetails } from "@app/components/assistant/AssistantDetails"; import { ConversationErrorDisplay } from "@app/components/assistant/conversation/ConversationError"; -import { ConversationsNavigationProvider } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; +import { + ConversationsNavigationProvider, + useConversationsNavigation, +} from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { ConversationTitle } from "@app/components/assistant/conversation/ConversationTitle"; import { FileDropProvider } from "@app/components/assistant/conversation/FileUploaderContext"; import { GenerationContextProvider } from "@app/components/assistant/conversation/GenerationContextProvider"; @@ -13,10 +16,7 @@ import { InputBarProvider } from "@app/components/assistant/conversation/input_b import { AssistantSidebarMenu } from "@app/components/assistant/conversation/SidebarMenu"; import AppLayout from "@app/components/sparkle/AppLayout"; import { useURLSheet } from "@app/hooks/useURLSheet"; -import { - useConversation, - useDeleteConversation, -} from "@app/lib/swr/conversations"; +import { useConversation } from "@app/lib/swr/conversations"; export interface ConversationLayoutProps { baseUrl: string; @@ -32,116 +32,81 @@ export default function ConversationLayout({ children: React.ReactNode; pageProps: ConversationLayoutProps; }) { - const { baseUrl, conversationId, owner, subscription } = pageProps; - - const router = useRouter(); + const { baseUrl, owner, subscription } = pageProps; - const [detailViewContent, setDetailViewContent] = useState(""); - const [activeConversationId, setActiveConversationId] = useState( - conversationId !== "new" ? conversationId : null + return ( + + + + {children} + + + ); +} +const ConversationLayoutContent = ({ + owner, + subscription, + baseUrl, + children, +}: any) => { + const router = useRouter(); const { onOpenChange: onOpenChangeAssistantModal } = useURLSheet("assistantDetails"); - - useEffect(() => { - const handleRouteChange = () => { - const assistantSId = router.query.assistantDetails ?? []; - // We use shallow browsing when creating a new conversation. - // Monitor router to update conversation info. - const conversationId = router.query.cId ?? ""; - - if (assistantSId && typeof assistantSId === "string") { - setDetailViewContent(assistantSId); - } else { - setDetailViewContent(""); - } - - if ( - conversationId && - typeof conversationId === "string" && - conversationId !== activeConversationId - ) { - setActiveConversationId( - conversationId !== "new" ? conversationId : null - ); - } - }; - - // Initial check in case the component mounts with the query already set. - handleRouteChange(); - - router.events.on("routeChangeComplete", handleRouteChange); - return () => { - router.events.off("routeChangeComplete", handleRouteChange); - }; - }, [ - router.query, - router.events, - setActiveConversationId, - activeConversationId, - ]); - + const { activeConversationId } = useConversationsNavigation(); const { conversation, conversationError } = useConversation({ conversationId: activeConversationId, workspaceId: owner.sId, }); - const doDelete = useDeleteConversation(owner); - - const onDeleteConversation = useCallback(async () => { - const res = await doDelete(conversation); - if (res) { - void router.push(`/w/${owner.sId}/assistant/new`); + const assistantSId = useMemo(() => { + const sid = router.query.assistantDetails ?? []; + if (sid && typeof sid === "string") { + return sid; } - }, [conversation, doDelete, owner.sId, router]); + return null; + }, [router.query.assistantDetails]); return ( - - - - - ) - } - navChildren={} - > - {conversationError ? ( - - ) : ( - <> - onOpenChangeAssistantModal(false)} - /> - - - {children} - - - - )} - - - - + + + ) + } + navChildren={} + > + {conversationError ? ( + + ) : ( + <> + onOpenChangeAssistantModal(false)} + /> + + {children} + + + )} + + ); -} +}; diff --git a/front/components/assistant/conversation/ConversationTitle.tsx b/front/components/assistant/conversation/ConversationTitle.tsx index 4a2e182a92e9..d34da1e9ff3d 100644 --- a/front/components/assistant/conversation/ConversationTitle.tsx +++ b/front/components/assistant/conversation/ConversationTitle.tsx @@ -10,30 +10,47 @@ import { TrashIcon, XMarkIcon, } from "@dust-tt/sparkle"; -import type { ConversationType } from "@dust-tt/types"; import type { WorkspaceType } from "@dust-tt/types"; +import { useRouter } from "next/router"; import type { MouseEvent } from "react"; -import React, { useRef, useState } from "react"; +import React, { useCallback, useRef, useState } from "react"; import { useSWRConfig } from "swr"; import { ConversationParticipants } from "@app/components/assistant/conversation/ConversationParticipants"; +import { useConversationsNavigation } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { DeleteConversationsDialog } from "@app/components/assistant/conversation/DeleteConversationsDialog"; +import { + useConversation, + useDeleteConversation, +} from "@app/lib/swr/conversations"; import { classNames } from "@app/lib/utils"; export function ConversationTitle({ owner, - conversationId, - conversation, - shareLink, - onDelete, + baseUrl, }: { owner: WorkspaceType; - conversationId: string; - conversation: ConversationType | null; - shareLink: string; - onDelete?: (conversationId: string) => void; + baseUrl: string; }) { const { mutate } = useSWRConfig(); + const router = useRouter(); + const { activeConversationId } = useConversationsNavigation(); + + const { conversation } = useConversation({ + conversationId: activeConversationId, + workspaceId: owner.sId, + }); + + const shareLink = `${baseUrl}/w/${owner.sId}/assistant/${activeConversationId}`; + + const doDelete = useDeleteConversation(owner); + + const onDelete = useCallback(async () => { + const res = await doDelete(conversation); + if (res) { + void router.push(`/w/${owner.sId}/assistant/new`); + } + }, [conversation, doDelete, owner.sId, router]); const [copyLinkSuccess, setCopyLinkSuccess] = useState(false); const [isEditingTitle, setIsEditingTitle] = useState(false); @@ -43,57 +60,62 @@ export function ConversationTitle({ const titleInputFocused = useRef(false); const saveButtonFocused = useRef(false); - const handleClick = async () => { + const handleClick = useCallback(async () => { await navigator.clipboard.writeText(shareLink || ""); setCopyLinkSuccess(true); setTimeout(() => { setCopyLinkSuccess(false); }, 1000); - }; + }, [shareLink]); - const onTitleChange = async (title: string) => { - try { - const res = await fetch( - `/api/w/${owner.sId}/assistant/conversations/${conversationId}`, - { - method: "PATCH", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - title, - visibility: conversation?.visibility, - }), + const onTitleChange = useCallback( + async (title: string) => { + try { + const res = await fetch( + `/api/w/${owner.sId}/assistant/conversations/${activeConversationId}`, + { + method: "PATCH", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + title, + visibility: conversation?.visibility, + }), + } + ); + await mutate( + `/api/w/${owner.sId}/assistant/conversations/${activeConversationId}` + ); + void mutate(`/api/w/${owner.sId}/assistant/conversations`); + if (!res.ok) { + throw new Error("Failed to update title"); } - ); - await mutate( - `/api/w/${owner.sId}/assistant/conversations/${conversationId}` - ); - void mutate(`/api/w/${owner.sId}/assistant/conversations`); - if (!res.ok) { - throw new Error("Failed to update title"); + setIsEditingTitle(false); + setEditedTitle(""); + } catch (e) { + alert("Failed to update title"); } - setIsEditingTitle(false); - setEditedTitle(""); - } catch (e) { - alert("Failed to update title"); - } - }; + }, + [activeConversationId, conversation?.visibility, mutate, owner.sId] + ); + + if (!activeConversationId) { + return null; + } return ( <> - {onDelete && ( - setShowDeleteDialog(false)} - onDelete={() => { - setShowDeleteDialog(false); - onDelete(conversationId); - }} - /> - )} + setShowDeleteDialog(false)} + onDelete={() => { + setShowDeleteDialog(false); + void onDelete(); + }} + />
{!isEditingTitle ? ( @@ -177,21 +199,19 @@ export function ConversationTitle({
- {onDelete && ( -
void; owner: WorkspaceType; user: UserType; - messageRankToScrollTo?: number | undefined; } /** @@ -63,7 +70,6 @@ const ConversationViewer = React.forwardRef< onStickyMentionsChange, isInModal = false, isFading = false, - messageRankToScrollTo, }, ref ) { @@ -93,10 +99,6 @@ const ConversationViewer = React.forwardRef< conversationId, workspaceId: owner.sId, limit: DEFAULT_PAGE_LIMIT, - // Make sure that the message rank to scroll to is in the middle of the page. - startAtRank: messageRankToScrollTo - ? Math.max(0, messageRankToScrollTo - Math.floor(DEFAULT_PAGE_LIMIT / 2)) - : undefined, }); const { mutateConversationParticipants } = useConversationParticipants({ @@ -320,57 +322,6 @@ const ConversationViewer = React.forwardRef< useLastMessageGroupObserver(typedGroupedMessages); - // Used for auto-scrolling to the message in the anchor. - const [hasScrolledToMessage, setHasScrolledToMessage] = useState(false); - const [messageShaking, setMessageShaking] = useState(false); - const scrollRef = React.useRef(null); - - // Track index of the group with message sId in anchor. - const groupIndexWithMessageIdInAnchor = useMemo(() => { - const messageToScrollTo = messages - .flatMap((messagePage) => messagePage.messages) - .find((message) => message.rank === messageRankToScrollTo); - if (!messageToScrollTo) { - return -1; - } - return typedGroupedMessages.findIndex((group) => { - return group.some((message) => message.sId === messageToScrollTo.sId); - }); - }, [typedGroupedMessages, messageRankToScrollTo, messages]); - - // Effect: scroll to the message and temporarily highlight if it is the anchor's target - useEffect(() => { - if ( - !messageRankToScrollTo || - !groupIndexWithMessageIdInAnchor || - !scrollRef.current || - hasScrolledToMessage - ) { - return; - } - setTimeout(() => { - if (scrollRef.current) { - setHasScrolledToMessage(true); - // Use ref to scroll to the message - scrollRef.current.scrollIntoView({ - behavior: "instant", - block: "center", - }); - setMessageShaking(true); - - // Have the message blink for a short time - setTimeout(() => { - setMessageShaking(false); - }, 1000); - } - }, 100); - }, [ - hasScrolledToMessage, - messageRankToScrollTo, - groupIndexWithMessageIdInAnchor, - scrollRef, - ]); - return (
)} {(isMessagesLoading || prevFirstMessageId) && ( -
+
)} {conversation && typedGroupedMessages.map((typedGroup, index) => { const isLastGroup = index === typedGroupedMessages.length - 1; - const isGroupInAnchor = index === groupIndexWithMessageIdInAnchor; return ( -
- -
+ messages={typedGroup} + isLastMessageGroup={isLastGroup} + conversationId={conversationId} + feedbacks={feedbacks} + isInModal={isInModal} + owner={owner} + prevFirstMessageId={prevFirstMessageId} + prevFirstMessageRef={prevFirstMessageRef} + user={user} + latestPage={latestPage} + /> ); })}
diff --git a/front/components/assistant/conversation/ConversationsNavigationProvider.tsx b/front/components/assistant/conversation/ConversationsNavigationProvider.tsx index 29d51ece88f5..c31bc166519f 100644 --- a/front/components/assistant/conversation/ConversationsNavigationProvider.tsx +++ b/front/components/assistant/conversation/ConversationsNavigationProvider.tsx @@ -1,22 +1,27 @@ +import { useRouter } from "next/router"; import type { RefObject } from "react"; -import { createContext, useContext, useRef } from "react"; +import { createContext, useCallback, useContext, useMemo, useRef } from "react"; interface ConversationsNavigationContextType { conversationsNavigationRef: RefObject; scrollConversationsToTop: () => void; + activeConversationId: string | null; } const ConversationsNavigationContext = createContext(null); export function ConversationsNavigationProvider({ + initialConversationId, children, }: { + initialConversationId?: string | null; children: React.ReactNode; }) { + const router = useRouter(); const conversationsNavigationRef = useRef(null); - const scrollConversationsToTop = () => { + const scrollConversationsToTop = useCallback(() => { if (conversationsNavigationRef.current) { // Find the ScrollArea viewport const viewport = conversationsNavigationRef.current.querySelector( @@ -29,13 +34,24 @@ export function ConversationsNavigationProvider({ }); } } - }; + }, []); + + const activeConversationId = useMemo(() => { + const conversationId = router.query.cId ?? ""; + + if (conversationId && typeof conversationId === "string") { + return conversationId === "new" ? null : conversationId; + } + + return initialConversationId ?? null; + }, [initialConversationId, router.query.cId]); return ( {children} diff --git a/sparkle/src/components/FeedbackSelector.tsx b/front/components/assistant/conversation/FeedbackSelector.tsx similarity index 87% rename from sparkle/src/components/FeedbackSelector.tsx rename to front/components/assistant/conversation/FeedbackSelector.tsx index fdf55b2b19fb..180bb1e137c2 100644 --- a/sparkle/src/components/FeedbackSelector.tsx +++ b/front/components/assistant/conversation/FeedbackSelector.tsx @@ -1,18 +1,13 @@ +import { Button } from "@dust-tt/sparkle"; +import { Checkbox } from "@dust-tt/sparkle"; +import { Page } from "@dust-tt/sparkle"; +import { PopoverContent, PopoverRoot, PopoverTrigger } from "@dust-tt/sparkle"; +import { Spinner } from "@dust-tt/sparkle"; +import { TextArea } from "@dust-tt/sparkle"; +import { Tooltip } from "@dust-tt/sparkle"; +import { HandThumbDownIcon, HandThumbUpIcon } from "@dust-tt/sparkle"; import React, { useCallback, useEffect, useRef } from "react"; -import { Button } from "@sparkle/components/Button"; -import { Checkbox } from "@sparkle/components/Checkbox"; -import { Page } from "@sparkle/components/Page"; -import { - PopoverContent, - PopoverRoot, - PopoverTrigger, -} from "@sparkle/components/Popover"; -import Spinner from "@sparkle/components/Spinner"; -import { TextArea } from "@sparkle/components/TextArea"; -import { Tooltip } from "@sparkle/components/Tooltip"; -import { HandThumbDownIcon, HandThumbUpIcon } from "@sparkle/icons/solid"; - export type ThumbReaction = "up" | "down"; export type FeedbackType = { @@ -134,10 +129,10 @@ export function FeedbackSelector({ ]); return ( -
+
-
+
{isSubmittingThumb ? ( -
+
) : ( -
+
{lastSelectedThumb === "up" ? "🎉 Glad you liked it! Tell us more?" @@ -196,14 +191,14 @@ export function FeedbackSelector({ ? "What did you like?" : "Tell us what went wrong so we can make this assistant better." } - className="s-mb-4 s-mt-4" + className="mb-4 mt-4" rows={3} value={localFeedbackContent ?? ""} onChange={handleTextAreaChange} /> {popOverInfo} -
+
{ @@ -214,7 +209,7 @@ export function FeedbackSelector({ By clicking, you accept to share your full conversation
-
+
diff --git a/front/components/assistant_builder/DataSourceSelectionSection.tsx b/front/components/assistant_builder/DataSourceSelectionSection.tsx index b3d755e53f22..c7f23f7f876c 100644 --- a/front/components/assistant_builder/DataSourceSelectionSection.tsx +++ b/front/components/assistant_builder/DataSourceSelectionSection.tsx @@ -148,20 +148,20 @@ export default function DataSourceSelectionSection({ size="xs" icon={BracesIcon} onClick={() => { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay( dsConfig.dataSourceView ); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} - variant="ghost" + disabled={node.type !== "file"} + variant="outline" />
} diff --git a/front/components/data_source/MultipleDocumentsUpload.tsx b/front/components/data_source/MultipleDocumentsUpload.tsx index b0b0baaec70e..118e11698381 100644 --- a/front/components/data_source/MultipleDocumentsUpload.tsx +++ b/front/components/data_source/MultipleDocumentsUpload.tsx @@ -19,6 +19,7 @@ import { import type { ChangeEvent } from "react"; import React, { useCallback, useEffect, useRef, useState } from "react"; +import { useFileDrop } from "@app/components/assistant/conversation/FileUploaderContext"; import { DocumentLimitPopup } from "@app/components/data_source/DocumentLimitPopup"; import type { FileBlob, @@ -74,12 +75,10 @@ export const MultipleDocumentsUpload = ({ completed: number; }>(null); - const handleFileChange = useCallback( - async ( - e: ChangeEvent & { target: { files: File[] } } - ) => { + const uploadFiles = useCallback( + async (files: File[]) => { // Empty file input - if (!e.target.files || e.target.files.length === 0) { + if (files.length === 0) { close(false); return; } @@ -87,21 +86,22 @@ export const MultipleDocumentsUpload = ({ // Open plan popup if limit is reached if ( plan.limits.dataSources.documents.count != -1 && - e.target.files.length + totalNodesCount > - plan.limits.dataSources.documents.count + files.length + totalNodesCount > plan.limits.dataSources.documents.count ) { setIsLimitPopupOpen(true); return; } setIsBulkFilesUploading({ - total: e.target.files.length, + total: files.length, completed: 0, }); // upload Files and get FileBlobs (only keep successful uploads) // Each individual error triggers a notification - const fileBlobs = (await fileUploaderService.handleFileChange(e))?.filter( + const fileBlobs = ( + await fileUploaderService.handleFilesUpload(files) + )?.filter( (fileBlob: FileBlob): fileBlob is FileBlobWithFileId => !!fileBlob.fileId ); @@ -122,7 +122,7 @@ export const MultipleDocumentsUpload = ({ // Have to use the filename to avoid fileId becoming apparent in the UI. upsertArgs: { title: blob.filename, - name: blob.filename, + document_id: blob.filename, }, }); @@ -148,6 +148,33 @@ export const MultipleDocumentsUpload = ({ ] ); + // Process dropped files if any. + const { droppedFiles, setDroppedFiles } = useFileDrop(); + useEffect(() => { + const handleDroppedFiles = async () => { + const droppedFilesCopy = [...droppedFiles]; + if (droppedFilesCopy.length > 0) { + // Make sure the files are cleared after processing + setDroppedFiles([]); + await uploadFiles(droppedFilesCopy); + } + }; + void handleDroppedFiles(); + }, [droppedFiles, setDroppedFiles, uploadFiles]); + + // Handle file change from file input. + const handleFileChange = useCallback( + async ( + e: ChangeEvent & { target: { files: File[] } } + ) => { + const selectedFiles = Array.from( + (e?.target as HTMLInputElement).files ?? [] + ); + await uploadFiles(selectedFiles); + }, + [uploadFiles] + ); + const handleFileInputBlur = useCallback(() => { close(false); }, [close]); diff --git a/front/components/home/ContentBlocks.tsx b/front/components/home/ContentBlocks.tsx index 81e91cfbe7f1..b048a4136f3c 100644 --- a/front/components/home/ContentBlocks.tsx +++ b/front/components/home/ContentBlocks.tsx @@ -173,13 +173,14 @@ export const HeaderContentBlock = ({ icon={RocketIcon} /> -
)}
diff --git a/front/components/home/HubSpotForm.tsx b/front/components/home/HubSpotForm.tsx index 048f18bc03cc..8c38913ea4c7 100644 --- a/front/components/home/HubSpotForm.tsx +++ b/front/components/home/HubSpotForm.tsx @@ -1,25 +1,45 @@ -import { useEffect } from "react"; +import React, { useEffect } from "react"; + +declare global { + interface Window { + hbspt?: { + forms: { + create: (config: { + region: string; + portalId: string; + formId: string; + target: string; + }) => void; + }; + }; + } +} export default function HubSpotForm() { useEffect(() => { - // Add script dynamically - const script = document.createElement("script"); - script.src = "https://js-eu1.hsforms.net/forms/embed/144442587.js"; - script.defer = true; - document.body.appendChild(script); - - // Cleanup on unmount - return () => { - document.body.removeChild(script); + const existingScript = document.getElementById("hubspot-script"); + const createForm = () => { + if (window.hbspt && window.hbspt.forms && window.hbspt.forms.create) { + window.hbspt.forms.create({ + region: "eu1", + portalId: "144442587", + formId: "31e790e5-f4d5-4c79-acc5-acd770fe8f84", + target: "#hubspotForm", + }); + } }; - }, []); - return ( -
- ); + if (!existingScript) { + const script = document.createElement("script"); + script.id = "hubspot-script"; + script.src = "https://js-eu1.hsforms.net/forms/v2.js"; + script.defer = true; + script.onload = createForm; + document.body.appendChild(script); + } else { + // If the script is already present, just recreate the form + createForm(); + } + }, []); + return
; } diff --git a/front/components/home/LandingLayout.tsx b/front/components/home/LandingLayout.tsx index 5f0de2fb30eb..62960d38f12f 100644 --- a/front/components/home/LandingLayout.tsx +++ b/front/components/home/LandingLayout.tsx @@ -99,14 +99,15 @@ export default function LandingLayout({
-
-
+ - {!isEmpty && ( - <> - { - setPagination( - { pageIndex: 0, pageSize: pagination.pageSize }, - "replace" - ); - setDataSourceSearch(s); - }} - /> - - )} - {isEmpty && emptyContent} - {isFolder(dataSourceView.dataSource) && ( - <> - {((viewType === "tables" && hasDocuments) || - (viewType === "documents" && hasTables)) && ( - - -
- {isNodesLoading && ( -
- + )} + {isManaged(dataSourceView.dataSource) && + connector && + !parentId && + space.kind === "system" && ( +
+ {!isNodesLoading && rows.length === 0 && ( +
Connection ready. Select the data to sync.
+ )} + + { + setShowConnectorPermissionsModal(false); + if (save) { + void mutateContentNodes(); + } + }} + readOnly={false} + isAdmin={isAdmin} + onManageButtonClick={() => { + setShowConnectorPermissionsModal(true); + }} + /> +
+ )}
- )} - {rows.length > 0 && ( - + +
+ )} + {rows.length > 0 && ( + + )} + - )} - - + + ); }; diff --git a/front/components/tables/TablePicker.tsx b/front/components/tables/TablePicker.tsx index 261769f6f857..593d335a6866 100644 --- a/front/components/tables/TablePicker.tsx +++ b/front/components/tables/TablePicker.tsx @@ -62,7 +62,7 @@ export default function TablePicker({ }); const currentTable = currentTableId - ? tables.find((t) => t.dustDocumentId === currentTableId) + ? tables.find((t) => t.internalId === currentTableId) : null; const [searchFilter, setSearchFilter] = useState(""); @@ -139,12 +139,12 @@ export default function TablePicker({ !excludeTables?.some( (et) => et.dataSourceId === dataSource.data_source_id && - et.tableId === t.dustDocumentId + et.tableId === t.internalId ) ) .map((t) => (
{ onTableUpdate(t); diff --git a/front/components/trackers/TrackerDataSourceSelectedTree.tsx b/front/components/trackers/TrackerDataSourceSelectedTree.tsx index 648e99bc5e95..e2f53af18b2e 100644 --- a/front/components/trackers/TrackerDataSourceSelectedTree.tsx +++ b/front/components/trackers/TrackerDataSourceSelectedTree.tsx @@ -109,20 +109,20 @@ export const TrackerDataSourceSelectedTree = ({ size="xs" icon={BracesIcon} onClick={() => { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay( dsConfig.dataSourceView ); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} - variant="ghost" + disabled={node.type !== "file"} + variant="outline" />
} diff --git a/front/components/workspace/ChangeMemberModal.tsx b/front/components/workspace/ChangeMemberModal.tsx index 1db3ac79e2d5..86964c6a61d6 100644 --- a/front/components/workspace/ChangeMemberModal.tsx +++ b/front/components/workspace/ChangeMemberModal.tsx @@ -129,7 +129,7 @@ export function ChangeMemberModal({ rightButtonProps={{ label: "Yes, revoke", variant: "warning", - onClick: () => async () => { + onClick: async () => { await handleMembersRoleChange({ members: [member], role: "none", diff --git a/front/hooks/useAppKeyboardShortcuts.ts b/front/hooks/useAppKeyboardShortcuts.ts index 82dbe737f202..23f65812ada3 100644 --- a/front/hooks/useAppKeyboardShortcuts.ts +++ b/front/hooks/useAppKeyboardShortcuts.ts @@ -20,7 +20,9 @@ export function useAppKeyboardShortcuts(owner: LightWorkspaceType) { case "/": event.preventDefault(); - void router.push(`/w/${owner.sId}/assistant/new`); + void router.push(`/w/${owner.sId}/assistant/new`, undefined, { + shallow: true, + }); break; } } diff --git a/front/lib/api/assistant/conversation.ts b/front/lib/api/assistant/conversation.ts index 64ac535d6acb..f6bea16d5ad6 100644 --- a/front/lib/api/assistant/conversation.ts +++ b/front/lib/api/assistant/conversation.ts @@ -32,20 +32,18 @@ import type { import { assertNever, ConversationError, - getSmallWhitelistedModel, - isContentFragmentType, - isProviderWhitelisted, - md5, - removeNulls, -} from "@dust-tt/types"; -import { Err, + getSmallWhitelistedModel, getTimeframeSecondsFromLiteral, isAgentMention, isAgentMessageType, + isContentFragmentType, + isProviderWhitelisted, isUserMessageType, + md5, Ok, rateLimiter, + removeNulls, } from "@dust-tt/types"; import { isEqual, sortBy } from "lodash"; import type { Transaction } from "sequelize"; @@ -426,28 +424,6 @@ export async function getConversation( }); } -export async function getMessageRank( - auth: Authenticator, - messageId: string -): Promise { - const owner = auth.workspace(); - if (!owner) { - throw new Error("Unexpected `auth` without `workspace`."); - } - - const message = await Message.findOne({ - where: { - sId: messageId, - }, - }); - - if (!message) { - return null; - } - - return message.rank; -} - export async function getConversationMessageType( auth: Authenticator, conversation: ConversationType | ConversationWithoutContentType, diff --git a/front/lib/api/config.ts b/front/lib/api/config.ts index e6b72be97ef0..63b856ca0e27 100644 --- a/front/lib/api/config.ts +++ b/front/lib/api/config.ts @@ -161,6 +161,26 @@ const config = { getStatusPageApiToken: (): string => { return EnvironmentConfig.getEnvVariable("STATUS_PAGE_API_TOKEN"); }, + getDustAppsSyncEnabled: (): boolean => { + return ( + EnvironmentConfig.getOptionalEnvVariable("DUST_APPS_SYNC_ENABLED") === + "true" + ); + }, + getDustAppsSyncMasterApiUrl: (): string => { + return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_API_URL"); + }, + getDustAppsSyncMasterWorkspaceId: (): string => { + return EnvironmentConfig.getEnvVariable( + "DUST_APPS_SYNC_MASTER_WORKSPACE_ID" + ); + }, + getDustAppsSyncMasterSpaceId: (): string => { + return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_SPACE_ID"); + }, + getDustAppsSyncMasterApiKey: (): string => { + return EnvironmentConfig.getEnvVariable("DUST_APPS_SYNC_MASTER_API_KEY"); + }, }; export default config; diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts index 7632170f5135..68528e0c42a4 100644 --- a/front/lib/api/data_source_view.ts +++ b/front/lib/api/data_source_view.ts @@ -203,7 +203,6 @@ async function getContentNodesForStaticDataSourceView( } } return { - dustDocumentId: doc.document_id, expandable: false, internalId: doc.document_id, lastUpdatedAt: doc.timestamp, @@ -238,7 +237,6 @@ async function getContentNodesForStaticDataSourceView( const tablesAsContentNodes: DataSourceViewContentNode[] = tablesRes.value.tables.map((table) => ({ - dustDocumentId: table.table_id, expandable: false, internalId: getContentNodeInternalIdFromTableId( dataSourceView, diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts index dba6cd1c18ee..a03478d79d7b 100644 --- a/front/lib/api/data_sources.ts +++ b/front/lib/api/data_sources.ts @@ -226,8 +226,9 @@ export async function augmentDataSourceWithConnectorDetails( fetchConnectorErrorMessage, }; } -export type UpsertDocumentArgs = { - name: string; + +export interface UpsertDocumentArgs { + document_id: string; source_url?: string | null; text?: string | null; section?: FrontDataSourceDocumentSectionType | null; @@ -240,9 +241,10 @@ export type UpsertDocumentArgs = { auth: Authenticator; mime_type: string; title: string; -}; +} + export async function upsertDocument({ - name, + document_id, source_url, text, section, @@ -268,6 +270,33 @@ export async function upsertDocument({ DustError > > { + // enforcing validation on the parents and parent_id + const documentId = document_id; + const documentParents = parents || [documentId]; + const documentParentId = parent_id ?? null; + + // parents must comply to the invariant parents[0] === document_id + if (documentParents[0] !== documentId) { + return new Err( + new DustError( + "invalid_parents", + "Invalid request body, parents[0] and document_id should be equal" + ) + ); + } + // parents and parentId must comply to the invariant parents[1] === parentId || (parentId === null && parents.length < 2) + if ( + (documentParents.length >= 2 || documentParentId !== null) && + documentParents[1] !== documentParentId + ) { + return new Err( + new DustError( + "invalid_parent_id", + "Invalid request body, parents[1] and parent_id should be equal" + ) + ); + } + let sourceUrl: string | null = null; if (source_url) { const { valid: isSourceUrlValid, standardized: standardizedSourceUrl } = @@ -317,15 +346,6 @@ export async function upsertDocument({ ); } - if (parent_id && parents && parents[1] !== parent_id) { - return new Err( - new DustError( - "invalid_parent_id", - "Invalid request body, parents[1] and parent_id should be equal" - ) - ); - } - const fullText = sectionFullText(generatedSection); const coreAPI = new CoreAPI(apiConfig.getCoreAPIConfig(), logger); @@ -384,24 +404,13 @@ export async function upsertDocument({ // Data source operations are performed with our credentials. const credentials = dustManagedCredentials(); - const documentId = name; - const documentParents = parents || []; - - // Ensure that the documentId is included in the parents as the first item. - // remove it if it's already present and add it as the first item. - const indexOfDocumentId = documentParents.indexOf(documentId); - if (indexOfDocumentId !== -1) { - documentParents.splice(indexOfDocumentId, 1); - } - documentParents.unshift(documentId); - // Create document with the Dust internal API. const upsertRes = await coreAPI.upsertDataSourceDocument({ projectId: dataSource.dustAPIProjectId, dataSourceId: dataSource.dustAPIDataSourceId, - documentId: documentId, + documentId, tags: nonNullTags, - parentId: parent_id ?? null, + parentId: documentParentId, parents: documentParents, sourceUrl, // TEMPORARY -- need to unstuck a specific entry @@ -426,8 +435,8 @@ export async function upsertDocument({ return new Ok(upsertRes.value); } -export type UpsertTableArgs = { - tableId?: string | null; +export interface UpsertTableArgs { + tableId: string; name: string; description: string; truncate: boolean; @@ -442,7 +451,8 @@ export type UpsertTableArgs = { useAppForHeaderDetection?: boolean; title: string; mimeType: string; -}; +} + export async function upsertTable({ tableId, name, @@ -460,16 +470,30 @@ export async function upsertTable({ title, mimeType, }: UpsertTableArgs) { - const nonNullTableId = tableId ?? generateRandomModelSId(); - const tableParents: string[] = parents ?? []; - - // Ensure that the nonNullTableId is included in the parents as the first item. - // remove it if it's already present and add it as the first item. - const indexOfTableId = tableParents.indexOf(nonNullTableId); - if (indexOfTableId !== -1) { - tableParents.splice(indexOfTableId, 1); + const tableParents = parents ?? [tableId]; + const tableParentId = parentId ?? null; + + // parents must comply to the invariant parents[0] === document_id + if (tableParents[0] !== tableId) { + return new Err( + new DustError( + "invalid_parents", + "Invalid request body, parents[0] and table_id should be equal" + ) + ); + } + // parents and parentId must comply to the invariant parents[1] === parentId + if ( + (tableParents.length >= 2 || tableParentId !== null) && + tableParents[1] !== tableParentId + ) { + return new Err( + new DustError( + "invalid_parent_id", + "Invalid request body, parents[1] and parent_id should be equal" + ) + ); } - tableParents.unshift(nonNullTableId); const flags = await getFeatureFlags(auth.getNonNullableWorkspace()); @@ -496,12 +520,12 @@ export async function upsertTable({ upsertTable: { workspaceId: auth.getNonNullableWorkspace().sId, dataSourceId: dataSource.sId, - tableId: nonNullTableId, + tableId, tableName: name, tableDescription: description, tableTimestamp: timestamp ?? null, tableTags: tags ?? [], - tableParentId: parentId ?? null, + tableParentId, tableParents, csv: csv ?? null, truncate, @@ -521,12 +545,12 @@ export async function upsertTable({ const tableRes = await upsertTableFromCsv({ auth, dataSource: dataSource, - tableId: nonNullTableId, + tableId, tableName: name, tableDescription: description, tableTimestamp: timestamp ?? null, tableTags: tags || [], - tableParentId: parentId ?? null, + tableParentId, tableParents, csv: csv ?? null, truncate, diff --git a/front/lib/api/files/upsert.ts b/front/lib/api/files/upsert.ts index 3295870b13d2..6f94fe30d4cf 100644 --- a/front/lib/api/files/upsert.ts +++ b/front/lib/api/files/upsert.ts @@ -207,14 +207,13 @@ const upsertDocumentToDatasource: ProcessingFunction = async ({ upsertArgs, }) => { // Use the file id as the document id to make it easy to track the document back to the file. - const documentId = file.sId; const sourceUrl = file.getPrivateUrl(auth); const upsertDocumentRes = await upsertDocument({ - name: documentId, + document_id: file.sId, source_url: sourceUrl, text: content, - parents: [documentId], + parents: [file.sId], tags: [`title:${file.fileName}`, `fileId:${file.sId}`], light_document_output: true, dataSource, @@ -245,7 +244,7 @@ const upsertTableToDatasource: ProcessingFunction = async ({ dataSource, upsertArgs, }) => { - const tableId = file.sId; // Use the file sId as the table id to make it easy to track the table back to the file. + const tableId = upsertArgs?.tableId ?? file.sId; // Use the file sId as a fallback for the table_id to make it easy to track the table back to the file. const upsertTableRes = await upsertTable({ tableId, name: slugify(file.fileName), diff --git a/front/lib/api/poke/plugins/index.ts b/front/lib/api/poke/plugins/index.ts index c4a160d94c4b..a32eefdb54a8 100644 --- a/front/lib/api/poke/plugins/index.ts +++ b/front/lib/api/poke/plugins/index.ts @@ -1,4 +1,5 @@ export * from "./data_source_views"; export * from "./data_sources"; export * from "./global"; +export * from "./spaces"; export * from "./workspaces"; diff --git a/front/lib/api/poke/plugins/spaces/index.ts b/front/lib/api/poke/plugins/spaces/index.ts new file mode 100644 index 000000000000..aefade5833dd --- /dev/null +++ b/front/lib/api/poke/plugins/spaces/index.ts @@ -0,0 +1 @@ +export * from "./sync_apps"; diff --git a/front/lib/api/poke/plugins/spaces/sync_apps.ts b/front/lib/api/poke/plugins/spaces/sync_apps.ts new file mode 100644 index 000000000000..4bd9dd83cf8b --- /dev/null +++ b/front/lib/api/poke/plugins/spaces/sync_apps.ts @@ -0,0 +1,40 @@ +import { Err, Ok } from "@dust-tt/types"; + +import { createPlugin } from "@app/lib/api/poke/types"; +import { SpaceResource } from "@app/lib/resources/space_resource"; +import { synchronizeDustApps } from "@app/lib/utils/apps"; + +export const syncAppsPlugin = createPlugin( + { + id: "sync-apps", + name: "Sync dust-apps", + description: "Synchronize dust-apps from production", + resourceTypes: ["spaces"], + args: {}, + }, + async (auth, spaceId) => { + if (!spaceId) { + return new Err(new Error("No space specified")); + } + + const space = await SpaceResource.fetchById(auth, spaceId); + if (!space) { + return new Err(new Error("Space not found")); + } + const result = await synchronizeDustApps(auth, space); + if (result.isErr()) { + return new Err(new Error(`Error when syncing: ${result.error.message}`)); + } + if (!result.value) { + return new Ok({ + display: "text", + value: "Sync not enabled.", + }); + } + + return new Ok({ + display: "json", + value: { importedApp: result.value }, + }); + } +); diff --git a/front/lib/api/tracker.ts b/front/lib/api/tracker.ts index 38627bf4f6c6..bddcb86984f9 100644 --- a/front/lib/api/tracker.ts +++ b/front/lib/api/tracker.ts @@ -1,5 +1,5 @@ import type { TrackerGenerationToProcess } from "@dust-tt/types"; -import { concurrentExecutor, CoreAPI } from "@dust-tt/types"; +import { concurrentExecutor, CoreAPI, removeNulls } from "@dust-tt/types"; import _ from "lodash"; import config from "@app/lib/api/config"; @@ -84,8 +84,8 @@ const sendTrackerEmail = async ({ const sendEmail = generations.length > 0 - ? _sendTrackerWithGenerationEmail - : _sendTrackerDefaultEmail; + ? sendTrackerWithGenerationEmail + : sendTrackerDefaultEmail; await Promise.all( Array.from(recipients).map((recipient) => @@ -94,7 +94,7 @@ const sendTrackerEmail = async ({ ); }; -const _sendTrackerDefaultEmail = async ({ +const sendTrackerDefaultEmail = async ({ name, recipient, }: { @@ -115,7 +115,7 @@ const _sendTrackerDefaultEmail = async ({ }); }; -const _sendTrackerWithGenerationEmail = async ({ +export const sendTrackerWithGenerationEmail = async ({ name, recipient, generations, @@ -127,15 +127,32 @@ const _sendTrackerWithGenerationEmail = async ({ localLogger: Logger; }): Promise => { const coreAPI = new CoreAPI(config.getCoreAPIConfig(), localLogger); - const generationsByDataSources = _.groupBy(generations, "dataSource.id"); - const documentsById = new Map(); + const dataSourceById = _.keyBy( + removeNulls( + generations.map((g) => [g.dataSource, g.maintainedDataSource]).flat() + ), + "id" + ); + const docsToFetchByDataSourceId = _.mapValues( + _.groupBy( + generations.map((g) => ({ + dataSourceId: g.dataSource.id, + documentIds: removeNulls([g.documentId, g.maintainedDocumentId]), + })), + "dataSourceId" + ), + (docs) => docs.map((d) => d.documentIds).flat() + ); + const documentsByIdentifier = new Map< + string, + { name: string; url: string | null } + >(); // Fetch documents for each data source in parallel. await concurrentExecutor( - Object.entries(generationsByDataSources), - async ([, generations]) => { - const dataSource = generations[0].dataSource; - const documentIds = [...new Set(generations.map((g) => g.documentId))]; + Object.entries(docsToFetchByDataSourceId), + async ([dataSourceId, documentIds]) => { + const dataSource = dataSourceById[dataSourceId]; const docsResult = await coreAPI.getDataSourceDocuments({ projectId: dataSource.dustAPIProjectId, @@ -156,7 +173,7 @@ const _sendTrackerWithGenerationEmail = async ({ } docsResult.value.documents.forEach((doc) => { - documentsById.set(doc.document_id, { + documentsByIdentifier.set(`${dataSource.id}__${doc.document_id}`, { name: doc.title ?? "Unknown document", url: doc.source_url ?? null, }); @@ -165,31 +182,56 @@ const _sendTrackerWithGenerationEmail = async ({ { concurrency: 5 } ); - const generationBody = generations.map((generation) => { - const doc = documentsById.get(generation.documentId) ?? { - name: "Unknown document", - url: null, - }; - - const title = doc.url - ? `${doc.name}` - : `[${doc.name}]`; - - return [ - `Changes in document ${title} from ${generation.dataSource.name}:`, - generation.thinking && ``, - `

${generation.content}.

`, - ] - .filter(Boolean) - .join(""); - }); + const generationBody = await Promise.all( + generations.map((g) => { + const doc = documentsByIdentifier.get( + `${g.dataSource.id}__${g.documentId}` + ) ?? { + name: "Unknown document", + url: null, + }; + const maintainedDoc = g.maintainedDataSource + ? documentsByIdentifier.get( + `${g.maintainedDataSource.id}__${g.maintainedDocumentId}` + ) ?? null + : null; + + const title = doc.url + ? `${doc.name}` + : `[${doc.name}]`; + + let maintainedTitle: string | null = null; + if (maintainedDoc) { + maintainedTitle = maintainedDoc.url + ? `${maintainedDoc.name}` + : `[${maintainedDoc.name}]`; + } + + let body = `Changes in document ${title} from ${g.dataSource.name}`; + if (maintainedTitle && g.maintainedDataSource) { + body += ` might affect ${maintainedTitle} from ${g.maintainedDataSource.name}`; + } + body += `:`; + + if (g.thinking) { + body += ` +
+ View thinking +

${g.thinking.replace(/\n/g, "
")}

+
`; + } + + body += `

${g.content.replace(/\n/g, "
")}.

`; + return body; + }) + ); const body = `

We have new suggestions for your tracker ${name}:

${generations.length} recommendations were generated due to changes in watched documents.



-${generationBody.join("
")} +${generationBody.join("
")} `; await sendEmailWithTemplate({ diff --git a/front/lib/error.ts b/front/lib/error.ts index b41bfc5e7fb3..abf36afc2cc7 100644 --- a/front/lib/error.ts +++ b/front/lib/error.ts @@ -10,6 +10,7 @@ export type DustErrorCode = | "data_source_quota_error" | "text_or_section_required" | "invalid_url" + | "invalid_parents" | "invalid_parent_id" // Table | "missing_csv" diff --git a/front/lib/models/doc_tracker.ts b/front/lib/models/doc_tracker.ts index 3b9af1c38b21..5e40b00c0265 100644 --- a/front/lib/models/doc_tracker.ts +++ b/front/lib/models/doc_tracker.ts @@ -237,14 +237,16 @@ export class TrackerGenerationModel extends SoftDeletableModel; declare dataSourceId: ForeignKey; declare documentId: string; - declare maintainedDocumentDataSourceId: ForeignKey; - declare maintainedDocumentId: string; + declare maintainedDocumentDataSourceId: ForeignKey< + DataSourceModel["id"] + > | null; + declare maintainedDocumentId: string | null; declare consumedAt: Date | null; declare trackerConfiguration: NonAttribute; declare dataSource: NonAttribute; - declare maintainedDocumentDataSource: NonAttribute; + declare maintainedDocumentDataSource: NonAttribute | null; } TrackerGenerationModel.init( diff --git a/front/lib/resources/tracker_resource.ts b/front/lib/resources/tracker_resource.ts index 7910e389db46..3a197b57c6c9 100644 --- a/front/lib/resources/tracker_resource.ts +++ b/front/lib/resources/tracker_resource.ts @@ -437,6 +437,11 @@ export class TrackerConfigurationResource extends ResourceWithSpace = - fetcher; + const conversationFetcher: Fetcher = fetcher; const { data, error, mutate } = useSWRWithDefaults( `/api/w/${workspaceId}/assistant/conversations`, @@ -121,7 +122,6 @@ export function useConversationMessages({ conversationId, workspaceId, limit, - startAtRank, }: { conversationId: string | null; workspaceId: string; @@ -147,10 +147,7 @@ export function useConversationMessages({ } if (previousPageData === null) { - const startAtRankParam = startAtRank - ? `&lastValue=${startAtRank}` - : ""; - return `/api/w/${workspaceId}/assistant/conversations/${conversationId}/messages?orderDirection=desc&orderColumn=rank&limit=${limit}${startAtRankParam}`; + return `/api/w/${workspaceId}/assistant/conversations/${conversationId}/messages?orderDirection=desc&orderColumn=rank&limit=${limit}`; } return `/api/w/${workspaceId}/assistant/conversations/${conversationId}/messages?lastValue=${previousPageData.lastValue}&orderDirection=desc&orderColumn=rank&limit=${limit}`; @@ -208,7 +205,9 @@ export const useDeleteConversation = (owner: LightWorkspaceType) => { workspaceId: owner.sId, }); - const doDelete = async (conversation: ConversationType | null) => { + const doDelete = async ( + conversation: ConversationWithoutContentType | null + ) => { if (!conversation) { return false; } diff --git a/front/lib/utils/apps.ts b/front/lib/utils/apps.ts new file mode 100644 index 000000000000..c767a3afcb6b --- /dev/null +++ b/front/lib/utils/apps.ts @@ -0,0 +1,416 @@ +import type { ApiAppType } from "@dust-tt/client"; +import { DustAPI } from "@dust-tt/client"; +import type { CoreAPIError, Result, TraceType } from "@dust-tt/types"; +import { + CoreAPI, + credentialsFromProviders, + Err, + Ok, + removeNulls, +} from "@dust-tt/types"; +import { createParser } from "eventsource-parser"; +import _ from "lodash"; + +import { default as apiConfig, default as config } from "@app/lib/api/config"; +import { getDustAppSecrets } from "@app/lib/api/dust_app_secrets"; +import type { Authenticator } from "@app/lib/auth"; +import { AppResource } from "@app/lib/resources/app_resource"; +import { RunResource } from "@app/lib/resources/run_resource"; +import type { SpaceResource } from "@app/lib/resources/space_resource"; +import { Dataset, Provider } from "@app/lib/resources/storage/models/apps"; +import { dumpSpecification } from "@app/lib/specification"; +import logger from "@app/logger/logger"; + +async function updateOrCreateApp( + auth: Authenticator, + { + appToImport, + space, + }: { + appToImport: ApiAppType; + space: SpaceResource; + } +): Promise< + Result<{ app: AppResource; updated: boolean }, Error | CoreAPIError> +> { + const existingApps = await AppResource.listBySpace(auth, space, { + includeDeleted: true, + }); + const existingApp = existingApps.find((a) => a.sId === appToImport.sId); + if (existingApp) { + // Check if existing app was deleted + if (existingApp.deletedAt) { + return new Err( + new Error("App has been deleted, it can't be reimported.") + ); + } + + // Now update if name/descriptions have been modified + if ( + existingApp.name !== appToImport.name || + existingApp.description !== appToImport.description + ) { + await existingApp.updateSettings(auth, { + name: appToImport.name, + description: appToImport.description, + }); + return new Ok({ app: existingApp, updated: true }); + } + return new Ok({ app: existingApp, updated: false }); + } else { + // An app with this sId exist, check workspace and space first to see if it matches + const existingApp = await AppResource.fetchById(auth, appToImport.sId); + if (existingApp) { + return new Err( + new Error("App with this sId already exists in another space.") + ); + } + + // App does not exist, create a new app + const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); + const p = await coreAPI.createProject(); + + if (p.isErr()) { + return p; + } + const dustAPIProject = p.value.project; + + const owner = auth.getNonNullableWorkspace(); + const newApp = await AppResource.makeNew( + { + sId: appToImport.sId, + name: appToImport.name, + description: appToImport.description, + visibility: "private", + dustAPIProjectId: dustAPIProject.project_id.toString(), + workspaceId: owner.id, + }, + space + ); + + return new Ok({ app: newApp, updated: true }); + } +} + +async function updateDatasets( + auth: Authenticator, + { + app, + datasetsToImport, + }: { + app: AppResource; + datasetsToImport: ApiAppType["datasets"]; + } +): Promise> { + if (datasetsToImport) { + const owner = auth.getNonNullableWorkspace(); + const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); + + // Getting all existing datasets for this app + const existingDatasets = await Dataset.findAll({ + where: { + workspaceId: owner.id, + appId: app.id, + }, + }); + + for (const datasetToImport of datasetsToImport) { + // First, create or update the dataset in core + const coreDataset = await coreAPI.createDataset({ + projectId: app.dustAPIProjectId, + datasetId: datasetToImport.name, + data: datasetToImport.data || [], + }); + if (coreDataset.isErr()) { + return coreDataset; + } + + // Now update the dataset in front if it exists, or create one + const dataset = existingDatasets.find( + (d) => d.name === datasetToImport.name + ); + if (dataset) { + if ( + !_.isEqual(dataset.schema, datasetToImport.schema) || + dataset.description !== datasetToImport.description + ) { + await dataset.update({ + description: datasetToImport.description, + schema: datasetToImport.schema, + }); + } + } else { + await Dataset.create({ + name: datasetToImport.name, + description: datasetToImport.description, + appId: app.id, + workspaceId: owner.id, + schema: datasetToImport.schema, + }); + } + } + } + return new Ok(true); +} + +async function updateAppSpecifications( + auth: Authenticator, + { + app, + savedSpecification, + savedConfig, + }: { + app: AppResource; + savedSpecification: string; + savedConfig: string; + } +): Promise> { + // Specification and config have been modified and need to be imported + if ( + savedSpecification !== app.savedSpecification && + savedConfig !== app.savedConfig + ) { + // Fetch all datasets from core for this app + const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); + const coreDatasets = await coreAPI.getDatasets({ + projectId: app.dustAPIProjectId, + }); + if (coreDatasets.isErr()) { + return coreDatasets; + } + + const latestDatasets: { [key: string]: string } = {}; + for (const d in coreDatasets.value.datasets) { + latestDatasets[d] = coreDatasets.value.datasets[d][0].hash; + } + + const [datasetId] = Object.keys(latestDatasets); + if (datasetId) { + const owner = auth.getNonNullableWorkspace(); + // Fetch providers and secrets + const [providers, secrets] = await Promise.all([ + Provider.findAll({ + where: { + workspaceId: owner.id, + }, + }), + getDustAppSecrets(auth, true), + ]); + + // Create a new run to save specifications and configs + const dustRun = await coreAPI.createRunStream(owner, auth.groups(), { + projectId: app.dustAPIProjectId, + runType: "local", + specification: dumpSpecification( + JSON.parse(savedSpecification), + latestDatasets + ), + config: { blocks: JSON.parse(savedConfig) }, + credentials: credentialsFromProviders(providers), + datasetId, + secrets, + storeBlocksResults: true, + }); + + if (dustRun.isErr()) { + logger.error(app, "Failed to create run for app"); + return dustRun; + } + + let error = undefined; + try { + // Intercept block_execution events to store token usages. + const parser = createParser((event) => { + if (event.type === "event") { + if (event.data) { + const data = JSON.parse(event.data); + if (data.type === "block_execution") { + const traces: TraceType[][] = data.content.execution; + const errs = traces.flatMap((trace) => + removeNulls(trace.map((t) => t.error)) + ); + if (errs.length > 0) { + throw new Error(errs[0]); + } + } + } + } + }); + + for await (const chunk of dustRun.value.chunkStream) { + parser.feed(new TextDecoder().decode(chunk)); + } + } catch (err) { + if (err instanceof Error) { + error = err.message; + } else { + error = String(err); + } + } + + const dustRunId = await dustRun.value.dustRunId; + + // Update app state + await Promise.all([ + RunResource.makeNew({ + dustRunId, + appId: app.id, + runType: "local", + workspaceId: owner.id, + }), + + app.updateState(auth, { + savedSpecification, + savedConfig, + savedRun: dustRunId, + }), + ]); + + if (error) { + return new Err(new Error(error)); + } + + return new Ok(true); + } + } + return new Ok(false); +} + +export async function importApp( + auth: Authenticator, + space: SpaceResource, + appToImport: ApiAppType +): Promise< + Result<{ app: AppResource; updated: boolean }, CoreAPIError | Error> +> { + const appRes = await updateOrCreateApp(auth, { + appToImport, + space, + }); + if (appRes.isErr()) { + logger.error( + { sId: appToImport.sId, name: appToImport.name, error: appRes.error }, + "Error when importing app config" + ); + return appRes; + } + + const { app, updated } = appRes.value; + + const datasetsRes = await updateDatasets(auth, { + app, + datasetsToImport: appToImport.datasets, + }); + if (datasetsRes.isErr()) { + logger.error( + { + sId: app.sId, + name: app.name, + error: datasetsRes.error, + }, + "Error when importing app datasets" + ); + return datasetsRes; + } + + if (appToImport.savedSpecification && appToImport.savedConfig) { + const updateSpecificationsRes = await updateAppSpecifications(auth, { + app, + savedSpecification: appToImport.savedSpecification, + savedConfig: appToImport.savedConfig, + }); + if (updateSpecificationsRes.isErr()) { + logger.error( + { + sId: app.sId, + name: app.name, + error: updateSpecificationsRes.error, + }, + "Error when importing app specifications" + ); + return updateSpecificationsRes; + } + + const specUpdated = updateSpecificationsRes.value; + if (updated || specUpdated) { + logger.info( + { sId: app.sId, appName: app.name }, + "App imported successfully" + ); + } + + return new Ok({ app, updated: updated || specUpdated }); + } + + if (updated) { + logger.info( + { sId: app.sId, appName: app.name }, + "App imported successfully" + ); + } + return new Ok({ app, hash: undefined, updated }); +} + +interface ImportRes { + sId: string; + name: string; + error?: string; +} + +export async function importApps( + auth: Authenticator, + space: SpaceResource, + appsToImport: ApiAppType[] +): Promise { + const apps: ImportRes[] = []; + + for (const appToImport of appsToImport) { + const res = await importApp(auth, space, appToImport); + if (res.isErr()) { + apps.push({ + sId: appToImport.sId, + name: appToImport.name, + error: res.error.message, + }); + } else { + const { app, updated } = res.value; + if (updated) { + apps.push({ sId: app.sId, name: app.name }); + } + } + } + + return apps; +} + +export async function synchronizeDustApps( + auth: Authenticator, + space: SpaceResource +): Promise> { + if (!apiConfig.getDustAppsSyncEnabled()) { + return new Ok([]); + } + + const syncMasterApi = new DustAPI( + apiConfig.getDustAPIConfig(), + { + apiKey: apiConfig.getDustAppsSyncMasterApiKey(), + workspaceId: apiConfig.getDustAppsSyncMasterWorkspaceId(), + }, + logger, + apiConfig.getDustAppsSyncMasterApiUrl() + ); + + const exportRes = await syncMasterApi.exportApps({ + appSpaceId: apiConfig.getDustAppsSyncMasterSpaceId(), + }); + + if (exportRes.isErr()) { + const e = exportRes.error; + return new Err(new Error(`Cannot export: ${e.message}`)); + } + + const importRes = await importApps(auth, space, exportRes.value); + logger.info({ importedApp: importRes }, "Apps imported"); + return new Ok(importRes); +} diff --git a/front/migrations/20250113_backfill_github_mime_types.ts b/front/migrations/20250113_backfill_github_mime_types.ts new file mode 100644 index 000000000000..bc5dff4b9e83 --- /dev/null +++ b/front/migrations/20250113_backfill_github_mime_types.ts @@ -0,0 +1,164 @@ +import type { GithubMimeType } from "@dust-tt/types"; +import { MIME_TYPES } from "@dust-tt/types"; +import assert from "assert"; +import type { Sequelize } from "sequelize"; +import { QueryTypes } from "sequelize"; + +import { getCorePrimaryDbConnection } from "@app/lib/production_checks/utils"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import type Logger from "@app/logger/logger"; +import { makeScript } from "@app/scripts/helpers"; + +const { CORE_DATABASE_URI } = process.env; +const BATCH_SIZE = 256; + +type GithubContentNodeType = "REPO_CODE" | "REPO_CODE_DIR" | "REPO_CODE_FILE"; + +/** + * Gets the type of the GitHub content node from its internal id. + * Copy-pasted from connectors/src/connectors/github/lib/utils.ts + */ +function matchGithubInternalIdType(internalId: string): { + type: GithubContentNodeType; + repoId: number; +} { + // All code from repo is selected, format = "github-code-12345678" + if (/^github-code-\d+$/.test(internalId)) { + return { + type: "REPO_CODE", + repoId: parseInt(internalId.replace(/^github-code-/, ""), 10), + }; + } + // A code directory is selected, format = "github-code-12345678-dir-s0Up1n0u" + if (/^github-code-\d+-dir-[a-f0-9]+$/.test(internalId)) { + return { + type: "REPO_CODE_DIR", + repoId: parseInt( + internalId.replace(/^github-code-(\d+)-dir-.*/, "$1"), + 10 + ), + }; + } + // A code file is selected, format = "github-code-12345678-file-s0Up1n0u" + if (/^github-code-\d+-file-[a-f0-9]+$/.test(internalId)) { + return { + type: "REPO_CODE_FILE", + repoId: parseInt( + internalId.replace(/^github-code-(\d+)-file-.*/, "$1"), + 10 + ), + }; + } + throw new Error(`Invalid Github internal id (code-only): ${internalId}`); +} + +function getMimeTypeForNodeId(nodeId: string): GithubMimeType { + switch (matchGithubInternalIdType(nodeId).type) { + case "REPO_CODE": + return MIME_TYPES.GITHUB.CODE_ROOT; + case "REPO_CODE_DIR": + return MIME_TYPES.GITHUB.CODE_DIRECTORY; + case "REPO_CODE_FILE": + return MIME_TYPES.GITHUB.CODE_FILE; + default: + throw new Error(`Unreachable: unrecognized node_id: ${nodeId}`); + } +} + +async function backfillDataSource( + frontDataSource: DataSourceModel, + coreSequelize: Sequelize, + execute: boolean, + logger: typeof Logger +) { + logger.info("Processing data source"); + + let nextId = 0; + let updatedRowsCount; + do { + const rows: { id: number; node_id: string; mime_type: string }[] = + await coreSequelize.query( + ` + SELECT dsn.id, dsn.node_id, dsn.mime_type + FROM data_sources_nodes dsn + JOIN data_sources ds ON ds.id = dsn.data_source + WHERE dsn.id > :nextId + AND ds.data_source_id = :dataSourceId + AND ds.project = :projectId + AND dsn.node_id LIKE 'github-code-%' -- leverages the btree + ORDER BY dsn.id + LIMIT :batchSize;`, + { + replacements: { + dataSourceId: frontDataSource.dustAPIDataSourceId, + projectId: frontDataSource.dustAPIProjectId, + batchSize: BATCH_SIZE, + nextId, + }, + type: QueryTypes.SELECT, + } + ); + + if (rows.length == 0) { + logger.info({ nextId }, `Finished processing data source.`); + break; + } + nextId = rows[rows.length - 1].id; + updatedRowsCount = rows.length; + + if (execute) { + await coreSequelize.query( + `WITH pairs AS ( + SELECT UNNEST(ARRAY[:ids]) as id, UNNEST(ARRAY[:mimeTypes]) as mime_type + ) + UPDATE data_sources_nodes dsn + SET mime_type = p.mime_type + FROM pairs p + WHERE dsn.id = p.id;`, + { + replacements: { + mimeTypes: rows.map((row) => getMimeTypeForNodeId(row.node_id)), + ids: rows.map((row) => row.id), + }, + } + ); + logger.info( + `Updated chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } else { + logger.info( + { + nodes: rows.map((row) => ({ + nodeId: row.node_id, + fromMimeType: row.mime_type, + toMimeType: getMimeTypeForNodeId(row.node_id), + })), + }, + `Would update chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } + } while (updatedRowsCount === BATCH_SIZE); +} + +makeScript({}, async ({ execute }, logger) => { + assert(CORE_DATABASE_URI, "CORE_DATABASE_URI is required"); + + const coreSequelize = getCorePrimaryDbConnection(); + + const frontDataSources = await DataSourceModel.findAll({ + where: { connectorProvider: "github" }, + }); + logger.info(`Found ${frontDataSources.length} GitHub data sources`); + + for (const frontDataSource of frontDataSources) { + await backfillDataSource( + frontDataSource, + coreSequelize, + execute, + logger.child({ + dataSourceId: frontDataSource.id, + connectorId: frontDataSource.connectorId, + }) + ); + } +}); diff --git a/front/migrations/20250113_backfill_zendesk_hc_mime_types.ts b/front/migrations/20250113_backfill_zendesk_hc_mime_types.ts new file mode 100644 index 000000000000..3395b6451ce4 --- /dev/null +++ b/front/migrations/20250113_backfill_zendesk_hc_mime_types.ts @@ -0,0 +1,96 @@ +import { MIME_TYPES } from "@dust-tt/types"; +import assert from "assert"; +import type { Sequelize } from "sequelize"; +import { QueryTypes } from "sequelize"; + +import { getCorePrimaryDbConnection } from "@app/lib/production_checks/utils"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import type Logger from "@app/logger/logger"; +import { makeScript } from "@app/scripts/helpers"; + +const { CORE_DATABASE_URI } = process.env; +const BATCH_SIZE = 16; + +async function backfillDataSource( + frontDataSource: DataSourceModel, + coreSequelize: Sequelize, + execute: boolean, + logger: typeof Logger +) { + logger.info("Processing data source"); + + let nextId = 0; + let updatedRowsCount; + do { + const rows: { id: number }[] = await coreSequelize.query( + ` + SELECT dsn.id + FROM data_sources_nodes dsn + JOIN data_sources ds ON ds.id = dsn.data_source + WHERE dsn.id > :nextId + AND ds.data_source_id = :dataSourceId + AND ds.project = :projectId + AND dsn.node_id LIKE 'zendesk-help-center-%' + ORDER BY dsn.id + LIMIT :batchSize;`, + { + replacements: { + dataSourceId: frontDataSource.dustAPIDataSourceId, + projectId: frontDataSource.dustAPIProjectId, + batchSize: BATCH_SIZE, + nextId, + }, + type: QueryTypes.SELECT, + } + ); + + if (rows.length == 0) { + logger.info({ nextId }, `Finished processing data source.`); + break; + } + nextId = rows[rows.length - 1].id; + updatedRowsCount = rows.length; + + if (execute) { + await coreSequelize.query( + `UPDATE data_sources_nodes SET mime_type = :mimeType WHERE id IN (:ids)`, + { + replacements: { + mimeType: MIME_TYPES.ZENDESK.HELP_CENTER, + ids: rows.map((row) => row.id), + }, + } + ); + logger.info( + `Updated chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } else { + logger.info( + `Would update chunk from ${rows[0].id} to ${rows[rows.length - 1].id}` + ); + } + } while (updatedRowsCount === BATCH_SIZE); +} + +makeScript({}, async ({ execute }, logger) => { + assert(CORE_DATABASE_URI, "CORE_DATABASE_URI is required"); + + const coreSequelize = getCorePrimaryDbConnection(); + + const frontDataSources = await DataSourceModel.findAll({ + where: { connectorProvider: "zendesk" }, + }); + logger.info(`Found ${frontDataSources.length} Zendesk data sources`); + + for (const frontDataSource of frontDataSources) { + await backfillDataSource( + frontDataSource, + coreSequelize, + execute, + logger.child({ + dataSourceId: frontDataSource.id, + connectorId: frontDataSource.connectorId, + }) + ); + } +}); diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/check.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/check.ts new file mode 100644 index 000000000000..1a92d00c4215 --- /dev/null +++ b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/check.ts @@ -0,0 +1,102 @@ +import type { AppsCheckResponseType } from "@dust-tt/client"; +import { AppsCheckRequestSchema } from "@dust-tt/client"; +import type { WithAPIErrorResponse } from "@dust-tt/types"; +import { concurrentExecutor, CoreAPI } from "@dust-tt/types"; +import type { NextApiRequest, NextApiResponse } from "next"; +import { fromError } from "zod-validation-error"; + +import { withPublicAPIAuthentication } from "@app/lib/api/auth_wrappers"; +import config from "@app/lib/api/config"; +import { withResourceFetchingFromRoute } from "@app/lib/api/resource_wrappers"; +import type { Authenticator } from "@app/lib/auth"; +import { AppResource } from "@app/lib/resources/app_resource"; +import type { SpaceResource } from "@app/lib/resources/space_resource"; +import logger from "@app/logger/logger"; +import { apiError } from "@app/logger/withlogging"; + +/** + * @ignoreswagger + * System API key only endpoint. Undocumented. + */ +async function handler( + req: NextApiRequest, + res: NextApiResponse>, + auth: Authenticator, + { space }: { space: SpaceResource } +): Promise { + if (!auth.isSystemKey()) { + return apiError(req, res, { + status_code: 403, + api_error: { + type: "invalid_oauth_token_error", + message: "Only system keys are allowed to use this endpoint.", + }, + }); + } + + if (!space.canRead(auth)) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "space_not_found", + message: "The space you requested was not found.", + }, + }); + } + + switch (req.method) { + case "POST": + const r = AppsCheckRequestSchema.safeParse(req.body); + + if (r.error) { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: fromError(r.error).toString(), + }, + }); + } + + const coreAPI = new CoreAPI(config.getCoreAPIConfig(), logger); + const apps = await concurrentExecutor( + r.data.apps, + async (appRequest) => { + const app = await AppResource.fetchById(auth, appRequest.appId); + if (!app) { + return { ...appRequest, deployed: false }; + } + const coreSpec = await coreAPI.getSpecification({ + projectId: app.dustAPIProjectId, + specificationHash: appRequest.appHash, + }); + if (coreSpec.isErr()) { + return { ...appRequest, deployed: false }; + } + + return { ...appRequest, deployed: true }; + }, + { concurrency: 5 } + ); + + res.status(200).json({ + apps, + }); + return; + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withPublicAPIAuthentication( + withResourceFetchingFromRoute(handler, { + space: { requireCanRead: true }, + }) +); diff --git a/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/export.ts b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/export.ts new file mode 100644 index 000000000000..7a416617ce66 --- /dev/null +++ b/front/pages/api/v1/w/[wId]/spaces/[spaceId]/apps/export.ts @@ -0,0 +1,87 @@ +import type { GetAppsResponseType } from "@dust-tt/client"; +import type { WithAPIErrorResponse } from "@dust-tt/types"; +import { concurrentExecutor } from "@dust-tt/types"; +import type { NextApiRequest, NextApiResponse } from "next"; + +import { withPublicAPIAuthentication } from "@app/lib/api/auth_wrappers"; +import { getDatasetHash, getDatasets } from "@app/lib/api/datasets"; +import { withResourceFetchingFromRoute } from "@app/lib/api/resource_wrappers"; +import type { Authenticator } from "@app/lib/auth"; +import { AppResource } from "@app/lib/resources/app_resource"; +import type { SpaceResource } from "@app/lib/resources/space_resource"; +import { apiError } from "@app/logger/withlogging"; + +/** + * @ignoreswagger + * System API key only endpoint. Undocumented. + */ +async function handler( + req: NextApiRequest, + res: NextApiResponse>, + auth: Authenticator, + { space }: { space: SpaceResource } +): Promise { + if (!auth.isSystemKey()) { + return apiError(req, res, { + status_code: 403, + api_error: { + type: "invalid_oauth_token_error", + message: "Only system keys are allowed to use this endpoint.", + }, + }); + } + + if (!space.canRead(auth)) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "space_not_found", + message: "The space you requested was not found.", + }, + }); + } + + switch (req.method) { + case "GET": + const apps = await AppResource.listBySpace(auth, space); + + const enhancedApps = await concurrentExecutor( + apps.filter((app) => app.canRead(auth)), + async (app) => { + const datasetsFromFront = await getDatasets(auth, app.toJSON()); + const datasets = []; + for (const dataset of datasetsFromFront) { + const fromCore = await getDatasetHash( + auth, + app, + dataset.name, + "latest" + ); + datasets.push(fromCore ?? dataset); + } + return { ...app.toJSON(), datasets }; + }, + { concurrency: 5 } + ); + + res.status(200).json({ + apps: enhancedApps, + }); + return; + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, GET is expected.", + }, + }); + } +} + +export default withPublicAPIAuthentication( + withResourceFetchingFromRoute(handler, { + space: { requireCanRead: true }, + }) +); diff --git a/front/pages/api/w/[wId]/data_sources/[dsId]/files.ts b/front/pages/api/w/[wId]/data_sources/[dsId]/files.ts index 78a49f135504..b45d4898b84d 100644 --- a/front/pages/api/w/[wId]/data_sources/[dsId]/files.ts +++ b/front/pages/api/w/[wId]/data_sources/[dsId]/files.ts @@ -1,6 +1,5 @@ import type { FileType, WithAPIErrorResponse } from "@dust-tt/types"; -import type { NextApiRequest } from "next"; -import type { NextApiResponse } from "next"; +import type { NextApiRequest, NextApiResponse } from "next"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrappers"; import type { @@ -17,16 +16,11 @@ import { apiError } from "@app/logger/withlogging"; export interface UpsertFileToDataSourceRequestBody { fileId: string; upsertArgs?: - | Pick - | Pick< + | Pick + | (Pick< UpsertTableArgs, - | "name" - | "title" - | "description" - | "tableId" - | "tags" - | "useAppForHeaderDetection" - >; + "name" | "title" | "description" | "tags" | "useAppForHeaderDetection" + > & { tableId: string | undefined }); // we actually don't always have a tableId, this is very dirty, but the refactoring should be done at the level of the whole upsertArgs mechanic } export interface UpsertFileToDataSourceResponseBody { diff --git a/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/[documentId]/index.ts b/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/[documentId]/index.ts index bff299183488..2ffb8f9fd7d0 100644 --- a/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/[documentId]/index.ts +++ b/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/[documentId]/index.ts @@ -118,7 +118,7 @@ async function handler( } = bodyValidation.right; const upsertResult = await upsertDocument({ - name: documentId, + document_id: documentId, source_url, text, section, diff --git a/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/index.ts b/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/index.ts index 4d7635cf23ec..a7dafafaede5 100644 --- a/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/index.ts +++ b/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/documents/index.ts @@ -111,7 +111,7 @@ async function handler( } = bodyValidation.right; const upsertResult = await upsertDocument({ - name, + document_id: name, // using the name as the document_id since we don't have one here source_url, text, section, diff --git a/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts b/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts index a207d2170d90..08bba7a8e192 100644 --- a/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts +++ b/front/pages/api/w/[wId]/spaces/[spaceId]/data_sources/[dsId]/tables/index.ts @@ -10,6 +10,7 @@ import { withResourceFetchingFromRoute } from "@app/lib/api/resource_wrappers"; import type { Authenticator } from "@app/lib/auth"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; import type { SpaceResource } from "@app/lib/resources/space_resource"; +import { generateRandomModelSId } from "@app/lib/resources/string_ids"; import { apiError } from "@app/logger/withlogging"; export const config = { @@ -93,11 +94,13 @@ async function handler( }); } + const tableId = generateRandomModelSId(); const upsertRes = await upsertTable({ ...bodyValidation.right, async: bodyValidation.right.async ?? false, dataSource, auth, + tableId, }); if (upsertRes.isErr()) { diff --git a/front/pages/home/contact.tsx b/front/pages/home/contact.tsx index 5f3f40695d8a..e6182d782417 100644 --- a/front/pages/home/contact.tsx +++ b/front/pages/home/contact.tsx @@ -1,7 +1,7 @@ -import dynamic from "next/dynamic"; import type { ReactElement } from "react"; import { HeaderContentBlock } from "@app/components/home/ContentBlocks"; +import HubSpotForm from "@app/components/home/HubSpotForm"; import type { LandingLayoutProps } from "@app/components/home/LandingLayout"; import LandingLayout from "@app/components/home/LandingLayout"; import { @@ -9,12 +9,6 @@ import { shapeNames, } from "@app/components/home/Particles"; import TrustedBy from "@app/components/home/TrustedBy"; -// import HubSpotForm from "@app/components/home/HubSpotForm"; - -const HubSpotForm = dynamic( - () => import("@app/components/home/HubSpotForm").then((mod) => mod.default) - // { ssr: false } -); export async function getServerSideProps() { return { diff --git a/front/pages/home/solutions/customer-support.tsx b/front/pages/home/solutions/customer-support.tsx index 1159fbe7144d..9e1d8c0e3460 100644 --- a/front/pages/home/solutions/customer-support.tsx +++ b/front/pages/home/solutions/customer-support.tsx @@ -132,13 +132,14 @@ export default function CustomerSupport() { icon={RocketIcon} /> -
+
diff --git a/front/pages/w/[wId]/assistant/[cId]/index.tsx b/front/pages/w/[wId]/assistant/[cId]/index.tsx index 5d1b3ac5ad72..f6e8c49fe21a 100644 --- a/front/pages/w/[wId]/assistant/[cId]/index.tsx +++ b/front/pages/w/[wId]/assistant/[cId]/index.tsx @@ -7,8 +7,8 @@ import { useEffect, useState } from "react"; import { ConversationContainer } from "@app/components/assistant/conversation/ConversationContainer"; import type { ConversationLayoutProps } from "@app/components/assistant/conversation/ConversationLayout"; import ConversationLayout from "@app/components/assistant/conversation/ConversationLayout"; +import { useConversationsNavigation } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { CONVERSATION_PARENT_SCROLL_DIV_ID } from "@app/components/assistant/conversation/lib"; -import { getMessageRank } from "@app/lib/api/assistant/conversation"; import config from "@app/lib/api/config"; import { withDefaultUserAuthRequirements } from "@app/lib/iam/session"; @@ -18,7 +18,6 @@ export const getServerSideProps = withDefaultUserAuthRequirements< conversationId: string | null; user: UserType; isBuilder: boolean; - messageRankToScrollTo: number | null; } >(async (context, auth) => { const owner = auth.workspace(); @@ -47,13 +46,6 @@ export const getServerSideProps = withDefaultUserAuthRequirements< const { cId } = context.params; - // Extract messageId from query params and fetch its rank - const { messageId } = context.query; - let messageRankToScrollTo: number | null = null; - if (typeof messageId === "string") { - messageRankToScrollTo = (await getMessageRank(auth, messageId)) ?? null; - } - return { props: { user, @@ -62,7 +54,6 @@ export const getServerSideProps = withDefaultUserAuthRequirements< subscription, baseUrl: config.getClientFacingUrl(), conversationId: getValidConversationId(cId), - messageRankToScrollTo: messageRankToScrollTo, }, }; }); @@ -73,23 +64,24 @@ export default function AssistantConversation({ subscription, user, isBuilder, - messageRankToScrollTo, }: InferGetServerSidePropsType) { const [conversationKey, setConversationKey] = useState(null); const [agentIdToMention, setAgentIdToMention] = useState(null); const router = useRouter(); - const { cId, assistant } = router.query; + + const { activeConversationId } = useConversationsNavigation(); + + const { assistant } = router.query; + // This useEffect handles whether to change the key of the ConversationContainer // or not. Altering the key forces a re-render of the component. A random number // is used in the key to maintain the component during the transition from new // to the conversation view. The key is reset when navigating to a new conversation. useEffect(() => { - const conversationId = getValidConversationId(cId); - - if (conversationId && initialConversationId) { + if (activeConversationId) { // Set conversation id as key if it exists. - setConversationKey(conversationId); - } else if (!conversationId && !initialConversationId) { + setConversationKey(activeConversationId); + } else if (!activeConversationId) { // Force re-render by setting a new key with a random number. setConversationKey(`new_${Math.random() * 1000}`); @@ -109,19 +101,22 @@ export default function AssistantConversation({ } else { setAgentIdToMention(null); } - }, [cId, assistant, setConversationKey, initialConversationId]); + }, [ + assistant, + setConversationKey, + initialConversationId, + activeConversationId, + ]); return ( ); } diff --git a/front/pages/w/[wId]/spaces/[spaceId]/apps/[aId]/specification.tsx b/front/pages/w/[wId]/spaces/[spaceId]/apps/[aId]/specification.tsx index 688d5f8f45d9..e88bdd73bf0a 100644 --- a/front/pages/w/[wId]/spaces/[spaceId]/apps/[aId]/specification.tsx +++ b/front/pages/w/[wId]/spaces/[spaceId]/apps/[aId]/specification.tsx @@ -20,6 +20,7 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ readOnly: boolean; app: AppType; specification: string; + specificationFromCore: { created: number; data: string; hash: string } | null; }>(async (context, auth) => { const owner = auth.workspace(); const subscription = auth.subscription(); @@ -50,6 +51,26 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ }; } + let specificationFromCore = null; + const specificationFromCoreHash = context.query?.hash; + + if ( + specificationFromCoreHash && + typeof specificationFromCoreHash === "string" + ) { + const coreSpec = await coreAPI.getSpecification({ + projectId: app.dustAPIProjectId, + specificationHash: specificationFromCoreHash, + }); + + if (coreSpec.isOk()) { + specificationFromCore = { + ...coreSpec.value.specification, + hash: specificationFromCoreHash, + }; + } + } + const latestDatasets = {} as { [key: string]: string }; for (const d in datasets.value.datasets) { latestDatasets[d] = datasets.value.datasets[d][0].hash; @@ -67,6 +88,7 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ readOnly, app: app.toJSON(), specification: spec, + specificationFromCore, }, }; }); @@ -76,6 +98,7 @@ export default function Specification({ subscription, app, specification, + specificationFromCore, }: InferGetServerSidePropsType) { const router = useRouter(); @@ -113,8 +136,19 @@ export default function Specification({ )} -
- {specification} +
+

Current specifications :

+
+ {specification} +
+ {specificationFromCore && ( + <> +

Saved specifications {specificationFromCore.hash}:

+
+ {specificationFromCore.data} +
+ + )}
diff --git a/front/public/swagger.json b/front/public/swagger.json index 7477c469ded6..85dfb622fde6 100644 --- a/front/public/swagger.json +++ b/front/public/swagger.json @@ -3296,8 +3296,8 @@ }, "/api/v1/w/{wId}/spaces": { "get": { - "summary": "List Workspace Spaces", - "description": "Retrieves a list of spaces for the authenticated workspace.", + "summary": "List Spaces accessible.", + "description": "Retrieves a list of accessible spaces for the authenticated workspace.", "tags": [ "Spaces" ], diff --git a/front/scripts/send_tracker_generations.ts b/front/scripts/send_tracker_generations.ts new file mode 100644 index 000000000000..2dba123e632b --- /dev/null +++ b/front/scripts/send_tracker_generations.ts @@ -0,0 +1,101 @@ +import { sendTrackerWithGenerationEmail } from "@app/lib/api/tracker"; +import { TrackerGenerationModel } from "@app/lib/models/doc_tracker"; +import { frontSequelize } from "@app/lib/resources/storage"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import { isEmailValid } from "@app/lib/utils"; +import { makeScript } from "@app/scripts/helpers"; + +makeScript( + { + generationIds: { + type: "array", + demandOption: true, + description: "List of generation IDs", + }, + email: { + type: "string", + demandOption: true, + description: "Email address to send to", + }, + }, + async ({ execute, generationIds, email }, logger) => { + try { + // Validate email + if (!isEmailValid(email)) { + throw new Error("Invalid email address"); + } + + // Parse and validate generation IDs + const ids = generationIds.map((id) => parseInt(id)); + if (ids.some((id) => isNaN(id))) { + throw new Error("Invalid generation IDs - must be numbers"); + } + + if (execute) { + // Fetch generations with their data sources + const generations = await TrackerGenerationModel.findAll({ + where: { + id: ids, + }, + include: [ + { + model: DataSourceModel, + required: true, + }, + { + model: DataSourceModel, + as: "maintainedDocumentDataSource", + required: false, + }, + ], + }); + + if (generations.length === 0) { + throw new Error("No generations found with the provided IDs"); + } + + // Convert to TrackerGenerationToProcess format + const generationsToProcess = generations.map((g) => ({ + id: g.id, + content: g.content, + thinking: g.thinking, + documentId: g.documentId, + dataSource: { + id: g.dataSource.id, + name: g.dataSource.name, + dustAPIProjectId: g.dataSource.dustAPIProjectId, + dustAPIDataSourceId: g.dataSource.dustAPIDataSourceId, + }, + maintainedDocumentId: g.maintainedDocumentId, + maintainedDataSource: g.maintainedDocumentDataSource + ? { + id: g.maintainedDocumentDataSource.id, + name: g.maintainedDocumentDataSource.name, + dustAPIProjectId: + g.maintainedDocumentDataSource.dustAPIProjectId, + dustAPIDataSourceId: + g.maintainedDocumentDataSource.dustAPIDataSourceId, + } + : null, + })); + + // Send email + await sendTrackerWithGenerationEmail({ + name: "Manual Generation Email", + recipient: email, + generations: generationsToProcess, + localLogger: logger, + }); + + logger.info({}, "Email sent successfully"); + } else { + logger.info( + { generationIds: ids, email }, + "Dry run - would send email with these parameters" + ); + } + } finally { + await frontSequelize.close(); + } + } +); diff --git a/front/temporal/tracker/activities.ts b/front/temporal/tracker/activities.ts index 35749331c931..d8e4f5b40982 100644 --- a/front/temporal/tracker/activities.ts +++ b/front/temporal/tracker/activities.ts @@ -11,6 +11,7 @@ import { Err, GPT_4O_MODEL_CONFIG, Ok, + removeNulls, } from "@dust-tt/types"; import { Context } from "@temporalio/activity"; import _ from "lodash"; @@ -484,18 +485,19 @@ async function getTrackersToRun( config.getConnectorsAPIConfig(), logger ); - const parentsResult = await connectorsAPI.getContentNodesParents({ + const parentsResult = await connectorsAPI.getContentNodes({ connectorId: dataSource.connectorId, internalIds: [documentId], + includeParents: true, }); if (parentsResult.isErr()) { throw parentsResult.error; } - docParentIds = [ + docParentIds = removeNulls([ documentId, - ...parentsResult.value.nodes.flatMap((node) => node.parents), - ]; + ...parentsResult.value.nodes.flatMap((node) => node.parentInternalIds), + ]); } return TrackerConfigurationResource.fetchAllWatchedForDocument(auth, { diff --git a/init_dev_container.sh b/init_dev_container.sh index d3a42eabf2db..cb27a25ffc06 100755 --- a/init_dev_container.sh +++ b/init_dev_container.sh @@ -14,5 +14,5 @@ cd - ## Initializing Elasticsearch indices cd core/ -cargo run --bin elasticsearch_create_index -- --index-name data_sources_nodes --index-version 1 --skip-confirmation +cargo run --bin elasticsearch_create_index -- --index-name data_sources_nodes --index-version 2 --skip-confirmation cd - diff --git a/sdks/js/src/index.ts b/sdks/js/src/index.ts index 31bc68e0d7a9..f8e38d37881a 100644 --- a/sdks/js/src/index.ts +++ b/sdks/js/src/index.ts @@ -7,6 +7,7 @@ import type { AgentErrorEvent, AgentMessageSuccessEvent, APIError, + AppsCheckRequestType, CancelMessageGenerationRequestType, ConversationPublicType, DataSourceViewType, @@ -32,6 +33,7 @@ import type { } from "./types"; import { APIErrorSchema, + AppsCheckResponseSchema, CancelMessageGenerationResponseSchema, CreateConversationResponseSchema, DataSourceViewResponseSchema, @@ -41,6 +43,7 @@ import { FileUploadUrlRequestSchema, GetActiveMemberEmailsInWorkspaceResponseSchema, GetAgentConfigurationsResponseSchema, + GetAppsResponseSchema, GetConversationResponseSchema, GetConversationsResponseSchema, GetDataSourcesResponseSchema, @@ -1038,6 +1041,35 @@ export class DustAPI { return new Ok(r.value.dataSourceView); } + async exportApps({ appSpaceId }: { appSpaceId: string }) { + const res = await this.request({ + method: "GET", + path: `spaces/${appSpaceId}/apps/export`, + }); + + const r = await this._resultFromResponse(GetAppsResponseSchema, res); + + if (r.isErr()) { + return r; + } + return new Ok(r.value.apps); + } + + async checkApps(apps: AppsCheckRequestType, appSpaceId: string) { + const res = await this.request({ + method: "POST", + path: `spaces/${appSpaceId}/apps/check`, + body: apps, + }); + + const r = await this._resultFromResponse(AppsCheckResponseSchema, res); + + if (r.isErr()) { + return r; + } + return new Ok(r.value.apps); + } + private async _fetchWithError( url: string, config?: AxiosRequestConfig diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index c9a9928b6090..1b83aa6ae914 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -1462,6 +1462,26 @@ const SpaceTypeSchema = z.object({ updatedAt: z.number(), }); +const DatasetSchemaEntryType = FlexibleEnumSchema< + "string" | "number" | "boolean" | "json" +>(); + +const DatasetSchema = z.object({ + name: z.string(), + description: z.string().nullable(), + data: z.array(z.record(z.any())).nullable().optional(), + schema: z + .array( + z.object({ + key: z.string(), + type: DatasetSchemaEntryType, + description: z.string().nullable(), + }) + ) + .nullable() + .optional(), +}); + const AppTypeSchema = z.object({ id: ModelIdSchema, sId: z.string(), @@ -1472,8 +1492,11 @@ const AppTypeSchema = z.object({ savedRun: z.string().nullable(), dustAPIProjectId: z.string(), space: SpaceTypeSchema, + datasets: z.array(DatasetSchema).optional(), }); +export type ApiAppType = z.infer; + export const RunAppResponseSchema = z.object({ run: RunTypeSchema, }); @@ -1757,7 +1780,11 @@ export type ValidateMemberResponseType = z.infer< typeof ValidateMemberResponseSchema >; -const GetAppsResponseSchema = z.object({ +export const GetAppsResponseSchema = z.object({ + apps: AppTypeSchema.array(), +}); + +export const PostAppsRequestSchema = z.object({ apps: AppTypeSchema.array(), }); @@ -2321,3 +2348,26 @@ export function getTitleFromRetrievedDocument( return document.documentId; } + +export const AppsCheckRequestSchema = z.object({ + apps: z.array( + z.object({ + appId: z.string(), + appHash: z.string(), + }) + ), +}); + +export type AppsCheckRequestType = z.infer; + +export const AppsCheckResponseSchema = z.object({ + apps: z.array( + z.object({ + appId: z.string(), + appHash: z.string(), + deployed: z.boolean(), + }) + ), +}); + +export type AppsCheckResponseType = z.infer; diff --git a/sparkle/package-lock.json b/sparkle/package-lock.json index 1d6666da5d7f..5ee3879a3bd6 100644 --- a/sparkle/package-lock.json +++ b/sparkle/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dust-tt/sparkle", - "version": "0.2.362", + "version": "0.2.363", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dust-tt/sparkle", - "version": "0.2.362", + "version": "0.2.363", "license": "ISC", "dependencies": { "@emoji-mart/data": "^1.1.2", diff --git a/sparkle/package.json b/sparkle/package.json index e80d72e4f283..19cb50bd0fe6 100644 --- a/sparkle/package.json +++ b/sparkle/package.json @@ -1,6 +1,6 @@ { "name": "@dust-tt/sparkle", - "version": "0.2.362", + "version": "0.2.363", "scripts": { "build": "rm -rf dist && npm run tailwind && npm run build:esm && npm run build:cjs", "tailwind": "tailwindcss -i ./src/styles/tailwind.css -o dist/sparkle.css", diff --git a/sparkle/src/components/index.ts b/sparkle/src/components/index.ts index db73d0adc667..ba27ba1a9f07 100644 --- a/sparkle/src/components/index.ts +++ b/sparkle/src/components/index.ts @@ -61,8 +61,6 @@ export { ElementModal } from "./ElementModal"; export type { EmojiMartData } from "./EmojiPicker"; export { DataEmojiMart, EmojiPicker } from "./EmojiPicker"; export { EmptyCTA, EmptyCTAButton } from "./EmptyCTA"; -export type { FeedbackSelectorProps } from "./FeedbackSelector"; -export { FeedbackSelector } from "./FeedbackSelector"; export { FilterChips } from "./FilterChips"; export { Div3D, Hover3D } from "./Hover3D"; export { Hoverable } from "./Hoverable"; diff --git a/sparkle/src/stories/FeedbackSelector.stories.tsx b/sparkle/src/stories/FeedbackSelector.stories.tsx deleted file mode 100644 index f47c75fba94f..000000000000 --- a/sparkle/src/stories/FeedbackSelector.stories.tsx +++ /dev/null @@ -1,72 +0,0 @@ -import type { Meta, StoryObj } from "@storybook/react"; -import React from "react"; - -import { - FeedbackSelector, - FeedbackSelectorProps, -} from "@sparkle/components/FeedbackSelector"; - -const meta = { - title: "Primitives/FeedbackSelector", - component: FeedbackSelector, - argTypes: { - feedback: { - thumb: "up", - feedbackContent: null, - }, - onSubmitThumb: { - description: "The submit function", - control: { - type: "object", - }, - }, - isSubmittingThumb: { - description: "Whether the thumb is submitting", - control: { - type: "boolean", - }, - }, - }, -} satisfies Meta>; - -export default meta; -type Story = StoryObj; - -// Wrap the story in a component that can use hooks -const ExampleFeedbackComponent = () => { - const [messageFeedback, setMessageFeedback] = - React.useState({ - feedback: null, - onSubmitThumb: async (element) => { - setMessageFeedback((prev) => ({ - ...prev, - feedback: element.shouldRemoveExistingFeedback - ? null - : { - thumb: element.thumb, - feedbackContent: element.feedbackContent, - isConversationShared: element.isConversationShared, - }, - })); - }, - isSubmittingThumb: false, - getPopoverInfo: () =>
Some info here, like the last author
, - }); - - return ; -}; - -export const ExamplePicker: Story = { - args: { - feedback: { - thumb: "up", - feedbackContent: null, - isConversationShared: true, - }, - onSubmitThumb: async (element) => { - console.log(element); - }, - isSubmittingThumb: false, - }, - render: () => , -}; diff --git a/types/src/front/api_handlers/public/spaces.ts b/types/src/front/api_handlers/public/spaces.ts index 20e51a4c8941..cd1a50f1b3b0 100644 --- a/types/src/front/api_handlers/public/spaces.ts +++ b/types/src/front/api_handlers/public/spaces.ts @@ -21,7 +21,6 @@ export type PatchDataSourceViewType = t.TypeOf< >; export type LightContentNode = { - dustDocumentId: string | null; expandable: boolean; internalId: string; lastUpdatedAt: number | null; diff --git a/types/src/front/data_source_view.ts b/types/src/front/data_source_view.ts index cf0f01c0cd44..82fc37c116cf 100644 --- a/types/src/front/data_source_view.ts +++ b/types/src/front/data_source_view.ts @@ -1,6 +1,11 @@ import { ModelId } from "../shared/model_id"; import { DataSourceViewCategory } from "./api_handlers/public/spaces"; -import { ConnectorStatusDetails, DataSourceType, DataSourceWithAgentsUsageType, EditedByUser } from "./data_source"; +import { + ConnectorStatusDetails, + DataSourceType, + DataSourceWithAgentsUsageType, + EditedByUser, +} from "./data_source"; import { ContentNode } from "./lib/connectors_api"; export interface DataSourceViewType { diff --git a/types/src/front/lib/assistant.ts b/types/src/front/lib/assistant.ts index c95a968df51b..ef445c28fab7 100644 --- a/types/src/front/lib/assistant.ts +++ b/types/src/front/lib/assistant.ts @@ -713,6 +713,8 @@ export const SUPPORTED_MODEL_CONFIGS: ModelConfigurationType[] = [ MISTRAL_CODESTRAL_MODEL_CONFIG, GEMINI_PRO_DEFAULT_MODEL_CONFIG, GEMINI_FLASH_DEFAULT_MODEL_CONFIG, + GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG, TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_CONFIG, TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_CONFIG, TOGETHERAI_QWEN_32B_PREVIEW_MODEL_CONFIG, diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index f31ee8d98ce7..863598f6bbe0 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -1,4 +1,7 @@ -import { AdminCommandType, AdminResponseType } from "../../connectors/admin/cli"; +import { + AdminCommandType, + AdminResponseType, +} from "../../connectors/admin/cli"; import { ConnectorsAPIError, isConnectorsAPIError } from "../../connectors/api"; import { UpdateConnectorConfigurationType } from "../../connectors/api_handlers/connector_configuration"; import { ConnectorCreateRequestBody } from "../../connectors/api_handlers/create_connector"; @@ -103,7 +106,6 @@ export interface ContentNode { expandable: boolean; preventSelection?: boolean; permission: ConnectorPermission; - dustDocumentId: string | null; lastUpdatedAt: number | null; providerVisibility?: "public" | "private"; } @@ -495,36 +497,6 @@ export class ConnectorsAPI { return this._resultFromResponse(res); } - async getContentNodesParents({ - connectorId, - internalIds, - }: { - connectorId: string; - internalIds: string[]; - }): Promise< - ConnectorsAPIResponse<{ - nodes: { - internalId: string; - parents: string[]; - }[]; - }> - > { - const res = await this._fetchWithError( - `${this._url}/connectors/${encodeURIComponent( - connectorId - )}/content_nodes/parents`, - { - method: "POST", - headers: this.getDefaultHeaders(), - body: JSON.stringify({ - internalIds, - }), - } - ); - - return this._resultFromResponse(res); - } - async getContentNodes({ connectorId, includeParents, diff --git a/types/src/front/tracker.ts b/types/src/front/tracker.ts index 9eb0e275b0a1..74f960c4e6fe 100644 --- a/types/src/front/tracker.ts +++ b/types/src/front/tracker.ts @@ -68,15 +68,19 @@ export type TrackerIdWorkspaceId = { workspaceId: string; }; +export type TrackerDataSource = { + id: ModelId; + name: string; + dustAPIProjectId: string; + dustAPIDataSourceId: string; +}; + export type TrackerGenerationToProcess = { id: ModelId; content: string; thinking: string | null; documentId: string; - dataSource: { - id: ModelId; - name: string; - dustAPIProjectId: string; - dustAPIDataSourceId: string; - }; + dataSource: TrackerDataSource; + maintainedDataSource: TrackerDataSource | null; + maintainedDocumentId: string | null; }; diff --git a/types/src/shared/internal_mime_types.ts b/types/src/shared/internal_mime_types.ts index cc48fae16609..f358b741293d 100644 --- a/types/src/shared/internal_mime_types.ts +++ b/types/src/shared/internal_mime_types.ts @@ -1,99 +1,152 @@ -export const CONFLUENCE_MIME_TYPES = { - SPACE: "application/vnd.dust.confluence.space", - PAGE: "application/vnd.dust.confluence.page", +import { ConnectorProvider } from "../front/data_source"; + +/** + * This is a utility type that indicates that we removed all underscores from a string. + * This is used because we don't want underscores in mime types and remove them from connector providers. + */ +type WithoutUnderscores = T extends `${infer A}_${infer B}` + ? WithoutUnderscores<`${A}${B}`> // operates recursively to remove all underscores + : T; + +/** + * This is a utility type that indicates that we replaced all underscores with dashes in a string. + * We don't want underscores in mime types but want to type out the type with one: MIME_TYPE.CAT.SOU_PI_NOU + */ +type UnderscoreToDash = T extends `${infer A}_${infer B}` + ? UnderscoreToDash<`${A}-${B}`> // operates recursively to replace all underscores + : T; + +/** + * This function generates mime types for a given provider and resource types. + * The mime types are in the format `application/vnd.dust.PROVIDER.RESOURCE_TYPE`. + * Notes: + * - The underscores in the provider name are stripped in the generated mime type. + * - The underscores in the resource type are replaced with dashes in the generated mime type. + */ +function getMimeTypes< + P extends ConnectorProvider, + T extends Uppercase[] +>({ + provider, + resourceTypes, +}: { + provider: P; + resourceTypes: T; +}): { + [K in T[number]]: `application/vnd.dust.${WithoutUnderscores

}.${Lowercase< + UnderscoreToDash + >}`; +} { + return resourceTypes.reduce( + (acc, s) => ({ + ...acc, + [s]: `application/vnd.dust.${provider.replace("_", "")}.${s + .replace("_", "-") + .toLowerCase()}`, + }), + {} as { + [K in T[number]]: `application/vnd.dust.${WithoutUnderscores

}.${Lowercase< + UnderscoreToDash + >}`; + } + ); +} + +export const MIME_TYPES = { + CONFLUENCE: getMimeTypes({ + provider: "confluence", + resourceTypes: ["SPACE", "PAGE"], + }), + GITHUB: getMimeTypes({ + provider: "github", + resourceTypes: [ + "REPOSITORY", + "CODE_ROOT", + "CODE_DIRECTORY", + "CODE_FILE", + "ISSUES", + "ISSUE", + "DISCUSSIONS", + "DISCUSSION", + ], + }), + GOOGLE_DRIVE: getMimeTypes({ + provider: "google_drive", + resourceTypes: ["FOLDER"], // for files and spreadsheets, we keep Google's mime types + }), + INTERCOM: getMimeTypes({ + provider: "intercom", + resourceTypes: [ + "COLLECTION", + "TEAMS_FOLDER", + "CONVERSATION", + "TEAM", + "HELP_CENTER", + "ARTICLE", + ], + }), + MICROSOFT: getMimeTypes({ + provider: "microsoft", + resourceTypes: ["FOLDER"], // for files and spreadsheets, we keep Microsoft's mime types + }), + NOTION: getMimeTypes({ + provider: "notion", + resourceTypes: ["UNKNOWN_FOLDER", "DATABASE", "PAGE"], + }), + SLACK: getMimeTypes({ + provider: "slack", + resourceTypes: ["CHANNEL", "THREAD", "MESSAGES"], + }), + SNOWFLAKE: getMimeTypes({ + provider: "snowflake", + resourceTypes: ["DATABASE", "SCHEMA", "TABLE"], + }), + WEBCRAWLER: getMimeTypes({ + provider: "webcrawler", + resourceTypes: ["FOLDER"], // pages are upserted as text/html, not an internal mime type + }), + ZENDESK: getMimeTypes({ + provider: "zendesk", + resourceTypes: [ + "BRAND", + "HELP_CENTER", + "CATEGORY", + "ARTICLE", + "TICKETS", + "TICKET", + ], + }), }; export type ConfluenceMimeType = - (typeof CONFLUENCE_MIME_TYPES)[keyof typeof CONFLUENCE_MIME_TYPES]; - -export const GITHUB_MIME_TYPES = { - REPOSITORY: "application/vnd.dust.github.repository", - CODE_ROOT: "application/vnd.dust.github.code.root", - CODE_DIRECTORY: "application/vnd.dust.github.code.directory", - CODE_FILE: "application/vnd.dust.github.code.file", - ISSUES: "application/vnd.dust.github.issues", - ISSUE: "application/vnd.dust.github.issue", - DISCUSSIONS: "application/vnd.dust.github.discussions", - DISCUSSION: "application/vnd.dust.github.discussion", -}; + (typeof MIME_TYPES.CONFLUENCE)[keyof typeof MIME_TYPES.CONFLUENCE]; export type GithubMimeType = - (typeof GITHUB_MIME_TYPES)[keyof typeof GITHUB_MIME_TYPES]; - -export const GOOGLE_DRIVE_MIME_TYPES = { - FOLDER: "application/vnd.dust.googledrive.folder", - // for files and spreadsheets, we keep Google's mime types -}; + (typeof MIME_TYPES.GITHUB)[keyof typeof MIME_TYPES.GITHUB]; export type GoogleDriveMimeType = - (typeof GOOGLE_DRIVE_MIME_TYPES)[keyof typeof GOOGLE_DRIVE_MIME_TYPES]; - -export const INTERCOM_MIME_TYPES = { - COLLECTION: "application/vnd.dust.intercom.collection", - CONVERSATIONS: "application/vnd.dust.intercom.teams-folder", - CONVERSATION: "application/vnd.dust.intercom.conversation", - TEAM: "application/vnd.dust.intercom.team", - HELP_CENTER: "application/vnd.dust.intercom.help-center", - ARTICLE: "application/vnd.dust.intercom.article", -}; + (typeof MIME_TYPES.GOOGLE_DRIVE)[keyof typeof MIME_TYPES.GOOGLE_DRIVE]; export type IntercomMimeType = - (typeof INTERCOM_MIME_TYPES)[keyof typeof INTERCOM_MIME_TYPES]; - -export const MICROSOFT_MIME_TYPES = { - FOLDER: "application/vnd.dust.microsoft.folder", - // for files and spreadsheets, we keep Microsoft's mime types -}; + (typeof MIME_TYPES.INTERCOM)[keyof typeof MIME_TYPES.INTERCOM]; export type MicrosoftMimeType = - (typeof MICROSOFT_MIME_TYPES)[keyof typeof MICROSOFT_MIME_TYPES]; - -export const NOTION_MIME_TYPES = { - UNKNOWN_FOLDER: "application/vnd.dust.notion.unknown-folder", - DATABASE: "application/vnd.dust.notion.database", - PAGE: "application/vnd.dust.notion.page", -}; + (typeof MIME_TYPES.MICROSOFT)[keyof typeof MIME_TYPES.MICROSOFT]; export type NotionMimeType = - (typeof NOTION_MIME_TYPES)[keyof typeof NOTION_MIME_TYPES]; - -export const SLACK_MIME_TYPES = { - CHANNEL: "application/vnd.dust.slack.channel", - THREAD: "application/vnd.dust.slack.thread", - MESSAGES: "application/vnd.dust.slack.messages", -}; + (typeof MIME_TYPES.NOTION)[keyof typeof MIME_TYPES.NOTION]; export type SlackMimeType = - (typeof SLACK_MIME_TYPES)[keyof typeof SLACK_MIME_TYPES]; - -export const SNOWFLAKE_MIME_TYPES = { - DATABASE: "application/vnd.snowflake.database", - SCHEMA: "application/vnd.snowflake.schema", - TABLE: "application/vnd.snowflake.table", -}; + (typeof MIME_TYPES.SLACK)[keyof typeof MIME_TYPES.SLACK]; export type SnowflakeMimeType = - (typeof SNOWFLAKE_MIME_TYPES)[keyof typeof SNOWFLAKE_MIME_TYPES]; - -export const WEBCRAWLER_MIME_TYPES = { - FOLDER: "application/vnd.dust.webcrawler.folder", - // pages are upserted as text/html, not an internal mime type -}; + (typeof MIME_TYPES.SNOWFLAKE)[keyof typeof MIME_TYPES.SNOWFLAKE]; export type WebcrawlerMimeType = - (typeof WEBCRAWLER_MIME_TYPES)[keyof typeof WEBCRAWLER_MIME_TYPES]; - -export const ZENDESK_MIME_TYPES = { - BRAND: "application/vnd.dust.zendesk.brand", - HELP_CENTER: "application/vnd.dust.zendesk.helpcenter", - CATEGORY: "application/vnd.dust.zendesk.category", - ARTICLE: "application/vnd.dust.zendesk.article", - TICKETS: "application/vnd.dust.zendesk.tickets", - TICKET: "application/vnd.dust.zendesk.ticket", -}; + (typeof MIME_TYPES.WEBCRAWLER)[keyof typeof MIME_TYPES.WEBCRAWLER]; export type ZendeskMimeType = - (typeof ZENDESK_MIME_TYPES)[keyof typeof ZENDESK_MIME_TYPES]; + (typeof MIME_TYPES.ZENDESK)[keyof typeof MIME_TYPES.ZENDESK]; export type DustMimeType = | ConfluenceMimeType