From 4722772930cd9ef1f96e8ff63ae379bb2ab67e6e Mon Sep 17 00:00:00 2001 From: Sebastien Flory Date: Fri, 10 Jan 2025 17:12:36 +0100 Subject: [PATCH 01/47] Upg: cleaner lock implementation (#9901) --- front/lib/api/data_sources.ts | 12 ++++------ front/lib/lock.ts | 45 +++++++++++++++++++++++++++++++++++ front/lib/wake_lock.ts | 14 +++-------- 3 files changed, 52 insertions(+), 19 deletions(-) create mode 100644 front/lib/lock.ts diff --git a/front/lib/api/data_sources.ts b/front/lib/api/data_sources.ts index f563970641ac..dba6cd1c18ee 100644 --- a/front/lib/api/data_sources.ts +++ b/front/lib/api/data_sources.ts @@ -45,13 +45,13 @@ import { getMembers } from "@app/lib/api/workspace"; import type { Authenticator } from "@app/lib/auth"; import { getFeatureFlags } from "@app/lib/auth"; import { DustError } from "@app/lib/error"; +import { Lock } from "@app/lib/lock"; import { DataSourceResource } from "@app/lib/resources/data_source_resource"; import { DataSourceViewResource } from "@app/lib/resources/data_source_view_resource"; import { SpaceResource } from "@app/lib/resources/space_resource"; import { generateRandomModelSId } from "@app/lib/resources/string_ids"; import { ServerSideTracking } from "@app/lib/tracking/server"; import { enqueueUpsertTable } from "@app/lib/upsert_queue"; -import { wakeLock, wakeLockIsFree } from "@app/lib/wake_lock"; import logger from "@app/logger/logger"; import { launchScrubDataSourceWorkflow } from "@app/poke/temporal/client"; @@ -936,12 +936,9 @@ async function getOrCreateConversationDataSource( } const lockName = "conversationDataSource" + conversation.id; - // Handle race condition when we try to create a conversation data source - while (!wakeLockIsFree(lockName)) { - await new Promise((resolve) => setTimeout(resolve, 100)); - } - const res = await wakeLock( + const res = await Lock.executeWithLock( + lockName, async (): Promise< Result< DataSourceResource, @@ -984,8 +981,7 @@ async function getOrCreateConversationDataSource( } return new Ok(dataSource); - }, - lockName + } ); return res; diff --git a/front/lib/lock.ts b/front/lib/lock.ts new file mode 100644 index 000000000000..d142fcad5ea3 --- /dev/null +++ b/front/lib/lock.ts @@ -0,0 +1,45 @@ +export class Lock { + private static locks = new Map>(); + + static async executeWithLock( + lockName: string, + callback: () => Promise, + timeoutMs: number = 30000 + ): Promise { + const start = Date.now(); + + if (Lock.locks.has(lockName)) { + const currentLock = Lock.locks.get(lockName); + if (currentLock) { + const remainingTime = timeoutMs - (Date.now() - start); + if (remainingTime <= 0) { + throw new Error(`Lock acquisition timed out for ${lockName}`); + } + + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => { + reject(new Error(`Lock acquisition timed out for ${lockName}`)); + }, remainingTime); + }); + + await Promise.race([currentLock, timeoutPromise]); + } + } + + // Initialize resolveLock with a no-op function to satisfy TypeScript + let resolveLock = () => {}; + const lockPromise = new Promise((resolve) => { + resolveLock = resolve; + }); + + Lock.locks.set(lockName, lockPromise); + + try { + const result = await callback(); + return result; + } finally { + Lock.locks.delete(lockName); + resolveLock(); + } + } +} diff --git a/front/lib/wake_lock.ts b/front/lib/wake_lock.ts index bc0f8ed93c62..1266492c2885 100644 --- a/front/lib/wake_lock.ts +++ b/front/lib/wake_lock.ts @@ -1,13 +1,10 @@ import { v4 as uuidv4 } from "uuid"; -export async function wakeLock( - autoCallback: () => Promise, - lockName?: string -): Promise { +export async function wakeLock(autoCallback: () => Promise): Promise { if (!global.wakeLocks) { global.wakeLocks = new Set(); } - lockName ??= uuidv4(); + const lockName = uuidv4(); global.wakeLocks.add(lockName); try { const r = await autoCallback(); @@ -17,11 +14,6 @@ export async function wakeLock( } } -// If a lockName is provided, checks if that lock is free, otherwise checks if all locks are free -export function wakeLockIsFree(lockName?: string): boolean { - if (lockName) { - return !global.wakeLocks || !global.wakeLocks.has(lockName); - } - +export function wakeLockIsFree(): boolean { return !global.wakeLocks || global.wakeLocks.size === 0; } From 4f5d5ed8de23eb68aee640ffb31a127ff157f582 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Fri, 10 Jan 2025 17:48:53 +0100 Subject: [PATCH 02/47] feat(tracker): handle multiple docs in maintained scope (#9888) * feat(tracker): handle multiple docs in maintained scope * enh * migrate --------- Co-authored-by: Henry Fontanier --- .../tracker/actions/doc_tracker_retrieval.ts | 18 +- .../tracker/actions/doc_tracker_score_docs.ts | 65 +++++ front/lib/models/doc_tracker.ts | 16 ++ front/lib/registry.ts | 14 + front/lib/resources/tracker_resource.ts | 14 + front/migrations/db/migration_140.sql | 3 + front/temporal/tracker/activities.ts | 240 ++++++++++++++---- 7 files changed, 321 insertions(+), 49 deletions(-) create mode 100644 front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts create mode 100644 front/migrations/db/migration_140.sql diff --git a/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts index 3b25db694ab2..526e43c30c25 100644 --- a/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts +++ b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_retrieval.ts @@ -69,6 +69,7 @@ const DocTrackerRetrievalActionValueSchema = t.array( created: t.Integer, document_id: t.string, timestamp: t.Integer, + title: t.union([t.string, t.null]), tags: t.array(t.string), parents: t.array(t.string), source_url: t.union([t.string, t.null]), @@ -77,12 +78,17 @@ const DocTrackerRetrievalActionValueSchema = t.array( text: t.union([t.string, t.null, t.undefined]), chunk_count: t.Integer, chunks: t.array( - t.type({ - text: t.string, - hash: t.string, - offset: t.Integer, - score: t.number, - }) + t.intersection([ + t.type({ + text: t.string, + hash: t.string, + offset: t.Integer, + score: t.number, + }), + t.partial({ + expanded_offsets: t.array(t.Integer), + }), + ]) ), token_count: t.Integer, }) diff --git a/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts new file mode 100644 index 000000000000..fcaf6876133f --- /dev/null +++ b/front/lib/document_upsert_hooks/hooks/tracker/actions/doc_tracker_score_docs.ts @@ -0,0 +1,65 @@ +import * as t from "io-ts"; + +import { callAction } from "@app/lib/actions/helpers"; +import type { Authenticator } from "@app/lib/auth"; +import { cloneBaseConfig, DustProdActionRegistry } from "@app/lib/registry"; + +export async function callDocTrackerScoreDocsAction( + auth: Authenticator, + { + watchedDocDiff, + maintainedDocuments, + prompt, + providerId, + modelId, + }: { + watchedDocDiff: string; + maintainedDocuments: Array<{ + content: string; + title: string | null; + sourceUrl: string | null; + dataSourceId: string; + documentId: string; + }>; + prompt: string | null; + providerId: string; + modelId: string; + } +): Promise { + const action = DustProdActionRegistry["doc-tracker-score-docs"]; + + const config = cloneBaseConfig(action.config); + config.MODEL.provider_id = providerId; + config.MODEL.model_id = modelId; + + const res = await callAction(auth, { + action, + config, + input: { + watched_diff: watchedDocDiff, + maintained_documents: maintainedDocuments, + prompt, + }, + responseValueSchema: DocTrackerScoreDocsActionResultSchema, + }); + + if (res.isErr()) { + throw res.error; + } + + return res.value; +} + +const DocTrackerScoreDocsActionResultSchema = t.array( + t.type({ + documentId: t.string, + dataSourceId: t.string, + score: t.number, + title: t.union([t.string, t.null, t.undefined]), + sourceUrl: t.union([t.string, t.null, t.undefined]), + }) +); + +type DocTrackerScoreDocsActionResult = t.TypeOf< + typeof DocTrackerScoreDocsActionResultSchema +>; diff --git a/front/lib/models/doc_tracker.ts b/front/lib/models/doc_tracker.ts index d42ffedf1745..3b9af1c38b21 100644 --- a/front/lib/models/doc_tracker.ts +++ b/front/lib/models/doc_tracker.ts @@ -237,11 +237,14 @@ export class TrackerGenerationModel extends SoftDeletableModel; declare dataSourceId: ForeignKey; declare documentId: string; + declare maintainedDocumentDataSourceId: ForeignKey; + declare maintainedDocumentId: string; declare consumedAt: Date | null; declare trackerConfiguration: NonAttribute; declare dataSource: NonAttribute; + declare maintainedDocumentDataSource: NonAttribute; } TrackerGenerationModel.init( @@ -275,6 +278,10 @@ TrackerGenerationModel.init( type: DataTypes.DATE, allowNull: true, }, + maintainedDocumentId: { + type: DataTypes.STRING, + allowNull: true, + }, }, { modelName: "tracker_generation", @@ -300,3 +307,12 @@ TrackerGenerationModel.belongsTo(DataSourceModel, { foreignKey: { allowNull: false }, as: "dataSource", }); + +DataSourceModel.hasMany(TrackerGenerationModel, { + foreignKey: { allowNull: true }, + as: "maintainedDocumentDataSource", +}); +TrackerGenerationModel.belongsTo(DataSourceModel, { + foreignKey: { allowNull: false }, + as: "maintainedDocumentDataSource", +}); diff --git a/front/lib/registry.ts b/front/lib/registry.ts index b51ae750152c..32f1558d44dd 100644 --- a/front/lib/registry.ts +++ b/front/lib/registry.ts @@ -136,6 +136,20 @@ export const DustProdActionRegistry = createActionRegistry({ }, }, }, + "doc-tracker-score-docs": { + app: { + workspaceId: PRODUCTION_DUST_APPS_WORKSPACE_ID, + appId: "N0RrhyTXfq", + appHash: + "ba5637f356c55676c7e175719bbd4fa5059c5a99a519ec75aea78b452e2168dc", + appSpaceId: PRODUCTION_DUST_APPS_SPACE_ID, + }, + config: { + MODEL: { + use_cache: true, + }, + }, + }, "doc-tracker-suggest-changes": { app: { workspaceId: PRODUCTION_DUST_APPS_WORKSPACE_ID, diff --git a/front/lib/resources/tracker_resource.ts b/front/lib/resources/tracker_resource.ts index 243077392f27..7910e389db46 100644 --- a/front/lib/resources/tracker_resource.ts +++ b/front/lib/resources/tracker_resource.ts @@ -236,22 +236,36 @@ export class TrackerConfigurationResource extends ResourceWithSpace 1) { + // TODO(DOC_TRACKER): mechanism to retry "manually" + throw new Error("Too many attempts"); + } + const localLogger = logger.child({ workspaceId, dataSourceId, @@ -66,6 +83,8 @@ export async function trackersGenerationActivity( throw new Error(`Could not find data source ${dataSourceId}`); } + // We start by finding all trackers that are watching the modified document. + const trackers = await getTrackersToRun(auth, dataSource, documentId); if (!trackers.length) { @@ -114,6 +133,9 @@ export async function trackersGenerationActivity( return; } + // We compute the diff between the current version of the document and the version right before the edit + // that triggered the tracker. + const documentDiff = await getDocumentDiff({ dataSource, documentId, @@ -185,6 +207,21 @@ export async function trackersGenerationActivity( const targetMaintainedScopeTokens = TRACKER_TOTAL_TARGET_TOKENS - tokensInDiffCount; + // We don't want to retrieve more than targetMaintainedScopeTokens / CHUNK_SIZE chunks, + // in case all retrieved chunks are from the same document (in which case, we'd have + // more than targetMaintainedScopeTokens tokens for that document). + const maintainedScopeTopK = Math.min( + TRACKER_MAINTAINED_SCOPE_MAX_TOP_K, + Math.floor(targetMaintainedScopeTokens / CHUNK_SIZE) + ); + + if (maintainedScopeTopK === 0) { + throw new Error( + "Unreachable: targetMaintainedScopeTokens is less than CHUNK_SIZE." + ); + } + + // We run each tracker. for (const tracker of trackers) { const trackerLogger = localLogger.child({ trackerId: tracker.sId, @@ -213,63 +250,180 @@ export async function trackersGenerationActivity( (x) => x.filter?.parents?.in ?? null ); + // We retrieve content from the maintained scope based on the diff. const maintainedScopeRetrieval = await callDocTrackerRetrievalAction(auth, { inputText: diffString, targetDocumentTokens: targetMaintainedScopeTokens, - topK: TRACKER_MAINTAINED_DOCUMENT_TOP_K, + topK: maintainedScopeTopK, maintainedScope, parentsInMap, }); - // TODO(DOC_TRACKER): Right now we only handle the top match. - // We may want to support topK > 1 and process more than 1 doc if the top doc has less than - // $targetDocumentTokens. if (maintainedScopeRetrieval.length === 0) { trackerLogger.info("No content retrieved from maintained scope."); continue; } - const content = maintainedScopeRetrieval[0].chunks - .map((c) => c.text) - .join("\n"); - if (!content) { - trackerLogger.info("No content retrieved from maintained scope."); - continue; - } - - const suggestChangesResult = await callDocTrackerSuggestChangesAction( - auth, - { - watchedDocDiff: diffString, - maintainedDocContent: content, - prompt: tracker.prompt, - providerId: tracker.providerId, - modelId: tracker.modelId, + const maintainedDocuments: { + content: string; + sourceUrl: string | null; + title: string | null; + dataSourceId: string; + documentId: string; + }[] = []; + + // For each document retrieved from the maintained scope, we build the content of the document. + // We add "[...]" separators when there is a gap in the chunks (so the model understands that parts of the document are missing). + for (const retrievalDoc of maintainedScopeRetrieval) { + let docContent: string = ""; + const sortedChunks = _.sortBy(retrievalDoc.chunks, (c) => c.offset); + + for (const [i, chunk] of sortedChunks.entries()) { + if (i === 0) { + // If we are at index 0 (i.e the first retrieved chunk), we check whether our chunk includes + // the beginning of the document. If it doesn't, we add a "[...]"" separator. + const allOffsetsInChunk = [ + chunk.offset, + ...(chunk.expanded_offsets ?? []), + ]; + const isBeginningOfDocument = allOffsetsInChunk.includes(0); + if (!isBeginningOfDocument) { + docContent += "[...]\n"; + } + } else { + // If we are not at index 0, we check whether the current chunk is a direct continuation of the previous chunk. + // We do this by checking that the first offset of the current chunk is the last offset of the previous chunk + 1. + const previousChunk = sortedChunks[i - 1]; + const allOffsetsInCurrentChunk = [ + chunk.offset, + ...(chunk.expanded_offsets ?? []), + ]; + const firstOffsetInCurrentChunk = _.min(allOffsetsInCurrentChunk)!; + const allOffsetsInPreviousChunk = [ + previousChunk.offset, + ...(previousChunk.expanded_offsets ?? []), + ]; + const lastOffsetInPreviousChunk = _.max(allOffsetsInPreviousChunk)!; + const hasGap = + firstOffsetInCurrentChunk !== lastOffsetInPreviousChunk + 1; + + if (hasGap) { + docContent += "[...]\n"; + } + } + + // Add the chunk text to the document. + docContent += chunk.text + "\n"; + + if (i === sortedChunks.length - 1) { + // If we are at the last chunk, we check if we have the last offset of the doc. + // If not, we add a "[...]" separator. + const lastChunk = sortedChunks[sortedChunks.length - 1]; + if (lastChunk.offset !== retrievalDoc.chunk_count - 1) { + docContent += "[...]\n"; + } + } } - ); - if (!suggestChangesResult.suggestion) { - trackerLogger.info("No changes suggested."); - continue; + maintainedDocuments.push({ + content: docContent, + sourceUrl: retrievalDoc.source_url, + title: retrievalDoc.title, + dataSourceId: retrievalDoc.data_source_id, + documentId: retrievalDoc.document_id, + }); } - const suggestedChanges = suggestChangesResult.suggestion; - const thinking = suggestChangesResult.thinking; - const confidenceScore = suggestChangesResult.confidence_score; - - trackerLogger.info( - { - confidenceScore, - }, - "Changes suggested." + const contentByDocumentIdentifier = _.mapValues( + _.keyBy( + maintainedDocuments, + (doc) => `${doc.dataSourceId}__${doc.documentId}` + ), + (doc) => doc.content ); - await tracker.addGeneration({ - generation: suggestedChanges, - thinking: thinking ?? null, - dataSourceId, - documentId, + // We find documents for which to run the change suggestion. + // We do this by asking which documents are most relevant to the diff and using the + // logprobs as a score. + const scoreDocsResult = await callDocTrackerScoreDocsAction(auth, { + watchedDocDiff: diffString, + maintainedDocuments, + prompt: tracker.prompt, + providerId: TRACKER_SCORE_DOCS_MODEL_CONFIG.providerId, + modelId: TRACKER_SCORE_DOCS_MODEL_CONFIG.modelId, }); + + // The output of the Dust App above is a list of document for which we want to run the change suggestion. + + for (const { + documentId: maintainedDocumentId, + dataSourceId: maintainedDataSourceId, + score, + } of scoreDocsResult) { + logger.info( + { + maintainedDocumentId, + maintainedDataSourceId, + score, + }, + "Running document tracker suggest changes." + ); + + const content = + contentByDocumentIdentifier[ + `${maintainedDataSourceId}__${maintainedDocumentId}` + ]; + if (!content) { + continue; + } + + const suggestChangesResult = await callDocTrackerSuggestChangesAction( + auth, + { + watchedDocDiff: diffString, + maintainedDocContent: content, + prompt: tracker.prompt, + providerId: tracker.providerId, + modelId: tracker.modelId, + } + ); + + if (!suggestChangesResult.suggestion) { + trackerLogger.info("No changes suggested."); + continue; + } + + const maintainedDocumentDataSource = + await DataSourceResource.fetchByDustAPIDataSourceId( + auth, + maintainedDataSourceId + ); + if (!maintainedDocumentDataSource) { + throw new Error( + `Could not find maintained data source ${maintainedDataSourceId}` + ); + } + + const suggestedChanges = suggestChangesResult.suggestion; + const thinking = suggestChangesResult.thinking; + const confidenceScore = suggestChangesResult.confidence_score; + + trackerLogger.info( + { + confidenceScore, + }, + "Changes suggested." + ); + + await tracker.addGeneration({ + generation: suggestedChanges, + thinking: thinking ?? null, + dataSourceId, + documentId, + maintainedDocumentDataSourceId: maintainedDocumentDataSource.sId, + maintainedDocumentId, + }); + } } } From 2747b0245a7987443f0dfd535a2d218750655591 Mon Sep 17 00:00:00 2001 From: Lucas Massemin Date: Fri, 10 Jan 2025 18:10:02 +0100 Subject: [PATCH 03/47] Removed all occurrences of dustTableId (#9892) * Removed all occurrences of dustTableId * No unused import --- connectors/src/connectors/google_drive/index.ts | 5 ----- connectors/src/connectors/notion/index.ts | 3 --- connectors/src/connectors/slack/index.ts | 2 -- 3 files changed, 10 deletions(-) diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts index b504633b1d49..ec2caf0b40a0 100644 --- a/connectors/src/connectors/google_drive/index.ts +++ b/connectors/src/connectors/google_drive/index.ts @@ -8,7 +8,6 @@ import { Err, getGoogleIdsFromSheetContentNodeInternalId, getGoogleSheetContentNodeInternalId, - getGoogleSheetTableId, isGoogleSheetContentNodeInternalId, Ok, removeNulls, @@ -358,10 +357,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: false, permission: "read", - dustTableId: getGoogleSheetTableId( - s.driveFileId, - s.driveSheetId - ), }; }) ); diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts index fea202ad3086..63b285c753ea 100644 --- a/connectors/src/connectors/notion/index.ts +++ b/connectors/src/connectors/notion/index.ts @@ -1,7 +1,6 @@ import type { ContentNode, ContentNodesViewType, Result } from "@dust-tt/types"; import { Err, - getNotionDatabaseTableId, getOAuthConnectionAccessToken, NOTION_MIME_TYPES, Ok, @@ -566,7 +565,6 @@ export class NotionConnectorManager extends BaseConnectorManager { permission: "read", dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, - dustTableId: null, })) ); @@ -584,7 +582,6 @@ export class NotionConnectorManager extends BaseConnectorManager { permission: "read", dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: null, - dustTableId: getNotionDatabaseTableId(db.notionDatabaseId), })); const contentNodes = pageNodes.concat(dbNodes); diff --git a/connectors/src/connectors/slack/index.ts b/connectors/src/connectors/slack/index.ts index 9843b63e7686..86347d0d92ca 100644 --- a/connectors/src/connectors/slack/index.ts +++ b/connectors/src/connectors/slack/index.ts @@ -407,7 +407,6 @@ export class SlackConnectorManager extends BaseConnectorManager Date: Fri, 10 Jan 2025 19:04:27 +0100 Subject: [PATCH 04/47] [front/connectors] Deprecate `titleWithParentsContext` (#9898) * remove titleWithParentsContext (not sent anymore, not used anymore) * remove unused variables --- connectors/src/connectors/github/index.ts | 13 ------------- .../assistant/details/AssistantActionsSection.tsx | 2 +- .../DataSourceSelectionSection.tsx | 2 +- .../trackers/TrackerDataSourceSelectedTree.tsx | 2 +- types/src/front/api_handlers/public/spaces.ts | 1 - types/src/front/lib/connectors_api.ts | 1 - 6 files changed, 3 insertions(+), 18 deletions(-) diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts index 4ebfa01d7b52..4ff148eace5a 100644 --- a/connectors/src/connectors/github/index.ts +++ b/connectors/src/connectors/github/index.ts @@ -614,7 +614,6 @@ export class GithubConnectorManager extends BaseConnectorManager { parentInternalId: null, type: "folder", title: repo.name, - titleWithParentsContext: `[${repo.name}] - Full repository`, sourceUrl: repo.url, expandable: true, permission: "read", @@ -635,7 +634,6 @@ export class GithubConnectorManager extends BaseConnectorManager { parentInternalId: getRepositoryInternalId(repoId), type: "database", title: "Issues", - titleWithParentsContext: `[${repo.name}] Issues`, sourceUrl: repo.url + "/issues", expandable: false, permission: "read", @@ -654,7 +652,6 @@ export class GithubConnectorManager extends BaseConnectorManager { parentInternalId: getRepositoryInternalId(repoId), type: "channel", title: "Discussions", - titleWithParentsContext: `[${repo.name}] Discussions`, sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", @@ -705,14 +702,12 @@ export class GithubConnectorManager extends BaseConnectorManager { // Constructing Nodes for Code fullCodeInRepos.forEach((codeRepo) => { - const repo = uniqueRepos[parseInt(codeRepo.repoId)]; nodes.push({ provider: c.type, internalId: getCodeRootInternalId(codeRepo.repoId), parentInternalId: getRepositoryInternalId(codeRepo.repoId), type: "folder", title: "Code", - titleWithParentsContext: repo ? `[${repo.name}] Code` : "Code", sourceUrl: codeRepo.sourceUrl, expandable: true, permission: "read", @@ -723,16 +718,12 @@ export class GithubConnectorManager extends BaseConnectorManager { // Constructing Nodes for Code Directories codeDirectories.forEach((directory) => { - const repo = uniqueRepos[parseInt(directory.repoId)]; nodes.push({ provider: c.type, internalId: directory.internalId, parentInternalId: directory.parentInternalId, type: "folder", title: directory.dirName, - titleWithParentsContext: repo - ? `[${repo.name}] ${directory.dirName} (code)` - : directory.dirName, sourceUrl: directory.sourceUrl, expandable: true, permission: "read", @@ -743,16 +734,12 @@ export class GithubConnectorManager extends BaseConnectorManager { // Constructing Nodes for Code Files codeFiles.forEach((file) => { - const repo = uniqueRepos[parseInt(file.repoId)]; nodes.push({ provider: c.type, internalId: file.documentId, parentInternalId: file.parentInternalId, type: "file", title: file.fileName, - titleWithParentsContext: repo - ? `[${repo.name}] ${file.fileName} (code)` - : file.fileName, sourceUrl: file.sourceUrl, expandable: false, permission: "read", diff --git a/front/components/assistant/details/AssistantActionsSection.tsx b/front/components/assistant/details/AssistantActionsSection.tsx index 972302943618..0050ad73d074 100644 --- a/front/components/assistant/details/AssistantActionsSection.tsx +++ b/front/components/assistant/details/AssistantActionsSection.tsx @@ -422,7 +422,7 @@ function DataSourceViewSelectedNodes({ {nodes.map((node) => ( Date: Fri, 10 Jan 2025 19:18:14 +0100 Subject: [PATCH 05/47] Standardize and backfill Slack mimetypes (#9887) * add a mime types for non-threaded messages * use the correct mime type for non-threaded messages * add backfill script * replace count * add a default value * remove the default value * add missing FromStr * remove a slow query * rewrite the backfill in ts * use smaller chunks for the update --- .../connectors/slack/temporal/activities.ts | 2 +- .../20250110_backfill_slack_mime_types.ts | 117 ++++++++++++++++++ types/src/shared/internal_mime_types.ts | 3 +- 3 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 front/migrations/20250110_backfill_slack_mime_types.ts diff --git a/connectors/src/connectors/slack/temporal/activities.ts b/connectors/src/connectors/slack/temporal/activities.ts index 5cfa0691bfa5..f94ba8913612 100644 --- a/connectors/src/connectors/slack/temporal/activities.ts +++ b/connectors/src/connectors/slack/temporal/activities.ts @@ -636,7 +636,7 @@ export async function syncNonThreaded( sync_type: isBatchSync ? "batch" : "incremental", }, title: tags.find((t) => t.startsWith("title:"))?.split(":")[1] ?? "", - mimeType: SLACK_MIME_TYPES.THREAD, + mimeType: SLACK_MIME_TYPES.MESSAGES, async: true, }); } diff --git a/front/migrations/20250110_backfill_slack_mime_types.ts b/front/migrations/20250110_backfill_slack_mime_types.ts new file mode 100644 index 000000000000..de94cc0c039e --- /dev/null +++ b/front/migrations/20250110_backfill_slack_mime_types.ts @@ -0,0 +1,117 @@ +import assert from "assert"; +import _ from "lodash"; +import type { Sequelize } from "sequelize"; +import { QueryTypes } from "sequelize"; + +import { getCorePrimaryDbConnection } from "@app/lib/production_checks/utils"; +import { DataSourceModel } from "@app/lib/resources/storage/models/data_source"; +import type Logger from "@app/logger/logger"; +import { makeScript } from "@app/scripts/helpers"; + +const { CORE_DATABASE_URI } = process.env; +const SELECT_BATCH_SIZE = 512; +const UPDATE_BATCH_SIZE = 32; + +async function backfillDataSource( + frontDataSource: DataSourceModel, + coreSequelize: Sequelize, + nodeType: "thread" | "messages", + execute: boolean, + logger: typeof Logger +) { + const pattern = `^slack-[A-Z0-9]+-${nodeType}-[0-9.\\-]+$`; + const mimeType = `application/vnd.dust.slack.${nodeType}`; + + logger.info({ pattern, mimeType }, "Processing data source"); + + let nextId = 0; + let updatedRowsCount; + do { + const rows: { id: number }[] = await coreSequelize.query( + ` + SELECT dsn.id + FROM data_sources_nodes dsn + JOIN data_sources ds ON ds.id = dsn.data_source + WHERE dsn.id > :nextId + AND ds.data_source_id = :dataSourceId + AND ds.project = :projectId + AND dsn.node_id ~ :pattern + ORDER BY dsn.id + LIMIT :batchSize;`, + { + replacements: { + dataSourceId: frontDataSource.dustAPIDataSourceId, + projectId: frontDataSource.dustAPIProjectId, + batchSize: SELECT_BATCH_SIZE, + nextId, + pattern, + }, + type: QueryTypes.SELECT, + } + ); + + if (rows.length == 0) { + logger.info({ nextId }, `Finished processing data source.`); + break; + } + nextId = rows[rows.length - 1].id; + updatedRowsCount = rows.length; + + // doing smaller chunks to avoid long transactions + const chunks = _.chunk(rows, UPDATE_BATCH_SIZE); + + for (const chunk of chunks) { + if (execute) { + await coreSequelize.query( + `UPDATE data_sources_nodes SET mime_type = :mimeType WHERE id IN (:ids)`, + { + replacements: { + mimeType, + ids: chunk.map((row) => row.id), + }, + } + ); + logger.info( + `Updated chunk from ${chunk[0].id} to ${chunk[chunk.length - 1].id}` + ); + } else { + logger.info( + `Would update chunk from ${chunk[0].id} to ${chunk[chunk.length - 1].id}` + ); + } + } + } while (updatedRowsCount === SELECT_BATCH_SIZE); +} + +makeScript( + { + nodeType: { type: "string", choices: ["thread", "messages"] }, + }, + async ({ nodeType, execute }, logger) => { + assert(CORE_DATABASE_URI, "CORE_DATABASE_URI is required"); + + const coreSequelize = getCorePrimaryDbConnection(); + + if (!["thread", "messages"].includes(nodeType)) { + throw new Error(`Unknown node type: ${nodeType}`); + } + + const frontDataSources = await DataSourceModel.findAll({ + where: { connectorProvider: "slack" }, + }); + logger.info(`Found ${frontDataSources.length} Slack data sources`); + + for (const frontDataSource of frontDataSources) { + await backfillDataSource( + frontDataSource, + coreSequelize, + nodeType as "thread" | "messages", + execute, + logger.child({ + dataSourceId: frontDataSource.id, + connectorId: frontDataSource.connectorId, + }) + ); + } + } +); diff --git a/types/src/shared/internal_mime_types.ts b/types/src/shared/internal_mime_types.ts index 959afb2dd03a..cc48fae16609 100644 --- a/types/src/shared/internal_mime_types.ts +++ b/types/src/shared/internal_mime_types.ts @@ -59,7 +59,8 @@ export type NotionMimeType = export const SLACK_MIME_TYPES = { CHANNEL: "application/vnd.dust.slack.channel", - THREAD: "text/vnd.dust.slack.thread", + THREAD: "application/vnd.dust.slack.thread", + MESSAGES: "application/vnd.dust.slack.messages", }; export type SlackMimeType = From f42d27e57fc89a3291005c14f82625ad877a7fe0 Mon Sep 17 00:00:00 2001 From: Sebastien Flory Date: Fri, 10 Jan 2025 19:57:45 +0100 Subject: [PATCH 06/47] Add: title filter for conversations (#9905) --- .../assistant/conversation/SidebarMenu.tsx | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/front/components/assistant/conversation/SidebarMenu.tsx b/front/components/assistant/conversation/SidebarMenu.tsx index 780ebf7e1d61..3bfbc22063ce 100644 --- a/front/components/assistant/conversation/SidebarMenu.tsx +++ b/front/components/assistant/conversation/SidebarMenu.tsx @@ -15,6 +15,7 @@ import { NavigationListLabel, PlusIcon, RobotIcon, + SearchInput, TrashIcon, XMarkIcon, } from "@dust-tt/sparkle"; @@ -35,7 +36,7 @@ import { useConversations, useDeleteConversation, } from "@app/lib/swr/conversations"; -import { classNames } from "@app/lib/utils"; +import { classNames, removeDiacritics, subFilter } from "@app/lib/utils"; type AssistantSidebarMenuProps = { owner: WorkspaceType; @@ -67,6 +68,7 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { "all" | "selection" | null >(null); const [isDeleting, setIsDeleting] = useState(false); + const [titleFilter, setTitleFilter] = useState(""); const sendNotification = useSendNotification(); const toggleMultiSelect = useCallback(() => { @@ -139,6 +141,16 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { }; conversations.forEach((conversation: ConversationType) => { + if ( + titleFilter && + !subFilter( + removeDiacritics(titleFilter).toLowerCase(), + removeDiacritics(conversation.title ?? "").toLowerCase() + ) + ) { + return; + } + const updatedAt = moment(conversation.updated ?? conversation.created); if (updatedAt.isSameOrAfter(today)) { groups["Today"].push(conversation); @@ -204,6 +216,12 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { ) : (
+
-
- {testSuccessful ? ( -
- - - - - - - - ); -} diff --git a/front/components/providers/AzureOpenAISetup.tsx b/front/components/providers/AzureOpenAISetup.tsx deleted file mode 100644 index 34fb9f0a1466..000000000000 --- a/front/components/providers/AzureOpenAISetup.tsx +++ /dev/null @@ -1,220 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function AzureOpenAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [endpoint, setEndpoint] = useState(config ? config.endpoint : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - if (config && config.endpoint.length > 0 && endpoint.length == 0) { - setEndpoint(config.endpoint); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "azure_openai", { - api_key: apiKey, - endpoint, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/azure_openai`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - endpoint, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/azure_openai`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Azure OpenAI - -
-

- To use Azure OpenAI models you must provide your API key - and Endpoint. They can be found in the left menu of your - OpenAI Azure Resource portal (menu item `Keys and - Endpoint`). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setEndpoint(e.target.value); - setTestSuccessful(false); - }} - /> -
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Azure OpenAI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/BrowserlessAPISetup.tsx b/front/components/providers/BrowserlessAPISetup.tsx deleted file mode 100644 index ce93839e330e..000000000000 --- a/front/components/providers/BrowserlessAPISetup.tsx +++ /dev/null @@ -1,214 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function BrowserlessAPISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "browserlessapi", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/browserlessapi`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/browserlessapi`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Browserless API - -
-

- Browserless lets you use headless browsers to scrape web - content. To use Browserless, you must provide your API - key. It can be found{" "} - - here - - . -

-

- Note that it generally takes{" "} - 5 mins for the API - key to become active (an email is sent when it's ready). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable the Browserless API. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/DeepseekSetup.tsx b/front/components/providers/DeepseekSetup.tsx deleted file mode 100644 index aa9098d5bf04..000000000000 --- a/front/components/providers/DeepseekSetup.tsx +++ /dev/null @@ -1,199 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function DeepseekSetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "deepseek", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/deepseek`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/deepseek`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Deepseek - -
-

- To use Deepseek models you must provide your API key. -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError?.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Deepseek. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/GoogleAiStudioSetup.tsx b/front/components/providers/GoogleAiStudioSetup.tsx deleted file mode 100644 index fb4ec115aff5..000000000000 --- a/front/components/providers/GoogleAiStudioSetup.tsx +++ /dev/null @@ -1,208 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function GoogleAiStudioSetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "google_ai_studio", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/google_ai_studio`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/google_ai_studio`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Google AI Studio - -
-

- To use Google AI Studio models you must provide your API - key. It can be found{" "} - - here - -  (you can create a new key specifically for Dust). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Google AI Studio. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/MistralAISetup.tsx b/front/components/providers/MistralAISetup.tsx deleted file mode 100644 index 8477a384f179..000000000000 --- a/front/components/providers/MistralAISetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function MistralAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "mistral", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/mistral`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/mistral`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Mistral AI - -
-

- To use Mistral AI models you must provide your API key. - It can be found{" "} - - here - -  (you can create a new key specifically for Dust). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Mistral AI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/OpenAISetup.tsx b/front/components/providers/OpenAISetup.tsx deleted file mode 100644 index 6bc275eb0ada..000000000000 --- a/front/components/providers/OpenAISetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function OpenAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "openai", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/openai`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/openai`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup OpenAI - -
-

- To use OpenAI models you must provide your API key. It - can be found{" "} - - here - -  (you can create a new key specifically for Dust). -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable OpenAI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/ProviderSetup.tsx b/front/components/providers/ProviderSetup.tsx new file mode 100644 index 000000000000..2b2b6dfdee84 --- /dev/null +++ b/front/components/providers/ProviderSetup.tsx @@ -0,0 +1,428 @@ +import { + NewDialog, + NewDialogContainer, + NewDialogContent, + NewDialogDescription, + NewDialogFooter, + NewDialogHeader, + NewDialogTitle, +} from "@dust-tt/sparkle"; +import type { WorkspaceType } from "@dust-tt/types"; +import type { MouseEvent } from "react"; +import React, { useEffect, useState } from "react"; +import { useSWRConfig } from "swr"; + +import { checkProvider } from "@app/lib/providers"; + +export type ProviderField = { + name: string; + label?: string; + placeholder: string; + type?: string; +}; + +type ProviderConfig = { + title: string; + fields: { + name: string; + placeholder: string; + type?: string; + }[]; + instructions: React.ReactNode; +}; + +export const MODEL_PROVIDER_CONFIGS: Record = { + openai: { + title: "OpenAI", + fields: [{ name: "api_key", placeholder: "OpenAI API Key" }], + instructions: ( + <> +

+ To use OpenAI models you must provide your API key. It can be found{" "} + + here + + . +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + azure_openai: { + title: "Azure OpenAI", + fields: [ + { name: "endpoint", placeholder: "Azure OpenAI Endpoint" }, + { name: "api_key", placeholder: "Azure OpenAI API Key" }, + ], + instructions: ( + <> +

+ To use Azure OpenAI models you must provide your API key and Endpoint. + They can be found in the left menu of your OpenAI Azure Resource + portal (menu item `Keys and Endpoint`). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + anthropic: { + title: "Anthropic", + fields: [{ name: "api_key", placeholder: "Anthropic API Key" }], + instructions: ( + <> +

+ To use Anthropic models you must provide your API key. It can be found{" "} + + here + +  (you can create a new key specifically for Dust). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + mistral: { + title: "Mistral AI", + fields: [{ name: "api_key", placeholder: "Mistral AI API Key" }], + instructions: ( + <> +

+ To use Mistral AI models you must provide your API key. It can be + found{" "} + + here + +  (you can create a new key specifically for Dust). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + google_ai_studio: { + title: "Google AI Studio", + fields: [{ name: "api_key", placeholder: "Google AI Studio API Key" }], + instructions: ( + <> +

+ To use Google AI Studio models you must provide your API key. It can + be found{" "} + + here + +  (you can create a new key specifically for Dust). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + togetherai: { + title: "TogetherAI", + fields: [{ name: "api_key", placeholder: "TogetherAI API Key" }], + instructions: ( + <> +

To use TogetherAI models you must provide your API key.

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + deepseek: { + title: "Deepseek", + fields: [{ name: "api_key", placeholder: "Deepseek API Key" }], + instructions: ( + <> +

To use Deepseek models you must provide your API key.

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, +}; + +export const SERVICE_PROVIDER_CONFIGS: Record = { + serpapi: { + title: "SerpAPI Search", + fields: [{ name: "api_key", placeholder: "SerpAPI API Key" }], + instructions: ( + <> +

+ SerpAPI lets you search Google (and other search engines). To use + SerpAPI you must provide your API key. It can be found{" "} + + here + +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + serper: { + title: "Serper Search", + fields: [{ name: "api_key", placeholder: "Serper API Key" }], + instructions: ( + <> +

+ Serper lets you search Google (and other search engines). To use + Serper you must provide your API key. It can be found{" "} + + here + +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, + browserlessapi: { + title: "Browserless API", + fields: [{ name: "api_key", placeholder: "Browserless API Key" }], + instructions: ( + <> +

+ Browserless lets you use headless browsers to scrape web content. To + use Browserless, you must provide your API key. It can be found{" "} + + here + + . +

+

+ Note that it generally takes 5 mins{" "} + for the API key to become active (an email is sent when it's ready). +

+

+ We'll never use your API key for anything other than to run your apps. +

+ + ), + }, +}; + +export interface ProviderSetupProps { + owner: WorkspaceType; + providerId: string; + title: string; + instructions?: React.ReactNode; + fields: ProviderField[]; + config: { [key: string]: string }; + enabled: boolean; + testSuccessMessage?: string; + isOpen: boolean; + onClose: () => void; +} + +export function ProviderSetup({ + owner, + providerId, + title, + instructions, + fields, + config, + enabled, + testSuccessMessage, + isOpen, + onClose, +}: ProviderSetupProps) { + const { mutate } = useSWRConfig(); + const [values, setValues] = useState>({}); + const [testError, setTestError] = useState(""); + const [testSuccessful, setTestSuccessful] = useState(false); + const [testRunning, setTestRunning] = useState(false); + const [enableRunning, setEnableRunning] = useState(false); + + useEffect(() => { + const newValues: Record = {}; + for (const field of fields) { + newValues[field.name] = config[field.name] || ""; + } + setValues(newValues); + setTestSuccessful(false); + setTestError(""); + }, [config, fields]); + + const runTest = async () => { + setTestRunning(true); + setTestError(""); + setTestSuccessful(false); + + const partialConfig: Record = {}; + for (const field of fields) { + partialConfig[field.name] = values[field.name]; + } + + const check = await checkProvider(owner, providerId, partialConfig); + if (!check.ok) { + setTestError(check.error || "Unknown error"); + setTestSuccessful(false); + } else { + setTestError(""); + setTestSuccessful(true); + } + setTestRunning(false); + }; + + const handleEnable = async () => { + setEnableRunning(true); + const payload: Record = {}; + for (const field of fields) { + payload[field.name] = values[field.name]; + } + + await fetch(`/api/w/${owner.sId}/providers/${providerId}`, { + headers: { "Content-Type": "application/json" }, + method: "POST", + body: JSON.stringify({ config: JSON.stringify(payload) }), + }); + setEnableRunning(false); + await mutate(`/api/w/${owner.sId}/providers`); + onClose(); + }; + + const handleDisable = async () => { + await fetch(`/api/w/${owner.sId}/providers/${providerId}`, { + method: "DELETE", + }); + await mutate(`/api/w/${owner.sId}/providers`); + onClose(); + }; + + const renderFields = () => + fields.map((field) => ( +
+ {field.label && ( + + )} + { + setTestSuccessful(false); + const val = e.target.value; + setValues((prev) => ({ ...prev, [field.name]: val })); + }} + /> +
+ )); + + const testDisabled = + fields.some((field) => !values[field.name]) || testRunning; + + const rightButtonProps = testSuccessful + ? { + label: enabled + ? enableRunning + ? "Updating..." + : "Update" + : enableRunning + ? "Enabling..." + : "Enable", + variant: "primary" as const, + disabled: enableRunning, + onClick: handleEnable, + } + : { + label: testRunning ? "Testing..." : "Test", + variant: "primary" as const, + disabled: testDisabled, + onClick: async (event: MouseEvent) => { + event.preventDefault(); + await runTest(); + }, + }; + + return ( + !open && onClose()}> + + + {title} + + {instructions || ( +

Provide the necessary configuration for {title}.

+ )} + + + + +
+ {renderFields()} +
+ {testError ? ( + Error: {testError} + ) : testSuccessful ? ( + + {testSuccessMessage || + `Test succeeded! You can now enable ${title}.`} + + ) : ( +   + )} +
+
+ + + + + + ); +} diff --git a/front/components/providers/SerpAPISetup.tsx b/front/components/providers/SerpAPISetup.tsx deleted file mode 100644 index 21b88fb368b3..000000000000 --- a/front/components/providers/SerpAPISetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function SerpAPISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "serpapi", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/serpapi`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/serpapi`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup SerpAPI Search - -
-

- SerpAPI lets you search Google (and other search - engines). To use SerpAPI you must provide your API key. - It can be found{" "} - - here - -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable SerpAPI Search. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/SerperSetup.tsx b/front/components/providers/SerperSetup.tsx deleted file mode 100644 index 27ea09b16ab2..000000000000 --- a/front/components/providers/SerperSetup.tsx +++ /dev/null @@ -1,206 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function SerperSetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "serper", { api_key: apiKey }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/serper`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/serper`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup Serper Search - -
-

- Serper lets you search Google (and other search - engines). To use Serper you must provide your API key. - It can be found{" "} - - here - -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable Serper Search. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/components/providers/TogetherAISetup.tsx b/front/components/providers/TogetherAISetup.tsx deleted file mode 100644 index d25785738c08..000000000000 --- a/front/components/providers/TogetherAISetup.tsx +++ /dev/null @@ -1,199 +0,0 @@ -import { Button } from "@dust-tt/sparkle"; -import type { WorkspaceType } from "@dust-tt/types"; -import { Dialog, Transition } from "@headlessui/react"; -import { Fragment, useEffect, useState } from "react"; -import { useSWRConfig } from "swr"; - -import { checkProvider } from "@app/lib/providers"; - -export default function TogetherAISetup({ - owner, - open, - setOpen, - config, - enabled, -}: { - owner: WorkspaceType; - open: boolean; - setOpen: (open: boolean) => void; - config: { [key: string]: string }; - enabled: boolean; -}) { - const { mutate } = useSWRConfig(); - - const [apiKey, setApiKey] = useState(config ? config.api_key : ""); - const [testSuccessful, setTestSuccessful] = useState(false); - const [testRunning, setTestRunning] = useState(false); - const [testError, setTestError] = useState(""); - const [enableRunning, setEnableRunning] = useState(false); - - useEffect(() => { - if (config && config.api_key.length > 0 && apiKey.length == 0) { - setApiKey(config.api_key); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [config]); - - const runTest = async () => { - setTestRunning(true); - setTestError(""); - const check = await checkProvider(owner, "togetherai", { - api_key: apiKey, - }); - - if (!check.ok) { - setTestError(check.error); - setTestSuccessful(false); - setTestRunning(false); - } else { - setTestError(""); - setTestSuccessful(true); - setTestRunning(false); - } - }; - - const handleEnable = async () => { - setEnableRunning(true); - const res = await fetch(`/api/w/${owner.sId}/providers/togetherai`, { - headers: { - "Content-Type": "application/json", - }, - method: "POST", - body: JSON.stringify({ - config: JSON.stringify({ - api_key: apiKey, - }), - }), - }); - await res.json(); - setEnableRunning(false); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - const handleDisable = async () => { - const res = await fetch(`/api/w/${owner.sId}/providers/togetherai`, { - method: "DELETE", - }); - await res.json(); - setOpen(false); - await mutate(`/api/w/${owner.sId}/providers`); - }; - - return ( - - setOpen(false)}> - -
- - -
-
- - -
-
- - Setup TogetherAI - -
-

- To use TogetherAI models you must provide your API key. -

-

- We'll never use your API key for anything other than to - run your apps. -

-
-
- { - setApiKey(e.target.value); - setTestSuccessful(false); - }} - /> -
-
-
-
- {testError?.length > 0 ? ( - Error: {testError} - ) : testSuccessful ? ( - - Test succeeded! You can enable TogetherAI. - - ) : ( -   - )} -
-
- {enabled ? ( -
handleDisable()} - > - Disable -
- ) : ( - <> - )} -
-
-
-
- {testSuccessful ? ( -
-
- - -
-
-
- - ); -} diff --git a/front/pages/w/[wId]/developers/providers.tsx b/front/pages/w/[wId]/developers/providers.tsx index 68856f207b7c..9595a4a8e047 100644 --- a/front/pages/w/[wId]/developers/providers.tsx +++ b/front/pages/w/[wId]/developers/providers.tsx @@ -1,21 +1,14 @@ import { Button, Page, ShapesIcon } from "@dust-tt/sparkle"; -import type { UserType, WorkspaceType } from "@dust-tt/types"; -import type { SubscriptionType } from "@dust-tt/types"; +import type { SubscriptionType, UserType, WorkspaceType } from "@dust-tt/types"; import type { InferGetServerSidePropsType } from "next"; -import React from "react"; -import { useState } from "react"; +import React, { useState } from "react"; import { subNavigationAdmin } from "@app/components/navigation/config"; -import AnthropicSetup from "@app/components/providers/AnthropicSetup"; -import AzureOpenAISetup from "@app/components/providers/AzureOpenAISetup"; -import BrowserlessAPISetup from "@app/components/providers/BrowserlessAPISetup"; -import DeepseekSetup from "@app/components/providers/DeepseekSetup"; -import GoogleAiStudioSetup from "@app/components/providers/GoogleAiStudioSetup"; -import MistralAISetup from "@app/components/providers/MistralAISetup"; -import OpenAISetup from "@app/components/providers/OpenAISetup"; -import SerpAPISetup from "@app/components/providers/SerpAPISetup"; -import SerperSetup from "@app/components/providers/SerperSetup"; -import TogetherAISetup from "@app/components/providers/TogetherAISetup"; +import { + MODEL_PROVIDER_CONFIGS, + ProviderSetup, + SERVICE_PROVIDER_CONFIGS, +} from "@app/components/providers/ProviderSetup"; import AppLayout from "@app/components/sparkle/AppLayout"; import { withDefaultUserAuthRequirements } from "@app/lib/iam/session"; import { @@ -35,11 +28,8 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ const subscription = auth.getNonNullableSubscription(); const user = auth.getNonNullableUser(); if (!auth.isAdmin()) { - return { - notFound: true, - }; + return { notFound: true }; } - return { props: { owner, @@ -50,218 +40,84 @@ export const getServerSideProps = withDefaultUserAuthRequirements<{ }); export function Providers({ owner }: { owner: WorkspaceType }) { - const [openAIOpen, setOpenAIOpen] = useState(false); - const [azureOpenAIOpen, setAzureOpenAIOpen] = useState(false); - const [anthropicOpen, setAnthropicOpen] = useState(false); - const [mistalAIOpen, setMistralAiOpen] = useState(false); - const [googleAiStudioOpen, setGoogleAiStudioOpen] = useState(false); - const [serpapiOpen, setSerpapiOpen] = useState(false); - const [serperOpen, setSerperOpen] = useState(false); - const [browserlessapiOpen, setBrowserlessapiOpen] = useState(false); - const [togetherAiOpen, setTogetherAiOpen] = useState(false); - const [deepseekOpen, setDeepseekOpen] = useState(false); const { providers, isProvidersLoading, isProvidersError } = useProviders({ owner, }); + const [selectedProviderId, setSelectedProviderId] = useState( + null + ); + const [isModelProvider, setIsModelProvider] = useState(true); const appWhiteListedProviders = owner.whiteListedProviders ? [...owner.whiteListedProviders, "azure_openai"] : APP_MODEL_PROVIDER_IDS; + const filteredProvidersIdSet = new Set( modelProviders - .filter((provider) => { - return ( + .filter( + (provider) => APP_MODEL_PROVIDER_IDS.includes(provider.providerId) && appWhiteListedProviders.includes(provider.providerId) - ); - }) + ) .map((provider) => provider.providerId) ); const configs = {} as any; - if (!isProvidersLoading && !isProvidersError) { - for (let i = 0; i < providers.length; i++) { - // Extract API key and hide it from the config object to be displayed. - // Store the original API key in a separate property for display use. - const { api_key, ...rest } = JSON.parse(providers[i].config); - configs[providers[i].providerId] = { + for (const provider of providers) { + const { api_key, ...rest } = JSON.parse(provider.config); + configs[provider.providerId] = { ...rest, api_key: "", redactedApiKey: api_key, }; - filteredProvidersIdSet.add(providers[i].providerId); + filteredProvidersIdSet.add(provider.providerId); } } - const filteredProviders = modelProviders.filter((provider) => - filteredProvidersIdSet.has(provider.providerId) + + const filteredProviders = modelProviders.filter((p) => + filteredProvidersIdSet.has(p.providerId) ); - return ( - <> - - - - - - - - - - - <> - -
    - {filteredProviders.map((provider) => ( -
  • -
    -
    -
    -

    - {provider.name} -

    -
    -

    - {configs[provider.providerId] ? "enabled" : "disabled"} -

    -
    -
    - {configs[provider.providerId] && ( -

    - API Key:{" "} - 

    {configs[provider.providerId].redactedApiKey}
    -

    - )} -
    -
    -
    -
    -
    • - ))} -
+ const selectedConfig = selectedProviderId + ? isModelProvider + ? MODEL_PROVIDER_CONFIGS[selectedProviderId] + : SERVICE_PROVIDER_CONFIGS[selectedProviderId] + : null; + + const enabled = + selectedProviderId && configs[selectedProviderId] + ? !!configs[selectedProviderId] + : false; - + {selectedProviderId && selectedConfig && ( + setSelectedProviderId(null)} /> + )} -
    - {serviceProviders.map((provider) => ( -
  • -
    + +
      + {filteredProviders.map((provider) => ( +
    • +
      +

      -
      -
      +
      +
      • + ))} +
    + + +
      + {serviceProviders.map((provider) => ( +
    • +
      +
      +

      + {provider.name} +

      +
      +

      + ? "bg-green-100 text-green-800" + : "bg-gray-100 text-gray-800" + )} + > + {configs[provider.providerId] ? "enabled" : "disabled"} +

      -
      • - ))} -
    - +
    +
    • + ))} +
); } From eb116177a10ab8da0cf985d5c179a69b713d339d Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Mon, 13 Jan 2025 10:30:55 +0100 Subject: [PATCH 17/47] [Types] remove `provider` from `ContentNode`, rename `BaseContentNode` to `ContentNode` (#9895) * remove provider from ContentNode and replace BaseContentNode with ContentNode * stop sending provider in content nodes in connectors --- .../connectors/confluence/lib/permissions.ts | 2 -- connectors/src/connectors/github/index.ts | 14 -------------- .../src/connectors/google_drive/index.ts | 8 -------- connectors/src/connectors/intercom/index.ts | 19 ++++++++++--------- .../intercom/lib/conversation_permissions.ts | 5 ----- .../intercom/lib/help_center_permissions.ts | 6 ------ .../connectors/intercom/lib/permissions.ts | 3 --- .../connectors/microsoft/lib/content_nodes.ts | 9 --------- connectors/src/connectors/notion/index.ts | 6 ------ connectors/src/connectors/slack/index.ts | 10 +++++----- .../connectors/snowflake/lib/content_nodes.ts | 3 --- connectors/src/connectors/webcrawler/index.ts | 10 ++++------ .../src/connectors/zendesk/lib/permissions.ts | 4 ---- connectors/src/resources/zendesk_resources.ts | 6 ------ .../components/ConnectorPermissionsModal.tsx | 4 ++-- front/components/ContentNodeTree.tsx | 10 +++++----- .../assistant_builder/SlackIntegration.tsx | 4 ++-- front/lib/content_nodes.ts | 8 ++++---- types/src/front/data_source_view.ts | 11 +++-------- types/src/front/lib/connectors_api.ts | 11 ++--------- 20 files changed, 37 insertions(+), 116 deletions(-) diff --git a/connectors/src/connectors/confluence/lib/permissions.ts b/connectors/src/connectors/confluence/lib/permissions.ts index b286026d12b4..4efe4b8a8c15 100644 --- a/connectors/src/connectors/confluence/lib/permissions.ts +++ b/connectors/src/connectors/confluence/lib/permissions.ts @@ -46,7 +46,6 @@ export function createContentNodeFromSpace( const spaceId = isConfluenceSpaceModel(space) ? space.spaceId : space.id; return { - provider: "confluence", internalId: makeSpaceInternalId(spaceId), parentInternalId: null, type: "folder", @@ -66,7 +65,6 @@ export function createContentNodeFromPage( isExpandable = false ): ContentNode { return { - provider: "confluence", internalId: makePageInternalId(page.pageId), parentInternalId: parent.type === "space" diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts index 4ff148eace5a..ca4a9305e614 100644 --- a/connectors/src/connectors/github/index.ts +++ b/connectors/src/connectors/github/index.ts @@ -286,7 +286,6 @@ export class GithubConnectorManager extends BaseConnectorManager { nodes = nodes.concat( page.map((repo) => ({ - provider: c.type, internalId: getRepositoryInternalId(repo.id), parentInternalId: null, type: "folder", @@ -358,7 +357,6 @@ export class GithubConnectorManager extends BaseConnectorManager { if (latestIssue) { nodes.push({ - provider: c.type, internalId: getIssuesInternalId(repoId), parentInternalId, type: "database", @@ -373,7 +371,6 @@ export class GithubConnectorManager extends BaseConnectorManager { if (latestDiscussion) { nodes.push({ - provider: c.type, internalId: getDiscussionsInternalId(repoId), parentInternalId, type: "channel", @@ -388,7 +385,6 @@ export class GithubConnectorManager extends BaseConnectorManager { if (codeRepo) { nodes.push({ - provider: c.type, internalId: getCodeRootInternalId(repoId), parentInternalId, type: "folder", @@ -431,7 +427,6 @@ export class GithubConnectorManager extends BaseConnectorManager { directories.forEach((directory) => { nodes.push({ - provider: c.type, internalId: directory.internalId, parentInternalId, type: "folder", @@ -446,7 +441,6 @@ export class GithubConnectorManager extends BaseConnectorManager { files.forEach((file) => { nodes.push({ - provider: c.type, internalId: file.documentId, parentInternalId, type: "file", @@ -609,7 +603,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getRepositoryInternalId(repoId), parentInternalId: null, type: "folder", @@ -629,7 +622,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getIssuesInternalId(repoId), parentInternalId: getRepositoryInternalId(repoId), type: "database", @@ -647,7 +639,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getDiscussionsInternalId(repoId), parentInternalId: getRepositoryInternalId(repoId), type: "channel", @@ -667,7 +658,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getIssueInternalId(repoId, issueNumber), parentInternalId: getIssuesInternalId(repoId), type: "file", @@ -687,7 +677,6 @@ export class GithubConnectorManager extends BaseConnectorManager { return; } nodes.push({ - provider: c.type, internalId: getDiscussionInternalId(repoId, discussionNumber), parentInternalId: getDiscussionsInternalId(repoId), type: "file", @@ -703,7 +692,6 @@ export class GithubConnectorManager extends BaseConnectorManager { // Constructing Nodes for Code fullCodeInRepos.forEach((codeRepo) => { nodes.push({ - provider: c.type, internalId: getCodeRootInternalId(codeRepo.repoId), parentInternalId: getRepositoryInternalId(codeRepo.repoId), type: "folder", @@ -719,7 +707,6 @@ export class GithubConnectorManager extends BaseConnectorManager { // Constructing Nodes for Code Directories codeDirectories.forEach((directory) => { nodes.push({ - provider: c.type, internalId: directory.internalId, parentInternalId: directory.parentInternalId, type: "folder", @@ -735,7 +722,6 @@ export class GithubConnectorManager extends BaseConnectorManager { // Constructing Nodes for Code Files codeFiles.forEach((file) => { nodes.push({ - provider: c.type, internalId: file.documentId, parentInternalId: file.parentInternalId, type: "file", diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts index ec2caf0b40a0..d4341b8905f3 100644 --- a/connectors/src/connectors/google_drive/index.ts +++ b/connectors/src/connectors/google_drive/index.ts @@ -320,7 +320,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { const type = getPermissionViewType(f); return { - provider: c.type, internalId: getInternalId(f.driveFileId), parentInternalId: null, type, @@ -344,7 +343,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { nodes = nodes.concat( sheets.map((s) => { return { - provider: c.type, internalId: getGoogleSheetContentNodeInternalId( s.driveFileId, s.driveSheetId @@ -389,7 +387,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { ); } return { - provider: c.type, internalId: getInternalId(driveObject.id), parentInternalId: // note: if the parent is null, the drive object falls at top-level @@ -417,7 +414,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { // Adding a fake "Shared with me" node, to allow the user to see their shared files // that are not living in a shared drive. nodes.push({ - provider: c.type, internalId: getInternalId(GOOGLE_DRIVE_SHARED_WITH_ME_VIRTUAL_ID), parentInternalId: null, type: "folder" as const, @@ -486,7 +482,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { ); return { - provider: c.type, internalId: getInternalId(driveObject.id), parentInternalId: driveObject.parent && getInternalId(driveObject.parent), @@ -673,7 +668,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { const sourceUrl = getSourceUrlForGoogleDriveFiles(f); return { - provider: "google_drive", internalId: getInternalId(f.driveFileId), parentInternalId: null, type, @@ -713,7 +707,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { })(); const sheetNodes: ContentNode[] = sheets.map((s) => ({ - provider: "google_drive", internalId: getGoogleSheetContentNodeInternalId( s.driveFileId, s.driveSheetId @@ -974,7 +967,6 @@ async function getFoldersAsContentNodes({ } const sourceUrl = `https://drive.google.com/drive/folders/${f.folderId}`; return { - provider: "google_drive", internalId: getInternalId(f.folderId), parentInternalId: null, type: "folder", diff --git a/connectors/src/connectors/intercom/index.ts b/connectors/src/connectors/intercom/index.ts index 93ca8cbab6d5..6a61b6e7655b 100644 --- a/connectors/src/connectors/intercom/index.ts +++ b/connectors/src/connectors/intercom/index.ts @@ -1,5 +1,9 @@ -import type { ConnectorPermission, ContentNode, Result } from "@dust-tt/types"; -import type { ContentNodesViewType } from "@dust-tt/types"; +import type { + ConnectorPermission, + ContentNode, + ContentNodesViewType, + Result, +} from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; import { Op } from "sequelize"; @@ -39,8 +43,10 @@ import type { RetrievePermissionsErrorCode, UpdateConnectorErrorCode, } from "@connectors/connectors/interface"; -import { ConnectorManagerError } from "@connectors/connectors/interface"; -import { BaseConnectorManager } from "@connectors/connectors/interface"; +import { + BaseConnectorManager, + ConnectorManagerError, +} from "@connectors/connectors/interface"; import { dataSourceConfigFromConnector } from "@connectors/lib/api/data_source_config"; import { ExternalOAuthTokenError } from "@connectors/lib/error"; import { @@ -613,7 +619,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { const nodes: ContentNode[] = []; for (const helpCenter of helpCenters) { nodes.push({ - provider: "intercom", internalId: getHelpCenterInternalId( this.connectorId, helpCenter.helpCenterId @@ -630,7 +635,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { } for (const collection of collections) { nodes.push({ - provider: "intercom", internalId: getHelpCenterCollectionInternalId( this.connectorId, collection.collectionId @@ -652,7 +656,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { } for (const article of articles) { nodes.push({ - provider: "intercom", internalId: getHelpCenterArticleInternalId( this.connectorId, article.articleId @@ -674,7 +677,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { } if (isAllConversations) { nodes.push({ - provider: "intercom", internalId: getTeamsInternalId(this.connectorId), parentInternalId: null, type: "channel", @@ -691,7 +693,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { } for (const team of teams) { nodes.push({ - provider: "intercom", internalId: getTeamInternalId(this.connectorId, team.teamId), parentInternalId: getTeamsInternalId(this.connectorId), type: "channel", diff --git a/connectors/src/connectors/intercom/lib/conversation_permissions.ts b/connectors/src/connectors/intercom/lib/conversation_permissions.ts index a9b771705235..8e985bf589da 100644 --- a/connectors/src/connectors/intercom/lib/conversation_permissions.ts +++ b/connectors/src/connectors/intercom/lib/conversation_permissions.ts @@ -134,7 +134,6 @@ export async function retrieveIntercomConversationsPermissions({ if (isReadPermissionsOnly) { if (isRootLevel && isAllConversationsSynced) { nodes.push({ - provider: "intercom", internalId: allTeamsInternalId, parentInternalId: null, type: "channel", @@ -148,7 +147,6 @@ export async function retrieveIntercomConversationsPermissions({ }); } else if (isRootLevel && hasTeamsWithReadPermission) { nodes.push({ - provider: "intercom", internalId: allTeamsInternalId, parentInternalId: null, type: "channel", @@ -165,7 +163,6 @@ export async function retrieveIntercomConversationsPermissions({ if (parentInternalId === allTeamsInternalId) { teamsWithReadPermission.forEach((team) => { nodes.push({ - provider: connector.type, internalId: getTeamInternalId(connectorId, team.teamId), parentInternalId: allTeamsInternalId, type: "folder", @@ -183,7 +180,6 @@ export async function retrieveIntercomConversationsPermissions({ const teams = await fetchIntercomTeams({ accessToken }); if (isRootLevel) { nodes.push({ - provider: "intercom", internalId: allTeamsInternalId, parentInternalId: null, type: "channel", @@ -202,7 +198,6 @@ export async function retrieveIntercomConversationsPermissions({ return teamFromDb.teamId === team.id; }); nodes.push({ - provider: connector.type, internalId: getTeamInternalId(connectorId, team.id), parentInternalId: allTeamsInternalId, type: "folder", diff --git a/connectors/src/connectors/intercom/lib/help_center_permissions.ts b/connectors/src/connectors/intercom/lib/help_center_permissions.ts index 4898c9f37ab5..e494f19cf92b 100644 --- a/connectors/src/connectors/intercom/lib/help_center_permissions.ts +++ b/connectors/src/connectors/intercom/lib/help_center_permissions.ts @@ -364,7 +364,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }, }); nodes = helpCentersFromDb.map((helpCenter) => ({ - provider: connector.type, internalId: getHelpCenterInternalId( connectorId, helpCenter.helpCenterId @@ -381,7 +380,6 @@ export async function retrieveIntercomHelpCentersPermissions({ } else { const helpCenters = await fetchIntercomHelpCenters({ accessToken }); nodes = helpCenters.map((helpCenter) => ({ - provider: connector.type, internalId: getHelpCenterInternalId(connectorId, helpCenter.id), parentInternalId: null, type: "database", @@ -425,7 +423,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }); if (isReadPermissionsOnly) { nodes = collectionsInDb.map((collection) => ({ - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.collectionId @@ -452,7 +449,6 @@ export async function retrieveIntercomHelpCentersPermissions({ (c) => c.collectionId === collection.id ); return { - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.id @@ -493,7 +489,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }); const collectionNodes: ContentNode[] = collectionsInDb.map( (collection) => ({ - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.collectionId @@ -522,7 +517,6 @@ export async function retrieveIntercomHelpCentersPermissions({ }, }); const articleNodes: ContentNode[] = articlesInDb.map((article) => ({ - provider: connector.type, internalId: getHelpCenterArticleInternalId( connectorId, article.articleId diff --git a/connectors/src/connectors/intercom/lib/permissions.ts b/connectors/src/connectors/intercom/lib/permissions.ts index 30c27f54b6bd..961dd9fbf2a0 100644 --- a/connectors/src/connectors/intercom/lib/permissions.ts +++ b/connectors/src/connectors/intercom/lib/permissions.ts @@ -50,7 +50,6 @@ export async function retrieveSelectedNodes({ ); collectionsNodes.push({ - provider: connector.type, internalId: getHelpCenterCollectionInternalId( connectorId, collection.collectionId @@ -79,7 +78,6 @@ export async function retrieveSelectedNodes({ intercomWorkspace?.syncAllConversations === "scheduled_activate" ) { teamsNodes.push({ - provider: connector.type, internalId: getTeamsInternalId(connectorId), parentInternalId: null, type: "channel", @@ -100,7 +98,6 @@ export async function retrieveSelectedNodes({ }); teams.forEach((team) => { teamsNodes.push({ - provider: connector.type, internalId: getTeamInternalId(connectorId, team.teamId), parentInternalId: getTeamsInternalId(connectorId), type: "folder", diff --git a/connectors/src/connectors/microsoft/lib/content_nodes.ts b/connectors/src/connectors/microsoft/lib/content_nodes.ts index f65e83166eae..d71efb4db289 100644 --- a/connectors/src/connectors/microsoft/lib/content_nodes.ts +++ b/connectors/src/connectors/microsoft/lib/content_nodes.ts @@ -17,7 +17,6 @@ export function getRootNodes(): ContentNode[] { export function getSitesRootAsContentNode(): ContentNode { return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: "", nodeType: "sites-root", @@ -36,7 +35,6 @@ export function getSitesRootAsContentNode(): ContentNode { export function getTeamsRootAsContentNode(): ContentNode { return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: "", nodeType: "teams-root", @@ -54,7 +52,6 @@ export function getTeamsRootAsContentNode(): ContentNode { } export function getTeamAsContentNode(team: microsoftgraph.Team): ContentNode { return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: `/teams/${team.id}`, nodeType: "team", @@ -80,7 +77,6 @@ export function getSiteAsContentNode( throw new Error("Site id is required"); } return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: getSiteAPIPath(site), nodeType: "site", @@ -111,7 +107,6 @@ export function getChannelAsContentNode( } return { - provider: "microsoft", internalId: internalIdFromTypeAndPath({ itemAPIPath: `/teams/${parentInternalId}/channels/${channel.id}`, nodeType: "channel", @@ -136,7 +131,6 @@ export function getDriveAsContentNode( throw new Error("Drive id is required"); } return { - provider: "microsoft", internalId: getDriveInternalId(drive), parentInternalId, type: "folder", @@ -153,7 +147,6 @@ export function getFolderAsContentNode( parentInternalId: string ): ContentNode { return { - provider: "microsoft", internalId: getDriveItemInternalId(folder), parentInternalId, type: "folder", @@ -171,7 +164,6 @@ export function getFileAsContentNode( parentInternalId: string ): ContentNode { return { - provider: "microsoft", internalId: getDriveItemInternalId(file), parentInternalId, type: "file", @@ -207,7 +199,6 @@ export function getMicrosoftNodeAsContentNode( } return { - provider: "microsoft", internalId: node.internalId, parentInternalId: node.parentInternalId, type, diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts index 63b285c753ea..741b41772564 100644 --- a/connectors/src/connectors/notion/index.ts +++ b/connectors/src/connectors/notion/index.ts @@ -454,7 +454,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const expandable = Boolean(hasChildrenByPageId[page.notionPageId]); return { - provider: c.type, internalId: nodeIdFromNotionId(page.notionPageId), parentInternalId: !page.parentId || page.parentId === "workspace" @@ -478,7 +477,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const getDbNodes = async (db: NotionDatabase): Promise => { return { - provider: c.type, internalId: nodeIdFromNotionId(db.notionDatabaseId), parentInternalId: !db.parentId || db.parentId === "workspace" @@ -503,7 +501,6 @@ export class NotionConnectorManager extends BaseConnectorManager { // We also need to return a "fake" top-level folder call "Orphaned" to include resources // we haven't been able to find a parent for. folderNodes.push({ - provider: c.type, // Orphaned resources in the database will have "unknown" as their parentId. internalId: nodeIdFromNotionId("unknown"), parentInternalId: null, @@ -552,7 +549,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const hasChildrenByPageId = await hasChildren(pages, this.connectorId); const pageNodes: ContentNode[] = await Promise.all( pages.map(async (page) => ({ - provider: "notion", internalId: nodeIdFromNotionId(page.notionPageId), parentInternalId: !page.parentId || page.parentId === "workspace" @@ -569,7 +565,6 @@ export class NotionConnectorManager extends BaseConnectorManager { ); const dbNodes: ContentNode[] = dbs.map((db) => ({ - provider: "notion", internalId: nodeIdFromNotionId(db.notionDatabaseId), parentInternalId: !db.parentId || db.parentId === "workspace" @@ -590,7 +585,6 @@ export class NotionConnectorManager extends BaseConnectorManager { const orphanedCount = await getOrphanedCount(this.connectorId); if (orphanedCount > 0) { contentNodes.push({ - provider: "notion", // Orphaned resources in the database will have "unknown" as their parentId. internalId: nodeIdFromNotionId("unknown"), parentInternalId: null, diff --git a/connectors/src/connectors/slack/index.ts b/connectors/src/connectors/slack/index.ts index 86347d0d92ca..4f6dc5a6f28b 100644 --- a/connectors/src/connectors/slack/index.ts +++ b/connectors/src/connectors/slack/index.ts @@ -1,11 +1,11 @@ import type { ConnectorPermission, ContentNode, + ContentNodesViewType, ModelId, Result, SlackConfigurationType, } from "@dust-tt/types"; -import type { ContentNodesViewType } from "@dust-tt/types"; import { Err, isSlackAutoReadPatterns, @@ -20,8 +20,10 @@ import type { RetrievePermissionsErrorCode, UpdateConnectorErrorCode, } from "@connectors/connectors/interface"; -import { ConnectorManagerError } from "@connectors/connectors/interface"; -import { BaseConnectorManager } from "@connectors/connectors/interface"; +import { + BaseConnectorManager, + ConnectorManagerError, +} from "@connectors/connectors/interface"; import { getChannels } from "@connectors/connectors/slack//temporal/activities"; import { getBotEnabled } from "@connectors/connectors/slack/bot"; import { joinChannel } from "@connectors/connectors/slack/lib/channels"; @@ -397,7 +399,6 @@ export class SlackConnectorManager extends BaseConnectorManager ({ - provider: "slack", internalId: slackChannelInternalIdFromSlackChannelId(ch.slackChannelId), parentInternalId: null, type: "channel", @@ -620,7 +621,6 @@ export class SlackConnectorManager extends BaseConnectorManager ({ - provider: "slack", internalId: slackChannelInternalIdFromSlackChannelId(ch.slackChannelId), parentInternalId: null, type: "channel", diff --git a/connectors/src/connectors/snowflake/lib/content_nodes.ts b/connectors/src/connectors/snowflake/lib/content_nodes.ts index f94659e8c03d..919dcf0fceb7 100644 --- a/connectors/src/connectors/snowflake/lib/content_nodes.ts +++ b/connectors/src/connectors/snowflake/lib/content_nodes.ts @@ -34,7 +34,6 @@ export const getContentNodeFromInternalId = ( if (type === "database") { return { - provider: "snowflake", internalId: databaseName as string, parentInternalId: null, type: "folder", @@ -49,7 +48,6 @@ export const getContentNodeFromInternalId = ( } if (type === "schema") { return { - provider: "snowflake", internalId: `${databaseName}.${schemaName}`, parentInternalId: databaseName as string, type: "folder", @@ -64,7 +62,6 @@ export const getContentNodeFromInternalId = ( } if (type === "table") { return { - provider: "snowflake", internalId: `${databaseName}.${schemaName}.${tableName}`, parentInternalId: `${databaseName}.${schemaName}`, type: "database", diff --git a/connectors/src/connectors/webcrawler/index.ts b/connectors/src/connectors/webcrawler/index.ts index 99e06b576ef6..d5d492c4f9bb 100644 --- a/connectors/src/connectors/webcrawler/index.ts +++ b/connectors/src/connectors/webcrawler/index.ts @@ -19,8 +19,10 @@ import type { RetrievePermissionsErrorCode, UpdateConnectorErrorCode, } from "@connectors/connectors/interface"; -import { ConnectorManagerError } from "@connectors/connectors/interface"; -import { BaseConnectorManager } from "@connectors/connectors/interface"; +import { + BaseConnectorManager, + ConnectorManagerError, +} from "@connectors/connectors/interface"; import { getDisplayNameForFolder, getDisplayNameForPage, @@ -199,7 +201,6 @@ export class WebcrawlerConnectorManager extends BaseConnectorManager !excludedFoldersSet.has(f.url)) .map((folder): ContentNode => { return { - provider: "webcrawler", internalId: folder.internalId, parentInternalId: folder.parentUrl ? stableIdForUrl({ @@ -222,7 +223,6 @@ export class WebcrawlerConnectorManager extends BaseConnectorManager { nodes.push({ - provider: "webcrawler", internalId: folder.internalId, parentInternalId: folder.parentUrl, title: getDisplayNameForFolder(folder), @@ -288,7 +287,6 @@ export class WebcrawlerConnectorManager extends BaseConnectorManager { nodes.push({ - provider: "webcrawler", internalId: page.documentId, parentInternalId: page.parentUrl, title: getDisplayNameForPage(page), diff --git a/connectors/src/connectors/zendesk/lib/permissions.ts b/connectors/src/connectors/zendesk/lib/permissions.ts index 26385b08f72f..420c79b8a286 100644 --- a/connectors/src/connectors/zendesk/lib/permissions.ts +++ b/connectors/src/connectors/zendesk/lib/permissions.ts @@ -76,7 +76,6 @@ async function getRootLevelContentNodes( brandsInDatabase .find((b) => b.brandId === brand.id) ?.toContentNode(connectorId) ?? { - provider: "zendesk", internalId: getBrandInternalId({ connectorId, brandId: brand.id }), parentInternalId: null, type: "folder", @@ -134,7 +133,6 @@ async function getBrandChildren( const ticketsNode: ContentNode = brandInDb?.getTicketsContentNode( connector.id ) ?? { - provider: "zendesk", internalId: getTicketsInternalId({ connectorId: connector.id, brandId }), parentInternalId: parentInternalId, type: "folder", @@ -152,7 +150,6 @@ async function getBrandChildren( const helpCenterNode: ContentNode = brandInDb?.getHelpCenterContentNode( connector.id ) ?? { - provider: "zendesk", internalId: getHelpCenterInternalId({ connectorId: connector.id, brandId, @@ -213,7 +210,6 @@ async function getHelpCenterChildren( categoriesInDatabase .find((c) => c.categoryId === category.id) ?.toContentNode(connectorId) ?? { - provider: "zendesk", internalId: getCategoryInternalId({ connectorId, brandId, diff --git a/connectors/src/resources/zendesk_resources.ts b/connectors/src/resources/zendesk_resources.ts index de45cb15a23d..7cf52360f365 100644 --- a/connectors/src/resources/zendesk_resources.ts +++ b/connectors/src/resources/zendesk_resources.ts @@ -330,7 +330,6 @@ export class ZendeskBrandResource extends BaseResource { toContentNode(connectorId: number): ContentNode { const { brandId } = this; return { - provider: "zendesk", internalId: getBrandInternalId({ connectorId, brandId }), parentInternalId: null, type: "folder", @@ -353,7 +352,6 @@ export class ZendeskBrandResource extends BaseResource { ): ContentNode & { parentInternalId: string } { const { brandId } = this; return { - provider: "zendesk", internalId: getHelpCenterInternalId({ connectorId, brandId }), parentInternalId: getBrandInternalId({ connectorId, brandId }), type: "folder", @@ -375,7 +373,6 @@ export class ZendeskBrandResource extends BaseResource { ): ContentNode & { parentInternalId: string } { const { brandId } = this; return { - provider: "zendesk", internalId: getTicketsInternalId({ connectorId, brandId }), parentInternalId: getBrandInternalId({ connectorId, brandId }), type: "folder", @@ -641,7 +638,6 @@ export class ZendeskCategoryResource extends BaseResource { ): ContentNode { const { brandId, categoryId, permission } = this; return { - provider: "zendesk", internalId: getCategoryInternalId({ connectorId, brandId, categoryId }), parentInternalId: getHelpCenterInternalId({ connectorId, brandId }), type: "folder", @@ -727,7 +723,6 @@ export class ZendeskTicketResource extends BaseResource { toContentNode(connectorId: number): ContentNode { const { brandId, ticketId } = this; return { - provider: "zendesk", internalId: getTicketInternalId({ connectorId, brandId, ticketId }), parentInternalId: getTicketsInternalId({ connectorId, brandId }), type: "file", @@ -939,7 +934,6 @@ export class ZendeskArticleResource extends BaseResource { toContentNode(connectorId: number): ContentNode { const { brandId, categoryId, articleId } = this; return { - provider: "zendesk", internalId: getArticleInternalId({ connectorId, brandId, articleId }), parentInternalId: getCategoryInternalId({ connectorId, diff --git a/front/components/ConnectorPermissionsModal.tsx b/front/components/ConnectorPermissionsModal.tsx index e75e19edc248..7be2be126627 100644 --- a/front/components/ConnectorPermissionsModal.tsx +++ b/front/components/ConnectorPermissionsModal.tsx @@ -28,10 +28,10 @@ import { } from "@dust-tt/sparkle"; import type { APIError, - BaseContentNode, ConnectorPermission, ConnectorProvider, ConnectorType, + ContentNode, DataSourceType, LightWorkspaceType, UpdateConnectorRequestBody, @@ -946,7 +946,7 @@ export async function confirmPrivateNodesSync({ selectedNodes, confirm, }: { - selectedNodes: BaseContentNode[]; + selectedNodes: ContentNode[]; confirm: (n: ConfirmDataType) => Promise; }): Promise { // confirmation in case there are private nodes diff --git a/front/components/ContentNodeTree.tsx b/front/components/ContentNodeTree.tsx index 81b05bb24b2f..d41b07f481c2 100644 --- a/front/components/ContentNodeTree.tsx +++ b/front/components/ContentNodeTree.tsx @@ -8,7 +8,7 @@ import { Tooltip, Tree, } from "@dust-tt/sparkle"; -import type { APIError, BaseContentNode } from "@dust-tt/types"; +import type { APIError, ContentNode } from "@dust-tt/types"; import type { ReactNode } from "react"; import React, { useCallback, useContext, useState } from "react"; @@ -17,7 +17,7 @@ import { classNames, timeAgoFrom } from "@app/lib/utils"; const unselectedChildren = ( selection: Record, - node: BaseContentNode + node: ContentNode ) => Object.entries(selection).reduce((acc, [k, v]) => { const shouldUnselect = v.parents.includes(node.internalId); @@ -32,7 +32,7 @@ const unselectedChildren = ( }, {}); export type UseResourcesHook = (parentId: string | null) => { - resources: BaseContentNode[]; + resources: ContentNode[]; isResourcesLoading: boolean; isResourcesError: boolean; resourcesError?: APIError | null; @@ -40,7 +40,7 @@ export type UseResourcesHook = (parentId: string | null) => { export type ContentNodeTreeItemStatus = { isSelected: boolean; - node: BaseContentNode; + node: ContentNode; parents: string[]; }; @@ -121,7 +121,7 @@ function ContentNodeTreeChildren({ ); const getCheckedState = useCallback( - (node: BaseContentNode) => { + (node: ContentNode) => { if (!selectedNodes) { return false; } diff --git a/front/components/assistant_builder/SlackIntegration.tsx b/front/components/assistant_builder/SlackIntegration.tsx index 9bcfde333339..638809edbc7d 100644 --- a/front/components/assistant_builder/SlackIntegration.tsx +++ b/front/components/assistant_builder/SlackIntegration.tsx @@ -6,7 +6,7 @@ import { SlackLogo, } from "@dust-tt/sparkle"; import type { - BaseContentNode, + ContentNode, DataSourceType, WorkspaceType, } from "@dust-tt/types"; @@ -52,7 +52,7 @@ export function SlackIntegration({ }, [existingSelection, newSelection]); const customIsNodeChecked = useCallback( - (node: BaseContentNode) => { + (node: ContentNode) => { return ( newSelection?.some((c) => c.slackChannelId === node.internalId) || false ); diff --git a/front/lib/content_nodes.ts b/front/lib/content_nodes.ts index c33e4e29d1ff..68452c7b1ef1 100644 --- a/front/lib/content_nodes.ts +++ b/front/lib/content_nodes.ts @@ -6,10 +6,10 @@ import { LockIcon, Square3Stack3DIcon, } from "@dust-tt/sparkle"; -import type { BaseContentNode } from "@dust-tt/types"; +import type { ContentNode } from "@dust-tt/types"; import { assertNever } from "@dust-tt/types"; -function getVisualForFileContentNode(node: BaseContentNode & { type: "file" }) { +function getVisualForFileContentNode(node: ContentNode & { type: "file" }) { if (node.expandable) { return DocumentPileIcon; } @@ -17,7 +17,7 @@ function getVisualForFileContentNode(node: BaseContentNode & { type: "file" }) { return DocumentIcon; } -export function getVisualForContentNode(node: BaseContentNode) { +export function getVisualForContentNode(node: ContentNode) { switch (node.type) { case "channel": if (node.providerVisibility === "private") { @@ -30,7 +30,7 @@ export function getVisualForContentNode(node: BaseContentNode) { case "file": return getVisualForFileContentNode( - node as BaseContentNode & { type: "file" } + node as ContentNode & { type: "file" } ); case "folder": diff --git a/types/src/front/data_source_view.ts b/types/src/front/data_source_view.ts index 392d763b4d28..cf0f01c0cd44 100644 --- a/types/src/front/data_source_view.ts +++ b/types/src/front/data_source_view.ts @@ -1,12 +1,7 @@ import { ModelId } from "../shared/model_id"; import { DataSourceViewCategory } from "./api_handlers/public/spaces"; -import { - ConnectorStatusDetails, - DataSourceType, - DataSourceWithAgentsUsageType, - EditedByUser, -} from "./data_source"; -import { BaseContentNode } from "./lib/connectors_api"; +import { ConnectorStatusDetails, DataSourceType, DataSourceWithAgentsUsageType, EditedByUser } from "./data_source"; +import { ContentNode } from "./lib/connectors_api"; export interface DataSourceViewType { category: DataSourceViewCategory; @@ -26,7 +21,7 @@ export type DataSourceViewsWithDetails = DataSourceViewType & { usage: DataSourceWithAgentsUsageType; }; -export type DataSourceViewContentNode = BaseContentNode & { +export type DataSourceViewContentNode = ContentNode & { parentInternalIds: string[] | null; }; diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index d53ff73e0ff5..f31ee8d98ce7 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -1,7 +1,4 @@ -import { - AdminCommandType, - AdminResponseType, -} from "../../connectors/admin/cli"; +import { AdminCommandType, AdminResponseType } from "../../connectors/admin/cli"; import { ConnectorsAPIError, isConnectorsAPIError } from "../../connectors/api"; import { UpdateConnectorConfigurationType } from "../../connectors/api_handlers/connector_configuration"; import { ConnectorCreateRequestBody } from "../../connectors/api_handlers/create_connector"; @@ -96,7 +93,7 @@ export const contentNodeTypeSortOrder: Record = { * information. More details here: * https://www.notion.so/dust-tt/Design-Doc-Microsoft-ids-parents-c27726652aae45abafaac587b971a41d?pvs=4 */ -export interface BaseContentNode { +export interface ContentNode { internalId: string; // The direct parent ID of this content node parentInternalId: string | null; @@ -111,10 +108,6 @@ export interface BaseContentNode { providerVisibility?: "public" | "private"; } -export type ContentNode = BaseContentNode & { - provider: ConnectorProvider; -}; - export type ContentNodeWithParentIds = ContentNode & { // A list of all parent IDs up to the root node, including the direct parent // Note: When includeParents is true, this list will be populated From 51e344f3837c561c50fbd93d8ae42e657461cff9 Mon Sep 17 00:00:00 2001 From: Thomas Draier Date: Mon, 13 Jan 2025 10:45:08 +0100 Subject: [PATCH 18/47] [front] Compute and store active user stats (#9915) --- .../components/assistant/AssistantDetails.tsx | 112 +++++++++--------- .../components/assistant/AssistantsTable.tsx | 30 ++++- front/components/assistant/Usage.tsx | 29 +++++ front/lib/api/assistant/agent_usage.ts | 66 +++++++---- .../agent_configurations/[aId]/analytics.ts | 2 +- .../assistant/agent_configurations/index.ts | 5 +- sdks/js/src/types.ts | 2 + types/src/front/assistant/agent.ts | 2 + 8 files changed, 165 insertions(+), 83 deletions(-) diff --git a/front/components/assistant/AssistantDetails.tsx b/front/components/assistant/AssistantDetails.tsx index 4946b3ed9380..d867c631e495 100644 --- a/front/components/assistant/AssistantDetails.tsx +++ b/front/components/assistant/AssistantDetails.tsx @@ -2,7 +2,6 @@ import { Avatar, BarChartIcon, Button, - Card, CardGrid, ChatBubbleLeftRightIcon, ChatBubbleThoughtIcon, @@ -25,6 +24,7 @@ import { TabsList, TabsTrigger, Tooltip, + ValueCard, } from "@dust-tt/sparkle"; import type { AgentConfigurationScope, @@ -142,48 +142,49 @@ function AssistantDetailsPerformance({ ) : ( - -
-
-
Active Users
-
-
- {agentAnalytics?.users ? ( - <> -
- {agentAnalytics.users.length} -
+ +
+ {agentAnalytics?.users ? ( + <> +
+ {agentAnalytics.users.length} +
- - {removeNulls(agentAnalytics.users.map((top) => top.user)) - .slice(0, 5) - .map((user) => ( - - } - label={user.fullName} - /> - ))} - - - ) : ( - "-" - )} + + {removeNulls( + agentAnalytics.users.map((top) => top.user) + ) + .slice(0, 5) + .map((user) => ( + + } + label={user.fullName} + /> + ))} + + + ) : ( + "-" + )} +
-
- + } + className="h-32" + /> - -
-
-
Reactions
-
+ {agentConfiguration.scope !== "global" && agentAnalytics?.feedbacks ? ( @@ -205,13 +206,12 @@ function AssistantDetailsPerformance({ "-" )}
- - - -
-
-
Conversations
-
+ } + className="h-32" + /> +
@@ -224,13 +224,12 @@ function AssistantDetailsPerformance({
- - - -
-
-
Messages
-
+ } + className="h-32" + /> +
@@ -243,8 +242,9 @@ function AssistantDetailsPerformance({
- - + } + className="h-32" + /> )} {agentConfiguration.scope !== "global" && ( diff --git a/front/components/assistant/AssistantsTable.tsx b/front/components/assistant/AssistantsTable.tsx index 8800d3c0cbc5..8410147839d0 100644 --- a/front/components/assistant/AssistantsTable.tsx +++ b/front/components/assistant/AssistantsTable.tsx @@ -26,7 +26,10 @@ import { useRouter } from "next/router"; import { useMemo, useState } from "react"; import { DeleteAssistantDialog } from "@app/components/assistant/DeleteAssistantDialog"; -import { assistantUsageMessage } from "@app/components/assistant/Usage"; +import { + assistantActiveUsersMessage, + assistantUsageMessage, +} from "@app/components/assistant/Usage"; import { SCOPE_INFO } from "@app/components/assistant_builder/Sharing"; import { classNames, formatTimestampToFriendlyDate } from "@app/lib/utils"; @@ -133,6 +136,29 @@ const getTableColumns = () => { width: "6rem", }, }, + { + header: "Active Users", + accessorKey: "usage.userCount", + cell: (info: CellContext) => ( + + + {info.row.original.usage?.userCount ?? 0} + + } + /> + + ), + meta: { + width: "6rem", + }, + }, { header: "Feedbacks", accessorKey: "feedbacks", @@ -236,6 +262,8 @@ export function AssistantsTable({ name: agentConfiguration.name, usage: agentConfiguration.usage ?? { messageCount: 0, + conversationCount: 0, + userCount: 0, timePeriodSec: 30 * 24 * 60 * 60, }, description: agentConfiguration.description, diff --git a/front/components/assistant/Usage.tsx b/front/components/assistant/Usage.tsx index 32938eb779b0..04cd185f3c98 100644 --- a/front/components/assistant/Usage.tsx +++ b/front/components/assistant/Usage.tsx @@ -52,3 +52,32 @@ export function assistantUsageMessage({ ); } } + +export function assistantActiveUsersMessage({ + usage, + isLoading, + isError, +}: { + usage: AgentUsageType | null; + isLoading: boolean; + isError: boolean; +}): ReactNode { + if (isError) { + return "Error loading usage data."; + } + + if (isLoading) { + return "Loading usage data..."; + } + + if (usage) { + const days = usage.timePeriodSec / (60 * 60 * 24); + const nb = usage.userCount || 0; + + return ( + <> + {nb} active user{pluralize(nb)} over the last {days} days + + ); + } +} diff --git a/front/lib/api/assistant/agent_usage.ts b/front/lib/api/assistant/agent_usage.ts index 3a2504b71ec6..974c88930b9a 100644 --- a/front/lib/api/assistant/agent_usage.ts +++ b/front/lib/api/assistant/agent_usage.ts @@ -34,13 +34,8 @@ const TTL_KEY_NOT_SET = -1; type AgentUsageCount = { agentId: string; messageCount: number; - timePeriodSec: number; -}; - -type MentionCount = { - agentId: string; - count: number; conversationCount: number; + userCount: number; timePeriodSec: number; }; @@ -89,11 +84,16 @@ export async function getAgentsUsage({ // Retrieve and parse agents usage const agentsUsage = await redis.hGetAll(agentMessageCountKey); return Object.entries(agentsUsage) - .map(([agentId, count]) => ({ - agentId, - messageCount: parseInt(count), - timePeriodSec: RANKING_TIMEFRAME_SEC, - })) + .map(([agentId, value]) => { + const parsed = JSON.parse(value); + return { + agentId, + conversationCount: 0, + userCount: 0, + ...(typeof parsed === "object" ? parsed : { messageCount: parsed }), + timePeriodSec: RANKING_TIMEFRAME_SEC, + }; + }) .sort((a, b) => b.messageCount - a.messageCount) .slice(0, limit); } @@ -134,6 +134,8 @@ export async function getAgentUsage( ? { agentId: agentConfiguration.sId, messageCount: agentUsage, + conversationCount: 0, + userCount: 0, timePeriodSec: RANKING_TIMEFRAME_SEC, } : null; @@ -143,7 +145,7 @@ export async function agentMentionsCount( workspaceId: number, agentConfiguration?: LightAgentConfigurationType, rankingUsageDays: number = RANKING_USAGE_DAYS -): Promise { +): Promise { // We retrieve mentions from conversations in order to optimize the query // Since we need to filter out by workspace id, retrieving mentions first // would lead to retrieve every single messages @@ -155,12 +157,19 @@ export async function agentMentionsCount( ], [ Sequelize.fn("COUNT", Sequelize.literal('"messages->mentions"."id"')), - "count", + "messageCount", ], [ Sequelize.fn("COUNT", Sequelize.literal("DISTINCT conversation.id")), "conversationCount", ], + [ + Sequelize.fn( + "COUNT", + Sequelize.literal('DISTINCT "messages->userMessage"."userId"') + ), + "userCount", + ], ], where: { workspaceId, @@ -185,6 +194,12 @@ export async function agentMentionsCount( }, }, }, + { + model: UserMessage, + as: "userMessage", + required: true, + attributes: [], + }, ], }, ], @@ -196,13 +211,15 @@ export async function agentMentionsCount( return mentions.map((mention) => { const castMention = mention as unknown as { agentConfigurationId: string; - count: number; + messageCount: number; conversationCount: number; + userCount: number; }; return { agentId: castMention.agentConfigurationId, - count: castMention.count, + messageCount: castMention.messageCount, conversationCount: castMention.conversationCount, + userCount: castMention.userCount, timePeriodSec: rankingUsageDays * 24 * 60 * 60, }; }); @@ -210,7 +227,7 @@ export async function agentMentionsCount( export async function storeCountsInRedis( workspaceId: string, - agentMessageCounts: MentionCount[], + agentMessageCounts: AgentUsageCount[], redis: RedisClientType ) { const agentMessageCountKey = _getUsageKey(workspaceId); @@ -225,8 +242,9 @@ export async function storeCountsInRedis( if (!amcByAgentId[agentId]) { amcByAgentId[agentId] = { agentId, - count: 0, + messageCount: 0, conversationCount: 0, + userCount: 0, timePeriodSec: RANKING_TIMEFRAME_SEC, }; } @@ -234,9 +252,15 @@ export async function storeCountsInRedis( const transaction = redis.multi(); - Object.values(amcByAgentId).forEach(({ agentId, count }) => { - transaction.hSet(agentMessageCountKey, agentId, count); - }); + Object.values(amcByAgentId).forEach( + ({ agentId, messageCount, conversationCount, userCount }) => { + transaction.hSet( + agentMessageCountKey, + agentId, + JSON.stringify({ messageCount, conversationCount, userCount }) + ); + } + ); transaction.expire(agentMessageCountKey, MENTION_COUNT_TTL); @@ -273,7 +297,7 @@ type UsersUsageCount = { export async function getAgentUsers( owner: LightWorkspaceType, - agentConfiguration?: LightAgentConfigurationType, + agentConfiguration: LightAgentConfigurationType, rankingUsageDays: number = RANKING_USAGE_DAYS ): Promise { const mentions = await Conversation.findAll({ diff --git a/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts b/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts index 55e685bda05a..bf0fb46a61db 100644 --- a/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts +++ b/front/pages/api/w/[wId]/assistant/agent_configurations/[aId]/analytics.ts @@ -127,7 +127,7 @@ async function handler( })) .filter((r) => r.user), mentions: { - messageCount: mentionCounts?.count ?? 0, + messageCount: mentionCounts?.messageCount ?? 0, conversationCount: mentionCounts?.conversationCount ?? 0, timePeriodSec: mentionCounts?.timePeriodSec ?? period * 60 * 60 * 24, }, diff --git a/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts b/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts index 3cef3d3761a3..57ca3488cf5d 100644 --- a/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts +++ b/front/pages/api/w/[wId]/assistant/agent_configurations/index.ts @@ -120,10 +120,7 @@ async function handler( usageMap[agentConfiguration.sId] ? { ...agentConfiguration, - usage: { - messageCount: usageMap[agentConfiguration.sId].messageCount, - timePeriodSec: usageMap[agentConfiguration.sId].timePeriodSec, - }, + usage: _.omit(usageMap[agentConfiguration.sId], ["agentId"]), } : agentConfiguration ); diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index f18a9a84fbc9..4ac46c7f7e39 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -781,6 +781,8 @@ export type AgentConfigurationViewType = z.infer< const AgentUsageTypeSchema = z.object({ messageCount: z.number(), + conversationCount: z.number(), + userCount: z.number(), timePeriodSec: z.number(), }); diff --git a/types/src/front/assistant/agent.ts b/types/src/front/assistant/agent.ts index 4fe2a5195b05..420e6627ed90 100644 --- a/types/src/front/assistant/agent.ts +++ b/types/src/front/assistant/agent.ts @@ -173,6 +173,8 @@ export type AgentsGetViewType = export type AgentUsageType = { messageCount: number; + conversationCount: number; + userCount: number; timePeriodSec: number; }; From cb8c68b4325c864f9dec5768a409268a24b93cb9 Mon Sep 17 00:00:00 2001 From: Lucas Massemin Date: Mon, 13 Jan 2025 10:55:12 +0100 Subject: [PATCH 19/47] Co-edition with Alex -> Multiple font and spacing nits (#9871) * Co-edition with Alex -> Multiple font and spacing nits * feedbacks without s --- front/components/assistant/AssistantDetails.tsx | 6 +++--- .../assistant_builder/FeedbacksSection.tsx | 15 +++++++++------ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/front/components/assistant/AssistantDetails.tsx b/front/components/assistant/AssistantDetails.tsx index d867c631e495..d77e3d55d6a8 100644 --- a/front/components/assistant/AssistantDetails.tsx +++ b/front/components/assistant/AssistantDetails.tsx @@ -248,13 +248,13 @@ function AssistantDetailsPerformance({ )} {agentConfiguration.scope !== "global" && ( - <> - +
+ - +
)} ); diff --git a/front/components/assistant_builder/FeedbacksSection.tsx b/front/components/assistant_builder/FeedbacksSection.tsx index 1d6bfccc01ab..f6e684463822 100644 --- a/front/components/assistant_builder/FeedbacksSection.tsx +++ b/front/components/assistant_builder/FeedbacksSection.tsx @@ -82,7 +82,9 @@ export const FeedbacksSection = ({ !isAgentConfigurationFeedbacksLoading && (!agentConfigurationFeedbacks || agentConfigurationFeedbacks.length === 0) ) { - return
No feedbacks.
; + return ( +
No feedback yet.
+ ); } if (!agentConfigurationHistory) { @@ -164,7 +166,7 @@ function AgentConfigurationVersionHeader({ ); return ( -
+
{agentConfiguration ? getAgentConfigurationVersionString(agentConfiguration) : `v${agentConfigurationVersion}`} @@ -216,9 +218,9 @@ function FeedbackCard({ owner, feedback }: FeedbackCardProps) { {timeSinceFeedback} ago
{feedback.thumbDirection === "up" ? ( - + ) : ( - + )}
@@ -229,12 +231,13 @@ function FeedbackCard({ owner, feedback }: FeedbackCardProps) { href={conversationUrl ?? ""} icon={ExternalLinkIcon} disabled={!conversationUrl} + tooltip="View conversation" target="_blank" />
{feedback.content && ( -
-
+
+
{feedback.content}
From aad9a4e314894063473f36f1e3d4863e8fe0edd2 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 11:23:54 +0100 Subject: [PATCH 20/47] feat(tracker): improve debounce setup (#9918) Co-authored-by: Henry Fontanier --- front/temporal/tracker/workflows.ts | 31 ++++++++++++++--------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/front/temporal/tracker/workflows.ts b/front/temporal/tracker/workflows.ts index 68538eec4733..5feaf6767780 100644 --- a/front/temporal/tracker/workflows.ts +++ b/front/temporal/tracker/workflows.ts @@ -39,10 +39,10 @@ export async function trackersGenerationWorkflow( documentHash: string, dataSourceConnectorProvider: ConnectorProvider | null ) { - let signaled = true; + let lastUpsertAt = Date.now(); setHandler(newUpsertSignal, () => { - signaled = true; + lastUpsertAt = Date.now(); }); const shouldRun = await shouldRunTrackersActivity( @@ -57,22 +57,21 @@ export async function trackersGenerationWorkflow( const debounceMs = await getDebounceMsActivity(dataSourceConnectorProvider); - while (signaled) { - signaled = false; - await sleep(debounceMs); - - if (signaled) { - continue; - } + function getSleepTime() { + return Math.max(0, lastUpsertAt + debounceMs - Date.now()); + } - await trackersGenerationActivity( - workspaceId, - dataSourceId, - documentId, - documentHash, - dataSourceConnectorProvider - ); + while (getSleepTime() > 0) { + await sleep(getSleepTime()); } + + await trackersGenerationActivity( + workspaceId, + dataSourceId, + documentId, + documentHash, + dataSourceConnectorProvider + ); } /** From d7507da204e42a9c5a6d50949508c26b58bd294b Mon Sep 17 00:00:00 2001 From: Thomas Draier Date: Mon, 13 Jan 2025 11:32:51 +0100 Subject: [PATCH 21/47] [front] Fix usage update (#9919) --- front/lib/api/assistant/agent_usage.ts | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/front/lib/api/assistant/agent_usage.ts b/front/lib/api/assistant/agent_usage.ts index 974c88930b9a..b15903801859 100644 --- a/front/lib/api/assistant/agent_usage.ts +++ b/front/lib/api/assistant/agent_usage.ts @@ -285,7 +285,24 @@ export async function signalAgentUsage({ if (agentMessageCountTTL !== TTL_KEY_NOT_EXIST) { // We only want to increment if the counts have already been computed - await redis.hIncrBy(agentMessageCountKey, agentConfigurationId, 1); + const usage = await redis.hGet(agentMessageCountKey, agentConfigurationId); + if (usage) { + const value = JSON.parse(usage); + const newValue = + typeof value === "object" + ? { ...value, messageCount: value.messageCount + 1 } + : { + messageCount: value + 1, + conversationCount: 0, + userCount: 0, + }; + + await redis.hSet( + agentMessageCountKey, + agentConfigurationId, + JSON.stringify(newValue) + ); + } } } From 81bd640add37e4aa7e05c4eb2f73754df6980305 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 13:32:29 +0100 Subject: [PATCH 22/47] enh(tracker): skip workflow for slack non-threaded messages (#9920) Co-authored-by: Henry Fontanier --- front/temporal/tracker/activities.ts | 25 ++++++++++++++++++++----- front/temporal/tracker/workflows.ts | 7 ++++--- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/front/temporal/tracker/activities.ts b/front/temporal/tracker/activities.ts index 409981cdcabd..35749331c931 100644 --- a/front/temporal/tracker/activities.ts +++ b/front/temporal/tracker/activities.ts @@ -427,11 +427,26 @@ export async function trackersGenerationActivity( } } -export async function shouldRunTrackersActivity( - workspaceId: string, - dataSourceId: string, - documentId: string -): Promise { +export async function shouldRunTrackersActivity({ + workspaceId, + dataSourceId, + documentId, + dataSourceConnectorProvider, +}: { + workspaceId: string; + dataSourceId: string; + documentId: string; + dataSourceConnectorProvider: ConnectorProvider | null; +}): Promise { + if ( + dataSourceConnectorProvider === "slack" && + !documentId.includes("-thread-") + ) { + // Special case for Slack -- we only run trackers for threads (and not for "non-threaded messages" + // otherwise we end up running the tracker twice a a thread is initially a non-threaded message. + return false; + } + const auth = await Authenticator.internalBuilderForWorkspace(workspaceId); const dataSource = await DataSourceResource.fetchById(auth, dataSourceId); diff --git a/front/temporal/tracker/workflows.ts b/front/temporal/tracker/workflows.ts index 5feaf6767780..998c315d29bf 100644 --- a/front/temporal/tracker/workflows.ts +++ b/front/temporal/tracker/workflows.ts @@ -45,11 +45,12 @@ export async function trackersGenerationWorkflow( lastUpsertAt = Date.now(); }); - const shouldRun = await shouldRunTrackersActivity( + const shouldRun = await shouldRunTrackersActivity({ workspaceId, dataSourceId, - documentId - ); + documentId, + dataSourceConnectorProvider, + }); if (!shouldRun) { return; From 9b27ccaf4cf2cf84a57ba005436e608b714ff880 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 13:33:44 +0100 Subject: [PATCH 23/47] feat: add support for __dust_conversation_history magic input for dust apps (#9906) Co-authored-by: Henry Fontanier --- .../dust_app_run/DustAppRunActionDetails.tsx | 15 +++++---- front/lib/api/assistant/actions/constants.ts | 3 ++ .../lib/api/assistant/actions/dust_app_run.ts | 33 +++++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) diff --git a/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx b/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx index 37075dbebfbc..73f0d240d38c 100644 --- a/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx +++ b/front/components/actions/dust_app_run/DustAppRunActionDetails.tsx @@ -11,6 +11,7 @@ import { useMemo } from "react"; import { ActionDetailsWrapper } from "@app/components/actions/ActionDetailsWrapper"; import type { ActionDetailsComponentBaseProps } from "@app/components/actions/types"; +import { DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY } from "@app/lib/api/assistant/actions/constants"; export function DustAppRunActionDetails({ action, @@ -51,12 +52,14 @@ function DustAppRunParamsDetails({ action }: { action: DustAppRunActionType }) { return (
- {Object.entries(params).map(([k, v], idx) => ( -

- {capitalize(k)}: - {` ${v}`} -

- ))} + {Object.entries(params) + .filter(([k]) => k !== DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY) + .map(([k, v], idx) => ( +

+ {capitalize(k)}: + {` ${v}`} +

+ ))}
); } diff --git a/front/lib/api/assistant/actions/constants.ts b/front/lib/api/assistant/actions/constants.ts index f038fb8f2dd9..d74101224942 100644 --- a/front/lib/api/assistant/actions/constants.ts +++ b/front/lib/api/assistant/actions/constants.ts @@ -29,3 +29,6 @@ export const DEFAULT_CONVERSATION_QUERY_TABLES_ACTION_DATA_DESCRIPTION = `The ta export const DEFAULT_CONVERSATION_SEARCH_ACTION_NAME = "search_conversation_files"; export const DEFAULT_CONVERSATION_SEARCH_ACTION_DATA_DESCRIPTION = `Search within the 'searchable' conversation files as returned by \`${DEFAULT_CONVERSATION_LIST_FILES_ACTION_NAME}\``; + +export const DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY = + "__dust_conversation_history"; diff --git a/front/lib/api/assistant/actions/dust_app_run.ts b/front/lib/api/assistant/actions/dust_app_run.ts index b0fbb273fbd5..59002c0ca7d4 100644 --- a/front/lib/api/assistant/actions/dust_app_run.ts +++ b/front/lib/api/assistant/actions/dust_app_run.ts @@ -17,6 +17,7 @@ import type { Result } from "@dust-tt/types"; import { BaseAction, getHeaderFromGroupIds } from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; +import { DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY } from "@app/lib/api/assistant/actions/constants"; import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; import config from "@app/lib/api/config"; @@ -154,6 +155,13 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration if (datasetName) { // We have a dataset name we need to find associated schema. schema = await getDatasetSchema(auth, app, datasetName); + // remove from the schema the magic input key + if (schema) { + schema = schema.filter( + (s) => s.key !== DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY + ); + } + if (!schema) { return new Err( new Error( @@ -257,6 +265,25 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration } } + // Fetch the dataset schema again to check whether the magic input key is present. + const appSpec = JSON.parse( + app.savedSpecification || "[]" + ) as SpecificationType; + const inputSpec = appSpec.find((b) => b.type === "input"); + const inputConfig = inputSpec ? appConfig[inputSpec.name] : null; + const datasetName: string | null = inputConfig ? inputConfig.dataset : null; + + let schema: DatasetSchema | null = null; + if (datasetName) { + schema = await getDatasetSchema(auth, app, datasetName); + } + let shouldIncludeConversationHistory = false; + if ( + schema?.find((s) => s.key === DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY) + ) { + shouldIncludeConversationHistory = true; + } + // Create the AgentDustAppRunAction object in the database and yield an event for the generation // of the params. We store the action here as the params have been generated, if an error occurs // later on, the action won't have an output but the error will be stored on the parent agent @@ -313,6 +340,12 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration // As we run the app (using a system API key here), we do force using the workspace credentials so // that the app executes in the exact same conditions in which they were developed. + if (shouldIncludeConversationHistory) { + params[DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY] = JSON.stringify( + conversation.content + ); + } + const runRes = await api.runAppStreamed( { workspaceId: actionConfiguration.appWorkspaceId, From 460b321a1852b0c39a2334f6275332e06ec18f27 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 13:34:10 +0100 Subject: [PATCH 24/47] fix: bump tracker queue (#9925) Co-authored-by: Henry Fontanier --- front/temporal/tracker/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/front/temporal/tracker/config.ts b/front/temporal/tracker/config.ts index 0aab30c95634..50a9db704e3e 100644 --- a/front/temporal/tracker/config.ts +++ b/front/temporal/tracker/config.ts @@ -1,4 +1,4 @@ -const RUN_QUEUE_VERSION = 2; +const RUN_QUEUE_VERSION = 3; export const RUN_QUEUE_NAME = `document-tracker-queue-v${RUN_QUEUE_VERSION}`; const TRACKER_NOTIFICATION_QUEUE_VERSION = 1; From ae80b5c5b013610dd4eb44c652c0502f40655d80 Mon Sep 17 00:00:00 2001 From: thib-martin <168569391+thib-martin@users.noreply.github.com> Date: Mon, 13 Jan 2025 13:34:35 +0100 Subject: [PATCH 25/47] fixing contact form (#9913) * fixing contact form * removing unnecessary restrictions * linting * fixing default export * fixing default function --- front/components/home/HubSpotForm.tsx | 25 ++++++++++++ front/next.config.js | 6 +-- front/pages/home/contact.tsx | 56 +++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 front/components/home/HubSpotForm.tsx create mode 100644 front/pages/home/contact.tsx diff --git a/front/components/home/HubSpotForm.tsx b/front/components/home/HubSpotForm.tsx new file mode 100644 index 000000000000..048f18bc03cc --- /dev/null +++ b/front/components/home/HubSpotForm.tsx @@ -0,0 +1,25 @@ +import { useEffect } from "react"; + +export default function HubSpotForm() { + useEffect(() => { + // Add script dynamically + const script = document.createElement("script"); + script.src = "https://js-eu1.hsforms.net/forms/embed/144442587.js"; + script.defer = true; + document.body.appendChild(script); + + // Cleanup on unmount + return () => { + document.body.removeChild(script); + }; + }, []); + + return ( +
+ ); +} diff --git a/front/next.config.js b/front/next.config.js index 2d31949711bf..b8723723b7be 100644 --- a/front/next.config.js +++ b/front/next.config.js @@ -2,11 +2,11 @@ const path = require("path"); const CONTENT_SECURITY_POLICIES = [ "default-src 'none';", - `script-src 'self' 'unsafe-inline' 'unsafe-eval' *.googletagmanager.com *.google-analytics.com;`, + `script-src 'self' 'unsafe-inline' 'unsafe-eval' *.googletagmanager.com *.google-analytics.com *.hsforms.net;`, `style-src 'self' 'unsafe-inline' *.typekit.net;`, `img-src 'self' data: https:;`, - `connect-src 'self' *.google-analytics.com cdn.jsdelivr.net;`, - `frame-src 'self' *.wistia.net viz.dust.tt;`, + `connect-src 'self' *.google-analytics.com cdn.jsdelivr.net *.hsforms.com;`, + `frame-src 'self' *.wistia.net viz.dust.tt *.hsforms.net;`, `font-src 'self' data: *.typekit.net;`, `object-src 'none';`, `form-action 'self';`, diff --git a/front/pages/home/contact.tsx b/front/pages/home/contact.tsx new file mode 100644 index 000000000000..5f3f40695d8a --- /dev/null +++ b/front/pages/home/contact.tsx @@ -0,0 +1,56 @@ +import dynamic from "next/dynamic"; +import type { ReactElement } from "react"; + +import { HeaderContentBlock } from "@app/components/home/ContentBlocks"; +import type { LandingLayoutProps } from "@app/components/home/LandingLayout"; +import LandingLayout from "@app/components/home/LandingLayout"; +import { + getParticleShapeIndexByName, + shapeNames, +} from "@app/components/home/Particles"; +import TrustedBy from "@app/components/home/TrustedBy"; +// import HubSpotForm from "@app/components/home/HubSpotForm"; + +const HubSpotForm = dynamic( + () => import("@app/components/home/HubSpotForm").then((mod) => mod.default) + // { ssr: false } +); + +export async function getServerSideProps() { + return { + props: { + shape: getParticleShapeIndexByName(shapeNames.bigSphere), + }, + }; +} + +export default function Contact() { + return ( +
+ + To prepare for our demo call, please share a bit about yourself and + the challenges you're hoping to address with Dust. + + } + /> +
+
+
+ +
+
+
+ +
+ ); +} + +Contact.getLayout = (page: ReactElement, pageProps: LandingLayoutProps) => { + return {page}; +}; From 0f8f3f6ea59d9b9b8617979554971bf5a2329008 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daphn=C3=A9=20Popin?= Date: Mon, 13 Jan 2025 14:46:39 +0100 Subject: [PATCH 26/47] Assistant Detail modal displays error messag if you don't have access (#9927) --- .../components/assistant/AssistantDetails.tsx | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/front/components/assistant/AssistantDetails.tsx b/front/components/assistant/AssistantDetails.tsx index d77e3d55d6a8..914e0e087dc2 100644 --- a/front/components/assistant/AssistantDetails.tsx +++ b/front/components/assistant/AssistantDetails.tsx @@ -13,6 +13,7 @@ import { HandThumbDownIcon, HandThumbUpIcon, InformationCircleIcon, + LockIcon, Page, Sheet, SheetContainer, @@ -267,11 +268,14 @@ export function AssistantDetails({ }: AssistantDetailsProps) { const [isUpdatingScope, setIsUpdatingScope] = useState(false); const [selectedTab, setSelectedTab] = useState("info"); - const { agentConfiguration, isAgentConfigurationValidating } = - useAgentConfiguration({ - workspaceId: owner.sId, - agentConfigurationId: assistantId, - }); + const { + agentConfiguration, + isAgentConfigurationValidating, + isAgentConfigurationError, + } = useAgentConfiguration({ + workspaceId: owner.sId, + agentConfigurationId: assistantId, + }); const doUpdateScope = useUpdateAgentScope({ owner, @@ -380,6 +384,18 @@ export function AssistantDetails({ )}
)} + {isAgentConfigurationError?.error.type === + "agent_configuration_not_found" && ( + + This is a private assistant that can't be shared with other + workspace members. + + )} From 7679991ebd8b58b61a3df152a4a3846a0d5ad398 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 14:59:31 +0100 Subject: [PATCH 27/47] fix(magic dust app input): render conversation before passing to dust app (#9929) Co-authored-by: Henry Fontanier --- .../lib/api/assistant/actions/dust_app_run.ts | 56 ++++++++++++++++++- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/front/lib/api/assistant/actions/dust_app_run.ts b/front/lib/api/assistant/actions/dust_app_run.ts index 59002c0ca7d4..5b77b0971d16 100644 --- a/front/lib/api/assistant/actions/dust_app_run.ts +++ b/front/lib/api/assistant/actions/dust_app_run.ts @@ -14,12 +14,17 @@ import type { AgentActionSpecification } from "@dust-tt/types"; import type { SpecificationType } from "@dust-tt/types"; import type { DatasetSchema } from "@dust-tt/types"; import type { Result } from "@dust-tt/types"; -import { BaseAction, getHeaderFromGroupIds } from "@dust-tt/types"; +import { + BaseAction, + getHeaderFromGroupIds, + SUPPORTED_MODEL_CONFIGS, +} from "@dust-tt/types"; import { Err, Ok } from "@dust-tt/types"; import { DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY } from "@app/lib/api/assistant/actions/constants"; import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; +import { renderConversationForModel } from "@app/lib/api/assistant/generation"; import config from "@app/lib/api/config"; import { getDatasetSchema } from "@app/lib/api/datasets"; import type { Authenticator } from "@app/lib/auth"; @@ -341,9 +346,54 @@ export class DustAppRunConfigurationServerRunner extends BaseActionConfiguration // As we run the app (using a system API key here), we do force using the workspace credentials so // that the app executes in the exact same conditions in which they were developed. if (shouldIncludeConversationHistory) { - params[DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY] = JSON.stringify( - conversation.content + const model = SUPPORTED_MODEL_CONFIGS.find( + (m) => + m.modelId === agentConfiguration.model.modelId && + m.providerId === agentConfiguration.model.providerId ); + if (!model) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Model not found: ${agentConfiguration.model.modelId}`, + }, + }; + return; + } + const MIN_GENERATION_TOKENS = 2048; + const allowedTokenCount = model.contextSize - MIN_GENERATION_TOKENS; + const prompt = ""; + + const convoRes = await renderConversationForModel(auth, { + conversation, + model, + prompt, + allowedTokenCount, + excludeImages: true, + }); + if (convoRes.isErr()) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Error rendering conversation for model: ${convoRes.error.message}`, + }, + }; + return; + } + + const renderedConvo = convoRes.value; + const messages = renderedConvo.modelConversation.messages; + + params[DUST_CONVERSATION_HISTORY_MAGIC_INPUT_KEY] = + JSON.stringify(messages); } const runRes = await api.runAppStreamed( From ec143f6a5ea4e12340219f0ce28030d7baa0fe42 Mon Sep 17 00:00:00 2001 From: Philippe Rolet Date: Mon, 13 Jan 2025 15:53:54 +0100 Subject: [PATCH 28/47] [ElasticSearch] Script to run es commands (in particular migrations) (#9931) Description --- Allows running a command stored in a file on the elasticsearch cluster corresponding to env vars. Added mainly to allow migrations, see [ES runbook](https://www.notion.so/dust-tt/Runbook-Search-infrastructure-12d28599d941805ea09dc14fde7592ad?pvs=4#17728599d941803c982af90502ce4ca5) Check region is correct with a prompt. Risks --- na Deploy --- core --- .../elasticsearch_run_command_from_file.sh | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 core/admin/elasticsearch_run_command_from_file.sh diff --git a/core/admin/elasticsearch_run_command_from_file.sh b/core/admin/elasticsearch_run_command_from_file.sh new file mode 100755 index 000000000000..695ea2a4bfff --- /dev/null +++ b/core/admin/elasticsearch_run_command_from_file.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Runs a single command on the Elasticsearch cluster +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +if [ ! -f "$1" ]; then + echo "Error: File $1 not found" + exit 1 +fi + +# Extract method and path from first line +read -r ES_METHOD ES_PATH < "$1" +# Get JSON body (skip first line) +ES_BODY=$(sed '1d' "$1") + +# Validate JSON +if ! echo "$ES_BODY" | jq . >/dev/null 2>&1; then + echo "Error: Invalid JSON body" + exit 1 +fi + +read -p "Run command from file ${1} in region ${DUST_REGION}? [y/N] " response +if [[ ! $response =~ ^[Yy]$ ]]; then + echo "Operation cancelled" + exit 0 +fi + +curl -X "$ES_METHOD" "${ELASTICSEARCH_URL}/${ES_PATH}" \ + -H "Content-Type: application/json" \ + -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -d "$ES_BODY" \ No newline at end of file From cdaa817f60a98ea6f69b2dc3796e3d7e0c2728f9 Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:00:39 +0100 Subject: [PATCH 29/47] Revert "Log on parents that refer to missing `data_source_node` (#9808)" (#9932) This reverts commit 4fd2ddc25e3ba7f94078b0a87b9f22f997231851. --- core/src/stores/postgres.rs | 58 ------------------------------------- 1 file changed, 58 deletions(-) diff --git a/core/src/stores/postgres.rs b/core/src/stores/postgres.rs index ad85ed938dee..1f0c5d9acd66 100644 --- a/core/src/stores/postgres.rs +++ b/core/src/stores/postgres.rs @@ -11,7 +11,6 @@ use std::hash::Hasher; use std::str::FromStr; use tokio_postgres::types::ToSql; use tokio_postgres::{NoTls, Transaction}; -use tracing::info; use crate::data_sources::data_source::DocumentStatus; use crate::data_sources::node::{Node, NodeType}; @@ -155,27 +154,6 @@ impl PostgresStore { row_id: i64, tx: &Transaction<'_>, ) -> Result<()> { - // Check that all parents exist in data_sources_nodes. - let stmt_check = tx - .prepare( - "SELECT COUNT(*) FROM data_sources_nodes - WHERE data_source = $1 AND node_id = ANY($2)", - ) - .await?; - let count: i64 = tx - .query_one(&stmt_check, &[&data_source_row_id, &upsert_params.parents]) - .await? - .get(0); - if count != upsert_params.parents.len() as i64 { - info!( - data_source_id = data_source_row_id, - node_id = upsert_params.node_id, - parents = ?upsert_params.parents, - operation = "upsert_node", - "[KWSEARCH] invariant_parents_missing_in_nodes" - ); - } - let created = utils::now(); let (document_row_id, table_row_id, folder_row_id) = match upsert_params.node_type { @@ -1473,24 +1451,6 @@ impl Store for PostgresStore { let tx = c.transaction().await?; - // Check that all parents exist in data_sources_nodes. - let count: u64 = tx - .execute( - "SELECT COUNT(*) FROM data_sources_nodes - WHERE data_source = $1 AND node_id = ANY($2)", - &[&data_source_row_id, &parents], - ) - .await?; - if count != parents.len() as u64 { - info!( - data_source_id = data_source_row_id, - node_id = document_id, - parents = ?parents, - operation = "update_document_parents", - "[KWSEARCH] invariant_parents_missing_in_nodes" - ); - } - // Update parents on nodes table. tx.execute( "UPDATE data_sources_nodes SET parents = $1 \ @@ -2817,24 +2777,6 @@ impl Store for PostgresStore { let tx = c.transaction().await?; - // Check that all parents exist in data_sources_nodes. - let count: u64 = tx - .execute( - "SELECT COUNT(*) FROM data_sources_nodes - WHERE data_source = $1 AND node_id = ANY($2)", - &[&data_source_row_id, &parents], - ) - .await?; - if count != parents.len() as u64 { - info!( - data_source_id = data_source_row_id, - node_id = table_id, - parents = ?parents, - operation = "update_table_parents", - "[KWSEARCH] invariant_parents_missing_in_nodes" - ); - } - // Update parents on nodes table. let stmt = tx .prepare( From 29f3f4e0711b76cc0d98f8df2c4ffe3de68215e4 Mon Sep 17 00:00:00 2001 From: Philippe Rolet Date: Mon, 13 Jan 2025 16:03:48 +0100 Subject: [PATCH 30/47] [Nodes Core] Add source_url to Node (#9883) * adding source url * migration * spolu coms * es migration * update mapping --- core/bin/core_api.rs | 4 +++ core/src/data_sources/data_source.rs | 1 + core/src/data_sources/folder.rs | 7 ++++ core/src/data_sources/node.rs | 4 +++ core/src/databases/table.rs | 8 +++++ .../data_sources_nodes_2.mappings.json | 4 +++ .../migrations/20250113_add_source_url.http | 9 ++++++ .../20250109_nodes_add_source_url.sql | 1 + core/src/stores/postgres.rs | 32 ++++++++++++++----- core/src/stores/store.rs | 5 ++- .../data_source/DocumentUploadOrEditModal.tsx | 2 +- 11 files changed, 67 insertions(+), 10 deletions(-) create mode 100644 core/src/search_stores/migrations/20250113_add_source_url.http create mode 100644 core/src/stores/migrations/20250109_nodes_add_source_url.sql diff --git a/core/bin/core_api.rs b/core/bin/core_api.rs index 0732ae9fc7c3..ae5821916a02 100644 --- a/core/bin/core_api.rs +++ b/core/bin/core_api.rs @@ -2185,6 +2185,7 @@ struct DatabasesTablesUpsertPayload { tags: Vec, parent_id: Option, parents: Vec, + source_url: Option, // Remote DB specifics remote_database_table_id: Option, @@ -2257,6 +2258,7 @@ async fn tables_upsert( timestamp: payload.timestamp.unwrap_or(utils::now()), tags: payload.tags, parents: payload.parents, + source_url: payload.source_url, remote_database_table_id: payload.remote_database_table_id, remote_database_secret_id: payload.remote_database_secret_id, title: payload.title, @@ -2924,6 +2926,7 @@ struct FoldersUpsertPayload { parents: Vec, title: String, mime_type: String, + source_url: Option, } async fn folders_upsert( @@ -2987,6 +2990,7 @@ async fn folders_upsert( parents: payload.parents, title: payload.title, mime_type: payload.mime_type, + source_url: payload.source_url, }, ) .await diff --git a/core/src/data_sources/data_source.rs b/core/src/data_sources/data_source.rs index be0a79e8890a..affd9f9fbc68 100644 --- a/core/src/data_sources/data_source.rs +++ b/core/src/data_sources/data_source.rs @@ -231,6 +231,7 @@ impl From for Node { &document.mime_type, document.parent_id, document.parents.clone(), + document.source_url, ) } } diff --git a/core/src/data_sources/folder.rs b/core/src/data_sources/folder.rs index 027a3e1ec0a4..6ce4c0d8a3df 100644 --- a/core/src/data_sources/folder.rs +++ b/core/src/data_sources/folder.rs @@ -12,6 +12,7 @@ pub struct Folder { parent_id: Option, parents: Vec, mime_type: String, + source_url: Option, } impl Folder { @@ -24,6 +25,7 @@ impl Folder { parent_id: Option, parents: Vec, mime_type: String, + source_url: Option, ) -> Self { Folder { data_source_id, @@ -34,6 +36,7 @@ impl Folder { parent_id, parents, mime_type, + source_url, } } @@ -58,6 +61,9 @@ impl Folder { pub fn parents(&self) -> &Vec { &self.parents } + pub fn source_url(&self) -> &Option { + &self.source_url + } pub fn mime_type(&self) -> &str { &self.mime_type } @@ -75,6 +81,7 @@ impl From for Node { &folder.mime_type, folder.parent_id, folder.parents, + folder.source_url, ) } } diff --git a/core/src/data_sources/node.rs b/core/src/data_sources/node.rs index 177394d48c28..b45ba27acc1a 100644 --- a/core/src/data_sources/node.rs +++ b/core/src/data_sources/node.rs @@ -31,6 +31,7 @@ pub struct Node { pub mime_type: String, pub parent_id: Option, pub parents: Vec, + pub source_url: Option, } impl Node { @@ -44,6 +45,7 @@ impl Node { mime_type: &str, parent_id: Option, parents: Vec, + source_url: Option, ) -> Self { Node { data_source_id: data_source_id.to_string(), @@ -55,6 +57,7 @@ impl Node { mime_type: mime_type.to_string(), parent_id: parent_id.clone(), parents, + source_url, } } @@ -94,6 +97,7 @@ impl Node { self.parent_id, self.parents, self.mime_type, + self.source_url, ) } diff --git a/core/src/databases/table.rs b/core/src/databases/table.rs index 450164244b3b..607d6b0d9a62 100644 --- a/core/src/databases/table.rs +++ b/core/src/databases/table.rs @@ -64,6 +64,7 @@ pub struct Table { mime_type: String, parent_id: Option, parents: Vec, + source_url: Option, schema: Option, schema_stale_at: Option, @@ -87,6 +88,7 @@ impl Table { tags: Vec, parent_id: Option, parents: Vec, + source_url: Option, schema: Option, schema_stale_at: Option, remote_database_table_id: Option, @@ -106,6 +108,7 @@ impl Table { mime_type, parent_id, parents, + source_url, schema, schema_stale_at, remote_database_table_id, @@ -137,6 +140,9 @@ impl Table { pub fn parents(&self) -> &Vec { &self.parents } + pub fn source_url(&self) -> &Option { + &self.source_url + } pub fn name(&self) -> &str { &self.name } @@ -251,6 +257,7 @@ impl From for Node { &table.mime_type, table.parents.get(1).cloned(), table.parents, + table.source_url, ) } } @@ -599,6 +606,7 @@ mod tests { vec![], None, vec![], + None, Some(schema), None, None, diff --git a/core/src/search_stores/indices/data_sources_nodes_2.mappings.json b/core/src/search_stores/indices/data_sources_nodes_2.mappings.json index a7ff9c56e81d..816ccbeecfab 100644 --- a/core/src/search_stores/indices/data_sources_nodes_2.mappings.json +++ b/core/src/search_stores/indices/data_sources_nodes_2.mappings.json @@ -34,6 +34,10 @@ }, "mime_type": { "type": "keyword" + }, + "source_url": { + "type": "keyword", + "index": false } } } diff --git a/core/src/search_stores/migrations/20250113_add_source_url.http b/core/src/search_stores/migrations/20250113_add_source_url.http new file mode 100644 index 000000000000..dbe94e7a37fe --- /dev/null +++ b/core/src/search_stores/migrations/20250113_add_source_url.http @@ -0,0 +1,9 @@ +PUT core.data_sources_nodes/_mapping +{ + "properties": { + "source_url": { + "type": "keyword", + "index": false + } + } +} diff --git a/core/src/stores/migrations/20250109_nodes_add_source_url.sql b/core/src/stores/migrations/20250109_nodes_add_source_url.sql new file mode 100644 index 000000000000..da4c286a3bf6 --- /dev/null +++ b/core/src/stores/migrations/20250109_nodes_add_source_url.sql @@ -0,0 +1 @@ +ALTER TABLE data_sources_nodes ADD COLUMN source_url TEXT; diff --git a/core/src/stores/postgres.rs b/core/src/stores/postgres.rs index 1f0c5d9acd66..7b2e56fa58ad 100644 --- a/core/src/stores/postgres.rs +++ b/core/src/stores/postgres.rs @@ -53,6 +53,7 @@ pub struct UpsertNode<'a> { pub title: &'a str, pub mime_type: &'a str, pub parents: &'a Vec, + pub source_url: &'a Option, } impl PostgresStore { @@ -1917,6 +1918,7 @@ impl Store for PostgresStore { title: &document.title, mime_type: &document.mime_type, parents: &document.parents, + source_url: &document.source_url, }, data_source_row_id, document_row_id, @@ -2676,6 +2678,7 @@ impl Store for PostgresStore { upsert_params.tags, upsert_params.parents.get(1).cloned(), upsert_params.parents, + upsert_params.source_url, parsed_schema, table_schema_stale_at.map(|t| t as u64), upsert_params.remote_database_table_id, @@ -2690,6 +2693,7 @@ impl Store for PostgresStore { title: table.title(), mime_type: table.mime_type(), parents: table.parents(), + source_url: table.source_url(), }, data_source_row_id, table_row_id, @@ -2861,7 +2865,7 @@ impl Store for PostgresStore { let stmt = c .prepare( "SELECT t.created, t.table_id, t.name, t.description, \ - t.timestamp, t.tags_array, dsn.parents, \ + t.timestamp, t.tags_array, dsn.parents, dsn.source_url, \ t.schema, t.schema_stale_at, \ t.remote_database_table_id, t.remote_database_secret_id, \ dsn.title, dsn.mime_type \ @@ -2880,6 +2884,7 @@ impl Store for PostgresStore { Vec, Vec, Option, + Option, Option, Option, Option, @@ -2901,6 +2906,7 @@ impl Store for PostgresStore { r[0].get(10), r[0].get(11), r[0].get(12), + r[0].get(13), )), _ => unreachable!(), }; @@ -2915,6 +2921,7 @@ impl Store for PostgresStore { timestamp, tags, parents, + source_url, schema, schema_stale_at, remote_database_table_id, @@ -2947,6 +2954,7 @@ impl Store for PostgresStore { tags, parents.get(1).cloned(), parents, + source_url, parsed_schema, schema_stale_at.map(|t| t as u64), remote_database_table_id, @@ -3021,7 +3029,7 @@ impl Store for PostgresStore { t.timestamp, t.tags_array, dsn.parents, \ t.schema, t.schema_stale_at, \ t.remote_database_table_id, t.remote_database_secret_id, \ - dsn.title, dsn.mime_type \ + dsn.title, dsn.mime_type, dsn.source_url \ FROM tables t INNER JOIN data_sources_nodes dsn ON dsn.table=t.id \ WHERE {} ORDER BY t.timestamp DESC", where_clauses.join(" AND "), @@ -3063,7 +3071,7 @@ impl Store for PostgresStore { let remote_database_secret_id: Option = r.get(10); let title: String = r.get(11); let mime_type: String = r.get(12); - + let source_url: Option = r.get(13); let parsed_schema: Option = match schema { None => None, Some(schema) => { @@ -3089,6 +3097,7 @@ impl Store for PostgresStore { tags, parents.get(1).cloned(), parents, + source_url, parsed_schema, schema_stale_at.map(|t| t as u64), remote_database_table_id, @@ -3225,6 +3234,7 @@ impl Store for PostgresStore { upsert_params.parents.get(1).cloned(), upsert_params.parents, upsert_params.mime_type, + upsert_params.source_url, ); self.upsert_data_source_node( @@ -3235,6 +3245,7 @@ impl Store for PostgresStore { title: folder.title(), mime_type: folder.mime_type(), parents: folder.parents(), + source_url: folder.source_url(), }, data_source_row_id, folder_row_id, @@ -3344,7 +3355,7 @@ impl Store for PostgresStore { } let sql = format!( - "SELECT dsn.node_id, dsn.title, dsn.timestamp, dsn.parents, dsn.mime_type \ + "SELECT dsn.node_id, dsn.title, dsn.timestamp, dsn.parents, dsn.mime_type, dsn.source_url \ FROM data_sources_nodes dsn \ WHERE dsn.folder IS NOT NULL AND {} ORDER BY dsn.timestamp DESC", where_clauses.join(" AND "), @@ -3393,7 +3404,7 @@ impl Store for PostgresStore { let timestamp: i64 = r.get(2); let parents: Vec = r.get(3); let mime_type: String = r.get(4); - + let source_url: Option = r.get(5); Ok(Folder::new( data_source_id.clone(), data_source_internal_id.clone(), @@ -3403,6 +3414,7 @@ impl Store for PostgresStore { parents.get(1).cloned(), parents, mime_type, + source_url, )) }) .collect::>>()?; @@ -3475,7 +3487,7 @@ impl Store for PostgresStore { let stmt = c .prepare( - "SELECT timestamp, title, mime_type, parents, node_id, document, \"table\", folder \ + "SELECT timestamp, title, mime_type, parents, node_id, document, \"table\", folder, source_url \ FROM data_sources_nodes \ WHERE data_source = $1 AND node_id = $2 LIMIT 1", ) @@ -3499,6 +3511,7 @@ impl Store for PostgresStore { (None, None, Some(id)) => (NodeType::Folder, id), _ => unreachable!(), }; + let source_url: Option = row[0].get::<_, Option>(8); Ok(Some(( Node::new( &data_source_id, @@ -3510,6 +3523,7 @@ impl Store for PostgresStore { &mime_type, parents.get(1).cloned(), parents, + source_url, ), row_id, ))) @@ -3528,7 +3542,7 @@ impl Store for PostgresStore { let stmt = c .prepare( - "SELECT dsn.timestamp, dsn.title, dsn.mime_type, dsn.parents, dsn.node_id, dsn.document, dsn.\"table\", dsn.folder, ds.data_source_id, ds.internal_id, dsn.id \ + "SELECT dsn.timestamp, dsn.title, dsn.mime_type, dsn.parents, dsn.node_id, dsn.document, dsn.\"table\", dsn.folder, ds.data_source_id, ds.internal_id, dsn.source_url, dsn.id \ FROM data_sources_nodes dsn JOIN data_sources ds ON dsn.data_source = ds.id \ WHERE dsn.id > $1 ORDER BY dsn.id ASC LIMIT $2", ) @@ -3555,7 +3569,8 @@ impl Store for PostgresStore { (None, None, Some(id)) => (NodeType::Folder, id), _ => unreachable!(), }; - let row_id = row.get::<_, i64>(10); + let source_url: Option = row.get::<_, Option>(10); + let row_id = row.get::<_, i64>(11); ( Node::new( &data_source_id, @@ -3567,6 +3582,7 @@ impl Store for PostgresStore { &mime_type, parents.get(1).cloned(), parents, + source_url, ), row_id, element_row_id, diff --git a/core/src/stores/store.rs b/core/src/stores/store.rs index c998fc9551e8..93812ab1eaff 100644 --- a/core/src/stores/store.rs +++ b/core/src/stores/store.rs @@ -65,6 +65,7 @@ pub struct TableUpsertParams { pub timestamp: u64, pub tags: Vec, pub parents: Vec, + pub source_url: Option, pub remote_database_table_id: Option, pub remote_database_secret_id: Option, pub title: String, @@ -77,6 +78,7 @@ pub struct FolderUpsertParams { pub title: String, pub parents: Vec, pub mime_type: String, + pub source_url: Option, } #[async_trait] @@ -597,8 +599,9 @@ pub const POSTGRES_TABLES: [&'static str; 16] = [ title TEXT NOT NULL, mime_type TEXT NOT NULL, parents TEXT[] NOT NULL, + source_url TEXT, document BIGINT, - \"table\" BIGINT, + \"table\" BIGINT, folder BIGINT, FOREIGN KEY(data_source) REFERENCES data_sources(id), FOREIGN KEY(document) REFERENCES data_sources_documents(id), diff --git a/front/components/data_source/DocumentUploadOrEditModal.tsx b/front/components/data_source/DocumentUploadOrEditModal.tsx index e665a712aa52..d8aa7157386e 100644 --- a/front/components/data_source/DocumentUploadOrEditModal.tsx +++ b/front/components/data_source/DocumentUploadOrEditModal.tsx @@ -147,7 +147,7 @@ export const DocumentUploadOrEditModal = ({ const body = { name: initialId ?? document.name, title: initialId ?? document.name, - mime_type: document.mimeType ?? "application/octet-stream", + mime_type: document.mimeType ?? "text/plain", timestamp: null, parent_id: null, parents: [initialId ?? document.name], From d6ddf11fec0bda67b221b93a63df0c64a19bcd0b Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 16:06:28 +0100 Subject: [PATCH 31/47] enh(tracker): limit to 1 tracker per space (#9933) Co-authored-by: Henry Fontanier --- .../pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts b/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts index cfc315345632..ec4d930e5726 100644 --- a/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts +++ b/front/pages/api/w/[wId]/spaces/[spaceId]/trackers/index.ts @@ -12,6 +12,7 @@ import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrapper import { withResourceFetchingFromRoute } from "@app/lib/api/resource_wrappers"; import type { Authenticator } from "@app/lib/auth"; import { getFeatureFlags } from "@app/lib/auth"; +import { PRODUCTION_DUST_WORKSPACE_ID } from "@app/lib/registry"; import type { SpaceResource } from "@app/lib/resources/space_resource"; import { TrackerConfigurationResource } from "@app/lib/resources/tracker_resource"; import { apiError } from "@app/logger/withlogging"; @@ -106,12 +107,15 @@ async function handler( auth, space ); - if (existingTrackers.length >= 3) { + if ( + owner.sId !== PRODUCTION_DUST_WORKSPACE_ID && + existingTrackers.length >= 1 + ) { return apiError(req, res, { status_code: 400, api_error: { type: "invalid_request_error", - message: "You can't have more than 3 trackers in a space.", + message: "You can't have more than 1 tracker in a space.", }, }); } From 4a382ab1d544f76471de4c059b5702aab99abbed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daphn=C3=A9=20Popin?= Date: Mon, 13 Jan 2025 16:13:44 +0100 Subject: [PATCH 32/47] Bump sdk to add optionnal created on conversation schema (#9934) --- sdks/js/package-lock.json | 4 ++-- sdks/js/package.json | 2 +- sdks/js/src/types.ts | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/sdks/js/package-lock.json b/sdks/js/package-lock.json index 042f0a7fbb70..f5af04bb203c 100644 --- a/sdks/js/package-lock.json +++ b/sdks/js/package-lock.json @@ -1,12 +1,12 @@ { "name": "@dust-tt/client", - "version": "1.0.22", + "version": "1.0.23", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@dust-tt/client", - "version": "1.0.22", + "version": "1.0.23", "license": "ISC", "dependencies": { "axios": "^1.7.9", diff --git a/sdks/js/package.json b/sdks/js/package.json index e7a2061b790f..c74c731e563d 100644 --- a/sdks/js/package.json +++ b/sdks/js/package.json @@ -1,6 +1,6 @@ { "name": "@dust-tt/client", - "version": "1.0.22", + "version": "1.0.23", "description": "Client for Dust API", "repository": { "type": "git", diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 4ac46c7f7e39..cdeb639d3268 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -947,6 +947,7 @@ export type ConversationVisibility = z.infer< const ConversationWithoutContentSchema = z.object({ id: ModelIdSchema, created: z.number(), + updated: z.number().optional(), owner: WorkspaceSchema, sId: z.string(), title: z.string().nullable(), From a062c9d25db95ea25fb70acf2f089ce5feb16b80 Mon Sep 17 00:00:00 2001 From: Henry Fontanier Date: Mon, 13 Jan 2025 16:52:52 +0100 Subject: [PATCH 33/47] feat: add gemini 2.0 experimental models (FF) (#9936) Co-authored-by: Henry Fontanier --- front/components/providers/types.ts | 4 ++ .../api/w/[wId]/providers/[pId]/models.ts | 12 +++++- sdks/js/src/types.ts | 3 ++ types/src/front/lib/assistant.ts | 38 +++++++++++++++++++ types/src/shared/feature_flags.ts | 1 + 5 files changed, 56 insertions(+), 2 deletions(-) diff --git a/front/components/providers/types.ts b/front/components/providers/types.ts index 7fb5e4bebe57..72611bbfc253 100644 --- a/front/components/providers/types.ts +++ b/front/components/providers/types.ts @@ -10,6 +10,8 @@ import { CLAUDE_3_5_HAIKU_DEFAULT_MODEL_CONFIG, CLAUDE_3_5_SONNET_DEFAULT_MODEL_CONFIG, DEEPSEEK_CHAT_MODEL_CONFIG, + GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG, GEMINI_FLASH_DEFAULT_MODEL_CONFIG, GEMINI_PRO_DEFAULT_MODEL_CONFIG, GPT_4_TURBO_MODEL_CONFIG, @@ -53,6 +55,8 @@ export const USED_MODEL_CONFIGS: readonly ModelConfig[] = [ MISTRAL_CODESTRAL_MODEL_CONFIG, GEMINI_PRO_DEFAULT_MODEL_CONFIG, GEMINI_FLASH_DEFAULT_MODEL_CONFIG, + GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG, TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_CONFIG, TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_CONFIG, TOGETHERAI_QWEN_32B_PREVIEW_MODEL_CONFIG, diff --git a/front/pages/api/w/[wId]/providers/[pId]/models.ts b/front/pages/api/w/[wId]/providers/[pId]/models.ts index a10f05446bbc..ae20245dcf68 100644 --- a/front/pages/api/w/[wId]/providers/[pId]/models.ts +++ b/front/pages/api/w/[wId]/providers/[pId]/models.ts @@ -1,4 +1,10 @@ import type { WithAPIErrorResponse } from "@dust-tt/types"; +import { + GEMINI_1_5_FLASH_LATEST_MODEL_ID, + GEMINI_1_5_PRO_LATEST_MODEL_ID, + GEMINI_2_FLASH_PREVIEW_MODEL_ID, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID, +} from "@dust-tt/types"; import type { NextApiRequest, NextApiResponse } from "next"; import { withSessionAuthenticationForWorkspace } from "@app/lib/api/auth_wrappers"; @@ -237,8 +243,10 @@ async function handler( case "google_ai_studio": return res.status(200).json({ models: [ - { id: "gemini-1.5-flash-latest" }, - { id: "gemini-1.5-pro-latest" }, + { id: GEMINI_1_5_FLASH_LATEST_MODEL_ID }, + { id: GEMINI_1_5_PRO_LATEST_MODEL_ID }, + { id: GEMINI_2_FLASH_PREVIEW_MODEL_ID }, + { id: GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID }, ], }); diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index cdeb639d3268..c9a9928b6090 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -43,6 +43,8 @@ const ModelLLMIdSchema = FlexibleEnumSchema< | "codestral-latest" | "gemini-1.5-pro-latest" | "gemini-1.5-flash-latest" + | "gemini-2.0-flash-exp" + | "gemini-2.0-flash-thinking-exp-1219" | "meta-llama/Llama-3.3-70B-Instruct-Turbo" | "Qwen/Qwen2.5-Coder-32B-Instruct" | "Qwen/QwQ-32B-Preview" @@ -666,6 +668,7 @@ const WhitelistableFeaturesSchema = FlexibleEnumSchema< | "openai_o1_custom_assistants_feature" | "openai_o1_high_reasoning_custom_assistants_feature" | "deepseek_feature" + | "google_ai_studio_experimental_models_feature" | "snowflake_connector_feature" | "index_private_slack_channel" | "conversations_jit_actions" diff --git a/types/src/front/lib/assistant.ts b/types/src/front/lib/assistant.ts index 6a5bdd1414c4..d5947cd06d75 100644 --- a/types/src/front/lib/assistant.ts +++ b/types/src/front/lib/assistant.ts @@ -114,6 +114,9 @@ export const MISTRAL_CODESTRAL_MODEL_ID = "codestral-latest" as const; export const GEMINI_1_5_PRO_LATEST_MODEL_ID = "gemini-1.5-pro-latest" as const; export const GEMINI_1_5_FLASH_LATEST_MODEL_ID = "gemini-1.5-flash-latest" as const; +export const GEMINI_2_FLASH_PREVIEW_MODEL_ID = "gemini-2.0-flash-exp" as const; +export const GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID = + "gemini-2.0-flash-thinking-exp-1219" as const; export const TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_ID = "meta-llama/Llama-3.3-70B-Instruct-Turbo" as const; export const TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_ID = @@ -145,6 +148,8 @@ export const MODEL_IDS = [ MISTRAL_CODESTRAL_MODEL_ID, GEMINI_1_5_PRO_LATEST_MODEL_ID, GEMINI_1_5_FLASH_LATEST_MODEL_ID, + GEMINI_2_FLASH_PREVIEW_MODEL_ID, + GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID, TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_ID, TOGETHERAI_QWEN_2_5_CODER_32B_INSTRUCT_MODEL_ID, TOGETHERAI_QWEN_32B_PREVIEW_MODEL_ID, @@ -570,6 +575,39 @@ export const GEMINI_FLASH_DEFAULT_MODEL_CONFIG: ModelConfigurationType = { supportsVision: false, }; +export const GEMINI_2_FLASH_PREVIEW_MODEL_CONFIG: ModelConfigurationType = { + providerId: "google_ai_studio", + modelId: GEMINI_2_FLASH_PREVIEW_MODEL_ID, + displayName: "Gemini Flash 2.0", + contextSize: 1_000_000, + recommendedTopK: 64, + recommendedExhaustiveTopK: 128, + largeModel: true, + description: + "Google's lightweight, fast and cost-efficient model (1m context).", + shortDescription: "Google's cost-effective model (preview).", + isLegacy: false, + supportsVision: true, + featureFlag: "google_ai_studio_experimental_models_feature", +}; + +export const GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_CONFIG: ModelConfigurationType = + { + providerId: "google_ai_studio", + modelId: GEMINI_2_FLASH_THINKING_PREVIEW_MODEL_ID, + displayName: "Gemini Flash 2.0 Thinking", + contextSize: 32_000, + recommendedTopK: 64, + recommendedExhaustiveTopK: 128, + largeModel: true, + description: + "Google's lightweight model optimized for reasoning (1m context).", + shortDescription: "Google's reasoning-focused model (preview).", + isLegacy: false, + supportsVision: true, + featureFlag: "google_ai_studio_experimental_models_feature", + }; + export const TOGETHERAI_LLAMA_3_3_70B_INSTRUCT_TURBO_MODEL_CONFIG: ModelConfigurationType = { providerId: "togetherai", diff --git a/types/src/shared/feature_flags.ts b/types/src/shared/feature_flags.ts index 6c9ffb5d82ec..e9c2d47200cf 100644 --- a/types/src/shared/feature_flags.ts +++ b/types/src/shared/feature_flags.ts @@ -12,6 +12,7 @@ export const WHITELISTABLE_FEATURES = [ "openai_o1_custom_assistants_feature", "openai_o1_high_reasoning_custom_assistants_feature", "deepseek_feature", + "google_ai_studio_experimental_models_feature", "index_private_slack_channel", "conversations_jit_actions", "disable_run_logs", From a62526e1e77f23d183e9b573c2fb86ab188a29b0 Mon Sep 17 00:00:00 2001 From: Aubin <60398825+aubin-tchoi@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:56:48 +0100 Subject: [PATCH 34/47] lint (#9940) --- types/src/front/data_source_view.ts | 7 ++++++- types/src/front/lib/connectors_api.ts | 5 ++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/types/src/front/data_source_view.ts b/types/src/front/data_source_view.ts index cf0f01c0cd44..82fc37c116cf 100644 --- a/types/src/front/data_source_view.ts +++ b/types/src/front/data_source_view.ts @@ -1,6 +1,11 @@ import { ModelId } from "../shared/model_id"; import { DataSourceViewCategory } from "./api_handlers/public/spaces"; -import { ConnectorStatusDetails, DataSourceType, DataSourceWithAgentsUsageType, EditedByUser } from "./data_source"; +import { + ConnectorStatusDetails, + DataSourceType, + DataSourceWithAgentsUsageType, + EditedByUser, +} from "./data_source"; import { ContentNode } from "./lib/connectors_api"; export interface DataSourceViewType { diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index f31ee8d98ce7..9625e6abc934 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -1,4 +1,7 @@ -import { AdminCommandType, AdminResponseType } from "../../connectors/admin/cli"; +import { + AdminCommandType, + AdminResponseType, +} from "../../connectors/admin/cli"; import { ConnectorsAPIError, isConnectorsAPIError } from "../../connectors/api"; import { UpdateConnectorConfigurationType } from "../../connectors/api_handlers/connector_configuration"; import { ConnectorCreateRequestBody } from "../../connectors/api_handlers/create_connector"; From e1b9428dbefca54aa01696cbe6f1f827e7e5a70b Mon Sep 17 00:00:00 2001 From: Lucas Massemin Date: Mon, 13 Jan 2025 17:04:04 +0100 Subject: [PATCH 35/47] Add file Drag and Drop in data source view (#9874) * Dropzone working at the datasource level * Removed redundant check, better comment wording * reordered imports * clean dropped files after processing to avoid race conditions * Moved setter back up because of freezing * Fixed freeze when uploading files --- .../data_source/MultipleDocumentsUpload.tsx | 45 ++- front/components/misc/DropzoneContainer.tsx | 6 + .../spaces/SpaceDataSourceViewContentList.tsx | 261 +++++++++--------- 3 files changed, 177 insertions(+), 135 deletions(-) diff --git a/front/components/data_source/MultipleDocumentsUpload.tsx b/front/components/data_source/MultipleDocumentsUpload.tsx index b0b0baaec70e..f71760fb5109 100644 --- a/front/components/data_source/MultipleDocumentsUpload.tsx +++ b/front/components/data_source/MultipleDocumentsUpload.tsx @@ -19,6 +19,7 @@ import { import type { ChangeEvent } from "react"; import React, { useCallback, useEffect, useRef, useState } from "react"; +import { useFileDrop } from "@app/components/assistant/conversation/FileUploaderContext"; import { DocumentLimitPopup } from "@app/components/data_source/DocumentLimitPopup"; import type { FileBlob, @@ -74,12 +75,10 @@ export const MultipleDocumentsUpload = ({ completed: number; }>(null); - const handleFileChange = useCallback( - async ( - e: ChangeEvent & { target: { files: File[] } } - ) => { + const uploadFiles = useCallback( + async (files: File[]) => { // Empty file input - if (!e.target.files || e.target.files.length === 0) { + if (files.length === 0) { close(false); return; } @@ -87,21 +86,22 @@ export const MultipleDocumentsUpload = ({ // Open plan popup if limit is reached if ( plan.limits.dataSources.documents.count != -1 && - e.target.files.length + totalNodesCount > - plan.limits.dataSources.documents.count + files.length + totalNodesCount > plan.limits.dataSources.documents.count ) { setIsLimitPopupOpen(true); return; } setIsBulkFilesUploading({ - total: e.target.files.length, + total: files.length, completed: 0, }); // upload Files and get FileBlobs (only keep successful uploads) // Each individual error triggers a notification - const fileBlobs = (await fileUploaderService.handleFileChange(e))?.filter( + const fileBlobs = ( + await fileUploaderService.handleFilesUpload(files) + )?.filter( (fileBlob: FileBlob): fileBlob is FileBlobWithFileId => !!fileBlob.fileId ); @@ -148,6 +148,33 @@ export const MultipleDocumentsUpload = ({ ] ); + // Process dropped files if any. + const { droppedFiles, setDroppedFiles } = useFileDrop(); + useEffect(() => { + const handleDroppedFiles = async () => { + const droppedFilesCopy = [...droppedFiles]; + if (droppedFilesCopy.length > 0) { + // Make sure the files are cleared after processing + setDroppedFiles([]); + await uploadFiles(droppedFilesCopy); + } + }; + void handleDroppedFiles(); + }, [droppedFiles, setDroppedFiles, uploadFiles]); + + // Handle file change from file input. + const handleFileChange = useCallback( + async ( + e: ChangeEvent & { target: { files: File[] } } + ) => { + const selectedFiles = Array.from( + (e?.target as HTMLInputElement).files ?? [] + ); + await uploadFiles(selectedFiles); + }, + [uploadFiles] + ); + const handleFileInputBlur = useCallback(() => { close(false); }, [close]); diff --git a/front/components/misc/DropzoneContainer.tsx b/front/components/misc/DropzoneContainer.tsx index 3552f83bd75f..e00a0737dd3c 100644 --- a/front/components/misc/DropzoneContainer.tsx +++ b/front/components/misc/DropzoneContainer.tsx @@ -7,12 +7,14 @@ interface DropzoneContainerProps { children: React.ReactNode; description: string; title: string; + disabled?: boolean; } export function DropzoneContainer({ children, description, title, + disabled, }: DropzoneContainerProps) { const { setDroppedFiles } = useFileDrop(); @@ -45,6 +47,10 @@ export function DropzoneContainer({ } }; + if (disabled) { + return children; + } + return (
-
+ - {!isEmpty && ( - <> - { - setPagination( - { pageIndex: 0, pageSize: pagination.pageSize }, - "replace" - ); - setDataSourceSearch(s); - }} - /> - - )} - {isEmpty && emptyContent} - {isFolder(dataSourceView.dataSource) && ( - <> - {((viewType === "tables" && hasDocuments) || - (viewType === "documents" && hasTables)) && ( - - -
- {isNodesLoading && ( -
- + )} + {isManaged(dataSourceView.dataSource) && + connector && + !parentId && + space.kind === "system" && ( +
+ {!isNodesLoading && rows.length === 0 && ( +
Connection ready. Select the data to sync.
+ )} + + { + setShowConnectorPermissionsModal(false); + if (save) { + void mutateContentNodes(); + } + }} + readOnly={false} + isAdmin={isAdmin} + onManageButtonClick={() => { + setShowConnectorPermissionsModal(true); + }} + /> +
+ )}
- )} - {rows.length > 0 && ( - + +
+ )} + {rows.length > 0 && ( + + )} + - )} - - + + ); }; From c14a7094c58de5a4768a8ac46b917d3a78e84fa5 Mon Sep 17 00:00:00 2001 From: Lucas Massemin Date: Mon, 13 Jan 2025 17:23:31 +0100 Subject: [PATCH 36/47] Get rid of dustDocumentId (#9921) * removed occurences of dustDocumentId, fixed invisible button, display content for nodes with 'file' type * Removed unsused function * Forgotten dustDocumentId s --- .../src/connectors/confluence/lib/permissions.ts | 2 -- connectors/src/connectors/github/index.ts | 14 -------------- connectors/src/connectors/google_drive/index.ts | 13 +------------ .../src/connectors/google_drive/lib/permissions.ts | 14 +------------- connectors/src/connectors/intercom/index.ts | 5 ----- .../intercom/lib/conversation_permissions.ts | 5 ----- .../intercom/lib/help_center_permissions.ts | 6 ------ .../src/connectors/intercom/lib/permissions.ts | 3 --- .../src/connectors/microsoft/lib/content_nodes.ts | 9 --------- connectors/src/connectors/notion/index.ts | 6 ------ connectors/src/connectors/slack/index.ts | 2 -- .../src/connectors/snowflake/lib/content_nodes.ts | 3 --- connectors/src/connectors/webcrawler/index.ts | 4 ---- .../src/connectors/zendesk/lib/permissions.ts | 4 ---- connectors/src/resources/zendesk_resources.ts | 6 ------ front/components/ContentNodeTree.tsx | 10 +++++----- front/components/app/blocks/Database.tsx | 2 +- .../assistant/details/AssistantActionsSection.tsx | 8 ++++---- .../DataSourceSelectionSection.tsx | 10 +++++----- front/components/spaces/ContentActions.tsx | 2 +- front/components/tables/TablePicker.tsx | 6 +++--- .../trackers/TrackerDataSourceSelectedTree.tsx | 10 +++++----- front/lib/api/data_source_view.ts | 2 -- types/src/front/api_handlers/public/spaces.ts | 1 - types/src/front/lib/connectors_api.ts | 1 - 25 files changed, 26 insertions(+), 122 deletions(-) diff --git a/connectors/src/connectors/confluence/lib/permissions.ts b/connectors/src/connectors/confluence/lib/permissions.ts index 4efe4b8a8c15..48ac475a6873 100644 --- a/connectors/src/connectors/confluence/lib/permissions.ts +++ b/connectors/src/connectors/confluence/lib/permissions.ts @@ -53,7 +53,6 @@ export function createContentNodeFromSpace( sourceUrl: `${baseUrl}/wiki${urlSuffix}`, expandable: isExpandable, permission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -75,7 +74,6 @@ export function createContentNodeFromPage( sourceUrl: `${baseUrl}/wiki${page.externalUrl}`, expandable: isExpandable, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }; } diff --git a/connectors/src/connectors/github/index.ts b/connectors/src/connectors/github/index.ts index ca4a9305e614..19ffd791ab8f 100644 --- a/connectors/src/connectors/github/index.ts +++ b/connectors/src/connectors/github/index.ts @@ -293,7 +293,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, })) ); @@ -364,7 +363,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/issues", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: latestIssue.updatedAt.getTime(), }); } @@ -378,7 +376,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: latestDiscussion.updatedAt.getTime(), }); } @@ -392,7 +389,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(), }); } @@ -434,7 +430,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: directory.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: directory.codeUpdatedAt.getTime(), }); }); @@ -448,7 +443,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: file.sourceUrl, expandable: false, permission: "read", - dustDocumentId: file.documentId, lastUpdatedAt: file.codeUpdatedAt.getTime(), }); }); @@ -610,7 +604,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -629,7 +622,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/issues", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -646,7 +638,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + "/discussions", expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -665,7 +656,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + `/issues/${issueNumber}`, expandable: false, permission: "read", - dustDocumentId: getIssueInternalId(repoId, issueNumber), lastUpdatedAt: issue.updatedAt.getTime(), }); }); @@ -684,7 +674,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: repo.url + `/discussions/${discussionNumber}`, expandable: false, permission: "read", - dustDocumentId: getDiscussionInternalId(repoId, discussionNumber), lastUpdatedAt: discussion.updatedAt.getTime(), }); }); @@ -699,7 +688,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: codeRepo.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: codeRepo.codeUpdatedAt.getTime(), }); }); @@ -714,7 +702,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: directory.sourceUrl, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: directory.codeUpdatedAt.getTime(), }); }); @@ -729,7 +716,6 @@ export class GithubConnectorManager extends BaseConnectorManager { sourceUrl: file.sourceUrl, expandable: false, permission: "read", - dustDocumentId: file.documentId, lastUpdatedAt: file.codeUpdatedAt.getTime(), }); }); diff --git a/connectors/src/connectors/google_drive/index.ts b/connectors/src/connectors/google_drive/index.ts index d4341b8905f3..4987259ccb79 100644 --- a/connectors/src/connectors/google_drive/index.ts +++ b/connectors/src/connectors/google_drive/index.ts @@ -24,10 +24,7 @@ import { } from "@connectors/connectors/google_drive/lib"; import { GOOGLE_DRIVE_SHARED_WITH_ME_VIRTUAL_ID } from "@connectors/connectors/google_drive/lib/consts"; import { getGoogleDriveObject } from "@connectors/connectors/google_drive/lib/google_drive_api"; -import { - getGoogleDriveEntityDocumentId, - getPermissionViewType, -} from "@connectors/connectors/google_drive/lib/permissions"; +import { getPermissionViewType } from "@connectors/connectors/google_drive/lib/permissions"; import { folderHasChildren, getDrives, @@ -324,7 +321,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: null, type, title: f.name || "", - dustDocumentId: getGoogleDriveEntityDocumentId(f), lastUpdatedAt: f.lastUpsertedTs?.getTime() || null, sourceUrl: getSourceUrlForGoogleDriveFiles(f), expandable: await isDriveObjectExpandable({ @@ -350,7 +346,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: getInternalId(s.driveFileId), type: "database" as const, title: s.name || "", - dustDocumentId: null, lastUpdatedAt: s.updatedAt.getTime() || null, sourceUrl: null, expandable: false, @@ -394,7 +389,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { type: "folder" as const, title: driveObject.name, sourceUrl: driveObject.webViewLink || null, - dustDocumentId: null, lastUpdatedAt: driveObject.updatedAtMs || null, expandable: await folderHasChildren( this.connectorId, @@ -420,7 +414,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { preventSelection: true, title: "Shared with me", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -492,7 +485,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { this.connectorId, driveObject.id ), - dustDocumentId: null, lastUpdatedAt: driveObject.updatedAtMs || null, permission: (await GoogleDriveFolders.findOne({ where: { @@ -672,7 +664,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: null, type, title: f.name || "", - dustDocumentId: getGoogleDriveEntityDocumentId(f), lastUpdatedAt: f.lastUpsertedTs?.getTime() || null, sourceUrl, expandable: await isDriveObjectExpandable({ @@ -714,7 +705,6 @@ export class GoogleDriveConnectorManager extends BaseConnectorManager { parentInternalId: getInternalId(s.driveFileId), type: "database", title: s.name || "", - dustDocumentId: null, lastUpdatedAt: s.updatedAt.getTime() || null, sourceUrl: `https://docs.google.com/spreadsheets/d/${s.driveFileId}/edit#gid=${s.driveSheetId}`, expandable: false, @@ -972,7 +962,6 @@ async function getFoldersAsContentNodes({ type: "folder", title: fd.name || "", sourceUrl, - dustDocumentId: null, lastUpdatedAt: fd.updatedAtMs || null, expandable: await isDriveObjectExpandable({ objectId: f.folderId, diff --git a/connectors/src/connectors/google_drive/lib/permissions.ts b/connectors/src/connectors/google_drive/lib/permissions.ts index 8983e8b3f722..226cd96dcea0 100644 --- a/connectors/src/connectors/google_drive/lib/permissions.ts +++ b/connectors/src/connectors/google_drive/lib/permissions.ts @@ -1,8 +1,4 @@ -import { - isGoogleDriveFolder, - isGoogleDriveSpreadSheetFile, -} from "@connectors/connectors/google_drive/temporal/mime_types"; -import { getInternalId } from "@connectors/connectors/google_drive/temporal/utils"; +import { isGoogleDriveFolder } from "@connectors/connectors/google_drive/temporal/mime_types"; import type { GoogleDriveFiles } from "@connectors/lib/models/google_drive"; export function getPermissionViewType(file: GoogleDriveFiles) { @@ -12,11 +8,3 @@ export function getPermissionViewType(file: GoogleDriveFiles) { return "file"; } - -export function getGoogleDriveEntityDocumentId(file: GoogleDriveFiles) { - if (isGoogleDriveSpreadSheetFile(file) || isGoogleDriveFolder(file)) { - return null; - } - - return getInternalId(file.driveFileId); -} diff --git a/connectors/src/connectors/intercom/index.ts b/connectors/src/connectors/intercom/index.ts index 6a61b6e7655b..ff13edc25a57 100644 --- a/connectors/src/connectors/intercom/index.ts +++ b/connectors/src/connectors/intercom/index.ts @@ -629,7 +629,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: helpCenter.permission, - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -650,7 +649,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.lastUpsertedTs?.getTime() || null, }); } @@ -671,7 +669,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: article.url, expandable: false, permission: article.permission, - dustDocumentId: null, lastUpdatedAt: article.lastUpsertedTs?.getTime() || null, }); } @@ -687,7 +684,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { intercomWorkspace.syncAllConversations === "activated" ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -700,7 +696,6 @@ export class IntercomConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: null, }); } diff --git a/connectors/src/connectors/intercom/lib/conversation_permissions.ts b/connectors/src/connectors/intercom/lib/conversation_permissions.ts index 8e985bf589da..622e70ae7992 100644 --- a/connectors/src/connectors/intercom/lib/conversation_permissions.ts +++ b/connectors/src/connectors/intercom/lib/conversation_permissions.ts @@ -142,7 +142,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: false, preventSelection: false, permission: isAllConversationsSynced ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } else if (isRootLevel && hasTeamsWithReadPermission) { @@ -155,7 +154,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: true, preventSelection: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -170,7 +168,6 @@ export async function retrieveIntercomConversationsPermissions({ sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: null, }); }); @@ -188,7 +185,6 @@ export async function retrieveIntercomConversationsPermissions({ expandable: true, preventSelection: false, permission: isAllConversationsSynced ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -205,7 +201,6 @@ export async function retrieveIntercomConversationsPermissions({ sourceUrl: null, expandable: false, permission: isTeamInDb ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: null, }); }); diff --git a/connectors/src/connectors/intercom/lib/help_center_permissions.ts b/connectors/src/connectors/intercom/lib/help_center_permissions.ts index e494f19cf92b..995ca834e38e 100644 --- a/connectors/src/connectors/intercom/lib/help_center_permissions.ts +++ b/connectors/src/connectors/intercom/lib/help_center_permissions.ts @@ -374,7 +374,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: null, expandable: true, permission: helpCenter.permission, - dustDocumentId: null, lastUpdatedAt: helpCenter.updatedAt.getTime(), })); } else { @@ -388,7 +387,6 @@ export async function retrieveIntercomHelpCentersPermissions({ expandable: true, preventSelection: true, permission: "none", - dustDocumentId: null, lastUpdatedAt: null, })); } @@ -435,7 +433,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.updatedAt.getTime() || null, })); } else { @@ -464,7 +461,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: false, // WE DO NOT LET EXPAND BELOW LEVEL 1 WHEN SELECTING NODES permission: matchingCollectionInDb ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: matchingCollectionInDb?.updatedAt.getTime() || null, }; }); @@ -504,7 +500,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: collection.url, expandable: true, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.lastUpsertedTs?.getTime() || null, }) ); @@ -529,7 +524,6 @@ export async function retrieveIntercomHelpCentersPermissions({ sourceUrl: article.url, expandable: false, permission: article.permission, - dustDocumentId: null, lastUpdatedAt: article.updatedAt.getTime(), })); diff --git a/connectors/src/connectors/intercom/lib/permissions.ts b/connectors/src/connectors/intercom/lib/permissions.ts index 961dd9fbf2a0..c4a5734c58ca 100644 --- a/connectors/src/connectors/intercom/lib/permissions.ts +++ b/connectors/src/connectors/intercom/lib/permissions.ts @@ -63,7 +63,6 @@ export async function retrieveSelectedNodes({ sourceUrl: collection.url, expandable, permission: collection.permission, - dustDocumentId: null, lastUpdatedAt: collection.updatedAt.getTime() || null, }); }); @@ -85,7 +84,6 @@ export async function retrieveSelectedNodes({ sourceUrl: null, expandable: false, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -105,7 +103,6 @@ export async function retrieveSelectedNodes({ sourceUrl: null, expandable: false, permission: team.permission, - dustDocumentId: null, lastUpdatedAt: team.updatedAt.getTime() || null, }); }); diff --git a/connectors/src/connectors/microsoft/lib/content_nodes.ts b/connectors/src/connectors/microsoft/lib/content_nodes.ts index d71efb4db289..fe665918991a 100644 --- a/connectors/src/connectors/microsoft/lib/content_nodes.ts +++ b/connectors/src/connectors/microsoft/lib/content_nodes.ts @@ -25,7 +25,6 @@ export function getSitesRootAsContentNode(): ContentNode { type: "folder", title: "Sites", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -43,7 +42,6 @@ export function getTeamsRootAsContentNode(): ContentNode { type: "folder", title: "Teams", sourceUrl: null, - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -60,7 +58,6 @@ export function getTeamAsContentNode(team: microsoftgraph.Team): ContentNode { type: "folder", title: team.displayName || "unnamed", sourceUrl: team.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -85,7 +82,6 @@ export function getSiteAsContentNode( type: "folder", title: site.displayName || site.name || "unnamed", sourceUrl: site.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, preventSelection: true, expandable: true, @@ -115,7 +111,6 @@ export function getChannelAsContentNode( type: "channel", title: channel.displayName || "unnamed", sourceUrl: channel.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: false, permission: "none", @@ -136,7 +131,6 @@ export function getDriveAsContentNode( type: "folder", title: drive.name || "unnamed", sourceUrl: drive.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -152,7 +146,6 @@ export function getFolderAsContentNode( type: "folder", title: folder.name || "unnamed", sourceUrl: folder.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: true, permission: "none", @@ -169,7 +162,6 @@ export function getFileAsContentNode( type: "file", title: file.name || "unnamed", sourceUrl: file.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: false, permission: "none", @@ -204,7 +196,6 @@ export function getMicrosoftNodeAsContentNode( type, title: node.name || "unnamed", sourceUrl: node.webUrl ?? "", - dustDocumentId: null, lastUpdatedAt: null, expandable: isExpandable, permission: "none", diff --git a/connectors/src/connectors/notion/index.ts b/connectors/src/connectors/notion/index.ts index 741b41772564..bb824f3183e0 100644 --- a/connectors/src/connectors/notion/index.ts +++ b/connectors/src/connectors/notion/index.ts @@ -464,7 +464,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: page.notionUrl || null, expandable, permission: "read", - dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, }; }; @@ -487,7 +486,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: db.notionUrl || null, expandable: true, permission: "read", - dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: db.structuredDataUpsertedTs?.getTime() ?? null, }; }; @@ -509,7 +507,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } @@ -559,7 +556,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: page.notionUrl || null, expandable: Boolean(hasChildrenByPageId[page.notionPageId]), permission: "read", - dustDocumentId: nodeIdFromNotionId(page.notionPageId), lastUpdatedAt: page.lastUpsertedTs?.getTime() || null, })) ); @@ -575,7 +571,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: db.notionUrl || null, expandable: true, permission: "read", - dustDocumentId: nodeIdFromNotionId(`database-${db.notionDatabaseId}`), lastUpdatedAt: null, })); @@ -593,7 +588,6 @@ export class NotionConnectorManager extends BaseConnectorManager { sourceUrl: null, expandable: true, permission: "read", - dustDocumentId: null, lastUpdatedAt: null, }); } diff --git a/connectors/src/connectors/slack/index.ts b/connectors/src/connectors/slack/index.ts index 4f6dc5a6f28b..f839596ad1b9 100644 --- a/connectors/src/connectors/slack/index.ts +++ b/connectors/src/connectors/slack/index.ts @@ -406,7 +406,6 @@ export class SlackConnectorManager extends BaseConnectorManager { this.ticketsPermission === "read" ? "read" : "none", - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -359,7 +358,6 @@ export class ZendeskBrandResource extends BaseResource { sourceUrl: null, expandable: true, permission: this.helpCenterPermission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -380,7 +378,6 @@ export class ZendeskBrandResource extends BaseResource { sourceUrl: null, expandable: expandable, permission: this.ticketsPermission, - dustDocumentId: null, lastUpdatedAt: null, }; } @@ -645,7 +642,6 @@ export class ZendeskCategoryResource extends BaseResource { sourceUrl: this.url, expandable: expandable, permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -730,7 +726,6 @@ export class ZendeskTicketResource extends BaseResource { sourceUrl: this.url, expandable: false, permission: this.permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } @@ -945,7 +940,6 @@ export class ZendeskArticleResource extends BaseResource { sourceUrl: this.url, expandable: false, permission: this.permission, - dustDocumentId: null, lastUpdatedAt: this.updatedAt.getTime(), }; } diff --git a/front/components/ContentNodeTree.tsx b/front/components/ContentNodeTree.tsx index d41b07f481c2..e595d5e6f19c 100644 --- a/front/components/ContentNodeTree.tsx +++ b/front/components/ContentNodeTree.tsx @@ -249,15 +249,15 @@ function ContentNodeTreeChildren({ size="xs" icon={BracesIcon} onClick={() => { - if (n.dustDocumentId) { - onDocumentViewClick(n.dustDocumentId); + if (n.type === "file") { + onDocumentViewClick(n.internalId); } }} className={classNames( - n.dustDocumentId ? "" : "pointer-events-none opacity-0" + n.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!n.dustDocumentId} - variant="ghost" + disabled={n.type !== "file"} + variant="outline" /> )} diff --git a/front/components/app/blocks/Database.tsx b/front/components/app/blocks/Database.tsx index c77a23483116..c23613eb4d93 100644 --- a/front/components/app/blocks/Database.tsx +++ b/front/components/app/blocks/Database.tsx @@ -141,7 +141,7 @@ export function TablesManager({ currentTableId={table.table_id} onTableUpdate={(selectedTable) => { updateTableConfig(index, { - table_id: selectedTable.dustDocumentId!, + table_id: selectedTable.internalId, }); }} excludeTables={getSelectedTables()} diff --git a/front/components/assistant/details/AssistantActionsSection.tsx b/front/components/assistant/details/AssistantActionsSection.tsx index 0050ad73d074..f17407c916fd 100644 --- a/front/components/assistant/details/AssistantActionsSection.tsx +++ b/front/components/assistant/details/AssistantActionsSection.tsx @@ -446,15 +446,15 @@ function DataSourceViewSelectedNodes({ size="xs" icon={BracesIcon} onClick={() => { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay(dataSourceView); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId ? "" : "pointer-events-none opacity-0" + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} + disabled={node.type !== "file"} variant="outline" /> diff --git a/front/components/assistant_builder/DataSourceSelectionSection.tsx b/front/components/assistant_builder/DataSourceSelectionSection.tsx index b3d755e53f22..c7f23f7f876c 100644 --- a/front/components/assistant_builder/DataSourceSelectionSection.tsx +++ b/front/components/assistant_builder/DataSourceSelectionSection.tsx @@ -148,20 +148,20 @@ export default function DataSourceSelectionSection({ size="xs" icon={BracesIcon} onClick={() => { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay( dsConfig.dataSourceView ); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} - variant="ghost" + disabled={node.type !== "file"} + variant="outline" /> } diff --git a/front/components/spaces/ContentActions.tsx b/front/components/spaces/ContentActions.tsx index dd91c30f8775..b44e078e4ad0 100644 --- a/front/components/spaces/ContentActions.tsx +++ b/front/components/spaces/ContentActions.tsx @@ -104,7 +104,7 @@ export const ContentActions = React.forwardRef< useEffect(() => { if (currentAction.action === "DocumentViewRawContent") { - setCurrentDocumentId(currentAction.contentNode?.dustDocumentId ?? ""); + setCurrentDocumentId(currentAction.contentNode?.internalId ?? ""); } }, [currentAction, setCurrentDocumentId]); diff --git a/front/components/tables/TablePicker.tsx b/front/components/tables/TablePicker.tsx index 261769f6f857..593d335a6866 100644 --- a/front/components/tables/TablePicker.tsx +++ b/front/components/tables/TablePicker.tsx @@ -62,7 +62,7 @@ export default function TablePicker({ }); const currentTable = currentTableId - ? tables.find((t) => t.dustDocumentId === currentTableId) + ? tables.find((t) => t.internalId === currentTableId) : null; const [searchFilter, setSearchFilter] = useState(""); @@ -139,12 +139,12 @@ export default function TablePicker({ !excludeTables?.some( (et) => et.dataSourceId === dataSource.data_source_id && - et.tableId === t.dustDocumentId + et.tableId === t.internalId ) ) .map((t) => (
{ onTableUpdate(t); diff --git a/front/components/trackers/TrackerDataSourceSelectedTree.tsx b/front/components/trackers/TrackerDataSourceSelectedTree.tsx index 648e99bc5e95..e2f53af18b2e 100644 --- a/front/components/trackers/TrackerDataSourceSelectedTree.tsx +++ b/front/components/trackers/TrackerDataSourceSelectedTree.tsx @@ -109,20 +109,20 @@ export const TrackerDataSourceSelectedTree = ({ size="xs" icon={BracesIcon} onClick={() => { - if (node.dustDocumentId) { + if (node.type === "file") { setDataSourceViewToDisplay( dsConfig.dataSourceView ); - setDocumentToDisplay(node.dustDocumentId); + setDocumentToDisplay(node.internalId); } }} className={classNames( - node.dustDocumentId + node.type === "file" ? "" : "pointer-events-none opacity-0" )} - disabled={!node.dustDocumentId} - variant="ghost" + disabled={node.type !== "file"} + variant="outline" />
} diff --git a/front/lib/api/data_source_view.ts b/front/lib/api/data_source_view.ts index 7632170f5135..68528e0c42a4 100644 --- a/front/lib/api/data_source_view.ts +++ b/front/lib/api/data_source_view.ts @@ -203,7 +203,6 @@ async function getContentNodesForStaticDataSourceView( } } return { - dustDocumentId: doc.document_id, expandable: false, internalId: doc.document_id, lastUpdatedAt: doc.timestamp, @@ -238,7 +237,6 @@ async function getContentNodesForStaticDataSourceView( const tablesAsContentNodes: DataSourceViewContentNode[] = tablesRes.value.tables.map((table) => ({ - dustDocumentId: table.table_id, expandable: false, internalId: getContentNodeInternalIdFromTableId( dataSourceView, diff --git a/types/src/front/api_handlers/public/spaces.ts b/types/src/front/api_handlers/public/spaces.ts index 20e51a4c8941..cd1a50f1b3b0 100644 --- a/types/src/front/api_handlers/public/spaces.ts +++ b/types/src/front/api_handlers/public/spaces.ts @@ -21,7 +21,6 @@ export type PatchDataSourceViewType = t.TypeOf< >; export type LightContentNode = { - dustDocumentId: string | null; expandable: boolean; internalId: string; lastUpdatedAt: number | null; diff --git a/types/src/front/lib/connectors_api.ts b/types/src/front/lib/connectors_api.ts index 9625e6abc934..550fb81f9a44 100644 --- a/types/src/front/lib/connectors_api.ts +++ b/types/src/front/lib/connectors_api.ts @@ -106,7 +106,6 @@ export interface ContentNode { expandable: boolean; preventSelection?: boolean; permission: ConnectorPermission; - dustDocumentId: string | null; lastUpdatedAt: number | null; providerVisibility?: "public" | "private"; } From 14450fa1bffa0587268b7492041d27bf5ed9f0c1 Mon Sep 17 00:00:00 2001 From: Philippe Rolet Date: Mon, 13 Jan 2025 17:32:57 +0100 Subject: [PATCH 37/47] Fix: ES core nodes index version in init-dev-container (#9939) Description --- Cf title. Index version had not been updated there. Risk --- na Deploy --- core --- init_dev_container.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/init_dev_container.sh b/init_dev_container.sh index d3a42eabf2db..cb27a25ffc06 100755 --- a/init_dev_container.sh +++ b/init_dev_container.sh @@ -14,5 +14,5 @@ cd - ## Initializing Elasticsearch indices cd core/ -cargo run --bin elasticsearch_create_index -- --index-name data_sources_nodes --index-version 1 --skip-confirmation +cargo run --bin elasticsearch_create_index -- --index-name data_sources_nodes --index-version 2 --skip-confirmation cd - From eef1178ee074ca9dd4eb2973804aac8139860638 Mon Sep 17 00:00:00 2001 From: Sebastien Flory Date: Mon, 13 Jan 2025 17:43:32 +0100 Subject: [PATCH 38/47] Upg: use shallow browsing to make conversations navigation a breeze (#9928) * Upg: use shallow browsing to make conversations navigation a breeze * Review fdbk --- .../conversation/ConversationContainer.tsx | 7 +- .../conversation/ConversationLayout.tsx | 175 +++++++----------- .../conversation/ConversationTitle.tsx | 140 ++++++++------ .../conversation/ConversationViewer.tsx | 2 +- .../ConversationsNavigationProvider.tsx | 22 ++- .../assistant/conversation/SidebarMenu.tsx | 30 +-- front/hooks/useAppKeyboardShortcuts.ts | 4 +- front/lib/swr/conversations.ts | 9 +- front/pages/w/[wId]/assistant/[cId]/index.tsx | 23 ++- 9 files changed, 213 insertions(+), 199 deletions(-) diff --git a/front/components/assistant/conversation/ConversationContainer.tsx b/front/components/assistant/conversation/ConversationContainer.tsx index 1b6e932b4075..b9b8ea8db4b3 100644 --- a/front/components/assistant/conversation/ConversationContainer.tsx +++ b/front/components/assistant/conversation/ConversationContainer.tsx @@ -35,7 +35,6 @@ import { } from "@app/lib/swr/conversations"; interface ConversationContainerProps { - conversationId: string | null; owner: WorkspaceType; subscription: SubscriptionType; user: UserType; @@ -45,7 +44,6 @@ interface ConversationContainerProps { } export function ConversationContainer({ - conversationId, owner, subscription, user, @@ -53,8 +51,8 @@ export function ConversationContainer({ agentIdToMention, messageRankToScrollTo, }: ConversationContainerProps) { - const [activeConversationId, setActiveConversationId] = - useState(conversationId); + const { activeConversationId } = useConversationsNavigation(); + const [planLimitReached, setPlanLimitReached] = useState(false); const [stickyMentions, setStickyMentions] = useState([]); @@ -242,7 +240,6 @@ export function ConversationContainer({ undefined, { shallow: true } ); - setActiveConversationId(conversationRes.value.sId); await mutateConversations(); await scrollConversationsToTop(); diff --git a/front/components/assistant/conversation/ConversationLayout.tsx b/front/components/assistant/conversation/ConversationLayout.tsx index c42908d1c2ee..7b5332fcb43b 100644 --- a/front/components/assistant/conversation/ConversationLayout.tsx +++ b/front/components/assistant/conversation/ConversationLayout.tsx @@ -1,11 +1,14 @@ import type { SubscriptionType, WorkspaceType } from "@dust-tt/types"; import { useRouter } from "next/router"; -import React, { useCallback, useEffect, useState } from "react"; +import React, { useMemo } from "react"; import RootLayout from "@app/components/app/RootLayout"; import { AssistantDetails } from "@app/components/assistant/AssistantDetails"; import { ConversationErrorDisplay } from "@app/components/assistant/conversation/ConversationError"; -import { ConversationsNavigationProvider } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; +import { + ConversationsNavigationProvider, + useConversationsNavigation, +} from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { ConversationTitle } from "@app/components/assistant/conversation/ConversationTitle"; import { FileDropProvider } from "@app/components/assistant/conversation/FileUploaderContext"; import { GenerationContextProvider } from "@app/components/assistant/conversation/GenerationContextProvider"; @@ -13,10 +16,7 @@ import { InputBarProvider } from "@app/components/assistant/conversation/input_b import { AssistantSidebarMenu } from "@app/components/assistant/conversation/SidebarMenu"; import AppLayout from "@app/components/sparkle/AppLayout"; import { useURLSheet } from "@app/hooks/useURLSheet"; -import { - useConversation, - useDeleteConversation, -} from "@app/lib/swr/conversations"; +import { useConversation } from "@app/lib/swr/conversations"; export interface ConversationLayoutProps { baseUrl: string; @@ -32,116 +32,81 @@ export default function ConversationLayout({ children: React.ReactNode; pageProps: ConversationLayoutProps; }) { - const { baseUrl, conversationId, owner, subscription } = pageProps; - - const router = useRouter(); + const { baseUrl, owner, subscription } = pageProps; - const [detailViewContent, setDetailViewContent] = useState(""); - const [activeConversationId, setActiveConversationId] = useState( - conversationId !== "new" ? conversationId : null + return ( + + + + {children} + + + ); +} +const ConversationLayoutContent = ({ + owner, + subscription, + baseUrl, + children, +}: any) => { + const router = useRouter(); const { onOpenChange: onOpenChangeAssistantModal } = useURLSheet("assistantDetails"); - - useEffect(() => { - const handleRouteChange = () => { - const assistantSId = router.query.assistantDetails ?? []; - // We use shallow browsing when creating a new conversation. - // Monitor router to update conversation info. - const conversationId = router.query.cId ?? ""; - - if (assistantSId && typeof assistantSId === "string") { - setDetailViewContent(assistantSId); - } else { - setDetailViewContent(""); - } - - if ( - conversationId && - typeof conversationId === "string" && - conversationId !== activeConversationId - ) { - setActiveConversationId( - conversationId !== "new" ? conversationId : null - ); - } - }; - - // Initial check in case the component mounts with the query already set. - handleRouteChange(); - - router.events.on("routeChangeComplete", handleRouteChange); - return () => { - router.events.off("routeChangeComplete", handleRouteChange); - }; - }, [ - router.query, - router.events, - setActiveConversationId, - activeConversationId, - ]); - + const { activeConversationId } = useConversationsNavigation(); const { conversation, conversationError } = useConversation({ conversationId: activeConversationId, workspaceId: owner.sId, }); - const doDelete = useDeleteConversation(owner); - - const onDeleteConversation = useCallback(async () => { - const res = await doDelete(conversation); - if (res) { - void router.push(`/w/${owner.sId}/assistant/new`); + const assistantSId = useMemo(() => { + const sid = router.query.assistantDetails ?? []; + if (sid && typeof sid === "string") { + return sid; } - }, [conversation, doDelete, owner.sId, router]); + return null; + }, [router.query.assistantDetails]); return ( - - - - - ) - } - navChildren={} - > - {conversationError ? ( - - ) : ( - <> - onOpenChangeAssistantModal(false)} - /> - - - {children} - - - - )} - - - - + + + ) + } + navChildren={} + > + {conversationError ? ( + + ) : ( + <> + onOpenChangeAssistantModal(false)} + /> + + {children} + + + )} + + ); -} +}; diff --git a/front/components/assistant/conversation/ConversationTitle.tsx b/front/components/assistant/conversation/ConversationTitle.tsx index 4a2e182a92e9..d34da1e9ff3d 100644 --- a/front/components/assistant/conversation/ConversationTitle.tsx +++ b/front/components/assistant/conversation/ConversationTitle.tsx @@ -10,30 +10,47 @@ import { TrashIcon, XMarkIcon, } from "@dust-tt/sparkle"; -import type { ConversationType } from "@dust-tt/types"; import type { WorkspaceType } from "@dust-tt/types"; +import { useRouter } from "next/router"; import type { MouseEvent } from "react"; -import React, { useRef, useState } from "react"; +import React, { useCallback, useRef, useState } from "react"; import { useSWRConfig } from "swr"; import { ConversationParticipants } from "@app/components/assistant/conversation/ConversationParticipants"; +import { useConversationsNavigation } from "@app/components/assistant/conversation/ConversationsNavigationProvider"; import { DeleteConversationsDialog } from "@app/components/assistant/conversation/DeleteConversationsDialog"; +import { + useConversation, + useDeleteConversation, +} from "@app/lib/swr/conversations"; import { classNames } from "@app/lib/utils"; export function ConversationTitle({ owner, - conversationId, - conversation, - shareLink, - onDelete, + baseUrl, }: { owner: WorkspaceType; - conversationId: string; - conversation: ConversationType | null; - shareLink: string; - onDelete?: (conversationId: string) => void; + baseUrl: string; }) { const { mutate } = useSWRConfig(); + const router = useRouter(); + const { activeConversationId } = useConversationsNavigation(); + + const { conversation } = useConversation({ + conversationId: activeConversationId, + workspaceId: owner.sId, + }); + + const shareLink = `${baseUrl}/w/${owner.sId}/assistant/${activeConversationId}`; + + const doDelete = useDeleteConversation(owner); + + const onDelete = useCallback(async () => { + const res = await doDelete(conversation); + if (res) { + void router.push(`/w/${owner.sId}/assistant/new`); + } + }, [conversation, doDelete, owner.sId, router]); const [copyLinkSuccess, setCopyLinkSuccess] = useState(false); const [isEditingTitle, setIsEditingTitle] = useState(false); @@ -43,57 +60,62 @@ export function ConversationTitle({ const titleInputFocused = useRef(false); const saveButtonFocused = useRef(false); - const handleClick = async () => { + const handleClick = useCallback(async () => { await navigator.clipboard.writeText(shareLink || ""); setCopyLinkSuccess(true); setTimeout(() => { setCopyLinkSuccess(false); }, 1000); - }; + }, [shareLink]); - const onTitleChange = async (title: string) => { - try { - const res = await fetch( - `/api/w/${owner.sId}/assistant/conversations/${conversationId}`, - { - method: "PATCH", - headers: { - "Content-Type": "application/json", - }, - body: JSON.stringify({ - title, - visibility: conversation?.visibility, - }), + const onTitleChange = useCallback( + async (title: string) => { + try { + const res = await fetch( + `/api/w/${owner.sId}/assistant/conversations/${activeConversationId}`, + { + method: "PATCH", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + title, + visibility: conversation?.visibility, + }), + } + ); + await mutate( + `/api/w/${owner.sId}/assistant/conversations/${activeConversationId}` + ); + void mutate(`/api/w/${owner.sId}/assistant/conversations`); + if (!res.ok) { + throw new Error("Failed to update title"); } - ); - await mutate( - `/api/w/${owner.sId}/assistant/conversations/${conversationId}` - ); - void mutate(`/api/w/${owner.sId}/assistant/conversations`); - if (!res.ok) { - throw new Error("Failed to update title"); + setIsEditingTitle(false); + setEditedTitle(""); + } catch (e) { + alert("Failed to update title"); } - setIsEditingTitle(false); - setEditedTitle(""); - } catch (e) { - alert("Failed to update title"); - } - }; + }, + [activeConversationId, conversation?.visibility, mutate, owner.sId] + ); + + if (!activeConversationId) { + return null; + } return ( <> - {onDelete && ( - setShowDeleteDialog(false)} - onDelete={() => { - setShowDeleteDialog(false); - onDelete(conversationId); - }} - /> - )} + setShowDeleteDialog(false)} + onDelete={() => { + setShowDeleteDialog(false); + void onDelete(); + }} + />
{!isEditingTitle ? ( @@ -177,21 +199,19 @@ export function ConversationTitle({
- {onDelete && ( -
)} {(isMessagesLoading || prevFirstMessageId) && ( -
+
)} diff --git a/front/components/assistant/conversation/ConversationsNavigationProvider.tsx b/front/components/assistant/conversation/ConversationsNavigationProvider.tsx index 29d51ece88f5..c31bc166519f 100644 --- a/front/components/assistant/conversation/ConversationsNavigationProvider.tsx +++ b/front/components/assistant/conversation/ConversationsNavigationProvider.tsx @@ -1,22 +1,27 @@ +import { useRouter } from "next/router"; import type { RefObject } from "react"; -import { createContext, useContext, useRef } from "react"; +import { createContext, useCallback, useContext, useMemo, useRef } from "react"; interface ConversationsNavigationContextType { conversationsNavigationRef: RefObject; scrollConversationsToTop: () => void; + activeConversationId: string | null; } const ConversationsNavigationContext = createContext(null); export function ConversationsNavigationProvider({ + initialConversationId, children, }: { + initialConversationId?: string | null; children: React.ReactNode; }) { + const router = useRouter(); const conversationsNavigationRef = useRef(null); - const scrollConversationsToTop = () => { + const scrollConversationsToTop = useCallback(() => { if (conversationsNavigationRef.current) { // Find the ScrollArea viewport const viewport = conversationsNavigationRef.current.querySelector( @@ -29,13 +34,24 @@ export function ConversationsNavigationProvider({ }); } } - }; + }, []); + + const activeConversationId = useMemo(() => { + const conversationId = router.query.cId ?? ""; + + if (conversationId && typeof conversationId === "string") { + return conversationId === "new" ? null : conversationId; + } + + return initialConversationId ?? null; + }, [initialConversationId, router.query.cId]); return ( {children} diff --git a/front/components/assistant/conversation/SidebarMenu.tsx b/front/components/assistant/conversation/SidebarMenu.tsx index 3bfbc22063ce..689d82af64da 100644 --- a/front/components/assistant/conversation/SidebarMenu.tsx +++ b/front/components/assistant/conversation/SidebarMenu.tsx @@ -20,7 +20,7 @@ import { XMarkIcon, } from "@dust-tt/sparkle"; import { useSendNotification } from "@dust-tt/sparkle"; -import type { ConversationType } from "@dust-tt/types"; +import type { ConversationWithoutContentType } from "@dust-tt/types"; import type { WorkspaceType } from "@dust-tt/types"; import { isBuilder, isOnlyUser } from "@dust-tt/types"; import moment from "moment"; @@ -60,7 +60,7 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { }); const [isMultiSelect, setIsMultiSelect] = useState(false); const [selectedConversations, setSelectedConversations] = useState< - ConversationType[] + ConversationWithoutContentType[] >([]); const doDelete = useDeleteConversation(owner); @@ -77,7 +77,7 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { }, [setIsMultiSelect, setSelectedConversations]); const toggleConversationSelection = useCallback( - (c: ConversationType) => { + (c: ConversationWithoutContentType) => { if (selectedConversations.includes(c)) { setSelectedConversations((prev) => prev.filter((id) => id !== c)); } else { @@ -124,14 +124,16 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { setShowDeleteDialog(null); }, [conversations, doDelete, sendNotification]); - const groupConversationsByDate = (conversations: ConversationType[]) => { + const groupConversationsByDate = ( + conversations: ConversationWithoutContentType[] + ) => { const today = moment().startOf("day"); const yesterday = moment().subtract(1, "days").startOf("day"); const lastWeek = moment().subtract(1, "weeks").startOf("day"); const lastMonth = moment().subtract(1, "months").startOf("day"); const lastYear = moment().subtract(1, "years").startOf("day"); - const groups: Record = { + const groups: Record = { Today: [], Yesterday: [], "Last Week": [], @@ -140,7 +142,7 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { Older: [], }; - conversations.forEach((conversation: ConversationType) => { + conversations.forEach((conversation: ConversationWithoutContentType) => { if ( titleFilter && !subFilter( @@ -172,7 +174,7 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { const conversationsByDate = conversations.length ? groupConversationsByDate(conversations) - : ({} as Record); + : ({} as Record); const { setAnimate } = useContext(InputBarContext); @@ -224,6 +226,7 @@ export function AssistantSidebarMenu({ owner }: AssistantSidebarMenuProps) { />