From 3266ef63217e42564090cbb6358a2d33e2903924 Mon Sep 17 00:00:00 2001 From: Chris Weaver <25087905+Weves@users.noreply.github.com> Date: Wed, 4 Dec 2024 20:28:30 -0800 Subject: [PATCH] Improve chat page performance (#3347) * Simplify /manage/indexing-status * Rename endpoint --- backend/danswer/server/documents/connector.py | 21 +----- .../search/NoCompleteSourceModal.tsx | 71 ------------------- .../assistants/fetchPersonaEditorInfoSS.ts | 2 +- web/src/lib/chat/fetchAssistantdata.ts | 2 +- web/src/lib/chat/fetchChatData.ts | 2 +- web/src/lib/chat/fetchSomeChatData.ts | 6 +- web/src/lib/types.ts | 1 - 7 files changed, 6 insertions(+), 99 deletions(-) delete mode 100644 web/src/components/initialSetup/search/NoCompleteSourceModal.tsx diff --git a/backend/danswer/server/documents/connector.py b/backend/danswer/server/documents/connector.py index 36af9d2baa2..fd9b552a653 100644 --- a/backend/danswer/server/documents/connector.py +++ b/backend/danswer/server/documents/connector.py @@ -1010,37 +1010,18 @@ def get_connector_by_id( class BasicCCPairInfo(BaseModel): - docs_indexed: int has_successful_run: bool source: DocumentSource -@router.get("/indexing-status") +@router.get("/connector-status") def get_basic_connector_indexing_status( _: User = Depends(current_user), db_session: Session = Depends(get_session), ) -> list[BasicCCPairInfo]: cc_pairs = get_connector_credential_pairs(db_session) - cc_pair_identifiers = [ - ConnectorCredentialPairIdentifier( - connector_id=cc_pair.connector_id, credential_id=cc_pair.credential_id - ) - for cc_pair in cc_pairs - ] - document_count_info = get_document_counts_for_cc_pairs( - db_session=db_session, - cc_pair_identifiers=cc_pair_identifiers, - ) - cc_pair_to_document_cnt = { - (connector_id, credential_id): cnt - for connector_id, credential_id, cnt in document_count_info - } return [ BasicCCPairInfo( - docs_indexed=cc_pair_to_document_cnt.get( - (cc_pair.connector_id, cc_pair.credential_id) - ) - or 0, has_successful_run=cc_pair.last_successful_index_time is not None, source=cc_pair.connector.source, ) diff --git a/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx b/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx deleted file mode 100644 index decf302e2f9..00000000000 --- a/web/src/components/initialSetup/search/NoCompleteSourceModal.tsx +++ /dev/null @@ -1,71 +0,0 @@ -"use client"; - -import { Modal } from "../../Modal"; -import Link from "next/link"; -import { useEffect, useState } from "react"; -import { CCPairBasicInfo } from "@/lib/types"; -import { useRouter } from "next/navigation"; - -export function NoCompleteSourcesModal({ - ccPairs, -}: { - ccPairs: CCPairBasicInfo[]; -}) { - const router = useRouter(); - const [isHidden, setIsHidden] = useState(false); - - useEffect(() => { - const interval = setInterval(() => { - router.refresh(); - }, 5000); - - return () => clearInterval(interval); - }, [router]); - - if (isHidden) { - return null; - } - - const totalDocs = ccPairs.reduce( - (acc, ccPair) => acc + ccPair.docs_indexed, - 0 - ); - - return ( - setIsHidden(true)} - > -
-
-
- You've connected some sources, but none of them have finished - syncing. Depending on the size of the knowledge base(s) you've - connected to Danswer, it can take anywhere between 30 seconds to a - few days for the initial sync to complete. So far we've synced{" "} - {totalDocs} documents. -
-
- To view the status of your syncing connectors, head over to the{" "} - - Existing Connectors page - - . -
-
-

{ - setIsHidden(true); - }} - > - Or, click here to continue and ask questions on the partially - synced knowledge set. -

-
-
-
- - ); -} diff --git a/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts b/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts index 1939ad038ac..2c00e0dfcb7 100644 --- a/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts +++ b/web/src/lib/assistants/fetchPersonaEditorInfoSS.ts @@ -26,7 +26,7 @@ export async function fetchAssistantEditorInfoSS( | [null, string] > { const tasks = [ - fetchSS("/manage/indexing-status"), + fetchSS("/manage/connector-status"), fetchSS("/manage/document-set"), fetchSS("/llm/provider"), // duplicate fetch, but shouldn't be too big of a deal diff --git a/web/src/lib/chat/fetchAssistantdata.ts b/web/src/lib/chat/fetchAssistantdata.ts index 363edff8ede..2ca126caffd 100644 --- a/web/src/lib/chat/fetchAssistantdata.ts +++ b/web/src/lib/chat/fetchAssistantdata.ts @@ -30,7 +30,7 @@ export async function fetchAssistantData(): Promise { // Parallel fetch of additional data const [ccPairsResponse, llmProviders] = await Promise.all([ - fetchSS("/manage/indexing-status").catch((error) => { + fetchSS("/manage/connector-status").catch((error) => { console.error("Failed to fetch connectors:", error); return null; }), diff --git a/web/src/lib/chat/fetchChatData.ts b/web/src/lib/chat/fetchChatData.ts index 751aa4d005a..6a8d4ac0e76 100644 --- a/web/src/lib/chat/fetchChatData.ts +++ b/web/src/lib/chat/fetchChatData.ts @@ -52,7 +52,7 @@ export async function fetchChatData(searchParams: { const tasks = [ getAuthTypeMetadataSS(), getCurrentUserSS(), - fetchSS("/manage/indexing-status"), + fetchSS("/manage/connector-status"), fetchSS("/manage/document-set"), fetchSS("/chat/get-user-chat-sessions"), fetchSS("/query/valid-tags"), diff --git a/web/src/lib/chat/fetchSomeChatData.ts b/web/src/lib/chat/fetchSomeChatData.ts index 47c4c22465d..9b78bc8ad7d 100644 --- a/web/src/lib/chat/fetchSomeChatData.ts +++ b/web/src/lib/chat/fetchSomeChatData.ts @@ -70,7 +70,7 @@ export async function fetchSomeChatData( const taskMap: Record Promise> = { user: getCurrentUserSS, chatSessions: () => fetchSS("/chat/get-user-chat-sessions"), - ccPairs: () => fetchSS("/manage/indexing-status"), + ccPairs: () => fetchSS("/manage/connector-status"), documentSets: () => fetchSS("/manage/document-set"), assistants: fetchAssistantsSS, tags: () => fetchSS("/query/valid-tags"), @@ -229,9 +229,7 @@ export async function fetchSomeChatData( result.shouldDisplaySourcesIncompleteModal = hasAnyConnectors && !result.shouldShowWelcomeModal && - !result.ccPairs.some( - (ccPair) => ccPair.has_successful_run && ccPair.docs_indexed > 0 - ) && + !result.ccPairs.some((ccPair) => ccPair.has_successful_run) && (!user || user.role === "admin"); } diff --git a/web/src/lib/types.ts b/web/src/lib/types.ts index fdd3ac80e62..f4366d0ebc2 100644 --- a/web/src/lib/types.ts +++ b/web/src/lib/types.ts @@ -136,7 +136,6 @@ export interface ConnectorIndexingStatus< } export interface CCPairBasicInfo { - docs_indexed: number; has_successful_run: boolean; source: ValidSources; }