diff --git a/connectors/src/connectors/slack/bot.ts b/connectors/src/connectors/slack/bot.ts index ea5e984ba391..39f1e69e9b38 100644 --- a/connectors/src/connectors/slack/bot.ts +++ b/connectors/src/connectors/slack/bot.ts @@ -213,6 +213,7 @@ async function botAnswerMessage( slackChannel, lastSlackChatBotMessage?.threadTs || slackThreadTs || slackMessageTs, lastSlackChatBotMessage?.messageTs || slackThreadTs || slackMessageTs, + slackChatBotMessage, connector ); @@ -618,8 +619,9 @@ async function makeContentFragment( channelId: string, threadTs: string, startingAtTs: string | null, + slackChatBotMessage: SlackChatBotMessage, connector: Connector -) { +): Promise> { let allMessages: MessageElement[] = []; let next_cursor = undefined; @@ -710,5 +712,6 @@ async function makeContentFragment( content: text, url: url, contentType: "slack_thread_content", - } as PostContentFragmentRequestBody); + context: null, + }); } diff --git a/connectors/src/lib/dust_api.ts b/connectors/src/lib/dust_api.ts index f1ba4ece2b9f..cc51e8a70ece 100644 --- a/connectors/src/lib/dust_api.ts +++ b/connectors/src/lib/dust_api.ts @@ -89,6 +89,15 @@ export const PostContentFragmentRequestBodySchemaIoTs = t.type({ content: t.string, url: t.union([t.string, t.null]), contentType: t.literal("slack_thread_content"), + context: t.union([ + t.type({ + username: t.string, + fullName: t.union([t.string, t.null]), + email: t.union([t.string, t.null]), + profilePictureUrl: t.union([t.string, t.null]), + }), + t.null, + ]), }); const PostConversationsRequestBodySchemaIoTs = t.type({ diff --git a/front/components/assistant/conversation/ContentFragment.tsx b/front/components/assistant/conversation/ContentFragment.tsx index 630a12bf4f16..81a4cdfcb70e 100644 --- a/front/components/assistant/conversation/ContentFragment.tsx +++ b/front/components/assistant/conversation/ContentFragment.tsx @@ -9,6 +9,9 @@ export function ContentFragment({ message }: { message: ContentFragmentType }) { case "slack_thread_content": logoType = "slack"; break; + case "file_attachment": + logoType = "document"; + break; default: assertNever(message.contentType); @@ -18,6 +21,7 @@ export function ContentFragment({ message }: { message: ContentFragmentType }) { title={message.title} type={logoType} href={message.url || undefined} + avatarUrl={message.context.profilePictureUrl ?? undefined} /> ); } diff --git a/front/components/assistant/conversation/InputBar.tsx b/front/components/assistant/conversation/InputBar.tsx index ed9219b1b29a..64c063edec86 100644 --- a/front/components/assistant/conversation/InputBar.tsx +++ b/front/components/assistant/conversation/InputBar.tsx @@ -1,9 +1,12 @@ import { + AttachmentStrokeIcon, Avatar, Button, + Citation, IconButton, PaperAirplaneIcon, StopIcon, + Tooltip, } from "@dust-tt/sparkle"; import { Transition } from "@headlessui/react"; import { @@ -22,7 +25,9 @@ import { mutate } from "swr"; import { AssistantPicker } from "@app/components/assistant/AssistantPicker"; import { GenerationContext } from "@app/components/assistant/conversation/GenerationContextProvider"; +import { SendNotificationsContext } from "@app/components/sparkle/Notification"; import { compareAgentsForSort } from "@app/lib/assistant"; +import { handleFileUploadToText } from "@app/lib/client/handle_file_upload"; import { useAgentConfigurations } from "@app/lib/swr"; import { classNames, subFilter } from "@app/lib/utils"; import { AgentConfigurationType } from "@app/types/assistant/agent"; @@ -202,7 +207,11 @@ export function AssistantInputBar({ stickyMentions, }: { owner: WorkspaceType; - onSubmit: (input: string, mentions: MentionType[]) => void; + onSubmit: ( + input: string, + mentions: MentionType[], + contentFragment?: { title: string; content: string } + ) => void; stickyMentions?: AgentMention[]; }) { const [agentListVisible, setAgentListVisible] = useState(false); @@ -214,6 +223,13 @@ export function AssistantInputBar({ bottom: 0, left: 0, }); + const [contentFragmentBody, setContentFragmentBody] = useState< + string | undefined + >(undefined); + const [contentFragmentFilename, setContentFragmentFilename] = useState< + string | undefined + >(undefined); + const fileInputRef = useRef(null); const inputRef = useRef(null); const agentListRef = useRef<{ prev: () => void; @@ -255,6 +271,7 @@ export function AssistantInputBar({ const { agentConfigurations } = useAgentConfigurations({ workspaceId: owner.sId, }); + const sendNotification = useContext(SendNotificationsContext); const activeAgents = agentConfigurations.filter((a) => a.status === "active"); activeAgents.sort(compareAgentsForSort); @@ -291,9 +308,27 @@ export function AssistantInputBar({ content = content.trim(); content = content.replace(/\u200B/g, ""); + let contentFragment: + | { + title: string; + content: string; + url: string | null; + contentType: string; + } + | undefined = undefined; + if (contentFragmentBody && contentFragmentFilename) { + contentFragment = { + title: contentFragmentFilename, + content: contentFragmentBody, + url: null, + contentType: "file_attachment", + }; + } - onSubmit(content, mentions); + onSubmit(content, mentions, contentFragment); contentEditable.innerHTML = ""; + setContentFragmentFilename(undefined); + setContentFragmentBody(undefined); } }; @@ -376,7 +411,7 @@ export function AssistantInputBar({ position={agentListPosition} />
-
+
-
- Ask a question or get some @help +
+ { + // focus on the input text after the file selection interaction is over + inputRef.current?.focus(); + const file = e?.target?.files?.[0]; + if (!file) return; + const res = await handleFileUploadToText(file); + if (res.isErr()) { + sendNotification({ + type: "error", + title: "Error uploading file.", + description: res.error.message, + }); + return; + } + setContentFragmentFilename(res.value.title); + setContentFragmentBody(res.value.content); + }} + /> + + + { + fileInputRef.current?.click(); + }} + /> +
-
+
+ Ask a question or get some @help +
+ + {contentFragmentFilename && contentFragmentBody && ( +
+ { + setContentFragmentBody(undefined); + setContentFragmentFilename(undefined); + }} + /> +
)} - contentEditable={true} - ref={inputRef} - id={"dust-input-bar"} - suppressContentEditableWarning={true} - onPaste={(e) => { - e.preventDefault(); - - // Get the plain text. - const text = e.clipboardData.getData("text/plain"); - - // If the text is single line. - if (text.indexOf("\n") === -1 && text.indexOf("\r") === -1) { - document.execCommand("insertText", false, text); - return; - } - - const selection = window.getSelection(); - if (!selection) { - return; - } - const range = selection.getRangeAt(0); - let node = range.endContainer; - let offset = range.endOffset; - - if ( - // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. - node.getAttribute && - // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. - node.getAttribute("id") === "dust-input-bar" - ) { - const textNode = document.createTextNode(""); - node.appendChild(textNode); - node = textNode; - offset = 0; - } - - if ( - node.parentNode && - // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. - node.parentNode.getAttribute && - // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. - node.parentNode.getAttribute("id") === "dust-input-bar" - ) { - // Inject the text at the cursor position. - node.textContent = - node.textContent?.slice(0, offset) + - text + - node.textContent?.slice(offset); - } - - // Scroll to the end of the input - if (inputRef.current) { - setTimeout(() => { - const element = inputRef.current; - if (element) { - element.scrollTop = element.scrollHeight; - } - }, 0); - } - - // Move the cursor to the end of the paste. - const newRange = document.createRange(); - newRange.setStart(node, offset + text.length); - newRange.setEnd(node, offset + text.length); - selection.removeAllRanges(); - selection.addRange(newRange); - }} - onKeyDown={(e) => { - // We prevent the content editable from creating italics, bold and underline. - if (e.ctrlKey || e.metaKey) { - if (e.key === "u" || e.key === "b" || e.key === "i") { - e.preventDefault(); - } - } - if (!e.shiftKey && e.key === "Enter") { + +
{ e.preventDefault(); - e.stopPropagation(); - void handleSubmit(); - } - }} - onInput={() => { - const selection = window.getSelection(); - if ( - selection && - selection.rangeCount !== 0 && - selection.isCollapsed - ) { - const range = selection.getRangeAt(0); - const node = range.endContainer; - const offset = range.endOffset; - const lastOne = node.textContent - ? node.textContent.slice(offset - 1, offset) - : null; - const preLastOne = node.textContent - ? node.textContent.slice(offset - 2, offset - 1) - : null; + // Get the plain text. + const text = e.clipboardData.getData("text/plain"); + + // If the text is single line. + if (text.indexOf("\n") === -1 && text.indexOf("\r") === -1) { + document.execCommand("insertText", false, text); + return; + } - // Mention selection logic. + const selection = window.getSelection(); + if (!selection) { + return; + } + const range = selection.getRangeAt(0); + let node = range.endContainer; + let offset = range.endOffset; + + if ( + // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. + node.getAttribute && + // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. + node.getAttribute("id") === "dust-input-bar" + ) { + const textNode = document.createTextNode(""); + node.appendChild(textNode); + node = textNode; + offset = 0; + } if ( - lastOne === "@" && - (preLastOne === " " || preLastOne === "") && - node.textContent && node.parentNode && // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. node.parentNode.getAttribute && // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. node.parentNode.getAttribute("id") === "dust-input-bar" ) { - const mentionSelectNode = document.createElement("div"); - - mentionSelectNode.style.display = "inline-block"; - mentionSelectNode.setAttribute("key", "mentionSelect"); - mentionSelectNode.className = "text-brand font-medium"; - mentionSelectNode.textContent = "@"; - mentionSelectNode.contentEditable = "false"; - - const inputNode = document.createElement("span"); - inputNode.setAttribute("ignore", "none"); - inputNode.className = classNames( - "min-w-0 px-0 py-0", - "border-none outline-none focus:outline-none focus:border-none ring-0 focus:ring-0", - "text-brand font-medium" - ); - inputNode.contentEditable = "true"; - - mentionSelectNode.appendChild(inputNode); - - const beforeTextNode = document.createTextNode( - node.textContent.slice(0, offset - 1) - ); - const afterTextNode = document.createTextNode( - node.textContent.slice(offset) - ); - - node.parentNode.replaceChild(beforeTextNode, node); - - beforeTextNode.parentNode?.insertBefore( - afterTextNode, - beforeTextNode.nextSibling - ); - beforeTextNode.parentNode?.insertBefore( - mentionSelectNode, - afterTextNode - ); - - const rect = mentionSelectNode.getBoundingClientRect(); - const position = { - left: Math.floor(rect.left) - 24, - bottom: Math.floor(window.innerHeight - rect.bottom) + 32, - }; - if (!isNaN(position.left) && !isNaN(position.bottom)) { - setAgentListPosition(position); - } - - setAgentListVisible(true); - inputNode.focus(); + // Inject the text at the cursor position. + node.textContent = + node.textContent?.slice(0, offset) + + text + + node.textContent?.slice(offset); + } - inputNode.onblur = () => { - let selected = agentListRef.current?.selected(); - setAgentListVisible(false); - setTimeout(() => { - setAgentListFilter(""); - agentListRef.current?.reset(); - }); + // Scroll to the end of the input + if (inputRef.current) { + setTimeout(() => { + const element = inputRef.current; + if (element) { + element.scrollTop = element.scrollHeight; + } + }, 0); + } - if (inputNode.getAttribute("ignore") !== "none") { - selected = null; + // Move the cursor to the end of the paste. + const newRange = document.createRange(); + newRange.setStart(node, offset + text.length); + newRange.setEnd(node, offset + text.length); + selection.removeAllRanges(); + selection.addRange(newRange); + }} + onKeyDown={(e) => { + // We prevent the content editable from creating italics, bold and underline. + if (e.ctrlKey || e.metaKey) { + if (e.key === "u" || e.key === "b" || e.key === "i") { + e.preventDefault(); + } + } + if (!e.shiftKey && e.key === "Enter") { + e.preventDefault(); + e.stopPropagation(); + void handleSubmit(); + } + }} + onInput={() => { + const selection = window.getSelection(); + if ( + selection && + selection.rangeCount !== 0 && + selection.isCollapsed + ) { + const range = selection.getRangeAt(0); + const node = range.endContainer; + const offset = range.endOffset; + + const lastOne = node.textContent + ? node.textContent.slice(offset - 1, offset) + : null; + const preLastOne = node.textContent + ? node.textContent.slice(offset - 2, offset - 1) + : null; + + // Mention selection logic. + + if ( + lastOne === "@" && + (preLastOne === " " || preLastOne === "") && + node.textContent && + node.parentNode && + // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. + node.parentNode.getAttribute && + // @ts-expect-error - parentNode is the contenteditable, it has a getAttribute. + node.parentNode.getAttribute("id") === "dust-input-bar" + ) { + const mentionSelectNode = document.createElement("div"); + + mentionSelectNode.style.display = "inline-block"; + mentionSelectNode.setAttribute("key", "mentionSelect"); + mentionSelectNode.className = "text-brand font-medium"; + mentionSelectNode.textContent = "@"; + mentionSelectNode.contentEditable = "false"; + + const inputNode = document.createElement("span"); + inputNode.setAttribute("ignore", "none"); + inputNode.className = classNames( + "min-w-0 px-0 py-0", + "border-none outline-none focus:outline-none focus:border-none ring-0 focus:ring-0", + "text-brand font-medium" + ); + inputNode.contentEditable = "true"; + + mentionSelectNode.appendChild(inputNode); + + const beforeTextNode = document.createTextNode( + node.textContent.slice(0, offset - 1) + ); + const afterTextNode = document.createTextNode( + node.textContent.slice(offset) + ); + + node.parentNode.replaceChild(beforeTextNode, node); + + beforeTextNode.parentNode?.insertBefore( + afterTextNode, + beforeTextNode.nextSibling + ); + beforeTextNode.parentNode?.insertBefore( + mentionSelectNode, + afterTextNode + ); + + const rect = mentionSelectNode.getBoundingClientRect(); + const position = { + left: Math.floor(rect.left) - 24, + bottom: + Math.floor(window.innerHeight - rect.bottom) + 32, + }; + if (!isNaN(position.left) && !isNaN(position.bottom)) { + setAgentListPosition(position); } - // console.log("SELECTED", selected); + setAgentListVisible(true); + inputNode.focus(); - // We received a selected agent configration, recover the state of the - // contenteditable and inject an AgentMention component. - if (selected) { - // Construct an AgentMention component and inject it as HTML. - const mentionNode = getAgentMentionNode(selected); + inputNode.onblur = () => { + let selected = agentListRef.current?.selected(); + setAgentListVisible(false); + setTimeout(() => { + setAgentListFilter(""); + agentListRef.current?.reset(); + }); - // This is mainly to please TypeScript. - if (!mentionNode || !mentionSelectNode.parentNode) { - return; + if (inputNode.getAttribute("ignore") !== "none") { + selected = null; } - // Replace mentionSelectNode with mentionNode. - mentionSelectNode.parentNode.replaceChild( - mentionNode, - mentionSelectNode - ); + // console.log("SELECTED", selected); - // Prepend a space to afterTextNode (this will be the space that comes after - // the mention). - afterTextNode.textContent = ` ${afterTextNode.textContent}`; + // We received a selected agent configration, recover the state of the + // contenteditable and inject an AgentMention component. + if (selected) { + // Construct an AgentMention component and inject it as HTML. + const mentionNode = getAgentMentionNode(selected); - // If afterTextNode is the last node add an invisible character to prevent a - // Chrome bugish behaviour ¯\_(ツ)_/¯ - if (afterTextNode.nextSibling === null) { - afterTextNode.textContent = `${afterTextNode.textContent}\u200B`; - } + // This is mainly to please TypeScript. + if (!mentionNode || !mentionSelectNode.parentNode) { + return; + } - // Restore the cursor, taking into account the added space. - range.setStart(afterTextNode, 1); - range.setEnd(afterTextNode, 1); - selection.removeAllRanges(); - selection.addRange(range); - } - - // We didn't receive a selected agent configuration, restore the state of the - // contenteditable and re-inject the content that was created during the - // selection process into the contenteditable. - if (!selected && mentionSelectNode.parentNode) { - mentionSelectNode.parentNode.removeChild( - mentionSelectNode - ); - - range.setStart(afterTextNode, 0); - range.setEnd(afterTextNode, 0); - selection.removeAllRanges(); - selection.addRange(range); - - // Insert the content of mentionSelectNode after beforeTextNode only if - // we're not in ignore mode unless we are in ingnore mode (the user - // backspaced into the @) - if ( - inputNode.getAttribute("ignore") === "none" || - inputNode.getAttribute("ignore") === "space" - ) { - const newTextNode = document.createTextNode( - (mentionSelectNode.textContent || "") + - (inputNode.getAttribute("ignore") === "space" - ? " " - : "") + // Replace mentionSelectNode with mentionNode. + mentionSelectNode.parentNode.replaceChild( + mentionNode, + mentionSelectNode ); - beforeTextNode.parentNode?.insertBefore( - newTextNode, - beforeTextNode.nextSibling + + // Prepend a space to afterTextNode (this will be the space that comes after + // the mention). + afterTextNode.textContent = ` ${afterTextNode.textContent}`; + + // If afterTextNode is the last node add an invisible character to prevent a + // Chrome bugish behaviour ¯\_(ツ)_/¯ + if (afterTextNode.nextSibling === null) { + afterTextNode.textContent = `${afterTextNode.textContent}\u200B`; + } + + // Restore the cursor, taking into account the added space. + range.setStart(afterTextNode, 1); + range.setEnd(afterTextNode, 1); + selection.removeAllRanges(); + selection.addRange(range); + } + + // We didn't receive a selected agent configuration, restore the state of the + // contenteditable and re-inject the content that was created during the + // selection process into the contenteditable. + if (!selected && mentionSelectNode.parentNode) { + mentionSelectNode.parentNode.removeChild( + mentionSelectNode ); + + range.setStart(afterTextNode, 0); + range.setEnd(afterTextNode, 0); + selection.removeAllRanges(); + selection.addRange(range); + + // Insert the content of mentionSelectNode after beforeTextNode only if + // we're not in ignore mode unless we are in ingnore mode (the user + // backspaced into the @) + if ( + inputNode.getAttribute("ignore") === "none" || + inputNode.getAttribute("ignore") === "space" + ) { + const newTextNode = document.createTextNode( + (mentionSelectNode.textContent || "") + + (inputNode.getAttribute("ignore") === "space" + ? " " + : "") + ); + beforeTextNode.parentNode?.insertBefore( + newTextNode, + beforeTextNode.nextSibling + ); + } } - } - }; - - // These are events on the small contentEditable that receives the user input - // and drives the agent list selection. - inputNode.onkeydown = (e) => { - // console.log("KEYDOWN", e.key); - if (e.key === "Escape") { - agentListRef.current?.reset(); - inputNode.setAttribute("ignore", "escape"); - inputNode.blur(); - e.preventDefault(); - } - if (e.key === "ArrowDown") { - agentListRef.current?.next(); - e.preventDefault(); - } - if (e.key === "ArrowUp") { - agentListRef.current?.prev(); - e.preventDefault(); - } - if (e.key === "Backspace") { - if (inputNode.textContent === "") { + }; + + // These are events on the small contentEditable that receives the user input + // and drives the agent list selection. + inputNode.onkeydown = (e) => { + // console.log("KEYDOWN", e.key); + if (e.key === "Escape") { agentListRef.current?.reset(); - inputNode.setAttribute("ignore", "backspace"); + inputNode.setAttribute("ignore", "escape"); inputNode.blur(); e.preventDefault(); } - } - if (e.key === " ") { - if (agentListRef.current?.perfectMatch()) { - inputNode.blur(); + if (e.key === "ArrowDown") { + agentListRef.current?.next(); e.preventDefault(); - } else { - agentListRef.current?.reset(); - inputNode.setAttribute("ignore", "space"); - inputNode.blur(); + } + if (e.key === "ArrowUp") { + agentListRef.current?.prev(); e.preventDefault(); } - } - if (e.key === "Enter") { - inputNode.blur(); - e.preventDefault(); - } - }; - - // These are the event that drive the selection of the the agent list, if we - // have no more match we just blur to exit the selection process. - inputNode.oninput = (e) => { - const target = e.target as HTMLInputElement; - // console.log("INPUT", target.textContent); - setAgentListFilter(target.textContent || ""); - e.stopPropagation(); - setTimeout(() => { - if (agentListRef.current?.noMatch()) { - agentListRef.current?.reset(); + if (e.key === "Backspace") { + if (inputNode.textContent === "") { + agentListRef.current?.reset(); + inputNode.setAttribute("ignore", "backspace"); + inputNode.blur(); + e.preventDefault(); + } + } + if (e.key === " ") { + if (agentListRef.current?.perfectMatch()) { + inputNode.blur(); + e.preventDefault(); + } else { + agentListRef.current?.reset(); + inputNode.setAttribute("ignore", "space"); + inputNode.blur(); + e.preventDefault(); + } + } + if (e.key === "Enter") { inputNode.blur(); + e.preventDefault(); } - }); - }; + }; + + // These are the event that drive the selection of the the agent list, if we + // have no more match we just blur to exit the selection process. + inputNode.oninput = (e) => { + const target = e.target as HTMLInputElement; + // console.log("INPUT", target.textContent); + setAgentListFilter(target.textContent || ""); + e.stopPropagation(); + setTimeout(() => { + if (agentListRef.current?.noMatch()) { + agentListRef.current?.reset(); + inputNode.blur(); + } + }); + }; + } } - } - }} - >
+ }} + >
+
void; + onSubmit: ( + input: string, + mentions: MentionType[], + contentFragment?: { title: string; content: string } + ) => void; stickyMentions?: AgentMention[]; conversationId: string | null; }) { diff --git a/front/lib/api/assistant/conversation.ts b/front/lib/api/assistant/conversation.ts index 94ff2de29f1e..3b2823c0b460 100644 --- a/front/lib/api/assistant/conversation.ts +++ b/front/lib/api/assistant/conversation.ts @@ -41,6 +41,7 @@ import logger from "@app/logger/logger"; import { AgentMessageType, ContentFragmentContentType, + ContentFragmentContextType, ContentFragmentType, ConversationType, ConversationVisibility, @@ -339,6 +340,12 @@ function renderContentFragment({ content: contentFragment.content, url: contentFragment.url, contentType: contentFragment.contentType, + context: { + profilePictureUrl: contentFragment.userContextProfilePictureUrl, + fullName: contentFragment.userContextFullName, + email: contentFragment.userContextEmail, + username: contentFragment.userContextUsername, + }, }; } @@ -1675,12 +1682,14 @@ export async function postNewContentFragment( content, url, contentType, + context, }: { conversation: ConversationType; title: string; content: string; url: string | null; contentType: ContentFragmentContentType; + context: ContentFragmentContextType; } ): Promise { const owner = auth.workspace(); @@ -1694,7 +1703,17 @@ export async function postNewContentFragment( await getConversationRankVersionLock(conversation, t); const contentFragmentRow = await ContentFragment.create( - { content, title, url, contentType }, + { + content, + title, + url, + contentType, + userId: auth.user()?.id, + userContextProfilePictureUrl: context.profilePictureUrl, + userContextEmail: context.email, + userContextFullName: context.fullName, + userContextUsername: context.username, + }, { transaction: t } ); const nextMessageRank = diff --git a/front/lib/client/handle_file_upload.ts b/front/lib/client/handle_file_upload.ts new file mode 100644 index 000000000000..c5dd55e37d9c --- /dev/null +++ b/front/lib/client/handle_file_upload.ts @@ -0,0 +1,93 @@ +// @ts-expect-error: type package doesn't load properly because of how we are loading pdfjs +import * as PDFJS from "pdfjs-dist/build/pdf"; + +import { Err, Ok, Result } from "../result"; +PDFJS.GlobalWorkerOptions.workerSrc = `//cdnjs.cloudflare.com/ajax/libs/pdf.js/${PDFJS.version}/pdf.worker.min.js`; + +export async function handleFileUploadToText( + file: File +): Promise> { + return new Promise((resolve) => { + const handleFileLoadedText = (e: ProgressEvent) => { + const content = e.target?.result; + if (content && typeof content === "string") { + return resolve(new Ok({ title: file.name, content })); + } else { + return resolve( + new Err( + new Error( + "Failed extracting text from file. Please check that your file is not empty." + ) + ) + ); + return; + } + }; + const handleFileLoadedPDF = async (e: ProgressEvent) => { + try { + const arrayBuffer = e.target?.result; + if (!(arrayBuffer instanceof ArrayBuffer)) { + return resolve( + new Err( + new Error("Failed extracting text from PDF. Unexpected error") + ) + ); + } + const loadingTask = PDFJS.getDocument({ data: arrayBuffer }); + const pdf = await loadingTask.promise; + let text = ""; + for (let pageNum = 1; pageNum <= pdf.numPages; pageNum++) { + const page = await pdf.getPage(pageNum); + const content = await page.getTextContent(); + const strings = content.items.map((item: unknown) => { + if ( + item && + typeof item === "object" && + "str" in item && + typeof item.str === "string" + ) { + return item.str; + } + }); + text += strings.join(" ") + "\n"; + } + return resolve(new Ok({ title: file.name, content: text })); + } catch (e) { + console.error("Failed extracting text from PDF", e); + const errorMessage = + e instanceof Error ? e.message : "Unexpected error"; + return resolve( + new Err(new Error(`Failed extracting text from PDF. ${errorMessage}`)) + ); + } + }; + + try { + if (file.type === "application/pdf") { + const fileReader = new FileReader(); + fileReader.onloadend = handleFileLoadedPDF; + fileReader.readAsArrayBuffer(file); + } else if ( + ["text/plain", "text/csv", "text/markdown"].includes(file.type) + ) { + const fileData = new FileReader(); + fileData.onloadend = handleFileLoadedText; + fileData.readAsText(file); + } else { + return resolve( + new Err( + new Error( + "File type not supported. Supported file types: .txt, .pdf, .md" + ) + ) + ); + } + } catch (e) { + console.error("Error handling file", e); + const errorMessage = e instanceof Error ? e.message : "Unexpected error"; + return resolve( + new Err(new Error(`Error handling file. ${errorMessage}`)) + ); + } + }); +} diff --git a/front/lib/models/assistant/conversation.ts b/front/lib/models/assistant/conversation.ts index 0f7f8534075f..bd0e35939720 100644 --- a/front/lib/models/assistant/conversation.ts +++ b/front/lib/models/assistant/conversation.ts @@ -351,6 +351,13 @@ export class ContentFragment extends Model< declare content: string; declare url: string | null; declare contentType: ContentFragmentContentType; + + declare userContextUsername: string | null; + declare userContextFullName: string | null; + declare userContextEmail: string | null; + declare userContextProfilePictureUrl: string | null; + + declare userId: ForeignKey | null; } ContentFragment.init( @@ -386,6 +393,22 @@ ContentFragment.init( type: DataTypes.STRING, allowNull: false, }, + userContextProfilePictureUrl: { + type: DataTypes.STRING, + allowNull: true, + }, + userContextUsername: { + type: DataTypes.STRING, + allowNull: true, + }, + userContextFullName: { + type: DataTypes.STRING, + allowNull: true, + }, + userContextEmail: { + type: DataTypes.STRING, + allowNull: true, + }, }, { modelName: "content_fragment", @@ -393,6 +416,13 @@ ContentFragment.init( } ); +User.hasMany(ContentFragment, { + foreignKey: { name: "userId", allowNull: true }, // null = ContentFragment is not associated with a user +}); +ContentFragment.belongsTo(User, { + foreignKey: { name: "userId", allowNull: true }, +}); + export class Message extends Model< InferAttributes, InferCreationAttributes diff --git a/front/pages/api/v1/w/[wId]/assistant/conversations/[cId]/content_fragments.ts b/front/pages/api/v1/w/[wId]/assistant/conversations/[cId]/content_fragments.ts index d3b2e08ddf72..40e71974fcfe 100644 --- a/front/pages/api/v1/w/[wId]/assistant/conversations/[cId]/content_fragments.ts +++ b/front/pages/api/v1/w/[wId]/assistant/conversations/[cId]/content_fragments.ts @@ -20,9 +20,25 @@ export const PostContentFragmentRequestBodySchema = t.type({ title: t.string, content: t.string, url: t.union([t.string, t.null]), - contentType: t.literal("slack_thread_content"), + contentType: t.union([ + t.literal("slack_thread_content"), + t.literal("file_attachment"), + ]), + context: t.union([ + t.type({ + profilePictureUrl: t.union([t.string, t.null]), + fullName: t.union([t.string, t.null]), + email: t.union([t.string, t.null]), + username: t.union([t.string, t.null]), + }), + t.null, + ]), }); +export type PostContentFragmentRequestBody = t.TypeOf< + typeof PostContentFragmentRequestBodySchema +>; + async function handler( req: NextApiRequest, res: NextApiResponse @@ -76,7 +92,8 @@ async function handler( }); } - const { content, title, url, contentType } = bodyValidation.right; + const { content, title, url, contentType, context } = + bodyValidation.right; if (content.length === 0 || content.length > 64 * 1024) { return apiError(req, res, { @@ -95,6 +112,12 @@ async function handler( content, url, contentType, + context: { + username: context?.username || null, + fullName: context?.fullName || null, + email: context?.email || null, + profilePictureUrl: context?.profilePictureUrl || null, + }, }); res.status(200).json({ contentFragment }); diff --git a/front/pages/api/v1/w/[wId]/assistant/conversations/index.ts b/front/pages/api/v1/w/[wId]/assistant/conversations/index.ts index aca1154eb4e2..cfdd3eb9643d 100644 --- a/front/pages/api/v1/w/[wId]/assistant/conversations/index.ts +++ b/front/pages/api/v1/w/[wId]/assistant/conversations/index.ts @@ -125,6 +125,13 @@ async function handler( content: contentFragment.content, url: contentFragment.url, contentType: contentFragment.contentType, + context: { + username: contentFragment.context?.username || null, + fullName: contentFragment.context?.fullName || null, + email: contentFragment.context?.email || null, + profilePictureUrl: + contentFragment.context?.profilePictureUrl || null, + }, }); newContentFragment = cf; diff --git a/front/pages/api/w/[wId]/assistant/conversations/[cId]/content_fragment/index.ts b/front/pages/api/w/[wId]/assistant/conversations/[cId]/content_fragment/index.ts new file mode 100644 index 000000000000..ff96929584e2 --- /dev/null +++ b/front/pages/api/w/[wId]/assistant/conversations/[cId]/content_fragment/index.ts @@ -0,0 +1,137 @@ +import { isLeft } from "fp-ts/lib/Either"; +import * as t from "io-ts"; +import * as reporter from "io-ts-reporters"; +import { NextApiRequest, NextApiResponse } from "next"; + +import { + getConversation, + postNewContentFragment, +} from "@app/lib/api/assistant/conversation"; +import { Authenticator, getSession } from "@app/lib/auth"; +import { ReturnedAPIErrorType } from "@app/lib/error"; +import { apiError, withLogging } from "@app/logger/withlogging"; +import { ContentFragmentType } from "@app/types/assistant/conversation"; + +export const PostContentFragmentRequestBodySchema = t.type({ + title: t.string, + content: t.string, + url: t.union([t.string, t.null]), + contentType: t.union([ + t.literal("slack_thread_content"), + t.literal("file_attachment"), + ]), + context: t.type({ + timezone: t.string, + profilePictureUrl: t.union([t.string, t.null]), + }), +}); + +export type PostContentFragmentRequestBody = t.TypeOf< + typeof PostContentFragmentRequestBodySchema +>; + +async function handler( + req: NextApiRequest, + res: NextApiResponse< + { contentFragment: ContentFragmentType } | ReturnedAPIErrorType + > +): Promise { + const session = await getSession(req, res); + const auth = await Authenticator.fromSession( + session, + req.query.wId as string + ); + + const owner = auth.workspace(); + if (!owner) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_not_found", + message: "The workspace you're trying to modify was not found.", + }, + }); + } + + const user = auth.user(); + if (!user || !auth.isUser()) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "workspace_user_not_found", + message: "Could not find the user of the current session.", + }, + }); + } + + if (!(typeof req.query.cId === "string")) { + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: "Invalid query parameters, `cId` (string) is required.", + }, + }); + } + + const conversationId = req.query.cId; + const conversation = await getConversation(auth, conversationId); + if (!conversation) { + return apiError(req, res, { + status_code: 404, + api_error: { + type: "conversation_not_found", + message: "Conversation not found.", + }, + }); + } + + switch (req.method) { + case "POST": + const bodyValidation = PostContentFragmentRequestBodySchema.decode( + req.body + ); + + if (isLeft(bodyValidation)) { + const pathError = reporter.formatValidationErrors(bodyValidation.left); + + return apiError(req, res, { + status_code: 400, + api_error: { + type: "invalid_request_error", + message: `Invalid request body: ${pathError}`, + }, + }); + } + + const contentFragmentPayload = bodyValidation.right; + + const contentFragment = await postNewContentFragment(auth, { + conversation, + title: contentFragmentPayload.title, + content: contentFragmentPayload.content, + url: contentFragmentPayload.url, + contentType: contentFragmentPayload.contentType, + context: { + username: user.username, + fullName: user.fullName, + email: user.email, + profilePictureUrl: contentFragmentPayload.context.profilePictureUrl, + }, + }); + + res.status(200).json({ contentFragment }); + return; + + default: + return apiError(req, res, { + status_code: 405, + api_error: { + type: "method_not_supported_error", + message: "The method passed is not supported, POST is expected.", + }, + }); + } +} + +export default withLogging(handler); diff --git a/front/pages/api/w/[wId]/assistant/conversations/index.ts b/front/pages/api/w/[wId]/assistant/conversations/index.ts index 12fc00440ede..0130a40abbf2 100644 --- a/front/pages/api/w/[wId]/assistant/conversations/index.ts +++ b/front/pages/api/w/[wId]/assistant/conversations/index.ts @@ -5,7 +5,9 @@ import { NextApiRequest, NextApiResponse } from "next"; import { createConversation, + getConversation, getUserConversations, + postNewContentFragment, } from "@app/lib/api/assistant/conversation"; import { postUserMessageWithPubSub } from "@app/lib/api/assistant/pubsub"; import { Authenticator, getSession } from "@app/lib/auth"; @@ -13,11 +15,14 @@ import { ReturnedAPIErrorType } from "@app/lib/error"; import { apiError, withLogging } from "@app/logger/withlogging"; import { PostMessagesRequestBodySchema } from "@app/pages/api/w/[wId]/assistant/conversations/[cId]/messages"; import { + ContentFragmentType, ConversationType, ConversationWithoutContentType, UserMessageType, } from "@app/types/assistant/conversation"; +import { PostContentFragmentRequestBodySchema } from "./[cId]/content_fragment"; + export const PostConversationsRequestBodySchema = t.type({ title: t.union([t.string, t.null]), visibility: t.union([ @@ -26,6 +31,7 @@ export const PostConversationsRequestBodySchema = t.type({ t.literal("deleted"), ]), message: t.union([PostMessagesRequestBodySchema, t.null]), + contentFragment: t.union([PostContentFragmentRequestBodySchema, t.undefined]), }); export type GetConversationsResponseBody = { @@ -34,6 +40,7 @@ export type GetConversationsResponseBody = { export type PostConversationsResponseBody = { conversation: ConversationType; message?: UserMessageType; + contentFragment?: ContentFragmentType; }; async function handler( @@ -117,15 +124,42 @@ async function handler( }); } - const { title, visibility, message } = bodyValidation.right; + const { title, visibility, message, contentFragment } = + bodyValidation.right; - const conversation = await createConversation(auth, { + let conversation = await createConversation(auth, { title, visibility, }); + let newContentFragment: ContentFragmentType | null = null; let newMessage: UserMessageType | null = null; + if (contentFragment) { + const cf = await postNewContentFragment(auth, { + conversation, + title: contentFragment.title, + content: contentFragment.content, + url: contentFragment.url, + contentType: contentFragment.contentType, + context: { + username: user.username, + fullName: user.fullName, + email: user.email, + profilePictureUrl: contentFragment.context.profilePictureUrl, + }, + }); + + newContentFragment = cf; + const updatedConversation = await getConversation( + auth, + conversation.sId + ); + if (updatedConversation) { + conversation = updatedConversation; + } + } + if (message) { /* If a message was provided we do await for the message to be created before returning the conversation along with the message. @@ -150,7 +184,26 @@ async function handler( newMessage = messageRes.value; } - res.status(200).json({ conversation, message: newMessage ?? undefined }); + if (newContentFragment || newMessage) { + // If we created a user message or a content fragment (or both) we retrieve the + // conversation. If a user message was posted, we know that the agent messages have been + // created as well, so pulling the conversation again will allow to have an up to date view + // of the conversation with agent messages included so that the user of the API can start + // streaming events from these agent messages directly. + const updated = await getConversation(auth, conversation.sId); + + if (!updated) { + throw `Conversation unexpectedly not found after creation: ${conversation.sId}`; + } + + conversation = updated; + } + + res.status(200).json({ + conversation, + message: newMessage ?? undefined, + contentFragment: newContentFragment ?? undefined, + }); return; default: diff --git a/front/pages/w/[wId]/assistant/[cId]/index.tsx b/front/pages/w/[wId]/assistant/[cId]/index.tsx index b6938097b8c2..9a4d2ff81f16 100644 --- a/front/pages/w/[wId]/assistant/[cId]/index.tsx +++ b/front/pages/w/[wId]/assistant/[cId]/index.tsx @@ -83,7 +83,49 @@ export default function AssistantConversation({ conversationId, workspaceId: owner.sId, }); - const handleSubmit = async (input: string, mentions: MentionType[]) => { + const handleSubmit = async ( + input: string, + mentions: MentionType[], + contentFragment?: { + title: string; + content: string; + } + ) => { + // Create a new content fragment. + if (contentFragment) { + const mcfRes = await fetch( + `/api/w/${owner.sId}/assistant/conversations/${conversationId}/content_fragment`, + { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + title: contentFragment.title, + content: contentFragment.content, + url: null, + contentType: "file_attachment", + context: { + timezone: + Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC", + profilePictureUrl: user.image, + }, + }), + } + ); + + if (!mcfRes.ok) { + const data = await mcfRes.json(); + console.error("Error creating content fragment", data); + sendNotification({ + title: "Error uploading file.", + description: data.error.message || "Please try again or contact us.", + type: "error", + }); + return; + } + } + // Create a new user message. const mRes = await fetch( `/api/w/${owner.sId}/assistant/conversations/${conversationId}/messages`, diff --git a/front/pages/w/[wId]/assistant/new.tsx b/front/pages/w/[wId]/assistant/new.tsx index 9d059aea1b66..0b2ead45b3bd 100644 --- a/front/pages/w/[wId]/assistant/new.tsx +++ b/front/pages/w/[wId]/assistant/new.tsx @@ -33,6 +33,7 @@ import type { PostConversationsResponseBody, } from "@app/pages/api/w/[wId]/assistant/conversations"; import { + ContentFragmentContentType, ConversationType, MentionType, } from "@app/types/assistant/conversation"; @@ -99,7 +100,14 @@ export default function AssistantNew({ : activeAgents.slice(0, 4); const { submit: handleSubmit } = useSubmitFunction( - async (input: string, mentions: MentionType[]) => { + async ( + input: string, + mentions: MentionType[], + contentFragment?: { + title: string; + content: string; + } + ) => { const body: t.TypeOf = { title: null, visibility: "unlisted", @@ -111,6 +119,18 @@ export default function AssistantNew({ }, mentions, }, + contentFragment: contentFragment + ? { + ...contentFragment, + contentType: "file_attachment", + url: null, + context: { + timezone: + Intl.DateTimeFormat().resolvedOptions().timeZone || "UTC", + profilePictureUrl: user.image, + }, + } + : undefined, }; // Create new conversation and post the initial message at the same time. @@ -379,7 +399,15 @@ function StartHelperConversationButton({ size = "xs", }: { content: string; - handleSubmit: (input: string, mentions: MentionType[]) => Promise; + handleSubmit: ( + input: string, + mentions: MentionType[], + contentFragment?: { + title: string; + content: string; + contentType: ContentFragmentContentType; + } + ) => Promise; variant?: "primary" | "secondary"; size?: "sm" | "xs"; }) { diff --git a/front/pages/w/[wId]/builder/data-sources/[name]/upsert.tsx b/front/pages/w/[wId]/builder/data-sources/[name]/upsert.tsx index b7bdc5f5d391..96ff49b73e3e 100644 --- a/front/pages/w/[wId]/builder/data-sources/[name]/upsert.tsx +++ b/front/pages/w/[wId]/builder/data-sources/[name]/upsert.tsx @@ -23,6 +23,7 @@ import { subNavigationAdmin } from "@app/components/sparkle/navigation"; import { SendNotificationsContext } from "@app/components/sparkle/Notification"; import { getDataSource } from "@app/lib/api/data_sources"; import { Authenticator, getSession, getUserFromSession } from "@app/lib/auth"; +import { handleFileUploadToText } from "@app/lib/client/handle_file_upload"; import { classNames } from "@app/lib/utils"; import { DataSourceType } from "@app/types/data_source"; import { PlanType } from "@app/types/plan"; @@ -138,66 +139,6 @@ export default function DataSourceUpsert({ }); }; - const handleFileLoadedText = (e: any) => { - const content = e.target.result; - setUploading(false); - - // Enforce plan limits: DataSource documents size. - if ( - plan.limits.dataSources.documents.sizeMb != -1 && - text.length > 1024 * 1024 * plan.limits.dataSources.documents.sizeMb - ) { - alertDataSourcesLimit(); - return; - } - setText(content); - }; - - const handleFileLoadedPDF = async (e: any) => { - const arrayBuffer = e.target.result; - const loadingTask = PDFJS.getDocument({ data: arrayBuffer }); - const pdf = await loadingTask.promise; - let text = ""; - for (let pageNum = 1; pageNum <= pdf.numPages; pageNum++) { - const page = await pdf.getPage(pageNum); - const content = await page.getTextContent(); - const strings = content.items.map((item: any) => item.str); - text += strings.join(" ") + "\n"; - } - - setUploading(false); - - // Enforce plan limits: DataSource documents size. - if ( - plan.limits.dataSources.documents.sizeMb != -1 && - text.length > 1024 * 1024 * plan.limits.dataSources.documents.sizeMb - ) { - alertDataSourcesLimit(); - return; - } - setText(text); - }; - - const handleFileUpload = async (file: File) => { - setUploading(true); - if (file.type === "application/pdf") { - const fileReader = new FileReader(); - fileReader.onloadend = handleFileLoadedPDF; - fileReader.readAsArrayBuffer(file); - } else if (file.type === "text/plain" || file.type === "text/csv") { - const fileData = new FileReader(); - fileData.onloadend = handleFileLoadedText; - fileData.readAsText(file); - } else { - sendNotification({ - type: "error", - title: "File type not supported", - description: `Supported file types are: .txt, .pdf, .md, .csv.`, - }); - setUploading(false); - } - }; - const router = useRouter(); const handleUpsert = async () => { @@ -340,7 +281,26 @@ export default function DataSourceUpsert({ ref={fileInputRef} onChange={async (e) => { if (e.target.files && e.target.files.length > 0) { - await handleFileUpload(e.target.files[0]); + if ( + plan.limits.dataSources.documents.sizeMb != -1 && + text.length > + 1024 * 1024 * plan.limits.dataSources.documents.sizeMb + ) { + alertDataSourcesLimit(); + return; + } + setUploading(true); + const res = await handleFileUploadToText(e.target.files[0]); + setUploading(false); + if (res.isErr()) { + sendNotification({ + type: "error", + title: "Error uploading file.", + description: res.error.message, + }); + return; + } + setText(res.value.content); } }} > diff --git a/front/types/assistant/conversation.ts b/front/types/assistant/conversation.ts index b30b3ddca668..7374d312bc98 100644 --- a/front/types/assistant/conversation.ts +++ b/front/types/assistant/conversation.ts @@ -124,7 +124,16 @@ export function isAgentMessageType( /** * Content Fragments */ -export type ContentFragmentContentType = "slack_thread_content"; +export type ContentFragmentContextType = { + username: string | null; + fullName: string | null; + email: string | null; + profilePictureUrl: string | null; +}; + +export type ContentFragmentContentType = + | "slack_thread_content" + | "file_attachment"; export type ContentFragmentType = { id: ModelId; @@ -138,6 +147,7 @@ export type ContentFragmentType = { content: string; url: string | null; contentType: ContentFragmentContentType; + context: ContentFragmentContextType; }; export function isContentFragmentType(