From c5270e49f77df3fa583d5f57dd617d092ca5e1d5 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 15:00:58 +0100 Subject: [PATCH 01/22] jit actions scafolding --- front/lib/api/assistant/actions/list_files.ts | 439 ++++++++++++++++++ front/lib/api/assistant/agent.ts | 2 + front/lib/api/assistant/generation.ts | 2 + types/src/front/assistant/agent.ts | 4 +- .../front/lib/api/assistant/actions/index.ts | 3 +- types/src/index.ts | 1 + 6 files changed, 449 insertions(+), 2 deletions(-) create mode 100644 front/lib/api/assistant/actions/list_files.ts diff --git a/front/lib/api/assistant/actions/list_files.ts b/front/lib/api/assistant/actions/list_files.ts new file mode 100644 index 000000000000..4eb6f48369a7 --- /dev/null +++ b/front/lib/api/assistant/actions/list_files.ts @@ -0,0 +1,439 @@ +import { DustAPI } from "@dust-tt/client"; +import type { + DustAppRunBlockEvent, + DustAppRunErrorEvent, + DustAppRunParamsEvent, + DustAppRunSuccessEvent, + FunctionCallType, + FunctionMessageTypeModel, + JITListFilesConfigurationType, + ModelId, +} from "@dust-tt/types"; +import type { DustAppParameters, DustAppRunActionType } from "@dust-tt/types"; +import type { AgentActionSpecification } from "@dust-tt/types"; +import type { SpecificationType } from "@dust-tt/types"; +import type { DatasetSchema } from "@dust-tt/types"; +import type { Result } from "@dust-tt/types"; +import { BaseAction } from "@dust-tt/types"; +import { Err, Ok } from "@dust-tt/types"; + +import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; +import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; +import config from "@app/lib/api/config"; +import { getDatasetSchema } from "@app/lib/api/datasets"; +import type { Authenticator } from "@app/lib/auth"; +import { prodAPICredentialsForOwner } from "@app/lib/auth"; +import { extractConfig } from "@app/lib/config"; +import { AgentDustAppRunAction } from "@app/lib/models/assistant/actions/dust_app_run"; +import { AppResource } from "@app/lib/resources/app_resource"; +import { sanitizeJSONOutput } from "@app/lib/utils"; +import logger from "@app/logger/logger"; + +interface JITListFilesActionBlob { + agentMessageId: ModelId; + functionCallId: string | null; + functionCallName: string | null; + files: string[]; + step: number; +} + +export class JITListFilesAction extends BaseAction { + readonly agentMessageId: ModelId; + readonly files: string[]; + readonly functionCallId: string | null; + readonly functionCallName: string | null; + readonly step: number; + readonly type = "jit_list_files_action"; + + constructor(blob: JITListFilesActionBlob) { + super(-1, "jit_list_files_action"); + + this.agentMessageId = blob.agentMessageId; + this.files = blob.files; + this.functionCallId = blob.functionCallId; + this.functionCallName = blob.functionCallName; + this.step = blob.step; + } + + renderForFunctionCall(): FunctionCallType { + return { + id: this.functionCallId ?? `call_${this.id.toString()}`, + name: this.functionCallName ?? "list_conversation_files", + arguments: JSON.stringify({}), + }; + } + + renderForMultiActionsModel(): FunctionMessageTypeModel { + let content = "CONVERSATION FILES:\n"; + for (const file of this.files) { + content += `${file}\n`; + } + + return { + role: "function" as const, + name: this.functionCallName ?? "list_conversation_files", + function_call_id: this.functionCallId ?? `call_${this.id.toString()}`, + content, + }; + } +} + +/** + * Params generation. + */ + +export class JITListFileConfigurationServerRunner extends BaseActionConfigurationServerRunner { + // Generates the action specification for generation of rawInputs passed to `run`. + async buildSpecification( + _auth: Authenticator, + { name, description }: { name: string; description: string | null } + ): Promise> { + return new Ok({ + name, + description: + description || + "Retrieve the list of files attached to the conversation", + inputs: [], + }); + } + + // This method is in charge of running a dust app and creating an AgentDustAppRunAction object in + // the database. It does not create any generic model related to the conversation. It is possible + // for an AgentDustAppRunAction to be stored (once the params are infered) but for the dust app run + // to fail, in which case an error event will be emitted and the AgentDustAppRunAction won't have + // any output associated. The error is expected to be stored by the caller on the parent agent + // message. + async *run( + auth: Authenticator, + { + agentConfiguration, + conversation, + agentMessage, + rawInputs, + functionCallId, + step, + }: BaseActionRunParams, + { + spec, + }: { + spec: AgentActionSpecification; + } + ): AsyncGenerator< + | DustAppRunParamsEvent + | DustAppRunBlockEvent + | DustAppRunSuccessEvent + | DustAppRunErrorEvent, + void + > { + const owner = auth.workspace(); + if (!owner) { + throw new Error("Unexpected unauthenticated call to `run`"); + } + + const { actionConfiguration } = this; + + const app = await AppResource.fetchById(auth, actionConfiguration.appId); + if (!app) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_parameters_generation_error", + message: + "Failed to retrieve Dust app " + + `${actionConfiguration.appWorkspaceId}/${actionConfiguration.appId}`, + }, + }; + return; + } + + const appConfig = extractConfig(JSON.parse(app.savedSpecification || `{}`)); + + // Check that all inputs are accounted for. + const params: DustAppParameters = {}; + + for (const k of spec.inputs) { + if (k.name in rawInputs && typeof rawInputs[k.name] === k.type) { + // As defined in dustAppRunActionSpecification, type is either "string", "number" or "boolean" + params[k.name] = rawInputs[k.name] as string | number | boolean; + } else { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_parameters_generation_error", + message: `Failed to generate input ${k.name} (expected type ${ + k.type + }, got ${rawInputs[k.name]})`, + }, + }; + return; + } + } + + // Create the AgentDustAppRunAction object in the database and yield an event for the generation + // of the params. We store the action here as the params have been generated, if an error occurs + // later on, the action won't have an output but the error will be stored on the parent agent + // message. + const action = await AgentDustAppRunAction.create({ + dustAppRunConfigurationId: actionConfiguration.sId, + appWorkspaceId: actionConfiguration.appWorkspaceId, + appId: actionConfiguration.appId, + appName: app.name, + params, + functionCallId, + functionCallName: actionConfiguration.name, + agentMessageId: agentMessage.agentMessageId, + step, + }); + + yield { + type: "dust_app_run_params", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + action: new DustAppRunAction({ + id: action.id, + appWorkspaceId: actionConfiguration.appWorkspaceId, + appId: actionConfiguration.appId, + appName: app.name, + params, + runningBlock: null, + output: null, + functionCallId, + functionCallName: actionConfiguration.name, + agentMessageId: agentMessage.agentMessageId, + step, + }), + }; + + // Let's run the app now. + const now = Date.now(); + + const prodCredentials = await prodAPICredentialsForOwner(owner, { + useLocalInDev: true, + }); + const requestedGroupIds = auth.groups().map((g) => g.sId); + const api = new DustAPI( + config.getDustAPIConfig(), + { ...prodCredentials, groupIds: requestedGroupIds }, + logger, + { + useLocalInDev: true, + } + ); + + // As we run the app (using a system API key here), we do force using the workspace credentials so + // that the app executes in the exact same conditions in which they were developed. + const runRes = await api.runAppStreamed( + { + workspaceId: actionConfiguration.appWorkspaceId, + appId: actionConfiguration.appId, + appSpaceId: app.space.sId, + appHash: "latest", + }, + appConfig, + [params], + { useWorkspaceCredentials: true } + ); + + if (runRes.isErr()) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Error running Dust app: ${runRes.error.message}`, + }, + }; + return; + } + + const { eventStream, dustRunId } = runRes.value; + let lastBlockOutput: unknown | null = null; + + for await (const event of eventStream) { + if (event.type === "error") { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Error running Dust app: ${event.content.message}`, + }, + }; + return; + } + + if (event.type === "block_status") { + yield { + type: "dust_app_run_block", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + action: new DustAppRunAction({ + id: action.id, + appWorkspaceId: actionConfiguration.appWorkspaceId, + appId: actionConfiguration.appId, + appName: app.name, + params, + functionCallId, + functionCallName: actionConfiguration.name, + runningBlock: { + type: event.content.block_type, + name: event.content.name, + status: event.content.status, + }, + output: null, + agentMessageId: agentMessage.agentMessageId, + step: action.step, + }), + }; + } + + if (event.type === "block_execution") { + const e = event.content.execution[0][0]; + if (e.error) { + yield { + type: "dust_app_run_error", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + error: { + code: "dust_app_run_error", + message: `Error running Dust app: ${e.error}`, + }, + }; + return; + } + + lastBlockOutput = e.value; + } + } + + const output = sanitizeJSONOutput(lastBlockOutput); + + // Update DustAppRunAction with the output of the last block. + await action.update({ + runId: await dustRunId, + output, + }); + + logger.info( + { + workspaceId: conversation.owner.sId, + conversationId: conversation.sId, + elapsed: Date.now() - now, + }, + "[ASSISTANT_TRACE] DustAppRun acion run execution" + ); + + yield { + type: "dust_app_run_success", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + action: new DustAppRunAction({ + id: action.id, + appWorkspaceId: actionConfiguration.appWorkspaceId, + appId: actionConfiguration.appId, + appName: app.name, + params, + functionCallId, + functionCallName: actionConfiguration.name, + runningBlock: null, + output, + agentMessageId: agentMessage.agentMessageId, + step: action.step, + }), + }; + } +} + +async function dustAppRunActionSpecification({ + schema, + name, + description, +}: { + schema: DatasetSchema | null; + name: string; + description: string; +}): Promise> { + // If we have no schema (aka no input block) there is no need to generate any input. + if (!schema) { + return new Ok({ + name, + description, + inputs: [], + }); + } + + const inputs: { + name: string; + description: string; + type: "string" | "number" | "boolean"; + }[] = []; + + for (const k of schema) { + if (k.type === "json") { + return new Err( + new Error( + `JSON type for Dust app parameters is not supported, string, number and boolean are.` + ) + ); + } + + inputs.push({ + name: k.key, + description: k.description || "", + type: k.type, + }); + } + + return new Ok({ + name, + description, + inputs, + }); +} + +/** + * Action rendering. + */ + +// Internal interface for the retrieval and rendering of a DustAppRun action. This should not be +// used outside of api/assistant. We allow a ModelId interface here because we don't have `sId` on +// actions (the `sId` is on the `Message` object linked to the `UserMessage` parent of this action). +export async function dustAppRunTypesFromAgentMessageIds( + agentMessageIds: ModelId[] +): Promise { + const actions = await AgentDustAppRunAction.findAll({ + where: { + agentMessageId: agentMessageIds, + }, + }); + + return actions.map((action) => { + return new DustAppRunAction({ + id: action.id, + appWorkspaceId: action.appWorkspaceId, + appId: action.appId, + appName: action.appName, + params: action.params, + runningBlock: null, + output: action.output, + functionCallId: action.functionCallId, + functionCallName: action.functionCallName, + agentMessageId: action.agentMessageId, + step: action.step, + }); + }); +} diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index fb88edfd1fa1..4b4735a1ca09 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -86,6 +86,8 @@ export async function* runAgent( throw new Error("Unreachable: could not find owner workspace for agent"); } + // console.log(JSON.stringify(conversation)); + const stream = runMultiActionsAgentLoop( auth, fullConfiguration, diff --git a/front/lib/api/assistant/generation.ts b/front/lib/api/assistant/generation.ts index 4f64713a02f5..abe018e39429 100644 --- a/front/lib/api/assistant/generation.ts +++ b/front/lib/api/assistant/generation.ts @@ -522,6 +522,8 @@ async function renderConversationForModelMultiActions({ "[ASSISTANT_TRACE] renderConversationForModelMultiActions" ); + // console.log("MESSAGES\n", selected.map((m) => JSON.stringify(m)).join("\n")); + return new Ok({ modelConversation: { messages: selected, diff --git a/types/src/front/assistant/agent.ts b/types/src/front/assistant/agent.ts index c6c13ffc3128..9eda6b292c38 100644 --- a/types/src/front/assistant/agent.ts +++ b/types/src/front/assistant/agent.ts @@ -5,6 +5,7 @@ import { TablesQueryConfigurationType } from "../../front/assistant/actions/tabl import { ModelIdType, ModelProviderIdType } from "../../front/lib/assistant"; import { ModelId } from "../../shared/model_id"; import { BrowseConfigurationType } from "./actions/browse"; +import { JITListFilesConfigurationType } from "./actions/jit/list_files"; import { WebsearchConfigurationType } from "./actions/websearch"; /** @@ -20,7 +21,8 @@ export type AgentActionConfigurationType = | DustAppRunConfigurationType | ProcessConfigurationType | WebsearchConfigurationType - | BrowseConfigurationType; + | BrowseConfigurationType + | JITListFilesConfigurationType; type UnsavedConfiguration = Omit; diff --git a/types/src/front/lib/api/assistant/actions/index.ts b/types/src/front/lib/api/assistant/actions/index.ts index 28a0da1f2b29..0af2a6fdf25d 100644 --- a/types/src/front/lib/api/assistant/actions/index.ts +++ b/types/src/front/lib/api/assistant/actions/index.ts @@ -8,7 +8,8 @@ type BaseActionType = | "process_action" | "websearch_action" | "browse_action" - | "visualization_action"; + | "visualization_action" + | "jit_list_files_action"; export abstract class BaseAction { readonly id: ModelId; diff --git a/types/src/index.ts b/types/src/index.ts index 9673a27a9d4d..319851cab40b 100644 --- a/types/src/index.ts +++ b/types/src/index.ts @@ -23,6 +23,7 @@ export * from "./front/app"; export * from "./front/assistant/actions/browse"; export * from "./front/assistant/actions/dust_app_run"; export * from "./front/assistant/actions/guards"; +export * from "./front/assistant/actions/jit/list_files"; export * from "./front/assistant/actions/process"; export * from "./front/assistant/actions/retrieval"; export * from "./front/assistant/actions/tables_query"; From b26fc95a0934c890ed8514943e2a7f15681a85c8 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 16:28:13 +0100 Subject: [PATCH 02/22] list_files types --- .../front/assistant/actions/jit/list_files.ts | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 types/src/front/assistant/actions/jit/list_files.ts diff --git a/types/src/front/assistant/actions/jit/list_files.ts b/types/src/front/assistant/actions/jit/list_files.ts new file mode 100644 index 000000000000..d3ef4aed74db --- /dev/null +++ b/types/src/front/assistant/actions/jit/list_files.ts @@ -0,0 +1,21 @@ +import { BaseAction } from "../../../../front/lib/api/assistant/actions/index"; +import { ModelId } from "../../../../shared/model_id"; + +export type JITListFilesConfigurationType = { + id: ModelId; + sId: string; + + type: "jit_list_files_configuration"; + + name: string; + description: string | null; +}; + +export interface JITListFilesActionType extends BaseAction { + agentMessageId: ModelId; + files: string[]; + functionCallId: string | null; + functionCallName: string | null; + step: number; + type: "jit_list_files_action"; +} From 8c3adf878c539fd0b1a8da171faa1f9cb0668efb Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 16:51:06 +0100 Subject: [PATCH 03/22] more --- .../api/assistant/actions/jit/list_files.ts | 153 ++++++ front/lib/api/assistant/actions/list_files.ts | 439 ------------------ .../front/assistant/actions/jit/list_files.ts | 16 +- 3 files changed, 168 insertions(+), 440 deletions(-) create mode 100644 front/lib/api/assistant/actions/jit/list_files.ts delete mode 100644 front/lib/api/assistant/actions/list_files.ts diff --git a/front/lib/api/assistant/actions/jit/list_files.ts b/front/lib/api/assistant/actions/jit/list_files.ts new file mode 100644 index 000000000000..9c8e54f9c91e --- /dev/null +++ b/front/lib/api/assistant/actions/jit/list_files.ts @@ -0,0 +1,153 @@ +import type { + FunctionCallType, + FunctionMessageTypeModel, + JITFileType, + JITListFilesConfigurationType, + JITListFilesSuccessEvent, + ModelId, +} from "@dust-tt/types"; +import type { AgentActionSpecification } from "@dust-tt/types"; +import type { Result } from "@dust-tt/types"; +import { + BaseAction, + getTablesQueryResultsFileTitle, + isAgentMessageType, + isContentFragmentType, + isTablesQueryActionType, +} from "@dust-tt/types"; +import { Ok } from "@dust-tt/types"; + +import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; +import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; +import type { Authenticator } from "@app/lib/auth"; + +interface JITListFilesActionBlob { + agentMessageId: ModelId; + functionCallId: string | null; + functionCallName: string | null; + files: JITFileType[]; + step: number; +} + +export class JITListFilesAction extends BaseAction { + readonly agentMessageId: ModelId; + readonly files: JITFileType[]; + readonly functionCallId: string | null; + readonly functionCallName: string | null; + readonly step: number; + readonly type = "jit_list_files_action"; + + constructor(blob: JITListFilesActionBlob) { + super(-1, "jit_list_files_action"); + + this.agentMessageId = blob.agentMessageId; + this.files = blob.files; + this.functionCallId = blob.functionCallId; + this.functionCallName = blob.functionCallName; + this.step = blob.step; + } + + renderForFunctionCall(): FunctionCallType { + return { + id: this.functionCallId ?? `call_${this.id.toString()}`, + name: this.functionCallName ?? "list_conversation_files", + arguments: JSON.stringify({}), + }; + } + + renderForMultiActionsModel(): FunctionMessageTypeModel { + let content = "CONVERSATION FILES:\n"; + for (const file of this.files) { + content += `${file}\n`; + } + + return { + role: "function" as const, + name: this.functionCallName ?? "list_conversation_files", + function_call_id: this.functionCallId ?? `call_${this.id.toString()}`, + content, + }; + } +} + +/** + * Params generation. + */ + +export class JITListFileConfigurationServerRunner extends BaseActionConfigurationServerRunner { + // Generates the action specification for generation of rawInputs passed to `run`. + async buildSpecification( + _auth: Authenticator, + { name, description }: { name: string; description: string | null } + ): Promise> { + return new Ok({ + name, + description: + description || + "Retrieve the list of files attached to the conversation", + inputs: [], + }); + } + + async *run( + _auth: Authenticator, + { + agentConfiguration, + conversation, + agentMessage, + functionCallId, + step, + }: BaseActionRunParams + ): AsyncGenerator { + const jitFiles: JITFileType[] = []; + + for (const m of conversation.content.flat(1)) { + if (isContentFragmentType(m)) { + if (m.fileId) { + jitFiles.push({ + fileId: m.fileId, + title: m.title, + contentType: m.contentType, + }); + } + } else if (isAgentMessageType(m)) { + for (const a of m.actions) { + if (isTablesQueryActionType(a)) { + if (a.resultsFileId && a.resultsFileSnippet) { + jitFiles.push({ + fileId: a.resultsFileId, + contentType: "text/csv", + title: getTablesQueryResultsFileTitle({ output: a.output }), + }); + } + } + } + } + } + + yield { + type: "jit_list_files_success", + created: Date.now(), + configurationId: agentConfiguration.sId, + messageId: agentMessage.sId, + action: new JITListFilesAction({ + functionCallId, + functionCallName: "list_conversation_files", + files: jitFiles, + agentMessageId: agentMessage.agentMessageId, + step: step, + }), + }; + } +} + +/** + * Action rendering. + */ + +// JITListFilesAction are never stored in DB so they are never rendered to the user. +export async function jitListFilesTypesFromAgentMessageIds(): Promise< + JITListFilesAction[] +> { + return []; +} diff --git a/front/lib/api/assistant/actions/list_files.ts b/front/lib/api/assistant/actions/list_files.ts deleted file mode 100644 index 4eb6f48369a7..000000000000 --- a/front/lib/api/assistant/actions/list_files.ts +++ /dev/null @@ -1,439 +0,0 @@ -import { DustAPI } from "@dust-tt/client"; -import type { - DustAppRunBlockEvent, - DustAppRunErrorEvent, - DustAppRunParamsEvent, - DustAppRunSuccessEvent, - FunctionCallType, - FunctionMessageTypeModel, - JITListFilesConfigurationType, - ModelId, -} from "@dust-tt/types"; -import type { DustAppParameters, DustAppRunActionType } from "@dust-tt/types"; -import type { AgentActionSpecification } from "@dust-tt/types"; -import type { SpecificationType } from "@dust-tt/types"; -import type { DatasetSchema } from "@dust-tt/types"; -import type { Result } from "@dust-tt/types"; -import { BaseAction } from "@dust-tt/types"; -import { Err, Ok } from "@dust-tt/types"; - -import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; -import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; -import config from "@app/lib/api/config"; -import { getDatasetSchema } from "@app/lib/api/datasets"; -import type { Authenticator } from "@app/lib/auth"; -import { prodAPICredentialsForOwner } from "@app/lib/auth"; -import { extractConfig } from "@app/lib/config"; -import { AgentDustAppRunAction } from "@app/lib/models/assistant/actions/dust_app_run"; -import { AppResource } from "@app/lib/resources/app_resource"; -import { sanitizeJSONOutput } from "@app/lib/utils"; -import logger from "@app/logger/logger"; - -interface JITListFilesActionBlob { - agentMessageId: ModelId; - functionCallId: string | null; - functionCallName: string | null; - files: string[]; - step: number; -} - -export class JITListFilesAction extends BaseAction { - readonly agentMessageId: ModelId; - readonly files: string[]; - readonly functionCallId: string | null; - readonly functionCallName: string | null; - readonly step: number; - readonly type = "jit_list_files_action"; - - constructor(blob: JITListFilesActionBlob) { - super(-1, "jit_list_files_action"); - - this.agentMessageId = blob.agentMessageId; - this.files = blob.files; - this.functionCallId = blob.functionCallId; - this.functionCallName = blob.functionCallName; - this.step = blob.step; - } - - renderForFunctionCall(): FunctionCallType { - return { - id: this.functionCallId ?? `call_${this.id.toString()}`, - name: this.functionCallName ?? "list_conversation_files", - arguments: JSON.stringify({}), - }; - } - - renderForMultiActionsModel(): FunctionMessageTypeModel { - let content = "CONVERSATION FILES:\n"; - for (const file of this.files) { - content += `${file}\n`; - } - - return { - role: "function" as const, - name: this.functionCallName ?? "list_conversation_files", - function_call_id: this.functionCallId ?? `call_${this.id.toString()}`, - content, - }; - } -} - -/** - * Params generation. - */ - -export class JITListFileConfigurationServerRunner extends BaseActionConfigurationServerRunner { - // Generates the action specification for generation of rawInputs passed to `run`. - async buildSpecification( - _auth: Authenticator, - { name, description }: { name: string; description: string | null } - ): Promise> { - return new Ok({ - name, - description: - description || - "Retrieve the list of files attached to the conversation", - inputs: [], - }); - } - - // This method is in charge of running a dust app and creating an AgentDustAppRunAction object in - // the database. It does not create any generic model related to the conversation. It is possible - // for an AgentDustAppRunAction to be stored (once the params are infered) but for the dust app run - // to fail, in which case an error event will be emitted and the AgentDustAppRunAction won't have - // any output associated. The error is expected to be stored by the caller on the parent agent - // message. - async *run( - auth: Authenticator, - { - agentConfiguration, - conversation, - agentMessage, - rawInputs, - functionCallId, - step, - }: BaseActionRunParams, - { - spec, - }: { - spec: AgentActionSpecification; - } - ): AsyncGenerator< - | DustAppRunParamsEvent - | DustAppRunBlockEvent - | DustAppRunSuccessEvent - | DustAppRunErrorEvent, - void - > { - const owner = auth.workspace(); - if (!owner) { - throw new Error("Unexpected unauthenticated call to `run`"); - } - - const { actionConfiguration } = this; - - const app = await AppResource.fetchById(auth, actionConfiguration.appId); - if (!app) { - yield { - type: "dust_app_run_error", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - error: { - code: "dust_app_run_parameters_generation_error", - message: - "Failed to retrieve Dust app " + - `${actionConfiguration.appWorkspaceId}/${actionConfiguration.appId}`, - }, - }; - return; - } - - const appConfig = extractConfig(JSON.parse(app.savedSpecification || `{}`)); - - // Check that all inputs are accounted for. - const params: DustAppParameters = {}; - - for (const k of spec.inputs) { - if (k.name in rawInputs && typeof rawInputs[k.name] === k.type) { - // As defined in dustAppRunActionSpecification, type is either "string", "number" or "boolean" - params[k.name] = rawInputs[k.name] as string | number | boolean; - } else { - yield { - type: "dust_app_run_error", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - error: { - code: "dust_app_run_parameters_generation_error", - message: `Failed to generate input ${k.name} (expected type ${ - k.type - }, got ${rawInputs[k.name]})`, - }, - }; - return; - } - } - - // Create the AgentDustAppRunAction object in the database and yield an event for the generation - // of the params. We store the action here as the params have been generated, if an error occurs - // later on, the action won't have an output but the error will be stored on the parent agent - // message. - const action = await AgentDustAppRunAction.create({ - dustAppRunConfigurationId: actionConfiguration.sId, - appWorkspaceId: actionConfiguration.appWorkspaceId, - appId: actionConfiguration.appId, - appName: app.name, - params, - functionCallId, - functionCallName: actionConfiguration.name, - agentMessageId: agentMessage.agentMessageId, - step, - }); - - yield { - type: "dust_app_run_params", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - action: new DustAppRunAction({ - id: action.id, - appWorkspaceId: actionConfiguration.appWorkspaceId, - appId: actionConfiguration.appId, - appName: app.name, - params, - runningBlock: null, - output: null, - functionCallId, - functionCallName: actionConfiguration.name, - agentMessageId: agentMessage.agentMessageId, - step, - }), - }; - - // Let's run the app now. - const now = Date.now(); - - const prodCredentials = await prodAPICredentialsForOwner(owner, { - useLocalInDev: true, - }); - const requestedGroupIds = auth.groups().map((g) => g.sId); - const api = new DustAPI( - config.getDustAPIConfig(), - { ...prodCredentials, groupIds: requestedGroupIds }, - logger, - { - useLocalInDev: true, - } - ); - - // As we run the app (using a system API key here), we do force using the workspace credentials so - // that the app executes in the exact same conditions in which they were developed. - const runRes = await api.runAppStreamed( - { - workspaceId: actionConfiguration.appWorkspaceId, - appId: actionConfiguration.appId, - appSpaceId: app.space.sId, - appHash: "latest", - }, - appConfig, - [params], - { useWorkspaceCredentials: true } - ); - - if (runRes.isErr()) { - yield { - type: "dust_app_run_error", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - error: { - code: "dust_app_run_error", - message: `Error running Dust app: ${runRes.error.message}`, - }, - }; - return; - } - - const { eventStream, dustRunId } = runRes.value; - let lastBlockOutput: unknown | null = null; - - for await (const event of eventStream) { - if (event.type === "error") { - yield { - type: "dust_app_run_error", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - error: { - code: "dust_app_run_error", - message: `Error running Dust app: ${event.content.message}`, - }, - }; - return; - } - - if (event.type === "block_status") { - yield { - type: "dust_app_run_block", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - action: new DustAppRunAction({ - id: action.id, - appWorkspaceId: actionConfiguration.appWorkspaceId, - appId: actionConfiguration.appId, - appName: app.name, - params, - functionCallId, - functionCallName: actionConfiguration.name, - runningBlock: { - type: event.content.block_type, - name: event.content.name, - status: event.content.status, - }, - output: null, - agentMessageId: agentMessage.agentMessageId, - step: action.step, - }), - }; - } - - if (event.type === "block_execution") { - const e = event.content.execution[0][0]; - if (e.error) { - yield { - type: "dust_app_run_error", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - error: { - code: "dust_app_run_error", - message: `Error running Dust app: ${e.error}`, - }, - }; - return; - } - - lastBlockOutput = e.value; - } - } - - const output = sanitizeJSONOutput(lastBlockOutput); - - // Update DustAppRunAction with the output of the last block. - await action.update({ - runId: await dustRunId, - output, - }); - - logger.info( - { - workspaceId: conversation.owner.sId, - conversationId: conversation.sId, - elapsed: Date.now() - now, - }, - "[ASSISTANT_TRACE] DustAppRun acion run execution" - ); - - yield { - type: "dust_app_run_success", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - action: new DustAppRunAction({ - id: action.id, - appWorkspaceId: actionConfiguration.appWorkspaceId, - appId: actionConfiguration.appId, - appName: app.name, - params, - functionCallId, - functionCallName: actionConfiguration.name, - runningBlock: null, - output, - agentMessageId: agentMessage.agentMessageId, - step: action.step, - }), - }; - } -} - -async function dustAppRunActionSpecification({ - schema, - name, - description, -}: { - schema: DatasetSchema | null; - name: string; - description: string; -}): Promise> { - // If we have no schema (aka no input block) there is no need to generate any input. - if (!schema) { - return new Ok({ - name, - description, - inputs: [], - }); - } - - const inputs: { - name: string; - description: string; - type: "string" | "number" | "boolean"; - }[] = []; - - for (const k of schema) { - if (k.type === "json") { - return new Err( - new Error( - `JSON type for Dust app parameters is not supported, string, number and boolean are.` - ) - ); - } - - inputs.push({ - name: k.key, - description: k.description || "", - type: k.type, - }); - } - - return new Ok({ - name, - description, - inputs, - }); -} - -/** - * Action rendering. - */ - -// Internal interface for the retrieval and rendering of a DustAppRun action. This should not be -// used outside of api/assistant. We allow a ModelId interface here because we don't have `sId` on -// actions (the `sId` is on the `Message` object linked to the `UserMessage` parent of this action). -export async function dustAppRunTypesFromAgentMessageIds( - agentMessageIds: ModelId[] -): Promise { - const actions = await AgentDustAppRunAction.findAll({ - where: { - agentMessageId: agentMessageIds, - }, - }); - - return actions.map((action) => { - return new DustAppRunAction({ - id: action.id, - appWorkspaceId: action.appWorkspaceId, - appId: action.appId, - appName: action.appName, - params: action.params, - runningBlock: null, - output: action.output, - functionCallId: action.functionCallId, - functionCallName: action.functionCallName, - agentMessageId: action.agentMessageId, - step: action.step, - }); - }); -} diff --git a/types/src/front/assistant/actions/jit/list_files.ts b/types/src/front/assistant/actions/jit/list_files.ts index d3ef4aed74db..c1c60070c39d 100644 --- a/types/src/front/assistant/actions/jit/list_files.ts +++ b/types/src/front/assistant/actions/jit/list_files.ts @@ -11,11 +11,25 @@ export type JITListFilesConfigurationType = { description: string | null; }; +export type JITFileType = { + fileId: string; + title: string; + contentType: string; +}; + export interface JITListFilesActionType extends BaseAction { agentMessageId: ModelId; - files: string[]; + files: JITFileType[]; functionCallId: string | null; functionCallName: string | null; step: number; type: "jit_list_files_action"; } + +export type JITListFilesSuccessEvent = { + type: "jit_list_files_success"; + created: number; + configurationId: string; + messageId: string; + action: JITListFilesActionType; +}; From ee51e661ad63d4b4b6eddb1741428abb97569ab0 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 16:57:07 +0100 Subject: [PATCH 04/22] simplify --- .../api/assistant/actions/jit/list_files.ts | 111 ++++++------------ 1 file changed, 37 insertions(+), 74 deletions(-) diff --git a/front/lib/api/assistant/actions/jit/list_files.ts b/front/lib/api/assistant/actions/jit/list_files.ts index 9c8e54f9c91e..ed0ace3d46b6 100644 --- a/front/lib/api/assistant/actions/jit/list_files.ts +++ b/front/lib/api/assistant/actions/jit/list_files.ts @@ -1,13 +1,12 @@ import type { + AgentMessageType, + ConversationType, FunctionCallType, FunctionMessageTypeModel, JITFileType, - JITListFilesConfigurationType, - JITListFilesSuccessEvent, + JITListFilesActionType, ModelId, } from "@dust-tt/types"; -import type { AgentActionSpecification } from "@dust-tt/types"; -import type { Result } from "@dust-tt/types"; import { BaseAction, getTablesQueryResultsFileTitle, @@ -15,11 +14,6 @@ import { isContentFragmentType, isTablesQueryActionType, } from "@dust-tt/types"; -import { Ok } from "@dust-tt/types"; - -import type { BaseActionRunParams } from "@app/lib/api/assistant/actions/types"; -import { BaseActionConfigurationServerRunner } from "@app/lib/api/assistant/actions/types"; -import type { Authenticator } from "@app/lib/auth"; interface JITListFilesActionBlob { agentMessageId: ModelId; @@ -57,8 +51,8 @@ export class JITListFilesAction extends BaseAction { renderForMultiActionsModel(): FunctionMessageTypeModel { let content = "CONVERSATION FILES:\n"; - for (const file of this.files) { - content += `${file}\n`; + for (const f of this.files) { + content += `\n`; } return { @@ -70,75 +64,44 @@ export class JITListFilesAction extends BaseAction { } } -/** - * Params generation. - */ - -export class JITListFileConfigurationServerRunner extends BaseActionConfigurationServerRunner { - // Generates the action specification for generation of rawInputs passed to `run`. - async buildSpecification( - _auth: Authenticator, - { name, description }: { name: string; description: string | null } - ): Promise> { - return new Ok({ - name, - description: - description || - "Retrieve the list of files attached to the conversation", - inputs: [], - }); - } - - async *run( - _auth: Authenticator, - { - agentConfiguration, - conversation, - agentMessage, - functionCallId, - step, - }: BaseActionRunParams - ): AsyncGenerator { - const jitFiles: JITFileType[] = []; +export function makeJITListFilesAction( + step: number, + agentMessage: AgentMessageType, + conversation: ConversationType +): JITListFilesActionType { + const jitFiles: JITFileType[] = []; - for (const m of conversation.content.flat(1)) { - if (isContentFragmentType(m)) { - if (m.fileId) { - jitFiles.push({ - fileId: m.fileId, - title: m.title, - contentType: m.contentType, - }); - } - } else if (isAgentMessageType(m)) { - for (const a of m.actions) { - if (isTablesQueryActionType(a)) { - if (a.resultsFileId && a.resultsFileSnippet) { - jitFiles.push({ - fileId: a.resultsFileId, - contentType: "text/csv", - title: getTablesQueryResultsFileTitle({ output: a.output }), - }); - } + for (const m of conversation.content.flat(1)) { + if (isContentFragmentType(m)) { + if (m.fileId) { + jitFiles.push({ + fileId: m.fileId, + title: m.title, + contentType: m.contentType, + }); + } + } else if (isAgentMessageType(m)) { + for (const a of m.actions) { + if (isTablesQueryActionType(a)) { + if (a.resultsFileId && a.resultsFileSnippet) { + jitFiles.push({ + fileId: a.resultsFileId, + contentType: "text/csv", + title: getTablesQueryResultsFileTitle({ output: a.output }), + }); } } } } - - yield { - type: "jit_list_files_success", - created: Date.now(), - configurationId: agentConfiguration.sId, - messageId: agentMessage.sId, - action: new JITListFilesAction({ - functionCallId, - functionCallName: "list_conversation_files", - files: jitFiles, - agentMessageId: agentMessage.agentMessageId, - step: step, - }), - }; } + + return new JITListFilesAction({ + functionCallId: "call_" + Math.random().toString(36).substring(7), + functionCallName: "list_conversation_files", + files: jitFiles, + agentMessageId: agentMessage.agentMessageId, + step: step, + }); } /** From 64c6d62de378a1db50e8e1b80e2ecdcaf15ca5a7 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 16:57:21 +0100 Subject: [PATCH 05/22] simplify --- types/src/front/assistant/actions/jit/list_files.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/types/src/front/assistant/actions/jit/list_files.ts b/types/src/front/assistant/actions/jit/list_files.ts index c1c60070c39d..fd9885041729 100644 --- a/types/src/front/assistant/actions/jit/list_files.ts +++ b/types/src/front/assistant/actions/jit/list_files.ts @@ -25,11 +25,3 @@ export interface JITListFilesActionType extends BaseAction { step: number; type: "jit_list_files_action"; } - -export type JITListFilesSuccessEvent = { - type: "jit_list_files_success"; - created: number; - configurationId: string; - messageId: string; - action: JITListFilesActionType; -}; From 74d904e8e1509ea3d4dba97f78911d60ae9f38eb Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 16:58:16 +0100 Subject: [PATCH 06/22] clean --- front/lib/api/assistant/agent.ts | 2 -- front/lib/api/assistant/generation.ts | 2 -- 2 files changed, 4 deletions(-) diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index 4b4735a1ca09..fb88edfd1fa1 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -86,8 +86,6 @@ export async function* runAgent( throw new Error("Unreachable: could not find owner workspace for agent"); } - // console.log(JSON.stringify(conversation)); - const stream = runMultiActionsAgentLoop( auth, fullConfiguration, diff --git a/front/lib/api/assistant/generation.ts b/front/lib/api/assistant/generation.ts index abe018e39429..4f64713a02f5 100644 --- a/front/lib/api/assistant/generation.ts +++ b/front/lib/api/assistant/generation.ts @@ -522,8 +522,6 @@ async function renderConversationForModelMultiActions({ "[ASSISTANT_TRACE] renderConversationForModelMultiActions" ); - // console.log("MESSAGES\n", selected.map((m) => JSON.stringify(m)).join("\n")); - return new Ok({ modelConversation: { messages: selected, From 00792716e39b6de3fe64f187504d6f0f24011e30 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 17:00:40 +0100 Subject: [PATCH 07/22] simplify --- types/src/front/assistant/actions/jit/list_files.ts | 10 ---------- types/src/front/assistant/agent.ts | 4 +--- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/types/src/front/assistant/actions/jit/list_files.ts b/types/src/front/assistant/actions/jit/list_files.ts index fd9885041729..6a400aa0e75f 100644 --- a/types/src/front/assistant/actions/jit/list_files.ts +++ b/types/src/front/assistant/actions/jit/list_files.ts @@ -1,16 +1,6 @@ import { BaseAction } from "../../../../front/lib/api/assistant/actions/index"; import { ModelId } from "../../../../shared/model_id"; -export type JITListFilesConfigurationType = { - id: ModelId; - sId: string; - - type: "jit_list_files_configuration"; - - name: string; - description: string | null; -}; - export type JITFileType = { fileId: string; title: string; diff --git a/types/src/front/assistant/agent.ts b/types/src/front/assistant/agent.ts index 9eda6b292c38..c6c13ffc3128 100644 --- a/types/src/front/assistant/agent.ts +++ b/types/src/front/assistant/agent.ts @@ -5,7 +5,6 @@ import { TablesQueryConfigurationType } from "../../front/assistant/actions/tabl import { ModelIdType, ModelProviderIdType } from "../../front/lib/assistant"; import { ModelId } from "../../shared/model_id"; import { BrowseConfigurationType } from "./actions/browse"; -import { JITListFilesConfigurationType } from "./actions/jit/list_files"; import { WebsearchConfigurationType } from "./actions/websearch"; /** @@ -21,8 +20,7 @@ export type AgentActionConfigurationType = | DustAppRunConfigurationType | ProcessConfigurationType | WebsearchConfigurationType - | BrowseConfigurationType - | JITListFilesConfigurationType; + | BrowseConfigurationType; type UnsavedConfiguration = Omit; From fabbf9b44bd8a873172537ac9eb4aba3ddd9a6e4 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 17:06:25 +0100 Subject: [PATCH 08/22] one way of doing it --- .../lib/api/assistant/actions/jit/list_files.ts | 6 +++++- front/lib/api/assistant/agent.ts | 16 ++++++++++++++++ types/src/front/assistant/conversation.ts | 5 ++++- 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/front/lib/api/assistant/actions/jit/list_files.ts b/front/lib/api/assistant/actions/jit/list_files.ts index ed0ace3d46b6..134b9dcc899a 100644 --- a/front/lib/api/assistant/actions/jit/list_files.ts +++ b/front/lib/api/assistant/actions/jit/list_files.ts @@ -68,7 +68,7 @@ export function makeJITListFilesAction( step: number, agentMessage: AgentMessageType, conversation: ConversationType -): JITListFilesActionType { +): JITListFilesActionType | null { const jitFiles: JITFileType[] = []; for (const m of conversation.content.flat(1)) { @@ -95,6 +95,10 @@ export function makeJITListFilesAction( } } + if (jitFiles.length === 0) { + return null; + } + return new JITListFilesAction({ functionCallId: "call_" + Math.random().toString(36).substring(7), functionCallName: "list_conversation_files", diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index fb88edfd1fa1..f1dd7b931b35 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -50,6 +50,7 @@ import { cloneBaseConfig, DustProdActionRegistry } from "@app/lib/registry"; import logger from "@app/logger/logger"; import { getCitationsCount } from "./actions/utils"; +import { makeJITListFilesAction } from "./actions/jit/list_files"; const CANCELLATION_CHECK_INTERVAL = 500; const MAX_ACTIONS_PER_STEP = 16; @@ -362,6 +363,16 @@ async function* runMultiActionsAgent( const MIN_GENERATION_TOKENS = 2048; + const fakeJITListFiles = makeJITListFilesAction( + 0, + agentMessage, + conversation + ); + + if (fakeJITListFiles) { + agentMessage.actions.unshift(fakeJITListFiles); + } + // Turn the conversation into a digest that can be presented to the model. const modelConversationRes = await renderConversationForModel(auth, { conversation, @@ -370,6 +381,11 @@ async function* runMultiActionsAgent( allowedTokenCount: model.contextSize - MIN_GENERATION_TOKENS, }); + // remove the fake JIT action from the agentMessage.actions + if (fakeJITListFiles) { + agentMessage.actions.shift(); + } + if (modelConversationRes.isErr()) { logger.error( { diff --git a/types/src/front/assistant/conversation.ts b/types/src/front/assistant/conversation.ts index fd82bf0db958..d1b2374f6f95 100644 --- a/types/src/front/assistant/conversation.ts +++ b/types/src/front/assistant/conversation.ts @@ -7,6 +7,7 @@ import { UserType, WorkspaceType } from "../../front/user"; import { ModelId } from "../../shared/model_id"; import { ContentFragmentType } from "../content_fragment"; import { BrowseActionType } from "./actions/browse"; +import { JITListFilesActionType } from "./actions/jit/list_files"; import { WebsearchActionType } from "./actions/websearch"; /** @@ -110,7 +111,8 @@ export type AgentActionType = | TablesQueryActionType | ProcessActionType | WebsearchActionType - | BrowseActionType; + | BrowseActionType + | JITListFilesActionType; export type AgentMessageStatus = | "created" @@ -125,6 +127,7 @@ export const ACTION_RUNNING_LABELS: Record = { tables_query_action: "Querying tables", websearch_action: "Searching the web", browse_action: "Browsing page", + jit_list_files_action: "Listing files", }; /** From b5a43606f5d69749af0a74d628187ea053777255 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Thu, 14 Nov 2024 17:11:41 +0100 Subject: [PATCH 09/22] fix imports --- front/lib/api/assistant/agent.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index f1dd7b931b35..c5585c4cdf00 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -31,7 +31,9 @@ import { } from "@dust-tt/types"; import { runActionStreamed } from "@app/lib/actions/server"; +import { makeJITListFilesAction } from "@app/lib/api/assistant/actions/jit/list_files"; import { getRunnerForActionConfiguration } from "@app/lib/api/assistant/actions/runners"; +import { getCitationsCount } from "@app/lib/api/assistant/actions/utils"; import { AgentMessageContentParser, getDelimitersConfiguration, @@ -49,9 +51,6 @@ import { AgentMessageContent } from "@app/lib/models/assistant/agent_message_con import { cloneBaseConfig, DustProdActionRegistry } from "@app/lib/registry"; import logger from "@app/logger/logger"; -import { getCitationsCount } from "./actions/utils"; -import { makeJITListFilesAction } from "./actions/jit/list_files"; - const CANCELLATION_CHECK_INTERVAL = 500; const MAX_ACTIONS_PER_STEP = 16; From 84b3b3297b3942b3a65180cd77479461d9ba4ba2 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 07:17:13 +0100 Subject: [PATCH 10/22] typing work --- front/components/actions/types.ts | 17 +++++++++++------ .../api/assistant/actions/jit/list_files.ts | 11 ----------- sdks/js/src/types.ts | 19 +++++++++++++++++++ types/src/front/assistant/conversation.ts | 10 ++++++---- 4 files changed, 36 insertions(+), 21 deletions(-) diff --git a/front/components/actions/types.ts b/front/components/actions/types.ts index 9a280c405a9d..953c86d46270 100644 --- a/front/components/actions/types.ts +++ b/front/components/actions/types.ts @@ -1,4 +1,7 @@ -import type { AgentActionType, LightWorkspaceType } from "@dust-tt/types"; +import type { + ConfigurableAgentActionType, + LightWorkspaceType, +} from "@dust-tt/types"; import { ACTION_RUNNING_LABELS } from "@dust-tt/types"; import { BrowseActionDetails } from "@app/components/actions/browse/BrowseActionDetails"; @@ -9,22 +12,24 @@ import { TablesQueryActionDetails } from "@app/components/actions/tables_query/T import { WebsearchActionDetails } from "@app/components/actions/websearch/WebsearchActionDetails"; export interface ActionDetailsComponentBaseProps< - T extends AgentActionType = AgentActionType, + T extends ConfigurableAgentActionType = ConfigurableAgentActionType, > { action: T; owner: LightWorkspaceType; defaultOpen: boolean; } -interface ActionSpecification { +interface ActionSpecification { runningLabel: string; detailsComponent: React.ComponentType>; } -type ActionType = AgentActionType["type"]; +type ActionType = ConfigurableAgentActionType["type"]; type ActionSpecifications = { - [K in ActionType]: ActionSpecification>; + [K in ActionType]: ActionSpecification< + Extract + >; }; const actionsSpecification: ActionSpecifications = { @@ -56,6 +61,6 @@ const actionsSpecification: ActionSpecifications = { export function getActionSpecification( actionType: T -): ActionSpecification> { +): ActionSpecification> { return actionsSpecification[actionType]; } diff --git a/front/lib/api/assistant/actions/jit/list_files.ts b/front/lib/api/assistant/actions/jit/list_files.ts index 134b9dcc899a..a9d792583052 100644 --- a/front/lib/api/assistant/actions/jit/list_files.ts +++ b/front/lib/api/assistant/actions/jit/list_files.ts @@ -107,14 +107,3 @@ export function makeJITListFilesAction( step: step, }); } - -/** - * Action rendering. - */ - -// JITListFilesAction are never stored in DB so they are never rendered to the user. -export async function jitListFilesTypesFromAgentMessageIds(): Promise< - JITListFilesAction[] -> { - return []; -} diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 12347c932618..b83d876bc563 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -610,6 +610,24 @@ const TablesQueryActionTypeSchema = BaseActionSchema.extend({ }); type TablesQueryActionPublicType = z.infer; +const JITFileTypeSchema = z.object({ + fileId: z.string(), + title: z.string(), + contentType: SupportedContentFragmentTypeSchema, +}); + +const JITListFilesActionTypeSchema = BaseActionSchema.extend({ + files: z.array(JITFileTypeSchema).nullable(), + functionCallId: z.string().nullable(), + functionCallName: z.string().nullable(), + agentMessageId: ModelIdSchema, + step: z.number(), + type: z.literal("tables_query_action"), +}); +type JITListFIlesActionPublicType = z.infer< + typeof JITListFilesActionTypeSchema +>; + const WhitelistableFeaturesSchema = FlexibleEnumSchema([ "usage_data_api", "okta_enterprise_connection", @@ -839,6 +857,7 @@ const AgentActionTypeSchema = z.union([ ProcessActionTypeSchema, WebsearchActionTypeSchema, BrowseActionTypeSchema, + JITListFilesActionTypeSchema, ]); export type AgentActionPublicType = z.infer; diff --git a/types/src/front/assistant/conversation.ts b/types/src/front/assistant/conversation.ts index d1b2374f6f95..01869a90271f 100644 --- a/types/src/front/assistant/conversation.ts +++ b/types/src/front/assistant/conversation.ts @@ -104,15 +104,17 @@ export function isUserMessageType(arg: MessageType): arg is UserMessageType { /** * Agent messages */ - -export type AgentActionType = +export type ConfigurableAgentActionType = | RetrievalActionType | DustAppRunActionType | TablesQueryActionType | ProcessActionType | WebsearchActionType - | BrowseActionType - | JITListFilesActionType; + | BrowseActionType; + +export type JITAgentActionType = JITListFilesActionType; + +export type AgentActionType = ConfigurableAgentActionType | JITAgentActionType; export type AgentMessageStatus = | "created" From 53892e2c572c765480e3168d30e6d97b551d24be Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 07:39:34 +0100 Subject: [PATCH 11/22] fixes --- front/components/actions/types.ts | 21 ++++++++++----------- sdks/js/src/types.ts | 12 ++++++------ types/src/front/assistant/conversation.ts | 2 +- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/front/components/actions/types.ts b/front/components/actions/types.ts index 953c86d46270..915e6bcb31b9 100644 --- a/front/components/actions/types.ts +++ b/front/components/actions/types.ts @@ -1,7 +1,4 @@ -import type { - ConfigurableAgentActionType, - LightWorkspaceType, -} from "@dust-tt/types"; +import type { AgentActionType, LightWorkspaceType } from "@dust-tt/types"; import { ACTION_RUNNING_LABELS } from "@dust-tt/types"; import { BrowseActionDetails } from "@app/components/actions/browse/BrowseActionDetails"; @@ -12,24 +9,22 @@ import { TablesQueryActionDetails } from "@app/components/actions/tables_query/T import { WebsearchActionDetails } from "@app/components/actions/websearch/WebsearchActionDetails"; export interface ActionDetailsComponentBaseProps< - T extends ConfigurableAgentActionType = ConfigurableAgentActionType, + T extends AgentActionType = AgentActionType, > { action: T; owner: LightWorkspaceType; defaultOpen: boolean; } -interface ActionSpecification { +interface ActionSpecification { runningLabel: string; detailsComponent: React.ComponentType>; } -type ActionType = ConfigurableAgentActionType["type"]; +type ActionType = AgentActionType["type"]; type ActionSpecifications = { - [K in ActionType]: ActionSpecification< - Extract - >; + [K in ActionType]: ActionSpecification>; }; const actionsSpecification: ActionSpecifications = { @@ -57,10 +52,14 @@ const actionsSpecification: ActionSpecifications = { detailsComponent: BrowseActionDetails, runningLabel: ACTION_RUNNING_LABELS.browse_action, }, + jit_list_files_action: { + detailsComponent: () => null, + runningLabel: ACTION_RUNNING_LABELS.jit_list_files_action, + }, }; export function getActionSpecification( actionType: T -): ActionSpecification> { +): ActionSpecification> { return actionsSpecification[actionType]; } diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index b83d876bc563..3b5086967dc4 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -613,20 +613,20 @@ type TablesQueryActionPublicType = z.infer; const JITFileTypeSchema = z.object({ fileId: z.string(), title: z.string(), - contentType: SupportedContentFragmentTypeSchema, + contentType: z.string(), }); const JITListFilesActionTypeSchema = BaseActionSchema.extend({ - files: z.array(JITFileTypeSchema).nullable(), + files: z.array(JITFileTypeSchema), functionCallId: z.string().nullable(), functionCallName: z.string().nullable(), agentMessageId: ModelIdSchema, step: z.number(), - type: z.literal("tables_query_action"), + type: z.literal("jit_list_files_action"), }); -type JITListFIlesActionPublicType = z.infer< - typeof JITListFilesActionTypeSchema ->; +// type JITListFIlesActionPublicType = z.infer< +// typeof JITListFilesActionTypeSchema +// >; const WhitelistableFeaturesSchema = FlexibleEnumSchema([ "usage_data_api", diff --git a/types/src/front/assistant/conversation.ts b/types/src/front/assistant/conversation.ts index 01869a90271f..6c4364879af3 100644 --- a/types/src/front/assistant/conversation.ts +++ b/types/src/front/assistant/conversation.ts @@ -129,7 +129,7 @@ export const ACTION_RUNNING_LABELS: Record = { tables_query_action: "Querying tables", websearch_action: "Searching the web", browse_action: "Browsing page", - jit_list_files_action: "Listing files", + jit_list_files_action: "Listing conversation files", }; /** From 1ad4f02faba17d81e8da6778945668e742fe04b0 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 07:54:40 +0100 Subject: [PATCH 12/22] rename --- front/components/actions/types.ts | 2 +- .../{jit => conversation}/list_files.ts | 31 +++++++-------- front/lib/api/assistant/agent.ts | 39 ++++++++++++------- sdks/js/src/types.ts | 38 +++++++++--------- .../{jit => conversation}/list_files.ts | 9 ++--- types/src/front/assistant/conversation.ts | 10 +++-- .../front/lib/api/assistant/actions/index.ts | 2 +- types/src/index.ts | 2 +- 8 files changed, 70 insertions(+), 63 deletions(-) rename front/lib/api/assistant/actions/{jit => conversation}/list_files.ts (80%) rename types/src/front/assistant/actions/{jit => conversation}/list_files.ts (61%) diff --git a/front/components/actions/types.ts b/front/components/actions/types.ts index 915e6bcb31b9..2ebb5865e018 100644 --- a/front/components/actions/types.ts +++ b/front/components/actions/types.ts @@ -52,7 +52,7 @@ const actionsSpecification: ActionSpecifications = { detailsComponent: BrowseActionDetails, runningLabel: ACTION_RUNNING_LABELS.browse_action, }, - jit_list_files_action: { + conversation_list_files_action: { detailsComponent: () => null, runningLabel: ACTION_RUNNING_LABELS.jit_list_files_action, }, diff --git a/front/lib/api/assistant/actions/jit/list_files.ts b/front/lib/api/assistant/actions/conversation/list_files.ts similarity index 80% rename from front/lib/api/assistant/actions/jit/list_files.ts rename to front/lib/api/assistant/actions/conversation/list_files.ts index a9d792583052..d78899805a8f 100644 --- a/front/lib/api/assistant/actions/jit/list_files.ts +++ b/front/lib/api/assistant/actions/conversation/list_files.ts @@ -1,10 +1,10 @@ import type { AgentMessageType, + ConversationFileType, + ConversationListFilesActionType, ConversationType, FunctionCallType, FunctionMessageTypeModel, - JITFileType, - JITListFilesActionType, ModelId, } from "@dust-tt/types"; import { @@ -15,30 +15,27 @@ import { isTablesQueryActionType, } from "@dust-tt/types"; -interface JITListFilesActionBlob { +interface ConversationListFilesActionBlob { agentMessageId: ModelId; functionCallId: string | null; functionCallName: string | null; - files: JITFileType[]; - step: number; + files: ConversationFileType[]; } -export class JITListFilesAction extends BaseAction { +export class ConversationListFilesAction extends BaseAction { readonly agentMessageId: ModelId; - readonly files: JITFileType[]; + readonly files: ConversationFileType[]; readonly functionCallId: string | null; readonly functionCallName: string | null; - readonly step: number; - readonly type = "jit_list_files_action"; + readonly type = "conversation_list_files_action"; - constructor(blob: JITListFilesActionBlob) { - super(-1, "jit_list_files_action"); + constructor(blob: ConversationListFilesActionBlob) { + super(-1, "conversation_list_files_action"); this.agentMessageId = blob.agentMessageId; this.files = blob.files; this.functionCallId = blob.functionCallId; this.functionCallName = blob.functionCallName; - this.step = blob.step; } renderForFunctionCall(): FunctionCallType { @@ -64,12 +61,11 @@ export class JITListFilesAction extends BaseAction { } } -export function makeJITListFilesAction( - step: number, +export function makeConversationListFilesAction( agentMessage: AgentMessageType, conversation: ConversationType -): JITListFilesActionType | null { - const jitFiles: JITFileType[] = []; +): ConversationListFilesActionType | null { + const jitFiles: ConversationFileType[] = []; for (const m of conversation.content.flat(1)) { if (isContentFragmentType(m)) { @@ -99,11 +95,10 @@ export function makeJITListFilesAction( return null; } - return new JITListFilesAction({ + return new ConversationListFilesAction({ functionCallId: "call_" + Math.random().toString(36).substring(7), functionCallName: "list_conversation_files", files: jitFiles, agentMessageId: agentMessage.agentMessageId, - step: step, }); } diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index c5585c4cdf00..9e89cbbe3c48 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -4,6 +4,7 @@ import type { AgentActionSpecification, AgentActionSpecificEvent, AgentActionSuccessEvent, + AgentActionType, AgentChainOfThoughtEvent, AgentConfigurationType, AgentContentEvent, @@ -51,6 +52,9 @@ import { AgentMessageContent } from "@app/lib/models/assistant/agent_message_con import { cloneBaseConfig, DustProdActionRegistry } from "@app/lib/registry"; import logger from "@app/logger/logger"; +import { makeConversationListFilesAction } from "./actions/conversation/list_files"; +import { isJITActionsEnabled } from "./jit_actions"; + const CANCELLATION_CHECK_INTERVAL = 500; const MAX_ACTIONS_PER_STEP = 16; @@ -292,6 +296,23 @@ async function* runMultiActionsAgentLoop( } } +async function getEmulatedAgentMessageActions( + auth: Authenticator, + { + agentMessage, + conversation, + }: { agentMessage: AgentMessageType; conversation: ConversationType } +): Promise { + const actions: AgentActionType[] = []; + if (await isJITActionsEnabled(auth)) { + const a = makeConversationListFilesAction(agentMessage, conversation); + if (a) { + actions.push(a); + } + } + return actions; +} + // This method is used by the multi-actions execution loop to pick the next action to execute and // generate its inputs. async function* runMultiActionsAgent( @@ -362,15 +383,10 @@ async function* runMultiActionsAgent( const MIN_GENERATION_TOKENS = 2048; - const fakeJITListFiles = makeJITListFilesAction( - 0, - agentMessage, - conversation - ); - - if (fakeJITListFiles) { - agentMessage.actions.unshift(fakeJITListFiles); - } + // const emulatedActions = await getEmulatedAgentMessageActions(auth, { + // agentMessage, + // conversation, + // }); // Turn the conversation into a digest that can be presented to the model. const modelConversationRes = await renderConversationForModel(auth, { @@ -380,11 +396,6 @@ async function* runMultiActionsAgent( allowedTokenCount: model.contextSize - MIN_GENERATION_TOKENS, }); - // remove the fake JIT action from the agentMessage.actions - if (fakeJITListFiles) { - agentMessage.actions.shift(); - } - if (modelConversationRes.isErr()) { logger.error( { diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 3b5086967dc4..0c4723e57905 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -460,6 +460,24 @@ const BrowseActionTypeSchema = BaseActionSchema.extend({ }); type BrowseActionPublicType = z.infer; +const ConversationFileTypeSchema = z.object({ + fileId: z.string(), + title: z.string(), + contentType: z.string(), +}); + +const ConversationListFilesActionTypeSchema = BaseActionSchema.extend({ + files: z.array(ConversationFileTypeSchema), + functionCallId: z.string().nullable(), + functionCallName: z.string().nullable(), + agentMessageId: ModelIdSchema, + step: z.number(), + type: z.literal("conversation_list_files_action"), +}); +// type ConversationListFIlesActionPublicType = z.infer< +// typeof ConversationListFilesActionTypeSchema +// >; + const DustAppParametersSchema = z.record( z.union([z.string(), z.number(), z.boolean()]) ); @@ -610,24 +628,6 @@ const TablesQueryActionTypeSchema = BaseActionSchema.extend({ }); type TablesQueryActionPublicType = z.infer; -const JITFileTypeSchema = z.object({ - fileId: z.string(), - title: z.string(), - contentType: z.string(), -}); - -const JITListFilesActionTypeSchema = BaseActionSchema.extend({ - files: z.array(JITFileTypeSchema), - functionCallId: z.string().nullable(), - functionCallName: z.string().nullable(), - agentMessageId: ModelIdSchema, - step: z.number(), - type: z.literal("jit_list_files_action"), -}); -// type JITListFIlesActionPublicType = z.infer< -// typeof JITListFilesActionTypeSchema -// >; - const WhitelistableFeaturesSchema = FlexibleEnumSchema([ "usage_data_api", "okta_enterprise_connection", @@ -857,7 +857,7 @@ const AgentActionTypeSchema = z.union([ ProcessActionTypeSchema, WebsearchActionTypeSchema, BrowseActionTypeSchema, - JITListFilesActionTypeSchema, + ConversationListFilesActionTypeSchema, ]); export type AgentActionPublicType = z.infer; diff --git a/types/src/front/assistant/actions/jit/list_files.ts b/types/src/front/assistant/actions/conversation/list_files.ts similarity index 61% rename from types/src/front/assistant/actions/jit/list_files.ts rename to types/src/front/assistant/actions/conversation/list_files.ts index 6a400aa0e75f..ed1ebbc1235f 100644 --- a/types/src/front/assistant/actions/jit/list_files.ts +++ b/types/src/front/assistant/actions/conversation/list_files.ts @@ -1,17 +1,16 @@ import { BaseAction } from "../../../../front/lib/api/assistant/actions/index"; import { ModelId } from "../../../../shared/model_id"; -export type JITFileType = { +export type ConversationFileType = { fileId: string; title: string; contentType: string; }; -export interface JITListFilesActionType extends BaseAction { +export interface ConversationListFilesActionType extends BaseAction { agentMessageId: ModelId; - files: JITFileType[]; + files: ConversationFileType[]; functionCallId: string | null; functionCallName: string | null; - step: number; - type: "jit_list_files_action"; + type: "conversation_list_files_action"; } diff --git a/types/src/front/assistant/conversation.ts b/types/src/front/assistant/conversation.ts index 6c4364879af3..1bf83a1f18ae 100644 --- a/types/src/front/assistant/conversation.ts +++ b/types/src/front/assistant/conversation.ts @@ -7,7 +7,7 @@ import { UserType, WorkspaceType } from "../../front/user"; import { ModelId } from "../../shared/model_id"; import { ContentFragmentType } from "../content_fragment"; import { BrowseActionType } from "./actions/browse"; -import { JITListFilesActionType } from "./actions/jit/list_files"; +import { ConversationListFilesActionType } from "./actions/conversation/list_files"; import { WebsearchActionType } from "./actions/websearch"; /** @@ -112,9 +112,11 @@ export type ConfigurableAgentActionType = | WebsearchActionType | BrowseActionType; -export type JITAgentActionType = JITListFilesActionType; +export type ConversationAgentActionType = ConversationListFilesActionType; -export type AgentActionType = ConfigurableAgentActionType | JITAgentActionType; +export type AgentActionType = + | ConfigurableAgentActionType + | ConversationAgentActionType; export type AgentMessageStatus = | "created" @@ -129,7 +131,7 @@ export const ACTION_RUNNING_LABELS: Record = { tables_query_action: "Querying tables", websearch_action: "Searching the web", browse_action: "Browsing page", - jit_list_files_action: "Listing conversation files", + conversation_list_files_action: "Listing conversation files", }; /** diff --git a/types/src/front/lib/api/assistant/actions/index.ts b/types/src/front/lib/api/assistant/actions/index.ts index 0af2a6fdf25d..ceaf4c7c0bbe 100644 --- a/types/src/front/lib/api/assistant/actions/index.ts +++ b/types/src/front/lib/api/assistant/actions/index.ts @@ -9,7 +9,7 @@ type BaseActionType = | "websearch_action" | "browse_action" | "visualization_action" - | "jit_list_files_action"; + | "conversation_list_files_action"; export abstract class BaseAction { readonly id: ModelId; diff --git a/types/src/index.ts b/types/src/index.ts index 319851cab40b..57c97ba80867 100644 --- a/types/src/index.ts +++ b/types/src/index.ts @@ -21,9 +21,9 @@ export * from "./front/api_handlers/public/data_sources"; export * from "./front/api_handlers/public/spaces"; export * from "./front/app"; export * from "./front/assistant/actions/browse"; +export * from "./front/assistant/actions/conversation/list_files"; export * from "./front/assistant/actions/dust_app_run"; export * from "./front/assistant/actions/guards"; -export * from "./front/assistant/actions/jit/list_files"; export * from "./front/assistant/actions/process"; export * from "./front/assistant/actions/retrieval"; export * from "./front/assistant/actions/tables_query"; From fe06c9cdbcb1a5602ddea961a3f0a923520d5310 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 08:04:05 +0100 Subject: [PATCH 13/22] emulated actions injection/scrub --- front/lib/api/assistant/agent.ts | 17 +++++++++++++---- sdks/js/src/types.ts | 1 - 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index 9e89cbbe3c48..957780d68760 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -383,10 +383,14 @@ async function* runMultiActionsAgent( const MIN_GENERATION_TOKENS = 2048; - // const emulatedActions = await getEmulatedAgentMessageActions(auth, { - // agentMessage, - // conversation, - // }); + const emulatedActions = await getEmulatedAgentMessageActions(auth, { + agentMessage, + conversation, + }); + + // Prepend emulated actions to the current agent message before rendering the conversation for the + // model. + agentMessage.actions = emulatedActions.concat(agentMessage.actions); // Turn the conversation into a digest that can be presented to the model. const modelConversationRes = await renderConversationForModel(auth, { @@ -396,6 +400,11 @@ async function* runMultiActionsAgent( allowedTokenCount: model.contextSize - MIN_GENERATION_TOKENS, }); + // Scrub emulated actions from the agent message after rendering. + agentMessage.actions = agentMessage.actions.filter( + (a) => !emulatedActions.includes(a) + ); + if (modelConversationRes.isErr()) { logger.error( { diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 0c4723e57905..d0cdf9b04889 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -471,7 +471,6 @@ const ConversationListFilesActionTypeSchema = BaseActionSchema.extend({ functionCallId: z.string().nullable(), functionCallName: z.string().nullable(), agentMessageId: ModelIdSchema, - step: z.number(), type: z.literal("conversation_list_files_action"), }); // type ConversationListFIlesActionPublicType = z.infer< From 17eea888910fff9dbadbed79fa804aba8908a2dc Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 08:15:55 +0100 Subject: [PATCH 14/22] reintroduce step --- front/components/actions/types.ts | 2 +- front/lib/api/assistant/actions/conversation/list_files.ts | 1 + sdks/js/src/types.ts | 1 + types/src/front/assistant/actions/conversation/list_files.ts | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/front/components/actions/types.ts b/front/components/actions/types.ts index 2ebb5865e018..c4b3cf74dbff 100644 --- a/front/components/actions/types.ts +++ b/front/components/actions/types.ts @@ -54,7 +54,7 @@ const actionsSpecification: ActionSpecifications = { }, conversation_list_files_action: { detailsComponent: () => null, - runningLabel: ACTION_RUNNING_LABELS.jit_list_files_action, + runningLabel: ACTION_RUNNING_LABELS.conversation_list_files_action, }, }; diff --git a/front/lib/api/assistant/actions/conversation/list_files.ts b/front/lib/api/assistant/actions/conversation/list_files.ts index d78899805a8f..5f2f922f4822 100644 --- a/front/lib/api/assistant/actions/conversation/list_files.ts +++ b/front/lib/api/assistant/actions/conversation/list_files.ts @@ -27,6 +27,7 @@ export class ConversationListFilesAction extends BaseAction { readonly files: ConversationFileType[]; readonly functionCallId: string | null; readonly functionCallName: string | null; + readonly step: number = -1; readonly type = "conversation_list_files_action"; constructor(blob: ConversationListFilesActionBlob) { diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index d0cdf9b04889..0c4723e57905 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -471,6 +471,7 @@ const ConversationListFilesActionTypeSchema = BaseActionSchema.extend({ functionCallId: z.string().nullable(), functionCallName: z.string().nullable(), agentMessageId: ModelIdSchema, + step: z.number(), type: z.literal("conversation_list_files_action"), }); // type ConversationListFIlesActionPublicType = z.infer< diff --git a/types/src/front/assistant/actions/conversation/list_files.ts b/types/src/front/assistant/actions/conversation/list_files.ts index ed1ebbc1235f..3d098c87ee2a 100644 --- a/types/src/front/assistant/actions/conversation/list_files.ts +++ b/types/src/front/assistant/actions/conversation/list_files.ts @@ -12,5 +12,6 @@ export interface ConversationListFilesActionType extends BaseAction { files: ConversationFileType[]; functionCallId: string | null; functionCallName: string | null; + step: number; type: "conversation_list_files_action"; } From bbbfb4d1457459097220adcc7528f9da220f81fb Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 08:21:01 +0100 Subject: [PATCH 15/22] fix imports --- front/lib/api/assistant/agent.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index 957780d68760..47bdec89c560 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -32,7 +32,8 @@ import { } from "@dust-tt/types"; import { runActionStreamed } from "@app/lib/actions/server"; -import { makeJITListFilesAction } from "@app/lib/api/assistant/actions/jit/list_files"; +import { isJITActionsEnabled } from "@app/lib/api/assistant//jit_actions"; +import { makeConversationListFilesAction } from "@app/lib/api/assistant/actions/conversation/list_files"; import { getRunnerForActionConfiguration } from "@app/lib/api/assistant/actions/runners"; import { getCitationsCount } from "@app/lib/api/assistant/actions/utils"; import { @@ -52,9 +53,6 @@ import { AgentMessageContent } from "@app/lib/models/assistant/agent_message_con import { cloneBaseConfig, DustProdActionRegistry } from "@app/lib/registry"; import logger from "@app/logger/logger"; -import { makeConversationListFilesAction } from "./actions/conversation/list_files"; -import { isJITActionsEnabled } from "./jit_actions"; - const CANCELLATION_CHECK_INTERVAL = 500; const MAX_ACTIONS_PER_STEP = 16; From d771ae12bf8ab93f83390f0a496849d88cb7b2b7 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 08:35:20 +0100 Subject: [PATCH 16/22] clean-up visualiation --- front/lib/api/assistant/visualization.ts | 101 +++++++++++------------ 1 file changed, 50 insertions(+), 51 deletions(-) diff --git a/front/lib/api/assistant/visualization.ts b/front/lib/api/assistant/visualization.ts index a9a0caed72af..b5c08770e97a 100644 --- a/front/lib/api/assistant/visualization.ts +++ b/front/lib/api/assistant/visualization.ts @@ -19,65 +19,64 @@ export async function getVisualizationPrompt({ auth: Authenticator; conversation: ConversationType; }) { - const isJITEnabled = await isJITActionsEnabled(auth); + // If `jit_conversations_actions` is enabled we rely on the `conversations_list_files` emulated + // actions to make the list of files available to the agent. + if (await isJITActionsEnabled(auth)) { + return visualizationSystemPrompt; + } - // When JIT is enabled, we return the visualization prompt directly without listing the files as the files will be made available to the model via another mechanism (simulated function call). - if (isJITEnabled) { - return visualizationSystemPrompt.trim(); - } else { - const contentFragmentMessages: Array = []; - for (const m of conversation.content.flat(1)) { - if (isContentFragmentType(m)) { - contentFragmentMessages.push(m); - } + const contentFragmentMessages: Array = []; + for (const m of conversation.content.flat(1)) { + if (isContentFragmentType(m)) { + contentFragmentMessages.push(m); } - const contentFragmentFileBySid = _.keyBy( - await FileResource.fetchByIds( - auth, - removeNulls(contentFragmentMessages.map((m) => m.fileId)) - ), - "sId" - ); - - let prompt = visualizationSystemPrompt.trim() + "\n\n"; - - const fileAttachments: string[] = []; - for (const m of conversation.content.flat(1)) { - if (isContentFragmentType(m)) { - if (!m.fileId || !contentFragmentFileBySid[m.fileId]) { - continue; - } - fileAttachments.push( - `` - ); - } else if (isAgentMessageType(m)) { - for (const a of m.actions) { - if (isTablesQueryActionType(a)) { - const attachment = getTablesQueryResultsFileAttachment({ - resultsFileId: a.resultsFileId, - resultsFileSnippet: a.resultsFileSnippet, - output: a.output, - includeSnippet: false, - }); - if (attachment) { - fileAttachments.push(attachment); - } + } + const contentFragmentFileBySid = _.keyBy( + await FileResource.fetchByIds( + auth, + removeNulls(contentFragmentMessages.map((m) => m.fileId)) + ), + "sId" + ); + + let prompt = visualizationSystemPrompt.trim() + "\n\n"; + + const fileAttachments: string[] = []; + for (const m of conversation.content.flat(1)) { + if (isContentFragmentType(m)) { + if (!m.fileId || !contentFragmentFileBySid[m.fileId]) { + continue; + } + fileAttachments.push( + `` + ); + } else if (isAgentMessageType(m)) { + for (const a of m.actions) { + if (isTablesQueryActionType(a)) { + const attachment = getTablesQueryResultsFileAttachment({ + resultsFileId: a.resultsFileId, + resultsFileSnippet: a.resultsFileSnippet, + output: a.output, + includeSnippet: false, + }); + if (attachment) { + fileAttachments.push(attachment); } } } } + } - if (fileAttachments.length > 0) { - prompt += - "Files accessible to the :::visualization directive environment:\n"; - prompt += fileAttachments.join("\n"); - } else { - prompt += - "No files are currently accessible to the :::visualization directive environment in this conversation."; - } - - return prompt; + if (fileAttachments.length > 0) { + prompt += + "Files accessible to the :::visualization directive environment:\n"; + prompt += fileAttachments.join("\n"); + } else { + prompt += + "No files are currently accessible to the :::visualization directive environment in this conversation."; } + + return prompt; } export const visualizationSystemPrompt = `\ From a6b234256296eda300a23cdbf1982a2a00f642fe Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 09:00:03 +0100 Subject: [PATCH 17/22] fix steps --- front/lib/api/assistant/jit_actions.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/front/lib/api/assistant/jit_actions.ts b/front/lib/api/assistant/jit_actions.ts index 49226eaeec92..f3c23d794478 100644 --- a/front/lib/api/assistant/jit_actions.ts +++ b/front/lib/api/assistant/jit_actions.ts @@ -88,7 +88,6 @@ export async function renderConversationForModelJIT({ if (isAgentMessageType(m)) { const actions = removeNulls(m.actions); - // This array is 2D, because we can have multiple calls per agent message (parallel calls). const steps = [] as Array<{ @@ -105,8 +104,8 @@ export async function renderConversationForModelJIT({ actions: [], }) satisfies (typeof steps)[number]; - for (const action of actions) { - const stepIndex = action.step; + for (let stepIndex = 0; stepIndex < actions.length; stepIndex++) { + const action = actions[stepIndex]; steps[stepIndex] = steps[stepIndex] || emptyStep(); steps[stepIndex].actions.push({ call: action.renderForFunctionCall(), From f7ae369c5825638857c63a8797ad29a228464f09 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 13:26:31 +0100 Subject: [PATCH 18/22] rebase and comments --- front/lib/api/assistant/agent.ts | 4 + front/lib/api/assistant/jit_actions.ts | 125 ++++++------------------- 2 files changed, 32 insertions(+), 97 deletions(-) diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index 47bdec89c560..70d6c4c98294 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -30,6 +30,7 @@ import { isWebsearchConfiguration, SUPPORTED_MODEL_CONFIGS, } from "@dust-tt/types"; +import assert from "assert"; import { runActionStreamed } from "@app/lib/actions/server"; import { isJITActionsEnabled } from "@app/lib/api/assistant//jit_actions"; @@ -308,6 +309,9 @@ async function getEmulatedAgentMessageActions( actions.push(a); } } + + // We ensure that all emulated actions are injected with step -1. + assert(actions.every((a) => a.step === -1)); return actions; } diff --git a/front/lib/api/assistant/jit_actions.ts b/front/lib/api/assistant/jit_actions.ts index f3c23d794478..ab49892a8a24 100644 --- a/front/lib/api/assistant/jit_actions.ts +++ b/front/lib/api/assistant/jit_actions.ts @@ -12,12 +12,10 @@ import type { import { assertNever, Err, - getTablesQueryResultsFileAttachment, isAgentMessageType, isContentFragmentMessageTypeModel, isContentFragmentType, isDevelopment, - isTablesQueryActionType, isTextContent, isUserMessageType, Ok, @@ -81,45 +79,56 @@ export async function renderConversationForModelJIT({ const now = Date.now(); const messages: ModelMessageTypeMultiActions[] = []; - // Render loop. - // Render all messages and all actions. + // Render loop: dender all messages and all actions. for (const versions of conversation.content) { const m = versions[versions.length - 1]; if (isAgentMessageType(m)) { const actions = removeNulls(m.actions); - // This array is 2D, because we can have multiple calls per agent message (parallel calls). - const steps = [] as Array<{ - contents: string[]; - actions: Array<{ - call: FunctionCallType; - result: FunctionMessageTypeModel; - }>; - }>; + // This is a record of arrays, because we can have multiple calls per agent message (parallel + // calls). Actions all have a step index which indicates how they should be grouped but some + // actions injected by `getEmulatedAgentMessageActions` have a step index of `-1`. We + // therefore group by index, then order and transform in a 2D array to present to the model. + const stepsByStepIndex = {} as Record< + string, + { + contents: string[]; + actions: Array<{ + call: FunctionCallType; + result: FunctionMessageTypeModel; + }>; + } + >; const emptyStep = () => ({ contents: [], actions: [], - }) satisfies (typeof steps)[number]; + }) satisfies (typeof stepsByStepIndex)[number]; - for (let stepIndex = 0; stepIndex < actions.length; stepIndex++) { - const action = actions[stepIndex]; - steps[stepIndex] = steps[stepIndex] || emptyStep(); - steps[stepIndex].actions.push({ + for (const action of actions) { + const stepIndex = action.step; + stepsByStepIndex[stepIndex] = + stepsByStepIndex[stepIndex] || emptyStep(); + stepsByStepIndex[stepIndex].actions.push({ call: action.renderForFunctionCall(), result: action.renderForMultiActionsModel(), }); } for (const content of m.rawContents) { - steps[content.step] = steps[content.step] || emptyStep(); + stepsByStepIndex[content.step] = + stepsByStepIndex[content.step] || emptyStep(); if (content.content.trim()) { - steps[content.step].contents.push(content.content); + stepsByStepIndex[content.step].contents.push(content.content); } } + const steps = Object.entries(stepsByStepIndex) + .sort(([a], [b]) => Number(a) - Number(b)) + .map(([, step]) => step); + if (excludeActions) { // In Exclude Actions mode, we only render the last step that has content. const stepsWithContent = steps.filter((s) => s?.contents.length); @@ -221,44 +230,6 @@ export async function renderConversationForModelJIT({ } } - // If we have messages... - if (messages.length > 0) { - const { filesAsXML, hasFiles } = listConversationFiles({ - conversation, - }); - - // ... and files, we simulate a function call to list the files at the end of the conversation. - if (hasFiles) { - const randomCallId = "tool_" + Math.random().toString(36).substring(7); - const functionName = "list_conversation_files"; - - const simulatedAgentMessages = [ - // 1. We add a message from the agent, asking to use the files listing function - { - role: "assistant", - function_calls: [ - { - id: randomCallId, - name: functionName, - arguments: "{}", - }, - ], - } as AssistantFunctionCallMessageTypeModel, - - // 2. We add a message with the resulting files listing - { - function_call_id: randomCallId, - role: "function", - name: functionName, - content: filesAsXML, - } as FunctionMessageTypeModel, - ]; - - // Append the simulated messages to the end of the conversation. - messages.push(...simulatedAgentMessages); - } - } - // Compute in parallel the token count for each message and the prompt. const res = await tokenCountForTexts( [prompt, ...getTextRepresentationFromMessages(messages)], @@ -401,43 +372,3 @@ export async function renderConversationForModelJIT({ tokensUsed, }); } - -function listConversationFiles({ - conversation, -}: { - conversation: ConversationType; -}) { - const fileAttachments: string[] = []; - for (const m of conversation.content.flat(1)) { - if (isContentFragmentType(m)) { - if (!m.fileId) { - continue; - } - fileAttachments.push( - `` - ); - } else if (isAgentMessageType(m)) { - for (const a of m.actions) { - if (isTablesQueryActionType(a)) { - const attachment = getTablesQueryResultsFileAttachment({ - resultsFileId: a.resultsFileId, - resultsFileSnippet: a.resultsFileSnippet, - output: a.output, - includeSnippet: false, - }); - if (attachment) { - fileAttachments.push(attachment); - } - } - } - } - } - let filesAsXML = "\n"; - - if (fileAttachments.length > 0) { - filesAsXML += fileAttachments.join("\n"); - } - filesAsXML += "\n"; - - return { filesAsXML, hasFiles: fileAttachments.length > 0 }; -} From 6eea7d79de97c67e412ee911ebd15a03d6e10aa5 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 14:37:28 +0100 Subject: [PATCH 19/22] rename jitFiles --- .../api/assistant/actions/conversation/list_files.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/front/lib/api/assistant/actions/conversation/list_files.ts b/front/lib/api/assistant/actions/conversation/list_files.ts index 5f2f922f4822..4a66694c14c5 100644 --- a/front/lib/api/assistant/actions/conversation/list_files.ts +++ b/front/lib/api/assistant/actions/conversation/list_files.ts @@ -66,12 +66,12 @@ export function makeConversationListFilesAction( agentMessage: AgentMessageType, conversation: ConversationType ): ConversationListFilesActionType | null { - const jitFiles: ConversationFileType[] = []; + const files: ConversationFileType[] = []; for (const m of conversation.content.flat(1)) { if (isContentFragmentType(m)) { if (m.fileId) { - jitFiles.push({ + files.push({ fileId: m.fileId, title: m.title, contentType: m.contentType, @@ -81,7 +81,7 @@ export function makeConversationListFilesAction( for (const a of m.actions) { if (isTablesQueryActionType(a)) { if (a.resultsFileId && a.resultsFileSnippet) { - jitFiles.push({ + files.push({ fileId: a.resultsFileId, contentType: "text/csv", title: getTablesQueryResultsFileTitle({ output: a.output }), @@ -92,14 +92,14 @@ export function makeConversationListFilesAction( } } - if (jitFiles.length === 0) { + if (files.length === 0) { return null; } return new ConversationListFilesAction({ functionCallId: "call_" + Math.random().toString(36).substring(7), functionCallName: "list_conversation_files", - files: jitFiles, + files, agentMessageId: agentMessage.agentMessageId, }); } From 37ce9470d7cfca0b95091c87b7d43584012bb258 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 14:38:20 +0100 Subject: [PATCH 20/22] assert message --- front/lib/api/assistant/agent.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/front/lib/api/assistant/agent.ts b/front/lib/api/assistant/agent.ts index 70d6c4c98294..9e57c810a246 100644 --- a/front/lib/api/assistant/agent.ts +++ b/front/lib/api/assistant/agent.ts @@ -311,7 +311,10 @@ async function getEmulatedAgentMessageActions( } // We ensure that all emulated actions are injected with step -1. - assert(actions.every((a) => a.step === -1)); + assert( + actions.every((a) => a.step === -1), + "Emulated actions must have step -1" + ); return actions; } From 7ab6c5fb4fd750fec08d3a0afa53a4271261d32e Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 14:39:43 +0100 Subject: [PATCH 21/22] stepByIndex --- front/lib/api/assistant/jit_actions.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/front/lib/api/assistant/jit_actions.ts b/front/lib/api/assistant/jit_actions.ts index ab49892a8a24..71dd1b1db1f6 100644 --- a/front/lib/api/assistant/jit_actions.ts +++ b/front/lib/api/assistant/jit_actions.ts @@ -90,7 +90,7 @@ export async function renderConversationForModelJIT({ // calls). Actions all have a step index which indicates how they should be grouped but some // actions injected by `getEmulatedAgentMessageActions` have a step index of `-1`. We // therefore group by index, then order and transform in a 2D array to present to the model. - const stepsByStepIndex = {} as Record< + const stepByStepIndex = {} as Record< string, { contents: string[]; @@ -105,27 +105,26 @@ export async function renderConversationForModelJIT({ ({ contents: [], actions: [], - }) satisfies (typeof stepsByStepIndex)[number]; + }) satisfies (typeof stepByStepIndex)[number]; for (const action of actions) { const stepIndex = action.step; - stepsByStepIndex[stepIndex] = - stepsByStepIndex[stepIndex] || emptyStep(); - stepsByStepIndex[stepIndex].actions.push({ + stepByStepIndex[stepIndex] = stepByStepIndex[stepIndex] || emptyStep(); + stepByStepIndex[stepIndex].actions.push({ call: action.renderForFunctionCall(), result: action.renderForMultiActionsModel(), }); } for (const content of m.rawContents) { - stepsByStepIndex[content.step] = - stepsByStepIndex[content.step] || emptyStep(); + stepByStepIndex[content.step] = + stepByStepIndex[content.step] || emptyStep(); if (content.content.trim()) { - stepsByStepIndex[content.step].contents.push(content.content); + stepByStepIndex[content.step].contents.push(content.content); } } - const steps = Object.entries(stepsByStepIndex) + const steps = Object.entries(stepByStepIndex) .sort(([a], [b]) => Number(a) - Number(b)) .map(([, step]) => step); From deb9b77e4a8e83276cd0f8bc6d380a9bc366ec15 Mon Sep 17 00:00:00 2001 From: Stanislas Polu Date: Fri, 15 Nov 2024 14:40:16 +0100 Subject: [PATCH 22/22] remove comments --- sdks/js/src/types.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/sdks/js/src/types.ts b/sdks/js/src/types.ts index 0c4723e57905..9c61f7196609 100644 --- a/sdks/js/src/types.ts +++ b/sdks/js/src/types.ts @@ -474,9 +474,6 @@ const ConversationListFilesActionTypeSchema = BaseActionSchema.extend({ step: z.number(), type: z.literal("conversation_list_files_action"), }); -// type ConversationListFIlesActionPublicType = z.infer< -// typeof ConversationListFilesActionTypeSchema -// >; const DustAppParametersSchema = z.record( z.union([z.string(), z.number(), z.boolean()])