Skip to content

Commit

Permalink
Inputs generation improvements
Browse files Browse the repository at this point in the history
  • Loading branch information
@spolu
spolu committed Oct 6, 2023
1 parent 11f48c4 commit 0783d9f
Show file tree
Hide file tree
Showing 7 changed files with 68 additions and 37 deletions.
12 changes: 6 additions & 6 deletions front/components/assistant_builder/AssistantBuilder.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,8 @@ import {
CLAUDE_DEFAULT_MODEL_CONFIG,
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
getSupportedModelConfig,
GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG,
GPT_4_DEFAULT_MODEL_CONFIG,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
SupportedModel,
} from "@app/lib/assistant";
import { ConnectorProvider } from "@app/lib/connectors_api";
Expand All @@ -46,8 +46,8 @@ import { DataSourceType } from "@app/types/data_source";
import { UserType, WorkspaceType } from "@app/types/user";

const usedModelConfigs = [
GPT_4_DEFAULT_MODEL_CONFIG,
GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
CLAUDE_DEFAULT_MODEL_CONFIG,
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
];
Expand Down Expand Up @@ -178,8 +178,8 @@ export default function AssistantBuilder({
generationSettings: {
...DEFAULT_ASSISTANT_STATE.generationSettings,
modelSettings: owner.plan.limits.largeModels
? GPT_4_DEFAULT_MODEL_CONFIG
: GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG,
? GPT_4_32K_MODEL_CONFIG
: GPT_3_5_TURBO_16K_MODEL_CONFIG,
},
});
const [showDataSourcesModal, setShowDataSourcesModal] = useState(false);
Expand Down
2 changes: 1 addition & 1 deletion front/lib/actions/registry.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export const DustProdActionRegistry = createActionRegistry({
config: {
MODEL: {
provider_id: "openai",
model_id: "gpt-3.5-turbo-16k",
model_id: "gpt-4-32k",
function_call: "auto",
use_cache: false,
},
Expand Down
7 changes: 4 additions & 3 deletions front/lib/api/assistant/actions/retrieval.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -161,16 +161,17 @@ export async function retrievalActionSpecification(
name: "query",
description:
"The string used to retrieve relevant chunks of information using semantic similarity" +
" based on the user request and conversation context.",
" based on the user request and conversation context." +
" Include as much semantic signal based on the entire conversation history, paraphrasing if necessary. longer queries are generally better.",
type: "string" as const,
});
}
if (configuration.relativeTimeFrame === "auto") {
inputs.push({
name: "relativeTimeFrame",
description:
"The time frame (relative to now) to restrict the search based on the user request and past conversation context." +
" Possible values are: `all`, `{k}h`, `{k}d`, `{k}w`, `{k}m`, `{k}y` where {k} is a number.",
"The time frame (relative to LOCAL_TIME) to restrict the search based on the user request and past conversation context." +
" Possible values are: `all`, `{k}h`, `{k}d`, `{k}w`, `{k}m`, `{k}y` where {k} is a number. Be strict, do not invent invalid values.",
type: "string" as const,
});
}
Expand Down
41 changes: 35 additions & 6 deletions front/lib/api/assistant/agent.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ import {
renderConversationForModel,
runGeneration,
} from "@app/lib/api/assistant/generation";
import {
GPT_3_5_TURBO_16K_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
GPT_4_MODEL_CONFIG,
} from "@app/lib/assistant";
import { Authenticator } from "@app/lib/auth";
import { Err, Ok, Result } from "@app/lib/result";
import logger from "@app/logger/logger";
Expand Down Expand Up @@ -49,23 +54,47 @@ export async function generateActionInputs(
"You are a conversational assistant with access to function calling."
);

const model = {
providerId: "openai",
modelId: "gpt-3.5-turbo-16k",
};
const allowedTokenCount = 12288; // for 16k model.
const MIN_GENERATION_TOKENS = 2048;

const useLargeModels = auth.workspace()?.plan.limits.largeModels
? true
: false;

let model: { providerId: string; modelId: string } = useLargeModels
? {
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
}
: {
providerId: GPT_3_5_TURBO_16K_MODEL_CONFIG.providerId,
modelId: GPT_3_5_TURBO_16K_MODEL_CONFIG.modelId,
};

// Turn the conversation into a digest that can be presented to the model.
const modelConversationRes = await renderConversationForModel({
conversation,
model,
allowedTokenCount,
allowedTokenCount:
GPT_4_32K_MODEL_CONFIG.contextSize - MIN_GENERATION_TOKENS,
});

if (modelConversationRes.isErr()) {
return modelConversationRes;
}

// If we use gpt-4-32k but tokens used is less than GPT_4_CONTEXT_SIZE-MIN_GENERATION_TOKENS then
// switch the model back to GPT_4 standard (8k context, cheaper).
if (
model.modelId === GPT_4_32K_MODEL_CONFIG.modelId &&
modelConversationRes.value.tokensUsed <
GPT_4_MODEL_CONFIG.contextSize - MIN_GENERATION_TOKENS
) {
model = {
providerId: GPT_4_MODEL_CONFIG.providerId,
modelId: GPT_4_MODEL_CONFIG.modelId,
};
}

const config = cloneBaseConfig(
DustProdActionRegistry["assistant-v2-inputs-generator"].config
);
Expand Down
7 changes: 4 additions & 3 deletions front/lib/api/assistant/generation.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,11 +309,12 @@ export async function* runGeneration(
return;
}

// if model is gpt4-32k but tokens used is less than 4k,
// then we override the model to gpt4 standard (cheaper)
// If model is gpt4-32k but tokens used is less than GPT_4_CONTEXT_SIZE-MIN_GENERATION_TOKENS,
// then we override the model to gpt4 standard (8k context, cheaper).
if (
model.modelId === GPT_4_32K_MODEL_ID &&
modelConversationRes.value.tokensUsed < 4000
modelConversationRes.value.tokensUsed <
GPT_4_MODEL_CONFIG.contextSize - MIN_GENERATION_TOKENS
) {
model = {
modelId: GPT_4_MODEL_CONFIG.modelId,
Expand Down
28 changes: 14 additions & 14 deletions front/lib/api/assistant/global_agents.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,8 @@ import {
CLAUDE_DEFAULT_MODEL_CONFIG,
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
getSupportedModelConfig,
GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG,
GPT_4_DEFAULT_MODEL_CONFIG,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
MISTRAL_7B_DEFAULT_MODEL_CONFIG,
} from "@app/lib/assistant";
import { GLOBAL_AGENTS_SID } from "@app/lib/assistant";
Expand Down Expand Up @@ -81,12 +81,12 @@ async function _getHelperGlobalAgent(
}
const model = owner.plan.limits.largeModels
? {
providerId: GPT_4_DEFAULT_MODEL_CONFIG.providerId,
modelId: GPT_4_DEFAULT_MODEL_CONFIG.modelId,
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
}
: {
providerId: GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG.providerId,
modelId: GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG.modelId,
providerId: GPT_3_5_TURBO_16K_MODEL_CONFIG.providerId,
modelId: GPT_3_5_TURBO_16K_MODEL_CONFIG.modelId,
};

return {
Expand Down Expand Up @@ -128,8 +128,8 @@ async function _getGPT35TurboGlobalAgent({
id: -1,
prompt: "",
model: {
providerId: GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG.providerId,
modelId: GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG.modelId,
providerId: GPT_3_5_TURBO_16K_MODEL_CONFIG.providerId,
modelId: GPT_3_5_TURBO_16K_MODEL_CONFIG.modelId,
},
temperature: 0.7,
},
Expand All @@ -155,8 +155,8 @@ async function _getGPT4GlobalAgent({
id: -1,
prompt: "",
model: {
providerId: GPT_4_DEFAULT_MODEL_CONFIG.providerId,
modelId: GPT_4_DEFAULT_MODEL_CONFIG.modelId,
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
},
temperature: 0.7,
},
Expand Down Expand Up @@ -327,8 +327,8 @@ async function _getManagedDataSourceAgent(
id: -1,
prompt,
model: {
providerId: GPT_4_DEFAULT_MODEL_CONFIG.providerId,
modelId: GPT_4_DEFAULT_MODEL_CONFIG.modelId,
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
},
temperature: 0.4,
},
Expand Down Expand Up @@ -502,8 +502,8 @@ async function _getDustGlobalAgent(
prompt:
"Assist the user based on the retrieved data from their workspace.",
model: {
providerId: GPT_4_DEFAULT_MODEL_CONFIG.providerId,
modelId: GPT_4_DEFAULT_MODEL_CONFIG.modelId,
providerId: GPT_4_32K_MODEL_CONFIG.providerId,
modelId: GPT_4_32K_MODEL_CONFIG.modelId,
},
temperature: 0.4,
},
Expand Down
8 changes: 4 additions & 4 deletions front/lib/assistant.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import { AgentConfigurationType } from "@app/types/assistant/agent";
export const GPT_4_32K_MODEL_ID = "gpt-4-32k" as const;
export const GPT_4_MODEL_ID = "gpt-4" as const;

export const GPT_4_DEFAULT_MODEL_CONFIG = {
export const GPT_4_32K_MODEL_CONFIG = {
providerId: "openai",
modelId: GPT_4_32K_MODEL_ID,
displayName: "GPT 4",
Expand All @@ -26,7 +26,7 @@ export const GPT_4_MODEL_CONFIG = {
recommendedTopK: 16,
};

export const GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG = {
export const GPT_3_5_TURBO_16K_MODEL_CONFIG = {
providerId: "openai",
modelId: "gpt-3.5-turbo-16k",
displayName: "GPT 3.5 Turbo",
Expand Down Expand Up @@ -72,9 +72,9 @@ export const MISTRAL_7B_DEFAULT_MODEL_CONFIG = {
} as const;

export const SUPPORTED_MODEL_CONFIGS = [
GPT_3_5_TURBO_DEFAULT_MODEL_CONFIG,
GPT_3_5_TURBO_16K_MODEL_CONFIG,
GPT_3_5_TURBO_MODEL_CONFIG,
GPT_4_DEFAULT_MODEL_CONFIG,
GPT_4_32K_MODEL_CONFIG,
GPT_4_MODEL_CONFIG,
CLAUDE_DEFAULT_MODEL_CONFIG,
CLAUDE_INSTANT_DEFAULT_MODEL_CONFIG,
Expand Down

0 comments on commit 0783d9f

Please sign in to comment.