Skip to content

Commit

Permalink
Fix and improve Nebius provider. Add LLMs and descriptions
Browse files Browse the repository at this point in the history
  • Loading branch information
vadjs committed Nov 11, 2024
1 parent e2a4725 commit d296f9b
Show file tree
Hide file tree
Showing 12 changed files with 136 additions and 49 deletions.
14 changes: 10 additions & 4 deletions core/config/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -628,6 +628,11 @@ declare global {
| "mistral-large-latest"
| "mistral-7b"
| "mistral-8x7b"
| "mistral-8x22b"
| "mistral-tiny"
| "mistral-small"
| "mistral-medium"
| "mistral-nemo"
// Llama 2
| "llama2-7b"
| "llama2-13b"
Expand All @@ -641,6 +646,8 @@ declare global {
| "llama3-70b"
// Other Open-source
| "phi2"
| "phi-3-mini"
| "phi-3-medium"
| "phind-codellama-34b"
| "wizardcoder-7b"
| "wizardcoder-13b"
Expand All @@ -649,9 +656,12 @@ declare global {
| "codeup-13b"
| "deepseek-7b"
| "deepseek-33b"
| "deepseek-2-lite"
| "neural-chat-7b"
| "gemma-7b-it"
| "gemma2-2b-it"
| "gemma2-9b-it"
| "olmo-7b"
// Anthropic
| "claude-3-5-sonnet-latest"
| "claude-3-5-sonnet-20240620"
Expand All @@ -669,10 +679,6 @@ declare global {
| "gemini-1.5-pro"
| "gemini-1.5-flash-latest"
| "gemini-1.5-flash"
// Mistral
| "mistral-tiny"
| "mistral-small"
| "mistral-medium"
// Tab autocomplete
| "deepseek-1b"
| "starcoder-1b"
Expand Down
33 changes: 23 additions & 10 deletions core/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -494,11 +494,14 @@ export interface IDE {
stepIndex: number,
): Promise<void>;
getOpenFiles(): Promise<string[]>;
getCurrentFile(): Promise<undefined | {
isUntitled: boolean
path: string
contents: string
}>;
getCurrentFile(): Promise<
| undefined
| {
isUntitled: boolean;
path: string;
contents: string;
}
>;
getPinnedFiles(): Promise<string[]>;
getSearchResults(query: string): Promise<string>;
subprocess(command: string, cwd?: string): Promise<[string, string]>;
Expand Down Expand Up @@ -677,10 +680,12 @@ export type ModelName =
| "mistral-large-latest"
| "mistral-7b"
| "mistral-8x7b"
| "mistral-8x22b"
| "mistral-tiny"
| "mistral-small"
| "mistral-medium"
| "mistral-embed"
| "mistral-nemo"
// Llama 2
| "llama2-7b"
| "llama2-13b"
Expand All @@ -705,6 +710,8 @@ export type ModelName =
| "grok-beta"
// Other Open-source
| "phi2"
| "phi-3-mini"
| "phi-3-medium"
| "phind-codellama-34b"
| "wizardcoder-7b"
| "wizardcoder-13b"
Expand All @@ -713,9 +720,13 @@ export type ModelName =
| "codeup-13b"
| "deepseek-7b"
| "deepseek-33b"
| "deepseek-2-lite"
| "neural-chat-7b"
| "gemma-7b-it"
| "gemma2-2b-it"
| "gemma2-9b-it"
| "olmo-7b"
| "qwen-coder2.5-7b"
// Anthropic
| "claude-3-5-sonnet-latest"
| "claude-3-5-sonnet-20240620"
Expand Down Expand Up @@ -785,11 +796,13 @@ export interface CustomCommand {
}

interface Prediction {
type: "content"
content: string | {
type: "text"
text: string
}[]
type: "content";
content:
| string
| {
type: "text";
text: string;
}[];
}

interface BaseCompletionOptions {
Expand Down
23 changes: 22 additions & 1 deletion core/llm/llms/Nebius.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,32 @@ import { LLMOptions, ModelProvider } from "../..";
import OpenAI from "./OpenAI";

class Nebius extends OpenAI {
static providerName: ModelProvider = "nvidia";
static providerName: ModelProvider = "nebius";
static defaultOptions: Partial<LLMOptions> = {
apiBase: "https://api.studio.nebius.ai/v1/",
useLegacyCompletionsEndpoint: false,
};

private static MODEL_IDS: { [name: string]: string } = {
"llama3.1-70b-nemotron": "nvidia/Llama-3.1-Nemotron-70B-Instruct-HF-fast",
"llama3.1-8b": "meta-llama/Meta-Llama-3.1-70B-Instruct-fast",
"llama3.1-70b": "meta-llama/Meta-Llama-3.1-70B-Instruct-fast",
"llama3.1-405b": "meta-llama/Meta-Llama-3.1-405B-Instruct",
"mistral-nemo": "mistralai/Mistral-Nemo-Instruct-2407-fast",
"mistral-8x7b": "mistralai/Mixtral-8x7B-Instruct-v0.1-fast",
"mistral-8x22b": "mistralai/Mixtral-8x22B-Instruct-v0.1-fast",
"qwen-coder2.5-7b": "Qwen/Qwen2.5-Coder-7B-Instruct-fast",
"deepseek-2-lite": "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct-fast",
"phi-3-mini": "microsoft/Phi-3-mini-4k-instruct-fast",
"phi-3-medium": "microsoft/Phi-3-medium-128k-instruct-fast",
"gemma2-2b-it": "google/gemma-2-2b-it-fast",
"gemma2-9b-it": "google/gemma-2-9b-it-fast",
"olmo-7b": "allenai/OLMo-7B-Instruct-hf",
};

protected _convertModelName(model: string) {
return Nebius.MODEL_IDS[model] || this.model;
}
}

export default Nebius;
3 changes: 3 additions & 0 deletions core/test/llm.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -107,4 +107,7 @@ describe.skip("LLM", () => {
// testLLM(
// new Flowise({ apiKey: process.env.FLOWISE_API_KEY, model: "gpt-3.5-turbo" })
// );
// testLLM(
// new Nebius({ apiKey: process.env.NEBIUS_API_KEY, model: "llama3.1-8b" })
// );
});
12 changes: 12 additions & 0 deletions docs/docs/chat/model-setup.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,18 @@ If you prefer to use an open-weight model, then Llama 3.1 405B from Meta is your
]
```
</TabItem>
<TabItem value="Nebius">
```json title="config.json"
"models": [
{
"title": "Llama 3.1 405B",
"provider": "nebius",
"model": "llama3.1-405b",
"apiKey": "[NEBIUS_API_KEY]"
}
]
```
</TabItem>
</Tabs>

### GPT-4o from OpenAI
Expand Down
36 changes: 36 additions & 0 deletions docs/docs/customize/model-providers/more/nebius.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# [Nebius AI Studio](https://nebius.com/studio/inference)

You can get an API key from the [Nebius AI Studio API keys page](https://studio.nebius.ai/settings/api-keys)

## Availible models

Availible models can be found on the [Nebius AI Studio models page](https://studio.nebius.ai)

## Chat model

```json title="config.json"
{
"models": [
{
"title": "Llama 3.1 405b",
"provider": "nebius",
"model": "llama3.1-405b",
"apiKey": "[API_KEY]"
}
]
}
```

## Embeddings model

We recommend configuring **text-embedding-3-large** as your embeddings model.

```json title="config.json"
{
"embeddingsProvider": {
"provider": "nebius",
"model": "BAAI/bge-en-icl",
"apiKey": "[API_KEY]"
}
}
```
4 changes: 4 additions & 0 deletions docs/docusaurus.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -384,6 +384,10 @@ const config = {
to: "/customize/model-providers/more/watsonx",
from: "/reference/Model Providers/watsonx",
},
{
to: "/customize/model-providers/more/nebius",
from: "/reference/Model Providers/nebius",
},
// Sidebar items that should route directly to a subpage
{
to: "/chat/how-to-use-it",
Expand Down
Binary file added gui/public/logos/qwen.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
40 changes: 8 additions & 32 deletions gui/src/pages/AddNewModel/configs/models.ts
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ export const models: { [key: string]: ModelPackage } = {
"replicate",
"sambanova",
"cerebras",
"nebius",
],
isOpenSource: true,
},
Expand Down Expand Up @@ -190,6 +191,10 @@ export const models: { [key: string]: ModelPackage } = {
model: "mistral-8x7b",
title: "Mixtral",
},
"8x22b (MoE)": {
model: "mistral-8x22b",
title: "Mixtral",
},
},
},
],
Expand Down Expand Up @@ -1005,52 +1010,23 @@ export const models: { [key: string]: ModelPackage } = {
icon: "openai.png",
isOpenSource: false,
},
MetaLlama3Large: {
title: "Llama 3.1 405b",
description:
"Llama 3 is an auto-regressive language model that uses an optimized transformer architecture.",
params: {
title: "Llama 3.1 405b",
model: "meta-llama/Meta-Llama-3.1-405B-Instruct",
contextLength: 128_000,
},
icon: "meta.png",
dimensions: [
{
name: "Parameter Count",
description: "The number of parameters in the model",
options: {
"70b": {
model: "meta-llama/Meta-Llama-3.1-70B-Instruct",
title: "Llama 3.1 70b",
},
"405bb": {
model: "meta-llama/Meta-Llama-3.1-405B-Instruct",
title: "Llama 3.1 405b",
},
},
},
],
providerOptions: ["nebius"],
isOpenSource: true,
},
Qwen2Coder: {
title: "Qwen 2.5 Coder 7b",
description:
"Qwen 2.5 is an auto-regressive language model that uses an optimized transformer architecture.",
params: {
title: "Qwen 2.5 Coder 7b",
model: "Qwen/Qwen2.5-Coder-7B-Instruct",
model: "qwen-coder2.5-7b",
contextLength: 32_000,
},
icon: "meta.png",
icon: "qwen.png",
dimensions: [
{
name: "Parameter Count",
description: "The number of parameters in the model",
options: {
"7b": {
model: "Qwen/Qwen2.5-Coder-7B-Instruct",
model: "qwen-coder2.5-7b",
title: "Qwen 2.5 Coder 7b",
},
},
Expand Down
4 changes: 2 additions & 2 deletions gui/src/pages/AddNewModel/configs/providers.ts
Original file line number Diff line number Diff line change
Expand Up @@ -726,7 +726,7 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo
nebius: {
title: "Nebius AI Studio",
provider: "nebius",
refPage: "nebiusllm",
refPage: "nebius",
description: "Use the Nebius API to run open-source models",
longDescription: `Nebius AI Studio is a cheap hosted service with $100 trial. To get started with Nebius AI Studio:\n1. Obtain an API key from [here](https://studio.nebius.ai)\n2. Paste below\n3. Select a model preset`,
params: {
Expand All @@ -744,7 +744,7 @@ To get started, [register](https://dataplatform.cloud.ibm.com/registration/stepo
],
icon: "nebius.png",
tags: [ModelProviderTags.RequiresApiKey, ModelProviderTags.OpenSource],
packages: [models.MetaLlama3Large, models.Qwen2Coder],
packages: [models.llama31Chat, models.Qwen2Coder, models.mistralOs],
apiKeyUrl: "https://studio.nebius.ai/settings/api-keys",
},
};
5 changes: 5 additions & 0 deletions packages/openai-adapters/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ export function constructLlmApi(config: LlmApiConfig): BaseLlmApi {
...config,
apiBase: "https://api.sambanova.ai/v1/",
});
case "nebius":
return new OpenAIApi({
...config,
apiBase: "https://api.studio.nebius.ai/v1/",
});
default:
throw new Error(`Unsupported LLM API format: ${config.provider}`);
}
Expand Down
11 changes: 11 additions & 0 deletions packages/openai-adapters/test/main.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -273,6 +273,12 @@ const COMPLETION_TESTS: ({ chatOnly?: boolean } & LlmApiConfig)[] = [
apiKey: process.env.SAMBANOVA_API_KEY!,
chatOnly: true,
},
{
provider: "nebius",
model: "llama3.1-8b",
apiKey: process.env.NEBIUS_API_KEY!,
chatOnly: true,
},
];

const FIM_TESTS: LlmApiConfig[] = [
Expand Down Expand Up @@ -311,6 +317,11 @@ const EMBEDDINGS_TESTS: LlmApiConfig[] = [
model: "models/text-embedding-004",
apiKey: process.env.GEMINI_API_KEY!,
},
{
provider: "nebius",
model: "BAAI/bge-en-icl",
apiKey: process.env.NEBIUS_API_KEY!,
},
];

const RERANK_TESTS: LlmApiConfig[] = [
Expand Down

0 comments on commit d296f9b

Please sign in to comment.