diff --git a/README.md b/README.md index 23ff710..85a1877 100644 --- a/README.md +++ b/README.md @@ -18,28 +18,20 @@ Function LLM is distributed on NPM. Open a terminal and run the following comman npm install @fxn/llm ``` -## Creating a Function LLM Instance -Function LLM works by setting up an in-process server that handles requests for an LLM API provider (e.g. OpenAI, Anthropic). +> [!IMPORTANT] +> Make sure to create an access key by signing onto [Function](https://fxn.ai/settings/developer). You'll need it to fetch the predictor at runtime. + +## Using the OpenAI Client Locally +To run text generation and embedding models locally using the OpenAI client, create a `FunctionLLM` instance then pass its `baseUrl` on the OpenAI client: ```js import { FunctionLLM } from "@fxn/llm" +import { OpenAI } from "openai" // Create Function LLM client const fxnllm = new FunctionLLM({ - provider: "openai", // or "anthropic" + provider: "openai", accessKey: "" }); -``` - -The client provides a `baseUrl` property that can be passed to LLM clients in your code. - -> [!IMPORTANT] -> Create an access key by signing onto [Function](https://fxn.ai/settings/developer). - -## Using the OpenAI Client Locally -To run text-generation and embedding models locally using the OpenAI client, pass the Function LLM `baseUrl` on the OpenAI client: -```js -import OpenAI from "openai" - // Create an OpenAI client const openai = new OpenAI({ baseUrl: fxnllm.baseUrl, @@ -51,8 +43,25 @@ const openai = new OpenAI({ > Currently, only `openai.embeddings.create` is supported. Text generation is coming soon! ## Using the Anthropic Client Locally -*Coming soon* +To run text generation models locally using the Anthopic client, create a `FunctionLLM` instance then pass its `baseUrl` on the Anthropic client: +```js +import { FunctionLLM } from "@fxn/llm" +import { Anthropic } from "@anthropic-ai/sdk" + +// Create Function LLM client +const fxnllm = new FunctionLLM({ + provider: "anthropic", + accessKey: "" +}); +// Create an Anthropic client +const anthropic = new Anthropic({ + baseUrl: fxnllm.baseUrl, + apiKey: "fxn" +}); +``` +> [!DANGER] +> Anthropic support is not functional as it is still a work-in-progress. ___ ## Useful Links diff --git a/package-lock.json b/package-lock.json index bcb046d..d1f7123 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@anthropic-ai/sdk": "^0.26.1", "@testdeck/mocha": "^0.1.2", "@types/chai": "^4.2.22", "@types/chai-as-promised": "^7.1.4", @@ -33,6 +34,39 @@ "node": ">=18.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.26.1.tgz", + "integrity": "sha512-HeMJP1bDFfQPQS3XTJAmfXkFBdZ88wvfkE05+vsoA9zGn5dHqEaHOPsqkazf/i0gXYg2XlLxxZrf6rUAarSqzw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { + "version": "18.19.45", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.45.tgz", + "integrity": "sha512-VZxPKNNhjKmaC1SUYowuXSRSMGyQGmQjvvA1xE4QZ0xce2kLtEhPDS+kqpCPBZYgqblCLQ2DAjSzmgCM5auvhA==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "dev": true, + "license": "MIT" + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", diff --git a/package.json b/package.json index fa8a4fd..e09bc65 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "zod": "^3.23.8" }, "devDependencies": { + "@anthropic-ai/sdk": "^0.26.1", "@testdeck/mocha": "^0.1.2", "@types/chai": "^4.2.22", "@types/chai-as-promised": "^7.1.4",