Skip to content

Commit

Permalink
Dev (#1780)
Browse files Browse the repository at this point in the history
* Nate/control plane client (#1691)

* docs: add docs and schema for "OS" provider (#1536)

* ignore .env

* fix(gui): ctx rendering w/ renderInlineAs: "" (#1541)

* ✨ use and cache imports for autocomplete (#1456)

* ✨ use and cache imports for autocomplete

* fix tsc

* fix codeqwen autocomplete leading space

* add voyage rerank-1

* feat: `--noEmit` for tsc checks in CI (#1559)

* docs: update CustomContextProvider docs (#1557)

* add stop tokens to qwen prompt

* update docs to reflect 3.5 sonnet being best

* docs: comment out unused providers (#1561)

* import Handlebars

* feat: toast notification for config updates (#1560)

* feat: toast notification for config updates

* feat: only trigger toast on config.json save

* displayRawMarkdown option

* feat: open pane on install (#1564)

* feat: open pane on activation

* comment out testing code

* update to reflect 16 stop words limit for deepseek

* feat: only trigger config update toast in vscode (#1571)

* docs(prompt-files): fix typos + clarify (#1575)

* doc: prompt file typo + clarifications

* fix: add back correct docs

* chore: add telemetry for pageviews (#1576)

* feat: update onboarding w/ embeddings model (#1570)

* chore(gui): remove unused pages

* feat: add embeddings step

* feat: update styles

* feat: copy button updates

* fix: correct pull command for embed model

* fix: remove commented code

* fix: remove commented code

* feat: simplify copy btn props

* chore: rename onboarding selection event

* feat: add provider config

* fix: undo msg name

* remove dead code

* fix: invalid mode check

* fix: remove testing logic

* docs(telemetry): add pageviews to tracking list (#1581)

* Add reranker configuration options to codebase embedding docs (#1584)

- Introduce reranker concept
- List available reranker options
- Provide configuration instructions
- Update keywords to include "reranker"

* chore: update pr template with screenshots (#1590)

* Refactor ConfirmationDialog to use SecondaryButton for cancel action (#1586)

* Added instructions for running docs server locally (#1578)

- Added NPM script method
- Added VS Code task method
- Update contributing guidelines

* Update branch policy (#1577)

- Change PR target to `dev` branch
- Update `CONTRIBUTING.md` instructions

* Consolidate example configurations into the main configuration guide (#1579)

- Moved examples to configuration.md
- Deleted the separate examples.md file
- Updated sidebar order and links
- Improved readability and structure in configuration.md

* fix: fullscreen gui retains context when hidden, fixed fullscreen focusing (#1582)

* Update completionProvider.ts (warning tab-autocomplete models) (#1566)

* feat: enhanced IndexingProgressBar with blinking dot feature

- Integrated BlinkingDot component
- Added STATUS_COLORS for various states
- Replaced CircleDiv with BlinkingDot in UI
- Updated status messages and layout

* small UI tweaks

* feat(gui): enhance ModelCard, ModelProviderTag, and Toggle components (#1595)

- add styling and adjustments to ModelCard
- update ModelProviderTag font size
- remove box shadow from Toggle component
- tweak icon styles in ModelCard
- improve alignment and spacing

* media query

* feat: add best experience onboarding

* fix: file rename

* stop movement on button hover by keeping same border thickness

* fix mistake in setting cursor: pointer

* fix when free trial option is shown

* Support Node.js versions below 20 for streaming response handling (#1591)

- Add fallback for Node < 20
- Implement toAsyncIterable for streaming
- Use TextDecoder for manual decoding
- Maintain existing streaming for Node 20+

* small fixes

* feat: add free trial card to onboarding (#1600)

* feat: add free trial card to onboarding

* add import

* fix hasPassedFTL

* fix /edit cancellation from UI

* feat: add `applyCodeBlock` experimental prop (#1601)

* feat: add new model styling improvements (#1609)

* feat: add new model styling improvements

* better gap size

* feat: update bug_report.yml (#1610)

* chore: update bug_report.yml

* typo fix

* feat: add labels to "Add docs" dialog (#1612)

* feat: add labels to "Add docs" dialog

* remove autofocus

* don't double load config

* small fixes

* speed up directory traversal, and use correct native path module

* option not to show config update toast

* merge air-gapped and recommended setup pages

* chore: add telemetry for full screen toggle (#1618)

* Fix headings in codebase-embeddings.md (#1617)

* mention jetbrains

* docs: update changie  (#1619)

* feat: updated changie config

* hide toc and autogenerate

* Update changelog.mdx

* link to deeper explanation of embeddings models

* ensure target="_blank" for all links in sidebar

* fix gif links in intellij README.md

* don't require rust in dependency installation

* chore: fix padding on gh button (#1620)

* chore: adjust button padding

* Update tasks.json

* escape colons in diff path

* smoother lancedb indexing reporting

* smooth progress updates for indexing

* fix tsc err

* rerank-lite-1

* remove doccs

* basic tests for VS Code extension

* improved testing of VS Code extension

* docs: add docs and schema for "OS" provider (#1536)

* ignore .env

* 🚑 fix constant warnings when onboarding with Ollama

* ✨ use and cache imports for autocomplete (#1456)

* ✨ use and cache imports for autocomplete

* fix tsc

* team analytics

* apply control plane settings

* workos auth

* ide protocol get session info

* UI for auth

* profile switching

* small fixes

* updates

* refresh tokens

* updates

* fix tsc errs

* model select in toolbar to make room for profile selector

* prod client id

* link to prod URL

* internal beta option

* profiles change listener

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
Co-authored-by: Priyash <38959321+priyashpatil@users.noreply.github.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>
Co-authored-by: YohannZe <99359799+YohannZe@users.noreply.github.com>
Co-authored-by: Dan Dascalescu <ddascalescu+github@gmail.com>

* temporary patch for type errs

* Dev (#1689)

* Fix an issue where CMD+K does not clear the terminal when the terminal has focus (#1671)

On MacOS, ⌘+K is bound, by default, to Terminal:Clear. Without this
change ⌘+K does not clear the terminal but instead iniates a chord
sequence and waits for the next stroke of the chord.

Co-authored-by: Rob Leidle <rleidle@tesla.com>

* Change treeSitter to cache the Language objects it loads from wasm (#1672)

Without this change, for a repository with 600 typescript files, the
indexer would fail to finish correctly and there would be many of the
following errors in the webview console log:

'Unable to load language for file ${path} RuntimeError: table index is out of bounds'

The following bash will create a repo that reproduces the problem:

current_path="."

for ((i=1; i<=20; i++)); do
    new_folder="folder-$i"
    mkdir -p "$current_path/$new_folder"
    current_path="$current_path/$new_folder"

    for ((a=1; a<=30; a++)); do
      head -c 10000 /dev/urandom | base64 > "$current_path/file-$a.ts"
    done
done

Co-authored-by: Rob Leidle <rleidle@tesla.com>

* acknowledge sourcemap flag in esbuild.js

* don't run jetbrains-release.yaml on vscode releases

* further testing for walkDir

* chore: add telemetry to commands (#1673)

* test: Add basic unit test to baseLLM (#1668)

* update version

* test: Add basic unit test to baseLLM

---------

Co-authored-by: Nate Sesti <sestinj@gmail.com>
Co-authored-by: inimaz <imaz@ai-verse.com>

* feat: add Quick Actions CodeLens feature (#1674)

* docs: add docs and schema for "OS" provider (#1536)

* ignore .env

* ✨ use and cache imports for autocomplete (#1456)

* ✨ use and cache imports for autocomplete

* fix tsc

* add voyage rerank-1

* import Handlebars

* feat: open pane on install (#1564)

* feat: open pane on activation

* comment out testing code

* chore: add telemetry for pageviews (#1576)

* feat: update onboarding w/ embeddings model (#1570)

* chore(gui): remove unused pages

* feat: add embeddings step

* feat: update styles

* feat: copy button updates

* fix: correct pull command for embed model

* fix: remove commented code

* fix: remove commented code

* feat: simplify copy btn props

* chore: rename onboarding selection event

* feat: add provider config

* fix: undo msg name

* remove dead code

* fix: invalid mode check

* fix: remove testing logic

* fix: fullscreen gui retains context when hidden, fixed fullscreen focusing (#1582)

* small UI tweaks

* media query

* feat: add best experience onboarding

* small fixes

* feat: add free trial card to onboarding (#1600)

* feat: add free trial card to onboarding

* add import

* chore: add telemetry for full screen toggle (#1618)

* rerank-lite-1

* remove doccs

* basic tests for VS Code extension

* improved testing of VS Code extension

* manually implement stop tokens for hf inference api

* chore: onboarding metrics (#1626)

* fix: pageview tracking

* feat: add onboarding telemetry

* create single `onboardingStatus` type

* improved var naming

* remove console logs

* fix windows performance issue

* rename vscodeExtension.ts

* migration of onboarding variables

* "stash" instead of "delete" in indexing progress

* fix preview.yaml

* also fix main.yaml

* Update troubleshooting.md (#1637)

* feat: add quick actions

* Update index.d.ts

* quick actions mvp

* update docs

* subscribe to vscode change settings

* Update commands.ts

* cleanup

* Update quick-actions.md

* Update VerticalPerLineCodeLensProvider.ts

* resolve feedback

---------

Co-authored-by: Nate Sesti <sestinj@gmail.com>
Co-authored-by: Nate Sesti <33237525+sestinj@users.noreply.github.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>

* chore: add `isCommandEvent` to command telemetry (#1675)

* chore: add `isCommandEvent` to command telemetry

* Update commands.ts

* Nate/better retrieval (#1677)

* deduplicatearray tests

* break out separate retrieval pipelines

* IConfigHandler

* tests for codebase indexer

* better .continueignore for continue

* indexing fixes

* ignore .gitignore and .continueignore when indexing

* retrieval pipeline improvements

* fix formatting err in out .continueignore

* add necessary filter to lance_db_cache

* update package.json version

* skip unused tests

* don't ignore .prompt files

* update version

* Update pull_request_template.md

* don't use multi-media format when there are multiple text items

* add free trial experience (#1685)

* fix: add code range for quick actions/fixes (#1687)

* fix: add code range for quick actions/fixes

* Update test.js

* add pathSep message type

* docs improvements

* jetbrains fix

* update package.json version

---------

Co-authored-by: Rob Leidle <rleidle@gmail.com>
Co-authored-by: Rob Leidle <rleidle@tesla.com>
Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
Co-authored-by: inimaz <49730431+inimaz@users.noreply.github.com>
Co-authored-by: inimaz <imaz@ai-verse.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>
Co-authored-by: Priyash <38959321+priyashpatil@users.noreply.github.com>

* update package.json version

* skip extension tests

* update package.json version

* handlebars import

* don't push package.json version change from CI

* fix: scroll issues w/ code blocks (#1688)

* fix: scroll issues w/ code blocks

* chore: name offset

* chore: remove docs start

* chore: update headings on troubleshooting.md (#1696)

* ignore .svn folder for indexing (#1699)

* chore: clearer naming for `useSuffix` (#1702)

* chore: clearer naming for `useSuffix`

* schema updates

* fix: cmd+shft+l closes sidebar if focused (#1638)

* docs: add docs and schema for "OS" provider (#1536)

* ignore .env

* ✨ use and cache imports for autocomplete (#1456)

* ✨ use and cache imports for autocomplete

* fix tsc

* add voyage rerank-1

* import Handlebars

* feat: open pane on install (#1564)

* feat: open pane on activation

* comment out testing code

* chore: add telemetry for pageviews (#1576)

* feat: update onboarding w/ embeddings model (#1570)

* chore(gui): remove unused pages

* feat: add embeddings step

* feat: update styles

* feat: copy button updates

* fix: correct pull command for embed model

* fix: remove commented code

* fix: remove commented code

* feat: simplify copy btn props

* chore: rename onboarding selection event

* feat: add provider config

* fix: undo msg name

* remove dead code

* fix: invalid mode check

* fix: remove testing logic

* fix: fullscreen gui retains context when hidden, fixed fullscreen focusing (#1582)

* small UI tweaks

* media query

* feat: add best experience onboarding

* small fixes

* feat: add free trial card to onboarding (#1600)

* feat: add free trial card to onboarding

* add import

* chore: add telemetry for full screen toggle (#1618)

* rerank-lite-1

* remove doccs

* basic tests for VS Code extension

* improved testing of VS Code extension

* manually implement stop tokens for hf inference api

* chore: onboarding metrics (#1626)

* fix: pageview tracking

* feat: add onboarding telemetry

* create single `onboardingStatus` type

* improved var naming

* remove console logs

* fix windows performance issue

* rename vscodeExtension.ts

* migration of onboarding variables

* "stash" instead of "delete" in indexing progress

* fix preview.yaml

* also fix main.yaml

* Update troubleshooting.md (#1637)

* feat: close panel if main input is focused

* add skip param

* Update TipTapEditor.tsx

* merge dev

* Update commands.ts

---------

Co-authored-by: Nate Sesti <sestinj@gmail.com>
Co-authored-by: Nate Sesti <33237525+sestinj@users.noreply.github.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>

* bugfix: quick actions undefined array (#1704)

* feat: add rich quick pick for quick edit (#1706)

* feat: multistep quick pick

* fix: multi step for context provider

* Update test.js

* Create deepseek.md (#1708)

* style: add max height to model selector (#1712)

* update styling

* remove testing code

* remove comments

* feat: add tutorial card (#1716)

* feat: create tutorial card

* Update TutorialCard.tsx

* feat: add file search to quick edit (#1714)

* feat: multistep quick pick

* fix: multi step for context provider

* Update test.js

* add file search

* complete file search

* remove unused import

* add comments

* Update utils.ts

* update naming

* chore: move quick pick instantiation (#1723)

* Break out into NPM modules (#1726)

* config.json types

* export all schemas

* break out request logic into packages

* update openai adapters to export body types

* update packaging of npm modules

* update packages

* continue proxy

* mistral

* Fix typo in documentation (#1722)

In  set-up-codestral.md
"Trobleshooting" -> "Troubleshooting"

* update package.json version

* 🪄 model dropdown, control plane (#1692)

* Nate/control plane client (#1691)

* docs: add docs and schema for "OS" provider (#1536)

* ignore .env

* fix(gui): ctx rendering w/ renderInlineAs: "" (#1541)

* ✨ use and cache imports for autocomplete (#1456)

* ✨ use and cache imports for autocomplete

* fix tsc

* fix codeqwen autocomplete leading space

* add voyage rerank-1

* feat: `--noEmit` for tsc checks in CI (#1559)

* docs: update CustomContextProvider docs (#1557)

* add stop tokens to qwen prompt

* update docs to reflect 3.5 sonnet being best

* docs: comment out unused providers (#1561)

* import Handlebars

* feat: toast notification for config updates (#1560)

* feat: toast notification for config updates

* feat: only trigger toast on config.json save

* displayRawMarkdown option

* feat: open pane on install (#1564)

* feat: open pane on activation

* comment out testing code

* update to reflect 16 stop words limit for deepseek

* feat: only trigger config update toast in vscode (#1571)

* docs(prompt-files): fix typos + clarify (#1575)

* doc: prompt file typo + clarifications

* fix: add back correct docs

* chore: add telemetry for pageviews (#1576)

* feat: update onboarding w/ embeddings model (#1570)

* chore(gui): remove unused pages

* feat: add embeddings step

* feat: update styles

* feat: copy button updates

* fix: correct pull command for embed model

* fix: remove commented code

* fix: remove commented code

* feat: simplify copy btn props

* chore: rename onboarding selection event

* feat: add provider config

* fix: undo msg name

* remove dead code

* fix: invalid mode check

* fix: remove testing logic

* docs(telemetry): add pageviews to tracking list (#1581)

* Add reranker configuration options to codebase embedding docs (#1584)

- Introduce reranker concept
- List available reranker options
- Provide configuration instructions
- Update keywords to include "reranker"

* chore: update pr template with screenshots (#1590)

* Refactor ConfirmationDialog to use SecondaryButton for cancel action (#1586)

* Added instructions for running docs server locally (#1578)

- Added NPM script method
- Added VS Code task method
- Update contributing guidelines

* Update branch policy (#1577)

- Change PR target to `dev` branch
- Update `CONTRIBUTING.md` instructions

* Consolidate example configurations into the main configuration guide (#1579)

- Moved examples to configuration.md
- Deleted the separate examples.md file
- Updated sidebar order and links
- Improved readability and structure in configuration.md

* fix: fullscreen gui retains context when hidden, fixed fullscreen focusing (#1582)

* Update completionProvider.ts (warning tab-autocomplete models) (#1566)

* feat: enhanced IndexingProgressBar with blinking dot feature

- Integrated BlinkingDot component
- Added STATUS_COLORS for various states
- Replaced CircleDiv with BlinkingDot in UI
- Updated status messages and layout

* small UI tweaks

* feat(gui): enhance ModelCard, ModelProviderTag, and Toggle components (#1595)

- add styling and adjustments to ModelCard
- update ModelProviderTag font size
- remove box shadow from Toggle component
- tweak icon styles in ModelCard
- improve alignment and spacing

* media query

* feat: add best experience onboarding

* fix: file rename

* stop movement on button hover by keeping same border thickness

* fix mistake in setting cursor: pointer

* fix when free trial option is shown

* Support Node.js versions below 20 for streaming response handling (#1591)

- Add fallback for Node < 20
- Implement toAsyncIterable for streaming
- Use TextDecoder for manual decoding
- Maintain existing streaming for Node 20+

* small fixes

* feat: add free trial card to onboarding (#1600)

* feat: add free trial card to onboarding

* add import

* fix hasPassedFTL

* fix /edit cancellation from UI

* feat: add `applyCodeBlock` experimental prop (#1601)

* feat: add new model styling improvements (#1609)

* feat: add new model styling improvements

* better gap size

* feat: update bug_report.yml (#1610)

* chore: update bug_report.yml

* typo fix

* feat: add labels to "Add docs" dialog (#1612)

* feat: add labels to "Add docs" dialog

* remove autofocus

* don't double load config

* small fixes

* speed up directory traversal, and use correct native path module

* option not to show config update toast

* merge air-gapped and recommended setup pages

* chore: add telemetry for full screen toggle (#1618)

* Fix headings in codebase-embeddings.md (#1617)

* mention jetbrains

* docs: update changie  (#1619)

* feat: updated changie config

* hide toc and autogenerate

* Update changelog.mdx

* link to deeper explanation of embeddings models

* ensure target="_blank" for all links in sidebar

* fix gif links in intellij README.md

* don't require rust in dependency installation

* chore: fix padding on gh button (#1620)

* chore: adjust button padding

* Update tasks.json

* escape colons in diff path

* smoother lancedb indexing reporting

* smooth progress updates for indexing

* fix tsc err

* rerank-lite-1

* remove doccs

* basic tests for VS Code extension

* improved testing of VS Code extension

* docs: add docs and schema for "OS" provider (#1536)

* ignore .env

* 🚑 fix constant warnings when onboarding with Ollama

* ✨ use and cache imports for autocomplete (#1456)

* ✨ use and cache imports for autocomplete

* fix tsc

* team analytics

* apply control plane settings

* workos auth

* ide protocol get session info

* UI for auth

* profile switching

* small fixes

* updates

* refresh tokens

* updates

* fix tsc errs

* model select in toolbar to make room for profile selector

* prod client id

* link to prod URL

* internal beta option

* profiles change listener

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
Co-authored-by: Priyash <38959321+priyashpatil@users.noreply.github.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>
Co-authored-by: YohannZe <99359799+YohannZe@users.noreply.github.com>
Co-authored-by: Dan Dascalescu <ddascalescu+github@gmail.com>

* temporary patch for type errs

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
Co-authored-by: Priyash <38959321+priyashpatil@users.noreply.github.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>
Co-authored-by: YohannZe <99359799+YohannZe@users.noreply.github.com>
Co-authored-by: Dan Dascalescu <ddascalescu+github@gmail.com>

* skip extension tests

* JetBrains status bar spinner for autocomplete

* Remove unused images

* partial accept jetbrains

* fix double-rendering of jetbrains completions

* fix offset of multi-line jetbrains completions

* enable/disable actions

* upgrade to version 0.0.54

* fix: wrap editor buttons when out of space (#1727)

* Add .continuerc file to config directory on activation to prevent it's indexing. (#1733)

* Updated the content links that were deprecated in the document (#1709)

Co-authored-by: Ihe Fan <ih.fan.1027@gmail.com>

* Nate/dev (#1739)

* JetBrains status bar spinner for autocomplete

* Remove unused images

* partial accept jetbrains

* fix double-rendering of jetbrains completions

* fix offset of multi-line jetbrains completions

* enable/disable actions

* upgrade to version 0.0.54

* llm-info package

* small profile switching improvements

* instruct cmd+I not to leave placeholders

* @ files in context-providers docs

* onboarding fix

* update jetbrains ci to build gui

* feat: single default quick pick to edit (#1743)

* feat: single default quick pick to edit

* Update quick-actions.md

* Update quick-actions.md

* use defaultQuickAction

* quick fixes

* fix: handle line removal diff (#1744)

* fix: handle line removal diff

* remove comments

* Configurable `maxChunkSize` for embedding providers (#1746)

* Add `maxChunkSize` to `EmbeddingsProvider`

* Add `maxChunkSize` to schema

* add support for docs via config (#1594)

* mistral mamba (#1748)

* 🐍 mistral mamba

* autoscale retrieval based on context length

* autoscale retrieval based on context length

* fix input box movement

* update jetbrains build

* test: add myers diff tests (#1754)

* feat: enhance help center (#1755)

* feat: enhance help center

* Update help.tsx

* feat: add azure provider config (#1764)

* autocomplete reload fix

* logout of control plane

* Update QuickEditQuickPick.ts (#1772)

* Update QuickEditQuickPick.ts

* updates

* Update QuickEditQuickPick.ts

* pass refreshed access token to continue-proxy model

* test: add dir read check for walkDir (#1773)

* test: add dir read check for walkDir

* Update ignore.ts

* Update walkDir.test.ts

* Update walkDir.test.ts

* gpt-4o-mini

* gpt-4o-mini

* gpt4omini to ui selector

* fix: layout alignment (#1779)

* fix: delete old docs index on force re-index (#1778)

* add "Codebase Force Re-Index" command (#1757)

* Add `HuggingFaceTEIReranker` (#1711)

* Add `HuggingFaceTEIReranker`

* Add `huggingface-tei` to `config_schema.json`

* update package.json version

* update config schemas

---------

Co-authored-by: Patrick Erichsen <patrick.a.erichsen@gmail.com>
Co-authored-by: Priyash <38959321+priyashpatil@users.noreply.github.com>
Co-authored-by: Jonah Wagner <jonahwagner231@gmail.com>
Co-authored-by: YohannZe <99359799+YohannZe@users.noreply.github.com>
Co-authored-by: Dan Dascalescu <ddascalescu+github@gmail.com>
Co-authored-by: Rob Leidle <rleidle@gmail.com>
Co-authored-by: Rob Leidle <rleidle@tesla.com>
Co-authored-by: inimaz <49730431+inimaz@users.noreply.github.com>
Co-authored-by: inimaz <imaz@ai-verse.com>
Co-authored-by: Anaël Bonnafous <71386173+AnaelBonnafous@users.noreply.github.com>
Co-authored-by: Huy Tran <119860259+tranquochuy645@users.noreply.github.com>
Co-authored-by: Gabriel Gordbegli <ggordbegli@gmail.com>
Co-authored-by: Raven-1027 <83693755+Raven-1027@users.noreply.github.com>
Co-authored-by: Ihe Fan <ih.fan.1027@gmail.com>
Co-authored-by: Lukas Kreussel <65088241+LLukas22@users.noreply.github.com>
  • Loading branch information
16 people authored Jul 19, 2024
1 parent 82ac619 commit 62fa541
Show file tree
Hide file tree
Showing 51 changed files with 746 additions and 366 deletions.
1 change: 1 addition & 0 deletions core/config/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -523,6 +523,7 @@ declare global {
| "gpt-4-32k"
| "gpt-4-turbo"
| "gpt-4o"
| "gpt-4o-mini"
| "gpt-4-turbo-preview"
| "gpt-4-vision-preview"
// Mistral
Expand Down
2 changes: 2 additions & 0 deletions core/context/rerankers/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ import { CohereReranker } from "./cohere.js";
import { FreeTrialReranker } from "./freeTrial.js";
import { LLMReranker } from "./llm.js";
import { VoyageReranker } from "./voyage.js";
import {HuggingFaceTEIReranker} from "./tei.js"

export const AllRerankers: { [key in RerankerName]: any } = {
cohere: CohereReranker,
llm: LLMReranker,
voyage: VoyageReranker,
"free-trial": FreeTrialReranker,
"huggingface-tei": HuggingFaceTEIReranker
};
49 changes: 49 additions & 0 deletions core/context/rerankers/tei.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import fetch from "node-fetch";
import { Chunk, Reranker } from "../../index.js";

export class HuggingFaceTEIReranker implements Reranker {
name = "huggingface-tei";

static defaultOptions = {
apiBase: "http://localhost:8080",
truncate: true,
truncation_direction: "Right"
};

constructor(
private readonly params: {
apiBase?: string;
truncate?: boolean;
truncation_direction?: string;
},
) {}

async rerank(query: string, chunks: Chunk[]): Promise<number[]> {
let apiBase = this.params.apiBase ?? HuggingFaceTEIReranker.defaultOptions.apiBase;
if (!apiBase.endsWith("/")) {
apiBase += "/";
}

const resp = await fetch(new URL("rerank", apiBase), {
method: "POST",
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
query: query,
return_text: false,
raw_scores: false,
texts: chunks.map((chunk) => chunk.content),
truncation_direction: this.params.truncation_direction ?? HuggingFaceTEIReranker.defaultOptions.truncation_direction,
truncate: this.params.truncate ?? HuggingFaceTEIReranker.defaultOptions.truncate
}),
});

if (!resp.ok) {
throw new Error(await resp.text());
}

const data = (await resp.json()) as any;
// Resort into original order and extract scores
const results = data.sort((a: any, b: any) => a.index - b.index);
return results.map((result: any) => result.score);
}
}
26 changes: 14 additions & 12 deletions core/diff/streamDiff.ts
Original file line number Diff line number Diff line change
@@ -1,35 +1,37 @@
import { DiffLine } from "../index.js";
import { DiffLine, DiffLineType } from "../index.js";
import { LineStream, matchLine } from "./util.js";

/**
* https://blog.jcoglan.com/2017/02/12/the-myers-diff-algorithm-part-1/
* Invariants:
* - new + same = newLines.length
* - old + same = oldLines.length
* - old + same = oldLinesCopy.length
* ^ (above two guarantee that all lines get represented)
* - Lines are always output in order, at least among old and new separately
*/
export async function* streamDiff(
oldLines: string[],
newLines: LineStream,
): AsyncGenerator<DiffLine> {
const oldLinesCopy = [...oldLines];

// If one indentation mistake is made, others are likely. So we are more permissive about matching
let seenIndentationMistake = false;

let newLineResult = await newLines.next();

while (oldLines.length > 0 && !newLineResult.done) {
while (oldLinesCopy.length > 0 && !newLineResult.done) {
const { matchIndex, isPerfectMatch, newLine } = matchLine(
newLineResult.value,
oldLines,
oldLinesCopy,
seenIndentationMistake,
);

if (!seenIndentationMistake && newLineResult.value !== newLine) {
seenIndentationMistake = true;
}

let type: DiffLine["type"];
let type: DiffLineType;

let isLineRemoval = false;
const isNewLine = matchIndex === -1;
Expand All @@ -39,7 +41,7 @@ export async function* streamDiff(
} else {
// Insert all deleted lines before match
for (let i = 0; i < matchIndex; i++) {
yield { type: "old", line: oldLines.shift()! };
yield { type: "old", line: oldLinesCopy.shift()! };
}

type = isPerfectMatch ? "same" : "old";
Expand All @@ -51,13 +53,13 @@ export async function* streamDiff(
break;

case "same":
yield { type, line: oldLines.shift()! };
yield { type, line: oldLinesCopy.shift()! };
break;

case "old":
yield { type, line: oldLines.shift()! };
yield { type, line: oldLinesCopy.shift()! };

if (oldLines[0] !== newLine) {
if (oldLinesCopy[0] !== newLine) {
yield { type: "new", line: newLine };
} else {
isLineRemoval = true;
Expand All @@ -75,13 +77,13 @@ export async function* streamDiff(
}

// Once at the edge, only one choice
if (newLineResult.done && oldLines.length > 0) {
for (const oldLine of oldLines) {
if (newLineResult.done && oldLinesCopy.length > 0) {
for (const oldLine of oldLinesCopy) {
yield { type: "old", line: oldLine };
}
}

if (!newLineResult.done && oldLines.length === 0) {
if (!newLineResult.done && oldLinesCopy.length === 0) {
yield { type: "new", line: newLineResult.value };
for await (const newLine of newLines) {
yield { type: "new", line: newLine };
Expand Down
2 changes: 2 additions & 0 deletions core/diff/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@ function linesMatch(lineA: string, lineB: string, linesBetween = 0): boolean {
}

/**
* Used to find a match for a new line in an array of old lines.
*
* Return the index of the first match and whether it is a perfect match
* Also return a version of the line with correct indentation if needs fixing
*/
Expand Down
7 changes: 5 additions & 2 deletions core/index.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -374,8 +374,10 @@ export type CustomLLM = RequireAtLeastOne<

// IDE

export type DiffLineType = "new" | "old" | "same";

export interface DiffLine {
type: "new" | "old" | "same";
type: DiffLineType;
line: string;
}

Expand Down Expand Up @@ -596,6 +598,7 @@ export type ModelName =
| "gpt-3.5-turbo-0613"
| "gpt-4-32k"
| "gpt-4o"
| "gpt-4o-mini"
| "gpt-4-turbo"
| "gpt-4-turbo-preview"
| "gpt-4-vision-preview"
Expand Down Expand Up @@ -756,7 +759,7 @@ export interface EmbeddingsProvider {
embed(chunks: string[]): Promise<number[][]>;
}

export type RerankerName = "cohere" | "voyage" | "llm" | "free-trial";
export type RerankerName = "cohere" | "voyage" | "llm" | "free-trial" | "huggingface-tei";

export interface RerankerDescription {
name: RerankerName;
Expand Down
18 changes: 13 additions & 5 deletions core/indexing/docs/DocsService.ts
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ export class DocsService {
return;
}

if (!reIndex && await this.has(startUrl.toString())) {
if (!reIndex && (await this.has(startUrl.toString()))) {
yield {
progress: 1,
desc: "Already indexed",
Expand Down Expand Up @@ -275,11 +275,13 @@ export class DocsService {

try {
const subpathEmbeddings = await embeddingsProvider.embed(
chunkArticle(article, embeddingsProvider.maxChunkSize).map((chunk) => {
chunks.push(chunk);
chunkArticle(article, embeddingsProvider.maxChunkSize).map(
(chunk) => {
chunks.push(chunk);

return chunk.content;
}),
return chunk.content;
},
),
);

embeddings.push(...subpathEmbeddings);
Expand All @@ -296,6 +298,12 @@ export class DocsService {
status: "indexing",
};

// Clear old index if re-indexing.
if (reIndex) {
console.log("Deleting old embeddings");
await this.delete(startUrl.toString());
}

await this.add(siteIndexingConfig.title, startUrl, chunks, embeddings);
this.docsIndexingQueue.delete(startUrl.toString());

Expand Down
1 change: 1 addition & 0 deletions core/indexing/ignore.ts
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ export const DEFAULT_IGNORE_FILETYPES = [
".gitkeep",
".continueignore",
"config.json",
".csv",
// "*.prompt", // can be incredibly confusing for the LLM to have another set of instructions injected into the prompt
];
export const defaultIgnoreFile = ignore().add(DEFAULT_IGNORE_FILETYPES);
Expand Down
9 changes: 3 additions & 6 deletions core/indexing/walkDir.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { EventEmitter } from "events";
import { Minimatch } from "minimatch";
import path from "node:path";
import { FileType, IDE } from "..";
import { DEFAULT_IGNORE_DIRS, DEFAULT_IGNORE_FILETYPES } from "./ignore";
import { FileType, IDE } from "../index.js";
import { DEFAULT_IGNORE_DIRS, DEFAULT_IGNORE_FILETYPES } from "./ignore.js";

export interface WalkerOptions {
isSymbolicLink?: boolean;
Expand Down Expand Up @@ -34,10 +34,7 @@ class Walker extends EventEmitter {
sawError: boolean;
exact: boolean | undefined;
onlyDirs: boolean | undefined;
constructor(
opts: WalkerOptions = {},
protected readonly ide: IDE,
) {
constructor(opts: WalkerOptions = {}, protected readonly ide: IDE) {
super(opts as any);
this.isSymbolicLink = opts.isSymbolicLink || false;
this.path = opts.path || process.cwd();
Expand Down
1 change: 1 addition & 0 deletions core/llm/autodetect.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ const MODEL_SUPPORTS_IMAGES: string[] = [
"llava",
"gpt-4-turbo",
"gpt-4o",
"gpt-4o-mini",
"gpt-4-vision",
"claude-3",
"gemini-ultra",
Expand Down
1 change: 1 addition & 0 deletions core/llm/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const CONTEXT_LENGTH_FOR_MODEL: { [name: string]: number } = {
"gpt-4-32k": GPT_4_CTX_LEN,
"gpt-4-turbo-preview": GPT_4_CTX_LEN,
"gpt-4o": GPT_4_CTX_LEN,
"gpt-4o-mini": GPT_4_CTX_LEN,
"gpt-4-vision": GPT_4_CTX_LEN,
"gpt-4-0125-preview": GPT_4_CTX_LEN,
"gpt-4-1106-preview": GPT_4_CTX_LEN,
Expand Down
8 changes: 8 additions & 0 deletions core/llm/llms/Azure.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
import { ModelProvider } from "../../index.js";
import OpenAI from "./OpenAI.js";

class Azure extends OpenAI {
static providerName: ModelProvider = "azure";
}

export default Azure;
1 change: 1 addition & 0 deletions core/llm/llms/OpenAI.ts
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ const CHAT_ONLY_MODELS = [
"gpt-4-vision",
"gpt-4-0125-preview",
"gpt-4-1106-preview",
"gpt-4o-mini",
];

class OpenAI extends BaseLLM {
Expand Down
2 changes: 2 additions & 0 deletions core/llm/llms/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import LlamaCpp from "./LlamaCpp.js";
import Llamafile from "./Llamafile.js";
import Mistral from "./Mistral.js";
import Msty from "./Msty.js";
import Azure from "./Azure.js";
import Ollama from "./Ollama.js";
import OpenAI from "./OpenAI.js";
import Replicate from "./Replicate.js";
Expand Down Expand Up @@ -124,6 +125,7 @@ const LLMs = [
Cloudflare,
Deepseek,
Msty,
Azure,
];

export async function llmFromDescription(
Expand Down
1 change: 1 addition & 0 deletions core/llm/llms/stubs/ContinueProxy.ts
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ class ContinueProxy extends OpenAI {
static defaultOptions: Partial<LLMOptions> = {
apiBase:
"https://control-plane-api-service-i3dqylpbqa-uc.a.run.app/model-proxy/v1",
useLegacyCompletionsEndpoint: false,
};

supportsFim(): boolean {
Expand Down
10 changes: 10 additions & 0 deletions core/package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions core/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
"eslint": "^8",
"eslint-plugin-import": "^2.29.1",
"jest": "^29.7.0",
"myers-diff": "^2.1.0",
"onnxruntime-common": "1.14.0",
"onnxruntime-web": "1.14.0",
"ts-jest": "^29.1.1"
Expand Down
1 change: 1 addition & 0 deletions core/protocol/ide.ts
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ export type ToIdeFromWebviewOrCoreProtocol = {
{ silent: boolean },
ControlPlaneSessionInfo | undefined,
];
logoutOfControlPlane: [undefined, void];
pathSep: [undefined, string];
};

Expand Down
Loading

0 comments on commit 62fa541

Please sign in to comment.