diff --git a/binary/package-lock.json b/binary/package-lock.json index 41875c6b22..d5108f7ade 100644 --- a/binary/package-lock.json +++ b/binary/package-lock.json @@ -109,6 +109,7 @@ "eslint": "^8", "eslint-plugin-import": "^2.29.1", "jest": "^29.7.0", + "myers-diff": "^2.1.0", "onnxruntime-common": "1.14.0", "onnxruntime-web": "1.14.0", "ts-jest": "^29.1.1" diff --git a/core/autocomplete/completionProvider.ts b/core/autocomplete/completionProvider.ts index 418a811317..1cd43bf4f0 100644 --- a/core/autocomplete/completionProvider.ts +++ b/core/autocomplete/completionProvider.ts @@ -581,6 +581,7 @@ export class CompletionProvider { suffix, filename, reponame, + language: lang.name, }); } else { // Let the template function format snippets diff --git a/core/autocomplete/languages.ts b/core/autocomplete/languages.ts index 3360c3a9b1..a558dee00d 100644 --- a/core/autocomplete/languages.ts +++ b/core/autocomplete/languages.ts @@ -1,6 +1,7 @@ import type { LineFilter } from "./lineStream"; export interface AutocompleteLanguageInfo { + name: string; topLevelKeywords: string[]; singleLineComment: string; endOfLine: string[]; @@ -14,6 +15,7 @@ export interface AutocompleteLanguageInfo { // TypeScript export const Typescript = { + name: "TypeScript", topLevelKeywords: ["function", "class", "module", "export", "import"], singleLineComment: "//", endOfLine: [";"], @@ -21,6 +23,7 @@ export const Typescript = { // Python export const Python = { + name: "Python", // """"#" is for .ipynb files, where we add '"""' surrounding markdown blocks. // This stops the model from trying to complete the start of a new markdown block topLevelKeywords: ["def", "class", '"""#'], @@ -30,6 +33,7 @@ export const Python = { // Java export const Java = { + name: "Java", topLevelKeywords: ["class", "function"], singleLineComment: "//", endOfLine: [";"], @@ -37,6 +41,7 @@ export const Java = { // C++ export const Cpp = { + name: "C++", topLevelKeywords: ["class", "namespace", "template"], singleLineComment: "//", endOfLine: [";"], @@ -44,6 +49,7 @@ export const Cpp = { // C# export const CSharp = { + name: "C#", topLevelKeywords: ["class", "namespace", "void"], singleLineComment: "//", endOfLine: [";"], @@ -51,6 +57,7 @@ export const CSharp = { // C export const C = { + name: "C", topLevelKeywords: ["if", "else", "while", "for", "switch", "case"], singleLineComment: "//", endOfLine: [";"], @@ -58,6 +65,7 @@ export const C = { // Scala export const Scala = { + name: "Scala", topLevelKeywords: ["def", "val", "var", "class", "object", "trait"], singleLineComment: "//", endOfLine: [";"], @@ -65,6 +73,7 @@ export const Scala = { // Go export const Go = { + name: "Go", topLevelKeywords: ["func", "package", "import", "type"], singleLineComment: "//", endOfLine: [], @@ -72,6 +81,7 @@ export const Go = { // Rust export const Rust = { + name: "Rust", topLevelKeywords: ["fn", "mod", "pub", "struct", "enum", "trait"], singleLineComment: "//", endOfLine: [";"], @@ -79,6 +89,7 @@ export const Rust = { // Haskell export const Haskell = { + name: "Haskell", topLevelKeywords: [ "data", "type", @@ -95,6 +106,7 @@ export const Haskell = { // PHP export const PHP = { + name: "PHP", topLevelKeywords: ["function", "class", "namespace", "use"], singleLineComment: "//", endOfLine: [";"], @@ -102,6 +114,7 @@ export const PHP = { // Ruby on Rails export const RubyOnRails = { + name: "Ruby on Rails", topLevelKeywords: ["def", "class", "module"], singleLineComment: "#", endOfLine: [], @@ -109,6 +122,7 @@ export const RubyOnRails = { // Swift export const Swift = { + name: "Swift", topLevelKeywords: ["func", "class", "struct", "import"], singleLineComment: "//", endOfLine: [";"], @@ -116,6 +130,7 @@ export const Swift = { // Kotlin export const Kotlin = { + name: "Kotlin", topLevelKeywords: ["fun", "class", "package", "import"], singleLineComment: "//", endOfLine: [";"], @@ -123,6 +138,7 @@ export const Kotlin = { // Ruby export const Ruby = { + name: "Ruby", topLevelKeywords: ["class", "module", "def"], singleLineComment: "#", endOfLine: [], @@ -130,6 +146,7 @@ export const Ruby = { // Clojure export const Clojure = { + name: "Clojure", topLevelKeywords: ["def", "fn", "let", "do", "if", "defn", "ns", "defmacro"], singleLineComment: ";", endOfLine: [], @@ -137,6 +154,7 @@ export const Clojure = { // Julia export const Julia = { + name: "Julia", topLevelKeywords: [ "function", "macro", @@ -155,6 +173,7 @@ export const Julia = { // F# export const FSharp = { + name: "F#", topLevelKeywords: [ "let", "type", @@ -173,6 +192,7 @@ export const FSharp = { // R export const R = { + name: "R", topLevelKeywords: [ "function", "if", @@ -189,6 +209,7 @@ export const R = { // Dart export const Dart = { + name: "Dart", topLevelKeywords: ["class", "import", "void", "enum"], singleLineComment: "//", endOfLine: [";"], @@ -196,6 +217,7 @@ export const Dart = { // Solidity export const Solidity = { + name: "Solidity", topLevelKeywords: [ "contract", "event", @@ -218,6 +240,7 @@ export const Solidity = { // YAML export const YAML: AutocompleteLanguageInfo = { + name: "YAML", topLevelKeywords: [], singleLineComment: "#", endOfLine: [], @@ -258,6 +281,7 @@ export const YAML: AutocompleteLanguageInfo = { }; export const Markdown: AutocompleteLanguageInfo = { + name: "Markdown", topLevelKeywords: [], singleLineComment: "", endOfLine: [], diff --git a/core/config/ConfigHandler.ts b/core/config/ConfigHandler.ts index e01b41e0e8..64669f68d8 100644 --- a/core/config/ConfigHandler.ts +++ b/core/config/ConfigHandler.ts @@ -1,4 +1,7 @@ -import { ControlPlaneClient } from "../control-plane/client.js"; +import { + ControlPlaneClient, + ControlPlaneSessionInfo, +} from "../control-plane/client.js"; import { BrowserSerializedContinueConfig, ContinueConfig, @@ -118,44 +121,20 @@ export class ConfigHandler { return this.profiles.filter((p) => p.profileId !== this.selectedProfileId); } - constructor( - private readonly ide: IDE, - private ideSettingsPromise: Promise, - private readonly writeLog: (text: string) => Promise, - private readonly controlPlaneClient: ControlPlaneClient, - ) { - this.ide = ide; - this.ideSettingsPromise = ideSettingsPromise; - this.writeLog = writeLog; - - // Set local profile as default - const localProfileLoader = new LocalProfileLoader( - ide, - ideSettingsPromise, - writeLog, - ); - this.profiles = [new ProfileLifecycleManager(localProfileLoader)]; - this.selectedProfileId = localProfileLoader.profileId; - - // Always load local profile immediately in case control plane doesn't load - try { - this.loadConfig(); - } catch (e) { - console.error("Failed to load config: ", e); - } - - // Load control plane profiles - // TODO + private async fetchControlPlaneProfiles() { // Get the profiles and create their lifecycle managers this.controlPlaneClient.listWorkspaces().then(async (workspaces) => { + this.profiles = this.profiles.filter( + (profile) => profile.profileId === "local", + ); workspaces.forEach((workspace) => { const profileLoader = new ControlPlaneProfileLoader( workspace.id, workspace.name, this.controlPlaneClient, - ide, - ideSettingsPromise, - writeLog, + this.ide, + this.ideSettingsPromise, + this.writeLog, this.reloadConfig.bind(this), ); this.profiles.push(new ProfileLifecycleManager(profileLoader)); @@ -184,6 +163,36 @@ export class ConfigHandler { }); } + constructor( + private readonly ide: IDE, + private ideSettingsPromise: Promise, + private readonly writeLog: (text: string) => Promise, + private controlPlaneClient: ControlPlaneClient, + ) { + this.ide = ide; + this.ideSettingsPromise = ideSettingsPromise; + this.writeLog = writeLog; + + // Set local profile as default + const localProfileLoader = new LocalProfileLoader( + ide, + ideSettingsPromise, + writeLog, + ); + this.profiles = [new ProfileLifecycleManager(localProfileLoader)]; + this.selectedProfileId = localProfileLoader.profileId; + + // Always load local profile immediately in case control plane doesn't load + try { + this.loadConfig(); + } catch (e) { + console.error("Failed to load config: ", e); + } + + // Load control plane profiles + this.fetchControlPlaneProfiles(); + } + async setSelectedProfile(profileId: string) { this.selectedProfileId = profileId; const newConfig = await this.loadConfig(); @@ -209,6 +218,15 @@ export class ConfigHandler { this.reloadConfig(); } + updateControlPlaneSessionInfo( + sessionInfo: ControlPlaneSessionInfo | undefined, + ) { + this.controlPlaneClient = new ControlPlaneClient( + Promise.resolve(sessionInfo), + ); + this.fetchControlPlaneProfiles(); + } + private profilesListeners: ((profiles: ProfileDescription[]) => void)[] = []; onDidChangeAvailableProfiles( listener: (profiles: ProfileDescription[]) => void, diff --git a/core/context/index.ts b/core/context/index.ts index 6e98fa4562..7ad48f1122 100644 --- a/core/context/index.ts +++ b/core/context/index.ts @@ -6,6 +6,7 @@ import type { IContextProvider, LoadSubmenuItemsArgs, } from "../index.js"; + export abstract class BaseContextProvider implements IContextProvider { options: { [key: string]: any }; diff --git a/core/context/providers/DocsContextProvider.ts b/core/context/providers/DocsContextProvider.ts index 83329a930f..c79ee48857 100644 --- a/core/context/providers/DocsContextProvider.ts +++ b/core/context/providers/DocsContextProvider.ts @@ -1,15 +1,16 @@ -import fetch from "node-fetch"; import { + Chunk, ContextItem, ContextProviderDescription, ContextProviderExtras, ContextSubmenuItem, + EmbeddingsProvider, LoadSubmenuItemsArgs, - SiteIndexingConfig, + Reranker, } from "../../index.js"; import { DocsService } from "../../indexing/docs/DocsService.js"; -import configs from "../../indexing/docs/preIndexedDocs.js"; -import TransformersJsEmbeddingsProvider from "../../indexing/embeddings/TransformersJsEmbeddingsProvider.js"; +import preIndexedDocs from "../../indexing/docs/preIndexedDocs.js"; +import { Telemetry } from "../../util/posthog.js"; import { BaseContextProvider } from "../index.js"; class DocsContextProvider extends BaseContextProvider { @@ -21,6 +22,7 @@ class DocsContextProvider extends BaseContextProvider { description: "Type to search docs", type: "submenu", }; + private docsService: DocsService; constructor(options: any) { @@ -28,34 +30,96 @@ class DocsContextProvider extends BaseContextProvider { this.docsService = DocsService.getInstance(); } - private async _getIconDataUrl(url: string): Promise { + private async _rerankChunks( + chunks: Chunk[], + reranker: NonNullable, + fullInput: ContextProviderExtras["fullInput"], + ) { + let chunksCopy = [...chunks]; + try { - const response = await fetch(url); - if (!response.headers.get("content-type")?.startsWith("image/")) { - console.log("Not an image: ", await response.text()); - return undefined; - } - const buffer = await response.buffer(); - const base64data = buffer.toString("base64"); - return `data:${response.headers.get("content-type")};base64,${base64data}`; + const scores = await reranker.rerank(fullInput, chunksCopy); + + chunksCopy.sort( + (a, b) => scores[chunksCopy.indexOf(b)] - scores[chunksCopy.indexOf(a)], + ); + + chunksCopy = chunksCopy.splice( + 0, + this.options?.nFinal ?? DocsContextProvider.DEFAULT_N_FINAL, + ); } catch (e) { - console.log("E: ", e); - return undefined; + console.warn(`Failed to rerank docs results: ${e}`); + + chunksCopy = chunksCopy.splice( + 0, + this.options?.nFinal ?? DocsContextProvider.DEFAULT_N_FINAL, + ); } + + return chunksCopy; + } + + private _sortByPreIndexedDocs( + submenuItems: ContextSubmenuItem[], + ): ContextSubmenuItem[] { + // Sort submenuItems such that the objects with titles which don't occur in configs occur first, and alphabetized + return submenuItems.sort((a, b) => { + const aTitleInConfigs = a.metadata?.preIndexed ?? false; + const bTitleInConfigs = b.metadata?.preIndexed ?? false; + + // Primary criterion: Items not in configs come first + if (!aTitleInConfigs && bTitleInConfigs) { + return -1; + } else if (aTitleInConfigs && !bTitleInConfigs) { + return 1; + } else { + // Secondary criterion: Alphabetical order when both items are in the same category + return a.title.toString().localeCompare(b.title.toString()); + } + }); } async getContextItems( query: string, extras: ContextProviderExtras, ): Promise { - // Not supported in JetBrains IDEs right now - if ((await extras.ide.getIdeInfo()).ideType === "jetbrains") { - throw new Error( - "The @docs context provider is not currently supported in JetBrains IDEs. We'll have an update soon!", + const ideInfo = await extras.ide.getIdeInfo(); + const isJetBrains = ideInfo.ideType === "jetbrains"; + + const isJetBrainsAndPreIndexedDocsProvider = + this.docsService.isJetBrainsAndPreIndexedDocsProvider( + ideInfo, + extras.embeddingsProvider.id, ); + + if (isJetBrainsAndPreIndexedDocsProvider) { + extras.ide.errorPopup( + `${DocsService.preIndexedDocsEmbeddingsProvider.id} is configured as ` + + "the embeddings provider, but it cannot be used with JetBrains. " + + "Please select a different embeddings provider to use the '@docs' " + + "context provider.", + ); + + return []; + } + + const preIndexedDoc = preIndexedDocs[query]; + + let embeddingsProvider: EmbeddingsProvider; + + if (!!preIndexedDoc && !isJetBrains) { + // Pre-indexed docs should be filtered out in `loadSubmenuItems`, + // for JetBrains users, but we sanity check that here + Telemetry.capture("docs_pre_indexed_doc_used", { + doc: preIndexedDoc["title"], + }); + + embeddingsProvider = DocsService.preIndexedDocsEmbeddingsProvider; + } else { + embeddingsProvider = extras.embeddingsProvider; } - const embeddingsProvider = new TransformersJsEmbeddingsProvider(); const [vector] = await embeddingsProvider.embed([extras.fullInput]); let chunks = await this.docsService.retrieve( @@ -66,22 +130,11 @@ class DocsContextProvider extends BaseContextProvider { ); if (extras.reranker) { - try { - const scores = await extras.reranker.rerank(extras.fullInput, chunks); - chunks.sort( - (a, b) => scores[chunks.indexOf(b)] - scores[chunks.indexOf(a)], - ); - chunks = chunks.splice( - 0, - this.options?.nFinal ?? DocsContextProvider.DEFAULT_N_FINAL, - ); - } catch (e) { - console.warn(`Failed to rerank docs results: ${e}`); - chunks = chunks.splice( - 0, - this.options?.nFinal ?? DocsContextProvider.DEFAULT_N_FINAL, - ); - } + chunks = await this._rerankChunks( + chunks, + extras.reranker, + extras.fullInput, + ); } return [ @@ -96,7 +149,7 @@ class DocsContextProvider extends BaseContextProvider { .slice(1) .join("/") : chunk.otherMetadata?.title || chunk.filepath, - description: chunk.filepath, // new URL(chunk.filepath, query).toString(), + description: chunk.filepath, content: chunk.content, })) .reverse(), @@ -109,64 +162,51 @@ class DocsContextProvider extends BaseContextProvider { ]; } - // Get combined site configs from preIndexedDocs and options.sites. - private _getDocsSitesConfig(): SiteIndexingConfig[] { - return [...configs, ...(this.options?.sites || [])]; - } - - // Get indexed docs as ContextSubmenuItems from database. - private async _getIndexedDocsContextSubmenuItems(): Promise { - return (await this.docsService.list()).map((doc) => ({ - title: doc.title, - description: new URL(doc.baseUrl).hostname, - id: doc.baseUrl, - })); - } - async loadSubmenuItems( args: LoadSubmenuItemsArgs, ): Promise { + const ideInfo = await args.ide.getIdeInfo(); + const isJetBrains = ideInfo.ideType === "jetbrains"; + const configSites = this.options?.sites || []; const submenuItemsMap = new Map(); - for (const item of await this._getIndexedDocsContextSubmenuItems()) { - submenuItemsMap.set(item.id, item); + if (!isJetBrains) { + // Currently, we generate and host embeddings for pre-indexed docs using transformers.js. + // However, we don't ship transformers.js with the JetBrains extension. + // So, we only include pre-indexed docs in the submenu for non-JetBrains IDEs. + for (const { startUrl, title } of Object.values(preIndexedDocs)) { + submenuItemsMap.set(startUrl, { + title, + id: startUrl, + description: new URL(startUrl).hostname, + metadata: { + preIndexed: true, + }, + }); + } } - for (const config of this._getDocsSitesConfig()) { - submenuItemsMap.set(config.startUrl, { - id: config.startUrl, - title: config.title, - description: new URL(config.startUrl).hostname, - metadata: { preIndexed: !!configs.find((cnf) => cnf.title === config.title), }, + for (const { title, baseUrl } of await this.docsService.list()) { + submenuItemsMap.set(baseUrl, { + title, + id: baseUrl, + description: new URL(baseUrl).hostname, }); } - const submenuItems = Array.from(submenuItemsMap.values()); - - // Sort submenuItems such that the objects with titles which don't occur in configs occur first, and alphabetized - submenuItems.sort((a, b) => { - const aTitleInConfigs = a.metadata?.preIndexed ?? false; - const bTitleInConfigs = b.metadata?.preIndexed ?? false; + for (const { startUrl, title } of configSites) { + submenuItemsMap.set(startUrl, { + title, + id: startUrl, + description: new URL(startUrl).hostname, + }); + } - // Primary criterion: Items not in configs come first - if (!aTitleInConfigs && bTitleInConfigs) { - return -1; - } else if (aTitleInConfigs && !bTitleInConfigs) { - return 1; - } else { - // Secondary criterion: Alphabetical order when both items are in the same category - return a.title.toString().localeCompare(b.title.toString()); - } - }); + const submenuItems = Array.from(submenuItemsMap.values()); - // const icons = await Promise.all( - // submenuItems.map(async (item) => - // item.iconUrl ? this._getIconDataUrl(item.iconUrl) : undefined, - // ), - // ); - // icons.forEach((icon, i) => { - // submenuItems[i].iconUrl = icon; - // }); + if (!isJetBrains) { + return this._sortByPreIndexedDocs(submenuItems); + } return submenuItems; } diff --git a/core/context/providers/index.ts b/core/context/providers/index.ts index 72c47f0446..ac1d0431ae 100644 --- a/core/context/providers/index.ts +++ b/core/context/providers/index.ts @@ -56,11 +56,5 @@ const Providers: (typeof BaseContextProvider)[] = [ export function contextProviderClassFromName( name: ContextProviderName, ): typeof BaseContextProvider | undefined { - const cls = Providers.find((cls) => cls.description.title === name); - - if (!cls) { - return undefined; - } - - return cls; + return Providers.find((cls) => cls.description.title === name); } diff --git a/core/control-plane/TeamAnalytics.ts b/core/control-plane/TeamAnalytics.ts index 96fafbbf78..302b80177d 100644 --- a/core/control-plane/TeamAnalytics.ts +++ b/core/control-plane/TeamAnalytics.ts @@ -1,28 +1,34 @@ import { Analytics } from "@continuedev/config-types"; import os from "node:os"; +import { IAnalyticsProvider } from "./analytics/IAnalyticsProvider"; +import PostHogAnalyticsProvider from "./analytics/PostHogAnalyticsProvider"; + +function createAnalyticsProvider( + config: Analytics, +): IAnalyticsProvider | undefined { + // @ts-ignore + switch (config.provider) { + case "posthog": + return new PostHogAnalyticsProvider(); + default: + return undefined; + } +} export class TeamAnalytics { - static client: any = undefined; + static provider: IAnalyticsProvider | undefined = undefined; static uniqueId = "NOT_UNIQUE"; static os: string | undefined = undefined; static extensionVersion: string | undefined = undefined; static async capture(event: string, properties: { [key: string]: any }) { - TeamAnalytics.client?.capture({ - distinctId: TeamAnalytics.uniqueId, - event, - properties: { - ...properties, - os: TeamAnalytics.os, - extensionVersion: TeamAnalytics.extensionVersion, - }, + TeamAnalytics.provider?.capture(event, { + ...properties, + os: TeamAnalytics.os, + extensionVersion: TeamAnalytics.extensionVersion, }); } - static shutdownPosthogClient() { - TeamAnalytics.client?.shutdown(); - } - static async setup( config: Analytics, uniqueId: string, @@ -32,17 +38,12 @@ export class TeamAnalytics { TeamAnalytics.os = os.platform(); TeamAnalytics.extensionVersion = extensionVersion; - if (!config || !config.clientKey || !config.url) { - TeamAnalytics.client = undefined; + if (!config) { + await TeamAnalytics.provider?.shutdown(); + TeamAnalytics.provider = undefined; } else { - try { - const { PostHog } = await import("posthog-node"); - TeamAnalytics.client = new PostHog(config.clientKey, { - host: config.url, - }); - } catch (e) { - console.error(`Failed to setup telemetry: ${e}`); - } + TeamAnalytics.provider = createAnalyticsProvider(config); + await TeamAnalytics.provider?.setup(config, uniqueId); } } } diff --git a/core/control-plane/analytics/ElasticAnalyticsProvider.ts b/core/control-plane/analytics/ElasticAnalyticsProvider.ts new file mode 100644 index 0000000000..447b517405 --- /dev/null +++ b/core/control-plane/analytics/ElasticAnalyticsProvider.ts @@ -0,0 +1,17 @@ +import { Analytics } from "@continuedev/config-types"; +import { IAnalyticsProvider } from "./IAnalyticsProvider"; + +export default class ElasticAnalyticsProvider implements IAnalyticsProvider { + async capture( + event: string, + properties: { [key: string]: any }, + ): Promise { + throw new Error("Method not implemented."); + } + + async setup(config: Analytics, uniqueId: string): Promise { + throw new Error("Method not implemented."); + } + + async shutdown(): Promise {} +} diff --git a/core/control-plane/analytics/IAnalyticsProvider.ts b/core/control-plane/analytics/IAnalyticsProvider.ts new file mode 100644 index 0000000000..8a04ba1ced --- /dev/null +++ b/core/control-plane/analytics/IAnalyticsProvider.ts @@ -0,0 +1,11 @@ +import { Analytics } from "@continuedev/config-types"; + +export interface AnalyticsMetadata { + extensionVersion: string; +} + +export interface IAnalyticsProvider { + capture(event: string, properties: { [key: string]: any }): Promise; + setup(config: Analytics, uniqueId: string): Promise; + shutdown(): Promise; +} diff --git a/core/control-plane/analytics/PostHogAnalyticsProvider.ts b/core/control-plane/analytics/PostHogAnalyticsProvider.ts new file mode 100644 index 0000000000..f32a0737b2 --- /dev/null +++ b/core/control-plane/analytics/PostHogAnalyticsProvider.ts @@ -0,0 +1,37 @@ +import { Analytics } from "@continuedev/config-types"; +import { IAnalyticsProvider } from "./IAnalyticsProvider"; + +export default class PostHogAnalyticsProvider implements IAnalyticsProvider { + client?: any; + uniqueId?: string; + + async capture( + event: string, + properties: { [key: string]: any }, + ): Promise { + this.client?.capture({ + distinctId: this.uniqueId, + event, + properties, + }); + } + + async setup(config: Analytics, uniqueId: string): Promise { + if (!config || !config.clientKey || !config.url) { + this.client = undefined; + } else { + try { + this.uniqueId = uniqueId; + + const { PostHog } = await import("posthog-node"); + this.client = new PostHog(config.clientKey, { + host: config.url, + }); + } catch (e) { + console.error(`Failed to setup telemetry: ${e}`); + } + } + } + + async shutdown(): Promise {} +} diff --git a/core/core.ts b/core/core.ts index 25e44a8499..5cf32478e3 100644 --- a/core/core.ts +++ b/core/core.ts @@ -1,6 +1,7 @@ import { v4 as uuidv4 } from "uuid"; import type { ContextItemId, + EmbeddingsProvider, IDE, IndexingProgressUpdate, SiteIndexingConfig, @@ -16,9 +17,9 @@ import { import { createNewPromptFile } from "./config/promptFile.js"; import { addModel, addOpenAIKey, deleteModel } from "./config/util.js"; import { ContinueServerClient } from "./continueServer/stubs/client.js"; -import { ControlPlaneClient } from "./control-plane/client"; +import { ControlPlaneClient } from "./control-plane/client.js"; import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer.js"; -import { DocsService } from "./indexing/docs/DocsService"; +import { DocsService } from "./indexing/docs/DocsService.js"; import TransformersJsEmbeddingsProvider from "./indexing/embeddings/TransformersJsEmbeddingsProvider.js"; import Ollama from "./llm/llms/Ollama.js"; import type { FromCoreProtocol, ToCoreProtocol } from "./protocol"; @@ -65,6 +66,14 @@ export class Core { return this.messenger.invoke(messageType, data); } + send( + messageType: T, + data: FromCoreProtocol[T][0], + messageId?: string, + ): string { + return this.messenger.send(messageType, data); + } + // TODO: It shouldn't actually need an IDE type, because this can happen // through the messenger (it does in the case of any non-VS Code IDEs already) constructor( @@ -73,20 +82,35 @@ export class Core { private readonly onWrite: (text: string) => Promise = async () => {}, ) { this.indexingState = { status: "loading", desc: "loading", progress: 0 }; + const ideSettingsPromise = messenger.request("getIdeSettings", undefined); const sessionInfoPromise = messenger.request("getControlPlaneSessionInfo", { silent: true, }); + this.controlPlaneClient = new ControlPlaneClient(sessionInfoPromise); + this.configHandler = new ConfigHandler( this.ide, ideSettingsPromise, this.onWrite, this.controlPlaneClient, ); + this.configHandler.onConfigUpdate( (() => this.messenger.send("configUpdate", undefined)).bind(this), ); + + this.configHandler.onConfigUpdate(async ({ embeddingsProvider }) => { + if ( + await this.shouldReindexDocsOnNewEmbeddingsProvider( + embeddingsProvider.id, + ) + ) { + await this.reindexDocsOnNewEmbeddingsProvider(embeddingsProvider); + } + }); + this.configHandler.onDidChangeAvailableProfiles((profiles) => this.messenger.send("didChangeAvailableProfiles", { profiles }), ); @@ -117,9 +141,23 @@ export class Core { continueServerClient, ), ); + + // Index on initialization this.ide .getWorkspaceDirs() - .then((dirs) => this.refreshCodebaseIndex(dirs)); + .then(async (dirs) => { + // Respect pauseCodebaseIndexOnStart user settings + if (ideSettings.pauseCodebaseIndexOnStart) { + await this.messenger.request("indexProgress", { + progress: 100, + desc: "Initial Indexing Skipped", + status: "paused", + }); + return; + } + + this.refreshCodebaseIndex(dirs); + }); }); const getLlm = async () => { @@ -221,19 +259,8 @@ export class Core { // Context providers on("context/addDocs", async (msg) => { - const siteIndexingConfig: SiteIndexingConfig = { - startUrl: msg.data.startUrl, - rootUrl: msg.data.rootUrl, - title: msg.data.title, - maxDepth: msg.data.maxDepth, - faviconUrl: new URL("/favicon.ico", msg.data.rootUrl).toString(), - }; + await this.getEmbeddingsProviderAndIndexDoc(msg.data); - for await (const _ of this.docsService.indexAndAdd( - siteIndexingConfig, - new TransformersJsEmbeddingsProvider(), - )) { - } this.ide.infoPopup(`Successfully indexed ${msg.data.title}`); this.messenger.send("refreshSubmenuItems", undefined); }); @@ -248,11 +275,18 @@ export class Core { (provider) => provider.description.title === "docs", ); - const siteIndexingOptions: SiteIndexingConfig[] = provider ? - (mProvider => mProvider?.options?.sites || [])({ ...provider }) : - []; + if (!provider) { + this.ide.infoPopup("No docs in configuration"); + return; + } + + const siteIndexingOptions: SiteIndexingConfig[] = ((mProvider) => + mProvider?.options?.sites || [])({ ...provider }); + + for (const site of siteIndexingOptions) { + await this.getEmbeddingsProviderAndIndexDoc(site, msg.data.reIndex); + } - await this.indexDocs(siteIndexingOptions, msg.data.reIndex); this.ide.infoPopup("Docs indexing completed"); }); on("context/loadSubmenuItems", async (msg) => { @@ -602,7 +636,7 @@ export class Core { }); on("index/forceReIndex", async (msg) => { const dirs = msg.data ? [msg.data] : await this.ide.getWorkspaceDirs(); - this.refreshCodebaseIndex(dirs); + await this.refreshCodebaseIndex(dirs); }); on("index/setPaused", (msg) => { new GlobalContext().update("indexingPaused", msg.data); @@ -619,6 +653,9 @@ export class Core { on("didChangeSelectedProfile", (msg) => { this.configHandler.setSelectedProfile(msg.data.id); }); + on("didChangeControlPlaneSessionInfo", async (msg) => { + this.configHandler.updateControlPlaneSessionInfo(msg.data.sessionInfo); + }); } private indexingCancellationController: AbortController | undefined; @@ -635,17 +672,97 @@ export class Core { this.messenger.request("indexProgress", update); this.indexingState = update; } + + this.messenger.send("refreshSubmenuItems", undefined); } - private async indexDocs(sites: SiteIndexingConfig[], reIndex: boolean): Promise { - for (const site of sites) { - for await (const update of this.docsService.indexAndAdd(site, new TransformersJsEmbeddingsProvider(), reIndex)) { - // Temporary disabled posting progress updates to the UI due to - // possible collision with code indexing progress updates. + private async shouldReindexDocsOnNewEmbeddingsProvider( + curEmbeddingsProviderId: EmbeddingsProvider["id"], + ): Promise { + const ideInfo = await this.ide.getIdeInfo(); + const isJetBrainsAndPreIndexedDocsProvider = + this.docsService.isJetBrainsAndPreIndexedDocsProvider( + ideInfo, + curEmbeddingsProviderId, + ); - // this.messenger.request("indexProgress", update); - // this.indexingState = update; - } + if (isJetBrainsAndPreIndexedDocsProvider) { + this.ide.errorPopup( + `${DocsService.preIndexedDocsEmbeddingsProvider.id} cannot be used as an embeddings provider with JetBrains. Please select a different embeddings provider.`, + ); + + this.globalContext.update( + "curEmbeddingsProviderId", + curEmbeddingsProviderId, + ); + + return false; + } + + const lastEmbeddingsProviderId = this.globalContext.get( + "curEmbeddingsProviderId", + ); + + if (!lastEmbeddingsProviderId) { + // If it's the first time we're setting the `curEmbeddingsProviderId` + // global state, we don't need to reindex docs + this.globalContext.update( + "curEmbeddingsProviderId", + curEmbeddingsProviderId, + ); + + return false; + } + + return lastEmbeddingsProviderId !== curEmbeddingsProviderId; + } + + private async getEmbeddingsProviderAndIndexDoc( + site: SiteIndexingConfig, + reIndex: boolean = false, + ): Promise { + const config = await this.config(); + const { embeddingsProvider } = config; + + for await (const update of this.docsService.indexAndAdd( + site, + embeddingsProvider, + reIndex, + )) { + // Temporary disabled posting progress updates to the UI due to + // possible collision with code indexing progress updates. + // this.messenger.request("indexProgress", update); + // this.indexingState = update; } } + + private async reindexDocsOnNewEmbeddingsProvider( + embeddingsProvider: EmbeddingsProvider, + ) { + const docs = await this.docsService.list(); + + if (docs.length === 0) { + return; + } + + this.ide.infoPopup("Reindexing docs with new embeddings provider"); + + for (const { title, baseUrl } of docs) { + await this.docsService.delete(baseUrl); + + const generator = this.docsService.indexAndAdd( + { title, startUrl: baseUrl, rootUrl: baseUrl }, + embeddingsProvider, + ); + + while (!(await generator.next()).done) {} + } + + // Important that this only is invoked after we have successfully + // cleared and reindex the docs so that the table cannot end up in an + // invalid state. + this.globalContext.update("curEmbeddingsProviderId", embeddingsProvider.id); + + this.ide.infoPopup("Completed reindexing of all docs"); + } } diff --git a/core/index.d.ts b/core/index.d.ts index 55969eed84..bd8f99401e 100644 --- a/core/index.d.ts +++ b/core/index.d.ts @@ -422,6 +422,8 @@ export interface IdeSettings { remoteConfigSyncPeriod: number; userToken: string; enableControlServerBeta: boolean; + pauseCodebaseIndexOnStart: boolean + enableDebugLogs: boolean } export interface IDE { @@ -759,7 +761,12 @@ export interface EmbeddingsProvider { embed(chunks: string[]): Promise; } -export type RerankerName = "cohere" | "voyage" | "llm" | "free-trial" | "huggingface-tei"; +export type RerankerName = + | "cohere" + | "voyage" + | "llm" + | "free-trial" + | "huggingface-tei"; export interface RerankerDescription { name: RerankerName; diff --git a/core/indexing/CodebaseIndexer.ts b/core/indexing/CodebaseIndexer.ts index 5a5b73bd59..939487aad1 100644 --- a/core/indexing/CodebaseIndexer.ts +++ b/core/indexing/CodebaseIndexer.ts @@ -6,7 +6,7 @@ import { FullTextSearchCodebaseIndex } from "./FullTextSearch.js"; import { LanceDbIndex } from "./LanceDbIndex.js"; import { ChunkCodebaseIndex } from "./chunk/ChunkCodebaseIndex.js"; import { getComputeDeleteAddRemove } from "./refreshIndex.js"; -import { CodebaseIndex } from "./types.js"; +import { CodebaseIndex, IndexResultType } from "./types.js"; import { walkDir } from "./walkDir.js"; export class PauseToken { @@ -112,7 +112,7 @@ export class CodebaseIndexer { branch, artifactId: codebaseIndex.artifactId, }; - const [results, markComplete] = await getComputeDeleteAddRemove( + const [results, lastUpdated, markComplete] = await getComputeDeleteAddRemove( tag, { ...stats }, (filepath) => this.ide.readFile(filepath), @@ -159,6 +159,10 @@ export class CodebaseIndexer { }; } + lastUpdated.forEach((lastUpdated, path) => { + markComplete([lastUpdated], IndexResultType.UpdateLastUpdated); + }); + completedRelativeExpectedTime += codebaseIndex.relativeExpectedTime; yield { progress: diff --git a/core/indexing/chunk/ChunkCodebaseIndex.ts b/core/indexing/chunk/ChunkCodebaseIndex.ts index 5dd08c895a..39404eeaa6 100644 --- a/core/indexing/chunk/ChunkCodebaseIndex.ts +++ b/core/indexing/chunk/ChunkCodebaseIndex.ts @@ -114,6 +114,24 @@ export class ChunkCodebaseIndex implements CodebaseIndex { handleChunk(chunk); } + accumulatedProgress = + (i / results.compute.length) * (1 - progressReservedForTagging); + yield { + progress: accumulatedProgress, + desc: `Chunking ${getBasename(item.path)}`, + status: "indexing", + }; + markComplete([item], IndexResultType.Compute); + // Insert chunks + for await (const chunk of chunkDocument( + item.path, + contents[i], + this.maxChunkSize, + item.cacheKey, + )) { + handleChunk(chunk); + } + accumulatedProgress = (i / results.compute.length) * (1 - progressReservedForTagging); yield { @@ -125,17 +143,20 @@ export class ChunkCodebaseIndex implements CodebaseIndex { } // Add tag - for (const item of results.addTag) { - const chunksWithPath = await db.all( - "SELECT * FROM chunks WHERE cacheKey = ?", - [item.cacheKey], - ); + const addContents = await Promise.all( + results.addTag.map(({ path }) => this.readFile(path)), + ); + for (let i = 0; i < results.addTag.length; i++) { + const item = results.addTag[i]; - for (const chunk of chunksWithPath) { - await db.run("INSERT INTO chunk_tags (chunkId, tag) VALUES (?, ?)", [ - chunk.id, - tagString, - ]); + // Insert chunks + for await (const chunk of chunkDocument( + item.path, + addContents[i], + this.maxChunkSize, + item.cacheKey, + )) { + handleChunk(chunk); } markComplete([item], IndexResultType.AddTag); diff --git a/core/indexing/docs/DocsService.ts b/core/indexing/docs/DocsService.ts index bbec8cc65e..8d2854b9ee 100644 --- a/core/indexing/docs/DocsService.ts +++ b/core/indexing/docs/DocsService.ts @@ -3,15 +3,16 @@ import sqlite3 from "sqlite3"; import { Chunk, EmbeddingsProvider, + IdeInfo, IndexingProgressUpdate, SiteIndexingConfig, } from "../../index.js"; import { getDocsSqlitePath, getLanceDbPath } from "../../util/paths.js"; - import { Article, chunkArticle, pageToArticle } from "./article.js"; import { crawlPage } from "./crawl.js"; import { downloadFromS3, SiteIndexingResults } from "./preIndexed.js"; -import { default as configs } from "./preIndexedDocs.js"; +import preIndexedDocs from "./preIndexedDocs.js"; +import TransformersJsEmbeddingsProvider from "../embeddings/TransformersJsEmbeddingsProvider.js"; // Purposefully lowercase because lancedb converts interface LanceDbDocsRow { @@ -29,6 +30,8 @@ interface LanceDbDocsRow { export class DocsService { private static instance: DocsService; private static DOCS_TABLE_NAME = "docs"; + public static preIndexedDocsEmbeddingsProvider = + new TransformersJsEmbeddingsProvider(); private _sqliteTable: Database | undefined; private docsIndexingQueue: Set = new Set(); @@ -70,32 +73,35 @@ export class DocsService { nested = false, ): Promise { const lance = await this.getLanceDb(); - const db = await this.getSqliteTable(); - - const downloadDocs = async () => { - const config = configs.find((config) => config.startUrl === baseUrl); - if (config) { - await this.downloadPreIndexedDocs(embeddingsProviderId, config.title); - return await this.retrieve( - baseUrl, - vector, - nRetrieve, - embeddingsProviderId, - true, - ); - } - return undefined; + const tableNames = await lance.tableNames(); + const preIndexedDoc = preIndexedDocs[baseUrl]; + const isPreIndexedDoc = !!preIndexedDoc; + let shouldDownloadPreIndexedDoc = + !tableNames.includes(DocsService.DOCS_TABLE_NAME) && isPreIndexedDoc; + + const downloadAndRetrievePreIndexedDoc = async ( + preIndexedDoc: SiteIndexingConfig, + ) => { + await this.downloadAndAddPreIndexedDocs( + embeddingsProviderId, + preIndexedDoc.title, + ); + + return await this.retrieve( + baseUrl, + vector, + nRetrieve, + embeddingsProviderId, + true, + ); }; - const tableNames = await lance.tableNames(); - if (!tableNames.includes(DocsService.DOCS_TABLE_NAME)) { - const downloaded = await downloadDocs(); - if (downloaded) { - return downloaded; - } + if (shouldDownloadPreIndexedDoc) { + return await downloadAndRetrievePreIndexedDoc(preIndexedDoc!); } const table = await lance.openTable(DocsService.DOCS_TABLE_NAME); + let docs: LanceDbDocsRow[] = await table .search(vector) .limit(nRetrieve) @@ -104,11 +110,11 @@ export class DocsService { docs = docs.filter((doc) => doc.baseurl === baseUrl); - if ((!docs || docs.length === 0) && !nested) { - const downloaded = await downloadDocs(); - if (downloaded) { - return downloaded; - } + shouldDownloadPreIndexedDoc = + (!docs || docs.length === 0) && !nested && isPreIndexedDoc; + + if (shouldDownloadPreIndexedDoc) { + return await downloadAndRetrievePreIndexedDoc(preIndexedDoc!); } return docs.map((doc) => ({ @@ -184,7 +190,7 @@ export class DocsService { return !!doc; } - private async downloadPreIndexedDocs( + private async downloadAndAddPreIndexedDocs( embeddingsProviderId: string, title: string, ) { @@ -264,7 +270,8 @@ export class DocsService { const embeddings: number[][] = []; // Create embeddings of retrieved articles - console.log("Creating Embeddings for ", articles.length, " articles"); + console.log(`Creating embeddings for ${articles.length} articles`); + for (let i = 0; i < articles.length; i++) { const article = articles[i]; yield { @@ -313,4 +320,15 @@ export class DocsService { status: "done", }; } + + public isJetBrainsAndPreIndexedDocsProvider( + ideInfo: IdeInfo, + embeddingsProviderId: EmbeddingsProvider["id"], + ): boolean { + const isJetBrains = ideInfo.ideType === "jetbrains"; + const isPreIndexedDocsProvider = + embeddingsProviderId === DocsService.preIndexedDocsEmbeddingsProvider.id; + + return isJetBrains && isPreIndexedDocsProvider; + } } diff --git a/core/indexing/docs/crawl.ts b/core/indexing/docs/crawl.ts index 376c7f40c2..838a922416 100644 --- a/core/indexing/docs/crawl.ts +++ b/core/indexing/docs/crawl.ts @@ -16,7 +16,7 @@ const IGNORE_PATHS_ENDING_IN = [ "changelog.html", ]; -const GITHUB_PATHS_TO_TRAVERSE = ["/blob/", "/tree/"]; +const markdownRegex = new RegExp(/\.(md|mdx)$/); async function getDefaultBranch(owner: string, repo: string): Promise { const octokit = new Octokit({ auth: undefined }); @@ -53,7 +53,10 @@ async function crawlGithubRepo(baseUrl: URL) { ); const paths = tree.data.tree - .filter((file: any) => file.type === "blob" && file.path?.endsWith(".md")) + .filter( + (file: any) => + file.type === "blob" && markdownRegex.test(file.path ?? ""), + ) .map((file: any) => baseUrl.pathname + "/tree/main/" + file.path); return paths; @@ -142,7 +145,10 @@ export async function* crawlPage( url: URL, maxDepth: number = 3, ): AsyncGenerator { - console.log("Starting crawl from: ", url, " - Max Depth: ", maxDepth); + console.log( + `Starting crawl from: ${url.toString()} - Max Depth: ${maxDepth}`, + ); + const { baseUrl, basePath } = splitUrl(url); let paths: { path: string; depth: number }[] = [{ path: basePath, depth: 0 }]; diff --git a/core/indexing/docs/preIndexedDocs.ts b/core/indexing/docs/preIndexedDocs.ts index 8277adec5e..1cb34ba6f2 100644 --- a/core/indexing/docs/preIndexedDocs.ts +++ b/core/indexing/docs/preIndexedDocs.ts @@ -1,301 +1,305 @@ +/* eslint-disable @typescript-eslint/naming-convention */ import { SiteIndexingConfig } from "../../index.js"; -const configs: SiteIndexingConfig[] = [ - { +const preIndexedDocs: Record< + SiteIndexingConfig["startUrl"], + SiteIndexingConfig +> = { + "https://jinja.palletsprojects.com/en/3.1.x/": { title: "Jinja", startUrl: "https://jinja.palletsprojects.com/en/3.1.x/", rootUrl: "https://jinja.palletsprojects.com/en/3.1.x/", faviconUrl: "https://jinja.palletsprojects.com/favicon.ico", }, - { + "https://react.dev/reference/": { title: "React", startUrl: "https://react.dev/reference/", rootUrl: "https://react.dev/reference/", faviconUrl: "https://react.dev/favicon.ico", }, - { + "https://posthog.com/docs": { title: "PostHog", startUrl: "https://posthog.com/docs", rootUrl: "https://posthog.com/docs", faviconUrl: "https://posthog.com/favicon.ico", }, - { + "https://expressjs.com/en/5x/api.html": { title: "Express", startUrl: "https://expressjs.com/en/5x/api.html", rootUrl: "https://expressjs.com/en/5x/", faviconUrl: "https://expressjs.com/favicon.ico", }, - { + "https://platform.openai.com/docs/": { title: "OpenAI", startUrl: "https://platform.openai.com/docs/", rootUrl: "https://platform.openai.com/docs/", faviconUrl: "https://platform.openai.com/favicon.ico", }, - { + "https://www.prisma.io/docs": { title: "Prisma", startUrl: "https://www.prisma.io/docs", rootUrl: "https://www.prisma.io/docs", faviconUrl: "https://www.prisma.io/favicon.ico", }, - { + "https://boto3.amazonaws.com/v1/documentation/api/latest/index.html": { title: "Boto3", startUrl: "https://boto3.amazonaws.com/v1/documentation/api/latest/index.html", rootUrl: "https://boto3.amazonaws.com/v1/documentation/api/latest/", faviconUrl: "https://boto3.amazonaws.com/favicon.ico", }, - { + "https://pytorch.org/docs/stable/": { title: "Pytorch", startUrl: "https://pytorch.org/docs/stable/", rootUrl: "https://pytorch.org/docs/stable/", faviconUrl: "https://pytorch.org/favicon.ico", }, - { + "https://redis.io/docs/": { title: "Redis", startUrl: "https://redis.io/docs/", rootUrl: "https://redis.io/docs/", faviconUrl: "https://redis.io/favicon.ico", }, - { + "https://axios-http.com/docs/intro": { title: "Axios", startUrl: "https://axios-http.com/docs/intro", rootUrl: "https://axios-http.com/docs", faviconUrl: "https://axios-http.com/favicon.ico", }, - { + "https://redwoodjs.com/docs/introduction": { title: "Redwood JS", startUrl: "https://redwoodjs.com/docs/introduction", rootUrl: "https://redwoodjs.com/docs", faviconUrl: "https://redwoodjs.com/favicon.ico", }, - { + "https://graphql.org/learn/": { title: "GraphQL", startUrl: "https://graphql.org/learn/", rootUrl: "https://graphql.org/learn/", faviconUrl: "https://graphql.org/favicon.ico", }, - { + "https://www.typescriptlang.org/docs/": { title: "Typescript", startUrl: "https://www.typescriptlang.org/docs/", rootUrl: "https://www.typescriptlang.org/docs/", faviconUrl: "https://www.typescriptlang.org/favicon.ico", }, - { + "https://jestjs.io/docs/getting-started": { title: "Jest", startUrl: "https://jestjs.io/docs/getting-started", rootUrl: "https://jestjs.io/docs", faviconUrl: "https://jestjs.io/favicon.ico", }, - { + "https://tailwindcss.com/docs/installation": { title: "Tailwind CSS", startUrl: "https://tailwindcss.com/docs/installation", rootUrl: "https://tailwindcss.com/docs", faviconUrl: "https://tailwindcss.com/favicon.ico", }, - { + "https://vuejs.org/guide/introduction.html": { title: "Vue.js", startUrl: "https://vuejs.org/guide/introduction.html", rootUrl: "https://vuejs.org", faviconUrl: "https://vuejs.org/favicon.ico", }, - { + "https://svelte.dev/docs/introduction": { title: "Svelte", startUrl: "https://svelte.dev/docs/introduction", rootUrl: "https://svelte.dev/docs", faviconUrl: "https://svelte.dev/favicon.ico", }, - { + "https://docs.github.com/en/actions": { title: "GitHub Actions", startUrl: "https://docs.github.com/en/actions", rootUrl: "https://docs.github.com/en/actions", faviconUrl: "https://docs.github.com/favicon.ico", }, - { + "https://nodejs.org/docs/latest/api/": { title: "NodeJS", startUrl: "https://nodejs.org/docs/latest/api/", rootUrl: "https://nodejs.org/docs/latest/api/", faviconUrl: "https://nodejs.org/favicon.ico", }, - { + "https://socket.io/docs/v4/": { title: "Socket.io", startUrl: "https://socket.io/docs/v4/", rootUrl: "https://socket.io/docs/v4/", faviconUrl: "https://socket.io/favicon.ico", }, - { + "https://docs.gradle.org/current/userguide/userguide.html": { title: "Gradle", startUrl: "https://docs.gradle.org/current/userguide/userguide.html", rootUrl: "https://docs.gradle.org/current", faviconUrl: "https://docs.gradle.org/favicon.ico", }, - { + "https://redux-toolkit.js.org/introduction/getting-started": { title: "Redux Toolkit", startUrl: "https://redux-toolkit.js.org/introduction/getting-started", rootUrl: "https://redux-toolkit.js.org", faviconUrl: "https://redux-toolkit.js.org/favicon.ico", }, - { + "https://docs.trychroma.com/": { title: "Chroma", startUrl: "https://docs.trychroma.com/", rootUrl: "https://docs.trychroma.com/", faviconUrl: "https://docs.trychroma.com/favicon.ico", }, - { + "https://www.sqlite.org/docs.html": { title: "SQLite", startUrl: "https://www.sqlite.org/docs.html", rootUrl: "https://www.sqlite.org", faviconUrl: "https://www.sqlite.org/favicon.ico", }, - { + "https://redux.js.org/introduction/getting-started": { title: "Redux", startUrl: "https://redux.js.org/introduction/getting-started", rootUrl: "https://redux.js.org", faviconUrl: "https://redux.js.org/favicon.ico", }, - { + "https://prettier.io/docs/en/": { title: "Prettier", startUrl: "https://prettier.io/docs/en/", rootUrl: "https://prettier.io/docs/en/", faviconUrl: "https://prettier.io/favicon.ico", }, - { + "https://code.visualstudio.com/api": { title: "VS Code Extension API", startUrl: "https://code.visualstudio.com/api", rootUrl: "https://code.visualstudio.com/api", faviconUrl: "https://code.visualstudio.com/favicon.ico", }, - { + "https://docs.continue.dev/intro": { title: "Continue", startUrl: "https://docs.continue.dev/intro", rootUrl: "https://docs.continue.dev", faviconUrl: "https://docs.continue.dev/favicon.ico", }, - { + "https://api.jquery.com/": { title: "jQuery", startUrl: "https://api.jquery.com/", rootUrl: "https://api.jquery.com/", faviconUrl: "https://api.jquery.com/favicon.ico", }, - { + "https://docs.python.org/3/": { title: "Python", startUrl: "https://docs.python.org/3/", rootUrl: "https://docs.python.org/3/", faviconUrl: "https://docs.python.org/favicon.ico", }, - { + "https://doc.rust-lang.org/book/": { title: "Rust", startUrl: "https://doc.rust-lang.org/book/", rootUrl: "https://doc.rust-lang.org/book/", faviconUrl: "https://doc.rust-lang.org/favicon.ico", }, - { + "https://plugins.jetbrains.com/docs/intellij/welcome.html": { title: "IntelliJ Platform SDK", startUrl: "https://plugins.jetbrains.com/docs/intellij/welcome.html", rootUrl: "https://plugins.jetbrains.com/docs/intellij", faviconUrl: "https://plugins.jetbrains.com/favicon.ico", }, - { + "https://docs.docker.com/": { title: "Docker", startUrl: "https://docs.docker.com/", rootUrl: "https://docs.docker.com/", faviconUrl: "https://docs.docker.com/favicon.ico", }, - { + "https://docs.npmjs.com/": { title: "NPM", startUrl: "https://docs.npmjs.com/", rootUrl: "https://docs.npmjs.com/", faviconUrl: "https://docs.npmjs.com/favicon.ico", }, - { + "https://tiptap.dev/docs/editor/introduction": { title: "TipTap", startUrl: "https://tiptap.dev/docs/editor/introduction", rootUrl: "https://tiptap.dev/docs", faviconUrl: "https://tiptap.dev/favicon.ico", }, - { + "https://esbuild.github.io/": { title: "esbuild", startUrl: "https://esbuild.github.io/", rootUrl: "https://esbuild.github.io/", faviconUrl: "https://esbuild.github.io/favicon.ico", }, - { + "https://tree-sitter.github.io/tree-sitter/": { title: "Tree Sitter", startUrl: "https://tree-sitter.github.io/tree-sitter/", rootUrl: "https://tree-sitter.github.io/tree-sitter/", faviconUrl: "https://tree-sitter.github.io/favicon.ico", }, - { + "https://docs.netlify.com/": { title: "Netlify", startUrl: "https://docs.netlify.com/", rootUrl: "https://docs.netlify.com/", faviconUrl: "https://docs.netlify.com/favicon.ico", }, - { + "https://replicate.com/docs": { title: "Replicate", startUrl: "https://replicate.com/docs", rootUrl: "https://replicate.com/docs", faviconUrl: "https://replicate.com/favicon.ico", }, - { + "https://www.w3schools.com/html/default.asp": { title: "HTML", startUrl: "https://www.w3schools.com/html/default.asp", rootUrl: "https://www.w3schools.com/html", faviconUrl: "https://www.w3schools.com/favicon.ico", }, - { + "https://www.w3schools.com/css/default.asp": { title: "CSS", startUrl: "https://www.w3schools.com/css/default.asp", rootUrl: "https://www.w3schools.com/css", faviconUrl: "https://www.w3schools.com/favicon.ico", }, - { + "https://python.langchain.com/docs/get_started/introduction": { title: "Langchain", startUrl: "https://python.langchain.com/docs/get_started/introduction", rootUrl: "https://python.langchain.com/docs", faviconUrl: "https://python.langchain.com/favicon.ico", }, - { + "https://developer.woocommerce.com/docs/": { title: "WooCommerce", startUrl: "https://developer.woocommerce.com/docs/", rootUrl: "https://developer.woocommerce.com/docs/", faviconUrl: "https://developer.woocommerce.com/favicon.ico", }, - { + "https://developer.wordpress.org/reference/": { title: "WordPress", startUrl: "https://developer.wordpress.org/reference/", rootUrl: "https://developer.wordpress.org/reference/", faviconUrl: "https://developer.wordpress.org/favicon.ico", }, - { + "https://doc.qt.io/qtforpython-6/quickstart.html": { title: "PySide6", startUrl: "https://doc.qt.io/qtforpython-6/quickstart.html", rootUrl: "https://doc.qt.io/qtforpython-6/api.html", faviconUrl: "https://doc.qt.io/favicon.ico", }, - { + "https://getbootstrap.com/docs/5.3/getting-started/introduction/": { title: "Bootstrap", startUrl: "https://getbootstrap.com/docs/5.3/getting-started/introduction/", rootUrl: "https://getbootstrap.com/docs/5.3/", faviconUrl: "https://getbootstrap.com/favicon.ico", }, - { + "https://alpinejs.dev/start-here": { title: "Alpine.js", startUrl: "https://alpinejs.dev/start-here", rootUrl: "https://alpinejs.dev/", faviconUrl: "https://alpinejs.dev/favicon.ico", }, - { + "https://learn.microsoft.com/en-us/dotnet/csharp/": { title: "C# Language Reference", startUrl: "https://learn.microsoft.com/en-us/dotnet/csharp/", rootUrl: "https://learn.microsoft.com/en-us/dotnet/csharp/", faviconUrl: "https://learn.microsoft.com/favicon.ico", }, - { + "https://docs.godotengine.org/en/latest/": { title: "Godot", startUrl: "https://docs.godotengine.org/en/latest/", rootUrl: "https://docs.godotengine.org/en/latest/", faviconUrl: "https://godotengine.org/favicon.ico", }, -]; +}; -export default configs; +export default preIndexedDocs; diff --git a/core/indexing/refreshIndex.ts b/core/indexing/refreshIndex.ts index c673799604..db69c87259 100644 --- a/core/indexing/refreshIndex.ts +++ b/core/indexing/refreshIndex.ts @@ -44,6 +44,36 @@ export class SqliteDb { artifactId STRING NOT NULL )`, ); + // Delete duplicate rows from tag_catalog + await db.exec(` + DELETE FROM tag_catalog + WHERE id NOT IN ( + SELECT MIN(id) + FROM tag_catalog + GROUP BY dir, branch, artifactId, path, cacheKey + ) + `); + + // Delete duplicate rows from global_cache + await db.exec(` + DELETE FROM global_cache + WHERE id NOT IN ( + SELECT MIN(id) + FROM global_cache + GROUP BY cacheKey, dir, branch, artifactId + ) + `); + + // Add unique constraints if they don't exist + await db.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_tag_catalog_unique + ON tag_catalog(dir, branch, artifactId, path, cacheKey)`, + ); + + await db.exec( + `CREATE UNIQUE INDEX IF NOT EXISTS idx_global_cache_unique + ON global_cache(cacheKey, dir, branch, artifactId)`, + ); } private static indexSqlitePath = getIndexSqlitePath(); @@ -90,14 +120,24 @@ enum AddRemoveResultType { Remove = "remove", UpdateNewVersion = "updateNewVersion", UpdateOldVersion = "updateOldVersion", + UpdateLastUpdated = "updateLastUpdated", + Compute = "compute" } async function getAddRemoveForTag( tag: IndexTag, currentFiles: LastModifiedMap, readFile: (path: string) => Promise, -): Promise<[PathAndCacheKey[], PathAndCacheKey[], MarkCompleteCallback]> { +): Promise< + [ + PathAndCacheKey[], + PathAndCacheKey[], + PathAndCacheKey[], + MarkCompleteCallback, + ] +> { const newLastUpdatedTimestamp = Date.now(); + const files = { ...currentFiles }; const saved = await getSavedItemsForTag(tag); @@ -105,35 +145,41 @@ async function getAddRemoveForTag( const updateNewVersion: PathAndCacheKey[] = []; const updateOldVersion: PathAndCacheKey[] = []; const remove: PathAndCacheKey[] = []; + const updateLastUpdated: PathAndCacheKey[] = []; for (const item of saved) { const { lastUpdated, ...pathAndCacheKey } = item; - if (currentFiles[item.path] === undefined) { + if (files[item.path] === undefined) { // Was indexed, but no longer exists. Remove remove.push(pathAndCacheKey); } else { // Exists in old and new, so determine whether it was updated - if (lastUpdated < currentFiles[item.path]) { + if (lastUpdated < files[item.path]) { // Change was made after last update - updateNewVersion.push({ - path: pathAndCacheKey.path, - cacheKey: calculateHash(await readFile(pathAndCacheKey.path)), - }); - updateOldVersion.push(pathAndCacheKey); + const newHash = calculateHash(await readFile(pathAndCacheKey.path)); + if (pathAndCacheKey.cacheKey !== newHash) { + updateNewVersion.push({ + path: pathAndCacheKey.path, + cacheKey: newHash, + }); + updateOldVersion.push(pathAndCacheKey); + } else { + updateLastUpdated.push(pathAndCacheKey); + } } else { // Already updated, do nothing } // Remove so we can check leftovers afterward - delete currentFiles[item.path]; + delete files[item.path]; } } // Any leftover in current files need to be added add.push( ...(await Promise.all( - Object.keys(currentFiles).map(async (path) => { + Object.keys(files).map(async (path) => { const fileContents = await readFile(path); return { path, cacheKey: calculateHash(fileContents) }; }), @@ -143,18 +189,43 @@ async function getAddRemoveForTag( // Create the markComplete callback function const db = await SqliteDb.get(); const itemToAction: { - [key: string]: [PathAndCacheKey, AddRemoveResultType]; - } = {}; - - async function markComplete(items: PathAndCacheKey[], _: IndexResultType) { - const actions = items.map( - (item) => - itemToAction[ - JSON.stringify({ path: item.path, cacheKey: item.cacheKey }) - ], - ); - for (const [{ path, cacheKey }, resultType] of actions) { - switch (resultType) { + [key in AddRemoveResultType]: PathAndCacheKey[]; + } = { + [AddRemoveResultType.Add]: [], + [AddRemoveResultType.Remove]: [], + [AddRemoveResultType.UpdateNewVersion]: [], + [AddRemoveResultType.UpdateOldVersion]: [], + [AddRemoveResultType.UpdateLastUpdated]: [], + [AddRemoveResultType.Compute]: [], + }; + + async function markComplete( + items: PathAndCacheKey[], + resultType: IndexResultType, + ) { + const addRemoveResultType = + mapIndexResultTypeToAddRemoveResultType(resultType); + + const actionItems = itemToAction[addRemoveResultType]; + if (!actionItems) { + console.warn(`No action items found for result type: ${resultType}`); + return; + } + + for (const item of items) { + const { path, cacheKey } = item; + switch (addRemoveResultType) { + case AddRemoveResultType.Compute: + await db.run( + "REPLACE INTO tag_catalog (path, cacheKey, lastUpdated, dir, branch, artifactId) VALUES (?, ?, ?, ?, ?, ?)", + path, + cacheKey, + newLastUpdatedTimestamp, + tag.directory, + tag.branch, + tag.artifactId, + ); + break; case AddRemoveResultType.Add: await db.run( "INSERT INTO tag_catalog (path, cacheKey, lastUpdated, dir, branch, artifactId) VALUES (?, ?, ?, ?, ?, ?)", @@ -182,6 +253,7 @@ async function getAddRemoveForTag( tag.artifactId, ); break; + case AddRemoveResultType.UpdateLastUpdated: case AddRemoveResultType.UpdateNewVersion: await db.run( `UPDATE tag_catalog SET @@ -208,27 +280,22 @@ async function getAddRemoveForTag( } for (const item of updateNewVersion) { - itemToAction[JSON.stringify(item)] = [ - item, - AddRemoveResultType.UpdateNewVersion, - ]; + itemToAction[AddRemoveResultType.UpdateNewVersion].push(item); } for (const item of add) { - itemToAction[JSON.stringify(item)] = [item, AddRemoveResultType.Add]; + itemToAction[AddRemoveResultType.Add].push(item); } for (const item of updateOldVersion) { - itemToAction[JSON.stringify(item)] = [ - item, - AddRemoveResultType.UpdateOldVersion, - ]; + itemToAction[AddRemoveResultType.UpdateOldVersion].push(item); } for (const item of remove) { - itemToAction[JSON.stringify(item)] = [item, AddRemoveResultType.Remove]; + itemToAction[AddRemoveResultType.Remove].push(item); } return [ [...add, ...updateNewVersion], [...remove, ...updateOldVersion], + updateLastUpdated, markComplete, ]; } @@ -255,13 +322,31 @@ function calculateHash(fileContents: string): string { return hash.digest("hex"); } +function mapIndexResultTypeToAddRemoveResultType( + resultType: IndexResultType, +): AddRemoveResultType { + switch (resultType) { + case "updateLastUpdated": + return AddRemoveResultType.UpdateLastUpdated; + case "compute": + return AddRemoveResultType.Compute; + case "addTag": + return AddRemoveResultType.Add; + case "del": + case "removeTag": + return AddRemoveResultType.Remove; + default: + throw new Error(`Unexpected result type: ${resultType}`); + } +} + export async function getComputeDeleteAddRemove( tag: IndexTag, currentFiles: LastModifiedMap, readFile: (path: string) => Promise, repoName: string | undefined, -): Promise<[RefreshIndexResults, MarkCompleteCallback]> { - const [add, remove, markComplete] = await getAddRemoveForTag( +): Promise<[RefreshIndexResults, PathAndCacheKey[], MarkCompleteCallback]> { + const [add, remove, lastUpdated, markComplete] = await getAddRemoveForTag( tag, currentFiles, readFile, @@ -305,6 +390,7 @@ export async function getComputeDeleteAddRemove( return [ results, + lastUpdated, async (items, resultType) => { // Update tag catalog markComplete(items, resultType); @@ -347,15 +433,15 @@ export class GlobalCacheCodeBaseIndex implements CodebaseIndex { _: MarkCompleteCallback, repoName: string | undefined, ): AsyncGenerator { - const add = [...results.compute, ...results.addTag]; + const add = results.addTag; const remove = [...results.del, ...results.removeTag]; await Promise.all([ - ...add.map(({ cacheKey }) => { - return this.computeOrAddTag(cacheKey, tag); - }), ...remove.map(({ cacheKey }) => { return this.deleteOrRemoveTag(cacheKey, tag); }), + ...add.map(({ cacheKey }) => { + return this.computeOrAddTag(cacheKey, tag); + }), ]); yield { progress: 1, desc: "Done updating global cache", status: "done" }; } diff --git a/core/indexing/types.ts b/core/indexing/types.ts index d636564bd6..f952ea8dfc 100644 --- a/core/indexing/types.ts +++ b/core/indexing/types.ts @@ -5,6 +5,7 @@ export enum IndexResultType { Delete = "del", AddTag = "addTag", RemoveTag = "removeTag", + UpdateLastUpdated = "updateLastUpdated" } export type MarkCompleteCallback = ( diff --git a/core/indexing/walkDir.ts b/core/indexing/walkDir.ts index a75cd0c1de..45dc5d1c06 100644 --- a/core/indexing/walkDir.ts +++ b/core/indexing/walkDir.ts @@ -1,17 +1,15 @@ -import { EventEmitter } from "events"; import { Minimatch } from "minimatch"; import path from "node:path"; -import { FileType, IDE } from "../index.js"; -import { DEFAULT_IGNORE_DIRS, DEFAULT_IGNORE_FILETYPES } from "./ignore.js"; +import { FileType, IDE } from "../index.d.js"; +import { + DEFAULT_IGNORE_DIRS, + DEFAULT_IGNORE_FILETYPES, + defaultIgnoreDir, + defaultIgnoreFile, +} from "./ignore.js"; export interface WalkerOptions { - isSymbolicLink?: boolean; - path?: string; ignoreFiles?: string[]; - parent?: Walker | null; - includeEmpty?: boolean; - follow?: boolean; - exact?: boolean; onlyDirs?: boolean; returnRelativePaths?: boolean; additionalIgnoreRules?: string[]; @@ -19,314 +17,194 @@ export interface WalkerOptions { type Entry = [string, FileType]; -class Walker extends EventEmitter { - isSymbolicLink: boolean; - path: string; - basename: string; - ignoreFiles: string[]; - ignoreRules: { [key: string]: Minimatch[] }; - parent: Walker | null; - includeEmpty: boolean; - root: string; - follow: boolean; - result: Set; - entries: Entry[] | null; - sawError: boolean; - exact: boolean | undefined; - onlyDirs: boolean | undefined; - constructor(opts: WalkerOptions = {}, protected readonly ide: IDE) { - super(opts as any); - this.isSymbolicLink = opts.isSymbolicLink || false; - this.path = opts.path || process.cwd(); - this.basename = path.basename(this.path); - this.ignoreFiles = [...(opts.ignoreFiles || [".ignore"]), ".defaultignore"]; - this.ignoreRules = {}; - this.parent = opts.parent || null; - this.includeEmpty = !!opts.includeEmpty; - this.root = this.parent ? this.parent.root : this.path; - this.follow = !!opts.follow; - this.result = this.parent ? this.parent.result : new Set(); - this.entries = null; - this.sawError = false; - this.exact = opts.exact; - this.onlyDirs = opts.onlyDirs; - - if (opts.additionalIgnoreRules) { - this.addIgnoreRules(opts.additionalIgnoreRules); - } - } - - sort(a: string, b: string): number { - return a.localeCompare(b, "en"); - } - - emit(ev: string, data: any): boolean { - let ret = false; - - if (!(this.sawError && ev === "error")) { - if (ev === "error") { - this.sawError = true; - } else if (ev === "done" && !this.parent) { - data = (Array.from(data) as any) - .map((e: string) => (/^@/.test(e) ? `./${e}` : e)) - .sort(this.sort); - this.result = new Set(data); - } - - if (ev === "error" && this.parent) { - ret = this.parent.emit("error", data); - } else { - ret = super.emit(ev, data); - } - } - return ret; - } - - async *start() { - try { - const entries = await this.ide.listDir(this.path); - - for await (const result of this.onReadDir(entries)) { - yield result; - } - } catch (err) { - this.emit("error", err); - } - } - - isIgnoreFile(e: Entry): boolean { - const p = e[0]; - return p !== "." && p !== ".." && this.ignoreFiles.indexOf(p) !== -1; - } - - async *onReadDir(entries: Entry[]) { - this.entries = entries; - - if (entries.length === 0) { - if (this.includeEmpty) { - this.result.add(this.path.slice(this.root.length + 1)); - } - this.emit("done", this.result); - yield this.result; - } else { - const hasIg = this.entries.some((e) => this.isIgnoreFile(e)); +// helper struct used for the DFS walk +type WalkableEntry = { + relPath: string; + absPath: string; + type: FileType; + entry: Entry; +}; - if (hasIg) { - await this.addIgnoreFiles(); - } +// helper struct used for the DFS walk +type WalkContext = { + walkableEntry: WalkableEntry; + ignoreFiles: IgnoreFile[]; +}; - yield* this.filterEntries(); - } - } +class IgnoreFile { + private _rules: Minimatch[]; - async addIgnoreFiles() { - const newIg = this.entries!.filter((e) => this.isIgnoreFile(e)); - await Promise.all(newIg.map((e) => this.addIgnoreFile(e))); + constructor( + public path: string, + public content: string, + ) { + this.path = path; + this.content = content; + this._rules = this.contentToRules(content); } - async addIgnoreFile(fileEntry: Entry) { - const ig = path.resolve(this.path, fileEntry[0]); - - try { - const file = await this.ide.readFile(ig); - this.onReadIgnoreFile(fileEntry, file); - } catch (err) { - this.emit("error", err); - } + public get rules() { + return this._rules; } - onReadIgnoreFile(file: Entry, data: string): void { - const mmopt = { + private contentToRules(content: string): Minimatch[] { + const options = { matchBase: true, dot: true, flipNegate: true, nocase: true, }; - - const rules = data + return content .split(/\r?\n/) - .filter((line) => !/^#|^$/.test(line.trim())) - .map((rule) => { - return new Minimatch(rule.trim(), mmopt); - }); - - this.ignoreRules[file[0]] = rules; + .map((l) => l.trim()) + .filter((l) => !/^#|^$/.test(l)) + .map((l) => new Minimatch(l, options)); } +} - addIgnoreRules(rules: string[]) { - const mmopt = { - matchBase: true, - dot: true, - flipNegate: true, - nocase: true, +class DFSWalker { + private readonly path: string; + private readonly ide: IDE; + private readonly options: WalkerOptions; + private readonly ignoreFileNames: Set; + + constructor(path: string, ide: IDE, options: WalkerOptions) { + this.path = path; + this.ide = ide; + this.options = options; + this.ignoreFileNames = new Set(options.ignoreFiles); + } + + // walk is a depth-first search implementation + public async *walk(): AsyncGenerator { + const root: WalkContext = { + walkableEntry: { + relPath: "", + absPath: this.path, + type: 2 as FileType.Directory, + entry: ["", 2 as FileType.Directory], + }, + ignoreFiles: [], }; - - const minimatchRules = rules - .filter((line) => !/^#|^$/.test(line.trim())) - .map((rule) => { - return new Minimatch(rule.trim(), mmopt); - }); - - this.ignoreRules[".defaultignore"] = minimatchRules; - } - - async *filterEntries() { - const filtered = (await Promise.all( - this.entries!.map(async (entry) => { - const passFile = await this.filterEntry(entry[0]); - const passDir = await this.filterEntry(entry[0], true); - return passFile || passDir ? [entry, passFile, passDir] : false; - }), - ).then((entries) => entries.filter((e) => e))) as [ - Entry, - boolean, - boolean, - ][]; - let entryCount = filtered.length; - if (entryCount === 0) { - this.emit("done", this.result); - yield this.result; - } else { - const then = () => { - if (--entryCount === 0) { - // Otherwise in onlyDirs mode, nothing would be returned - if (this.onlyDirs && this.path !== this.root) { - this.result.add(this.path.slice(this.root.length + 1)); - } - this.emit("done", this.result); + const stack = [root]; + for (let cur = stack.pop(); cur; cur = stack.pop()) { + const walkableEntries = await this.listDirForWalking(cur.walkableEntry); + const ignoreFiles = await this.getIgnoreFilesToApplyInDir( + cur.ignoreFiles, + walkableEntries, + ); + for (const w of walkableEntries) { + if (!this.shouldInclude(w, ignoreFiles)) { + continue; } - }; - - for (const [entry, file, dir] of filtered) { - for await (const statResult of this.stat(entry, file, dir, then)) { - yield statResult; + if (this.entryIsDirectory(w.entry)) { + stack.push({ + walkableEntry: w, + ignoreFiles: ignoreFiles, + }); + if (this.options.onlyDirs) { + // when onlyDirs is enabled the walker will only return directory names + yield w.relPath; + } + } else { + yield w.relPath; } } } } - entryIsDirectory(entry: Entry) { - const Directory = 2 as FileType.Directory; - return entry[1] === Directory; - } - - entryIsSymlink(entry: Entry) { - const Directory = 64 as FileType.SymbolicLink; - return entry[1] === Directory; + private async listDirForWalking( + walkableEntry: WalkableEntry, + ): Promise { + const entries = await this.ide.listDir(walkableEntry.absPath); + return entries.map((e) => { + return { + relPath: path.join(walkableEntry.relPath, e[0]), + absPath: path.join(walkableEntry.absPath, e[0]), + type: e[1], + entry: e, + }; + }); } - async *onstat(entry: Entry, file: boolean, dir: boolean, then: () => void) { - const abs = this.path + "/" + entry[0]; - const isSymbolicLink = this.entryIsSymlink(entry); - if (!this.entryIsDirectory(entry)) { - if (file && !this.onlyDirs) { - this.result.add(abs.slice(this.root.length + 1)); - } - then(); - yield this.result; - } else { - if (dir) { - yield* this.walker( - entry[0], - { isSymbolicLink, exact: await this.filterEntry(entry[0] + "/") }, - then, - ); - } else { - then(); - yield this.result; - } + private async getIgnoreFilesToApplyInDir( + parentIgnoreFiles: IgnoreFile[], + walkableEntries: WalkableEntry[], + ): Promise { + const ignoreFilesInDir = await this.loadIgnoreFiles(walkableEntries); + if (ignoreFilesInDir.length === 0) { + return parentIgnoreFiles; } + return Array.prototype.concat(parentIgnoreFiles, ignoreFilesInDir); } - async *stat( - entry: Entry, - file: boolean, - dir: boolean, - then: () => void, - ): any { - yield* this.onstat(entry, file, dir, then); - } - - walkerOpt(entry: string, opts: Partial): WalkerOptions { - return { - path: this.path + "/" + entry, - parent: this, - ignoreFiles: this.ignoreFiles, - follow: this.follow, - includeEmpty: this.includeEmpty, - onlyDirs: this.onlyDirs, - ...opts, - }; - } - - async *walker(entry: string, opts: Partial, then: () => void) { - const walker = new Walker(this.walkerOpt(entry, opts), this.ide); - - walker.on("done", then); - yield* walker.start(); + private async loadIgnoreFiles( + entries: WalkableEntry[], + ): Promise { + const ignoreEntries = entries.filter((w) => this.isIgnoreFile(w.entry)); + const promises = ignoreEntries.map(async (w) => { + const content = await this.ide.readFile(w.absPath); + return new IgnoreFile(w.relPath, content); + }); + return Promise.all(promises); } - async filterEntry( - entry: string, - partial?: boolean, - entryBasename?: string, - ): Promise { - let included = true; - - if (this.parent && this.parent.filterEntry) { - const parentEntry = this.basename + "/" + entry; - const parentBasename = entryBasename || entry; - included = await this.parent.filterEntry( - parentEntry, - partial, - parentBasename, - ); - if (!included && !this.exact) { + private isIgnoreFile(e: Entry): boolean { + const p = e[0]; + return this.ignoreFileNames.has(p); + } + + private shouldInclude( + walkableEntry: WalkableEntry, + ignoreFiles: IgnoreFile[], + ) { + if (this.entryIsSymlink(walkableEntry.entry)) { + // If called from the root, a symlink either links to a real file in this repository, + // and therefore will be walked OR it linksto something outside of the repository and + // we do not want to index it + return false; + } + let relPath = walkableEntry.relPath; + if (this.entryIsDirectory(walkableEntry.entry)) { + if (defaultIgnoreDir.ignores(walkableEntry.relPath)) { + return false; + } + relPath = `${relPath}/`; + } else { + if (this.options.onlyDirs) { + return false; + } + if (defaultIgnoreFile.ignores(walkableEntry.relPath)) { return false; } + relPath = `/${relPath}`; } - - for (const f of this.ignoreFiles) { - if (this.ignoreRules[f]) { - for (const rule of this.ignoreRules[f]) { - if (rule.negate !== included) { - const isRelativeRule = - entryBasename && - rule.globParts.some( - (part) => part.length <= (part.slice(-1)[0] ? 1 : 2), - ); - - const match = - rule.match("/" + entry) || - rule.match(entry) || - (!!partial && - (rule.match("/" + entry + "/") || - rule.match(entry + "/") || - (rule.negate && - (rule.match("/" + entry, true) || - rule.match(entry, true))) || - (isRelativeRule && - (rule.match("/" + entryBasename + "/") || - rule.match(entryBasename + "/") || - (rule.negate && - (rule.match("/" + entryBasename, true) || - rule.match(entryBasename, true))))))); - - if (match) { - included = rule.negate; - } - } + let included = true; + for (const ignoreFile of ignoreFiles) { + for (const r of ignoreFile.rules) { + if (r.negate === included) { + // no need to test when the file is already NOT to be included unless this is a negate rule and vice versa + continue; + } + if (r.match(relPath)) { + included = r.negate; } } } - return included; } + + private entryIsDirectory(entry: Entry) { + return entry[1] === (2 as FileType.Directory); + } + + private entryIsSymlink(entry: Entry) { + return entry[1] === (64 as FileType.SymbolicLink); + } } const defaultOptions: WalkerOptions = { ignoreFiles: [".gitignore", ".continueignore"], - onlyDirs: false, additionalIgnoreRules: [...DEFAULT_IGNORE_DIRS, ...DEFAULT_IGNORE_FILETYPES], }; @@ -337,39 +215,17 @@ export async function walkDir( ): Promise { let entries: string[] = []; const options = { ...defaultOptions, ..._options }; - - const walker = new Walker( - { - path, - ignoreFiles: options.ignoreFiles, - onlyDirs: options.onlyDirs, - follow: true, - includeEmpty: false, - additionalIgnoreRules: options.additionalIgnoreRules, - }, - ide, - ); - - try { - for await (const walkedEntries of walker.start()) { - entries = [...walkedEntries]; - } - } catch (err) { - console.error(`Error walking directories: ${err}`); - throw err; + const dfsWalker = new DFSWalker(path, ide, options); + let relativePaths: string[] = []; + for await (const e of dfsWalker.walk()) { + relativePaths.push(e); } - - const relativePaths = entries || []; - if (options?.returnRelativePaths) { return relativePaths; } - const pathSep = await ide.pathSep(); - if (pathSep === "/") { return relativePaths.map((p) => path + pathSep + p); } - return relativePaths.map((p) => path + pathSep + p.split("/").join(pathSep)); } diff --git a/core/llm/llms/stubs/ContinueProxy.ts b/core/llm/llms/stubs/ContinueProxy.ts index 850ac74103..99eba5f73e 100644 --- a/core/llm/llms/stubs/ContinueProxy.ts +++ b/core/llm/llms/stubs/ContinueProxy.ts @@ -21,6 +21,10 @@ class ContinueProxy extends OpenAI { useLegacyCompletionsEndpoint: false, }; + supportsCompletions(): boolean { + return false; + } + supportsFim(): boolean { return true; } diff --git a/core/package-lock.json b/core/package-lock.json index e52c1435eb..e25eed0333 100644 --- a/core/package-lock.json +++ b/core/package-lock.json @@ -11,7 +11,7 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "^3.574.0", "@aws-sdk/credential-providers": "^3.596.0", - "@continuedev/config-types": "^1.0.6", + "@continuedev/config-types": "^1.0.8", "@continuedev/llm-info": "^1.0.1", "@mozilla/readability": "^0.5.0", "@octokit/rest": "^20.0.2", @@ -3932,9 +3932,9 @@ } }, "node_modules/@continuedev/config-types": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@continuedev/config-types/-/config-types-1.0.6.tgz", - "integrity": "sha512-JTlaGtsNW9vfSPcDJBn+I2vcs0wWRFIQcN3sa6pBFg2fj8W6qcMZoYUdzoU/+X+2ZT8OKcbomWfP2+qW0uuy6A==", + "version": "1.0.8", + "resolved": "https://registry.npmjs.org/@continuedev/config-types/-/config-types-1.0.8.tgz", + "integrity": "sha512-AwrqMK+FSVY1vFaEujVbOI2ETAYmRv6dT9qs/FYhKNCUNIVDbeI0/Mhf6APBVfgK8czrB6XVbY1aJFiZHGqejw==", "dependencies": { "zod": "^3.23.8" } diff --git a/core/package.json b/core/package.json index 16c88a868a..fafef2f4cb 100644 --- a/core/package.json +++ b/core/package.json @@ -32,7 +32,7 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "^3.574.0", "@aws-sdk/credential-providers": "^3.596.0", - "@continuedev/config-types": "^1.0.6", + "@continuedev/config-types": "^1.0.8", "@continuedev/llm-info": "^1.0.1", "@mozilla/readability": "^0.5.0", "@octokit/rest": "^20.0.2", diff --git a/core/test/walkDir.test.ts b/core/test/walkDir.test.ts index ba0714a40c..c3f40128aa 100644 --- a/core/test/walkDir.test.ts +++ b/core/test/walkDir.test.ts @@ -104,12 +104,41 @@ describe("walkDir", () => { ); }); + test("should use gitignore in parent directory for subdirectory", async () => { + const files = [ + "a.txt", + "b.py", + "d/", + "d/e.txt", + "d/f.py", + "d/g/", + "d/g/h.ts", + "d/g/i.py", + [".gitignore", "*.py"], + ]; + addToTestDir(files); + await expectPaths(["a.txt", "d/e.txt", "d/g/h.ts"], ["d/f.py", "d/g/i.py"]); + }); + test("should handle leading slash in gitignore", async () => { const files = [[".gitignore", "/no.txt"], "a.txt", "b.py", "no.txt"]; addToTestDir(files); await expectPaths(["a.txt", "b.py"], ["no.txt"]); }); + test("should not ignore leading slash when in subfolder", async () => { + const files = [ + [".gitignore", "/no.txt"], + "a.txt", + "b.py", + "no.txt", + "sub/", + "sub/no.txt", + ]; + addToTestDir(files); + await expectPaths(["a.txt", "b.py", "sub/no.txt"], ["no.txt"]); + }); + test("should handle multiple .gitignore files in nested structure", async () => { const files = [ [".gitignore", "*.txt"], @@ -202,7 +231,7 @@ describe("walkDir", () => { await expectPaths( ["d", "d/g"], ["a.txt", "b.py", "c.ts", "d/e.txt", "d/f.py", "d/g/h.ts"], - { onlyDirs: true, includeEmpty: true }, + { onlyDirs: true }, ); }); diff --git a/core/util/GlobalContext.ts b/core/util/GlobalContext.ts index d3b931c872..3880b46099 100644 --- a/core/util/GlobalContext.ts +++ b/core/util/GlobalContext.ts @@ -1,10 +1,18 @@ import fs from "node:fs"; import { getGlobalContextFilePath } from "./paths.js"; +import { EmbeddingsProvider } from "../index.js"; export type GlobalContextType = { indexingPaused: boolean; selectedTabAutocompleteModel: string; lastSelectedProfileForWorkspace: { [workspaceIdentifier: string]: string }; + /** + * This is needed to handle the case where a JetBrains user has created + * docs embeddings using one provider, and then updates to a new provider. + * + * For VS Code users, it is unnecessary since we use transformers.js by default. + */ + curEmbeddingsProviderId: EmbeddingsProvider["id"]; }; /** diff --git a/core/util/filesystem.ts b/core/util/filesystem.ts index 7b849cb33f..978fa08298 100644 --- a/core/util/filesystem.ts +++ b/core/util/filesystem.ts @@ -38,6 +38,8 @@ class FileSystemIde implements IDE { remoteConfigSyncPeriod: 60, userToken: "", enableControlServerBeta: false, + pauseCodebaseIndexOnStart: false, + enableDebugLogs: false, }; } async getGitHubAuthToken(): Promise { diff --git a/docs/docs/customization/overview.md b/docs/docs/customization/overview.md index d36dbfffe2..c5791aede7 100644 --- a/docs/docs/customization/overview.md +++ b/docs/docs/customization/overview.md @@ -10,7 +10,7 @@ Continue can be deeply customized by editing `config.json` and `config.ts` on yo Currently, you can customize the following: -- [Models](../setup/select-model.md) and [providers](../setup/select-provider.md) +- [Models](../setup/select-model.md) and [Model Providers](../setup/model-providers.md) - [Context Providers](./context-providers.md) - [Slash Commands](./slash-commands.md) - [Other configuration options](../reference/config.mdx) diff --git a/docs/docs/reference/Model Providers/freetrial.md b/docs/docs/reference/Model Providers/freetrial.md index da4f583cb6..ea402cc58e 100644 --- a/docs/docs/reference/Model Providers/freetrial.md +++ b/docs/docs/reference/Model Providers/freetrial.md @@ -69,7 +69,7 @@ Groq provides lightning fast inference for open-source LLMs like Llama3, up to t ### ⏩ Other options -The above were only a few examples, but Continue can be used with any LLM or provider. You can find [a full list of providers here](../../setup/select-provider.md). +The above were only a few examples, but Continue can be used with any LLM or provider. You can find [a full list of model providers here](../../setup/model-providers.md). ## Sign in diff --git a/docs/docs/setup/configuration.md b/docs/docs/setup/configuration.md index 29c1110fd8..26965d87a0 100644 --- a/docs/docs/setup/configuration.md +++ b/docs/docs/setup/configuration.md @@ -295,7 +295,7 @@ You can find all existing templates for /edit in [`core/llm/templates/edit.ts`]( ## Defining a Custom LLM Provider -If you are using an LLM API that isn't already [supported by Continue](./select-provider.md), and is not an OpenAI-compatible API, you'll need to define a `CustomLLM` object in `config.ts`. This object only requires one of (or both of) a `streamComplete` or `streamChat` function. Here is an example: +If you are using an LLM API that isn't already [supported by Continue](./model-providers.md), and is not an OpenAI-compatible API, you'll need to define a `CustomLLM` object in `config.ts`. This object only requires one of (or both of) a `streamComplete` or `streamChat` function. Here is an example: ```typescript title="~/.continue/config.ts" export function modifyConfig(config: Config): Config { diff --git a/docs/docs/setup/select-provider.md b/docs/docs/setup/model-providers.md similarity index 82% rename from docs/docs/setup/select-provider.md rename to docs/docs/setup/model-providers.md index d17218a6ea..8bce892ed9 100644 --- a/docs/docs/setup/select-provider.md +++ b/docs/docs/setup/model-providers.md @@ -1,12 +1,29 @@ --- -title: Select providers -description: Configure LLM providers -keywords: [openai, anthropic, gemini, ollama, ggml] +title: Model Providers +description: Configure and integrate various LLM (Large Language Model) providers for chat, autocomplete, and embedding models, whether self-hosted, remote, or via SaaS. +keywords: + [ + large language models, + LLM providers, + open-source LLM, + commercial LLM, + self-hosted LLM, + remote LLM, + SaaS LLM, + AI model configuration, + AI providers, + OpenAI, + Anthropic, + Gemini, + Ollama, + HuggingFace, + AWS Bedrock, + ] --- -# Select providers +# Model Providers -Continue makes it easy to use different providers for serving your chat, autocomplete, and embeddings models. +Configure and integrate various LLM (Large Language Model) providers for chat, autocomplete, and embedding models, whether self-hosted, remote, or via SaaS. To select the ones you want to use, add them to your `config.json`. diff --git a/docs/docs/setup/overview.md b/docs/docs/setup/overview.md index 922389dc64..d8c1bc8dc7 100644 --- a/docs/docs/setup/overview.md +++ b/docs/docs/setup/overview.md @@ -9,5 +9,5 @@ You will need to decide which models and providers you use for [chat](select-mod Learn more: - [Configuration](configuration.md) -- [Select providers](select-provider.md) +- [Model Providers](model-providers.md) - [Select models](select-model.md) diff --git a/docs/docs/setup/select-model.md b/docs/docs/setup/select-model.md index a04cfa399e..7f46fe6a8e 100644 --- a/docs/docs/setup/select-model.md +++ b/docs/docs/setup/select-model.md @@ -87,4 +87,4 @@ We recommend the following embeddings models, which are used for codebase retrie _You can also use other embeddings models by adding them to your `config.json`._ -**In addition to selecting models, you will need to figure out [what providers to use](./select-provider.md).** +**In addition to selecting models, you will need to figure out [what model providers to use](./model-providers.md).** diff --git a/docs/docusaurus.config.js b/docs/docusaurus.config.js index 80ec065aef..fc3dfaffaf 100644 --- a/docs/docusaurus.config.js +++ b/docs/docusaurus.config.js @@ -185,6 +185,10 @@ const config = { from: "/model-setup/configuration", to: "/setup/configuration", }, + { + from: "/setup/select-provider", + to: "/setup/model-providers", + }, ], }, ], diff --git a/docs/sidebars.js b/docs/sidebars.js index 08ebe4d206..cdd4f87d52 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -25,7 +25,7 @@ const sidebars = { items: [ "setup/overview", "setup/configuration", - "setup/select-provider", + "setup/model-providers", "setup/select-model", ], }, diff --git a/docs/static/schemas/config.json b/docs/static/schemas/config.json index 19813eb6ae..ac7461d8fd 100644 --- a/docs/static/schemas/config.json +++ b/docs/static/schemas/config.json @@ -1395,6 +1395,46 @@ } }, "allOf": [ + { + "if": { + "properties": { + "name": { + "enum": ["docs"] + } + } + }, + "then": { + "properties": { + "params": { + "properties": { + "sites": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "startUrl": { + "type": "string" + }, + "rootUrl": { + "type": "string" + }, + "maxDepth": { + "type": "integer" + } + }, + "required": ["title", "startUrl"] + } + } + }, + "required": ["sites"] + } + }, + "required": ["params"] + } + }, { "if": { "properties": { @@ -1955,7 +1995,13 @@ "type": "object", "properties": { "name": { - "enum": ["cohere", "voyage", "llm", "free-trial", "huggingface-tei"] + "enum": [ + "cohere", + "voyage", + "llm", + "free-trial", + "huggingface-tei" + ] }, "params": { "type": "object" @@ -2070,7 +2116,7 @@ "default": false }, "truncation_direction": { - "enum": ["Right","Left"], + "enum": ["Right", "Left"], "markdownDescription": "Wether to truncate sequences from the `left` or `right`.", "default": "Right" } diff --git a/eval/repos/amplified-dev b/eval/repos/amplified-dev deleted file mode 160000 index 4cf94d7023..0000000000 --- a/eval/repos/amplified-dev +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 4cf94d7023f5063f912d376027194fbd67670ba5 diff --git a/eval/repos/continue b/eval/repos/continue deleted file mode 160000 index 2eafdff54b..0000000000 --- a/eval/repos/continue +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2eafdff54b1a77c6d403fa26a64c6793d7a4dfb9 diff --git a/eval/repos/the-x b/eval/repos/the-x deleted file mode 160000 index 9218fea448..0000000000 --- a/eval/repos/the-x +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 9218fea4483ffcbbdf1471c83f65c01f3c25f554 diff --git a/eval/repos/trayracer b/eval/repos/trayracer deleted file mode 160000 index 2f0e7fbe96..0000000000 --- a/eval/repos/trayracer +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2f0e7fbe962312a1b8bef8a29ed1ac4111ab11c4 diff --git a/extensions/intellij/src/main/resources/config_schema.json b/extensions/intellij/src/main/resources/config_schema.json index 19813eb6ae..ac7461d8fd 100644 --- a/extensions/intellij/src/main/resources/config_schema.json +++ b/extensions/intellij/src/main/resources/config_schema.json @@ -1395,6 +1395,46 @@ } }, "allOf": [ + { + "if": { + "properties": { + "name": { + "enum": ["docs"] + } + } + }, + "then": { + "properties": { + "params": { + "properties": { + "sites": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "startUrl": { + "type": "string" + }, + "rootUrl": { + "type": "string" + }, + "maxDepth": { + "type": "integer" + } + }, + "required": ["title", "startUrl"] + } + } + }, + "required": ["sites"] + } + }, + "required": ["params"] + } + }, { "if": { "properties": { @@ -1955,7 +1995,13 @@ "type": "object", "properties": { "name": { - "enum": ["cohere", "voyage", "llm", "free-trial", "huggingface-tei"] + "enum": [ + "cohere", + "voyage", + "llm", + "free-trial", + "huggingface-tei" + ] }, "params": { "type": "object" @@ -2070,7 +2116,7 @@ "default": false }, "truncation_direction": { - "enum": ["Right","Left"], + "enum": ["Right", "Left"], "markdownDescription": "Wether to truncate sequences from the `left` or `right`.", "default": "Right" } diff --git a/extensions/vscode/config_schema.json b/extensions/vscode/config_schema.json index 19813eb6ae..ac7461d8fd 100644 --- a/extensions/vscode/config_schema.json +++ b/extensions/vscode/config_schema.json @@ -1395,6 +1395,46 @@ } }, "allOf": [ + { + "if": { + "properties": { + "name": { + "enum": ["docs"] + } + } + }, + "then": { + "properties": { + "params": { + "properties": { + "sites": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "startUrl": { + "type": "string" + }, + "rootUrl": { + "type": "string" + }, + "maxDepth": { + "type": "integer" + } + }, + "required": ["title", "startUrl"] + } + } + }, + "required": ["sites"] + } + }, + "required": ["params"] + } + }, { "if": { "properties": { @@ -1955,7 +1995,13 @@ "type": "object", "properties": { "name": { - "enum": ["cohere", "voyage", "llm", "free-trial", "huggingface-tei"] + "enum": [ + "cohere", + "voyage", + "llm", + "free-trial", + "huggingface-tei" + ] }, "params": { "type": "object" @@ -2070,7 +2116,7 @@ "default": false }, "truncation_direction": { - "enum": ["Right","Left"], + "enum": ["Right", "Left"], "markdownDescription": "Wether to truncate sequences from the `left` or `right`.", "default": "Right" } diff --git a/extensions/vscode/continue_rc_schema.json b/extensions/vscode/continue_rc_schema.json index e1da75f392..2c67ce0c19 100644 --- a/extensions/vscode/continue_rc_schema.json +++ b/extensions/vscode/continue_rc_schema.json @@ -1541,6 +1541,55 @@ } }, "allOf": [ + { + "if": { + "properties": { + "name": { + "enum": [ + "docs" + ] + } + } + }, + "then": { + "properties": { + "params": { + "properties": { + "sites": { + "type": "array", + "items": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "startUrl": { + "type": "string" + }, + "rootUrl": { + "type": "string" + }, + "maxDepth": { + "type": "integer" + } + }, + "required": [ + "title", + "startUrl" + ] + } + } + }, + "required": [ + "sites" + ] + } + }, + "required": [ + "params" + ] + } + }, { "if": { "properties": { diff --git a/extensions/vscode/package-lock.json b/extensions/vscode/package-lock.json index c1382a7012..118fd459de 100644 --- a/extensions/vscode/package-lock.json +++ b/extensions/vscode/package-lock.json @@ -1,12 +1,12 @@ { "name": "continue", - "version": "0.9.185", + "version": "0.9.189", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "continue", - "version": "0.9.185", + "version": "0.9.189", "license": "Apache-2.0", "dependencies": { "@electron/rebuild": "^3.2.10", @@ -97,7 +97,7 @@ "dependencies": { "@aws-sdk/client-bedrock-runtime": "^3.574.0", "@aws-sdk/credential-providers": "^3.596.0", - "@continuedev/config-types": "^1.0.6", + "@continuedev/config-types": "^1.0.8", "@continuedev/llm-info": "^1.0.1", "@mozilla/readability": "^0.5.0", "@octokit/rest": "^20.0.2", diff --git a/extensions/vscode/package.json b/extensions/vscode/package.json index b3423829c0..2b8acfa76d 100644 --- a/extensions/vscode/package.json +++ b/extensions/vscode/package.json @@ -1,7 +1,7 @@ { "name": "continue", "icon": "media/icon.png", - "version": "0.9.186", + "version": "0.9.187", "repository": { "type": "git", "url": "https://github.com/continuedev/continue" @@ -94,6 +94,16 @@ "default": false, "markdownDescription": "Pause Continue's tab autocomplete feature when your battery is low." }, + "continue.pauseCodebaseIndexOnStart": { + "type": "boolean", + "default": false, + "markdownDescription": "Pause Continue's codebase index on start." + }, + "continue.enableDebugLogs": { + "type": "boolean", + "default": false, + "markdownDescription": "Enable Continue Debug Logs in the Output panel." + }, "continue.remoteConfigServerUrl": { "type": "string", "default": null, diff --git a/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts b/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts index c713244b0b..08fedd54e1 100644 --- a/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts +++ b/extensions/vscode/src/ContinueGUIWebviewViewProvider.ts @@ -12,12 +12,55 @@ export class ContinueGUIWebviewViewProvider public static readonly viewType = "continue.continueGUIView"; public webviewProtocol: VsCodeWebviewProtocol; + private updateDebugLogsStatus() { + const settings = vscode.workspace.getConfiguration("continue"); + this.enableDebugLogs = settings.get("enableDebugLogs", false); + if (this.enableDebugLogs) { + this.outputChannel.show(true); + } else { + this.outputChannel.hide(); + } + } + + // Show or hide the output channel on enableDebugLogs + private setupDebugLogsListener() { + vscode.workspace.onDidChangeConfiguration((event) => { + if (event.affectsConfiguration('continue.enableDebugLogs')) { + const settings = vscode.workspace.getConfiguration("continue"); + const enableDebugLogs = settings.get("enableDebugLogs", false); + if (enableDebugLogs) { + this.outputChannel.show(true); + } else { + this.outputChannel.hide(); + } + } + }); + } + + private async handleWebviewMessage(message: any) { + if (message.messageType === "log") { + const settings = vscode.workspace.getConfiguration("continue"); + const enableDebugLogs = settings.get("enableDebugLogs", false); + + if (message.level === "debug" && !enableDebugLogs) { + return; // Skip debug logs if enableDebugLogs is false + } + + const timestamp = new Date().toISOString().split(".")[0]; + const logMessage = `[${timestamp}] [${message.level.toUpperCase()}] ${message.text}`; + this.outputChannel.appendLine(logMessage); + } +} + resolveWebviewView( webviewView: vscode.WebviewView, _context: vscode.WebviewViewResolveContext, _token: vscode.CancellationToken, ): void | Thenable { this._webview = webviewView.webview; + this._webview.onDidReceiveMessage((message) => + this.handleWebviewMessage(message), + ); webviewView.webview.html = this.getSidebarContent( this.extensionContext, webviewView, @@ -26,6 +69,8 @@ export class ContinueGUIWebviewViewProvider private _webview?: vscode.Webview; private _webviewView?: vscode.WebviewView; + private outputChannel: vscode.OutputChannel; + private enableDebugLogs: boolean; get isVisible() { return this._webviewView?.visible; @@ -50,11 +95,17 @@ export class ContinueGUIWebviewViewProvider }); } + constructor( private readonly configHandlerPromise: Promise, private readonly windowId: string, private readonly extensionContext: vscode.ExtensionContext, ) { + this.outputChannel = vscode.window.createOutputChannel("Continue"); + this.enableDebugLogs = false; + this.updateDebugLogsStatus(); + this.setupDebugLogsListener(); + this.webviewProtocol = new VsCodeWebviewProtocol( (async () => { const configHandler = await this.configHandlerPromise; @@ -131,6 +182,22 @@ export class ContinueGUIWebviewViewProvider
+ ${``} ${ inDevelopmentMode ? `