From 16e7c68fcb8719c655f5f11dd0c82526e90288b3 Mon Sep 17 00:00:00 2001 From: Mark Sujew Date: Tue, 13 Aug 2024 16:59:07 +0200 Subject: [PATCH] Improve VS Code file system handling (#51) --- package-lock.json | 44 +--- .../src/connection.ts | 4 +- .../src/messages.ts | 2 +- .../open-collaboration-vscode/package.json | 8 +- .../src/collaboration-file-system.ts | 13 +- .../src/collaboration-instance.ts | 204 ++++++++++++------ .../src/utils/uri.ts | 49 +++++ 7 files changed, 213 insertions(+), 111 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8d67f37..0d6ecf1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1293,19 +1293,6 @@ "integrity": "sha512-dRLjCWHYg4oaA77cxO64oO+7JwCwnIzkZPdrrC71jQmQtlhM556pwKo5bUzqvZndkVbeFLIIi+9TC40JNF5hNQ==", "dev": true }, - "node_modules/@types/lodash": { - "version": "4.17.4", - "dev": true, - "license": "MIT" - }, - "node_modules/@types/lodash.debounce": { - "version": "4.0.9", - "dev": true, - "license": "MIT", - "dependencies": { - "@types/lodash": "*" - } - }, "node_modules/@types/mime": { "version": "1.3.5", "dev": true, @@ -1387,12 +1374,6 @@ "@types/passport": "*" } }, - "node_modules/@types/path-browserify": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/@types/path-browserify/-/path-browserify-1.0.2.tgz", - "integrity": "sha512-ZkC5IUqqIFPXx3ASTTybTzmQdwHwe2C0u3eL75ldQ6T9E9IWFJodn6hIfbZGab73DfyiHN4Xw15gNxUq2FbvBA==", - "dev": true - }, "node_modules/@types/qs": { "version": "6.9.11", "dev": true, @@ -4566,11 +4547,6 @@ }, "node_modules/lodash": { "version": "4.17.21", - "dev": true, - "license": "MIT" - }, - "node_modules/lodash.debounce": { - "version": "4.0.8", "license": "MIT" }, "node_modules/lodash.merge": { @@ -5216,12 +5192,6 @@ "node": ">= 0.4.0" } }, - "node_modules/path-browserify": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-browserify/-/path-browserify-1.0.1.tgz", - "integrity": "sha512-b7uo2UCUOYZcnF/3ID0lulOJi/bafxa1xPe7ZPsammBSpjSWQkjNxlt635YGS2MiR9GjvuXCtz2emr3jbsz98g==", - "dev": true - }, "node_modules/path-exists": { "version": "4.0.0", "dev": true, @@ -7425,24 +7395,28 @@ "dependencies": { "async-mutex": "^0.5.0", "inversify": "^6.0.2", - "lodash.debounce": "^4.0.8", + "lodash": "^4.17.21", "node-fetch": "^2.0.0", "open-collaboration-protocol": "0.1.0", "open-collaboration-yjs": "0.1.0", "reflect-metadata": "^0.2.2" }, "devDependencies": { - "@types/lodash.debounce": "^4.0.9", + "@types/lodash": "^4.17.7", "@types/node-fetch": "^2.0.0", - "@types/path-browserify": "^1.0.2", "@types/vscode": "^1.73.0", - "@vscode/l10n-dev": "^0.0.35", - "path-browserify": "1.0.1" + "@vscode/l10n-dev": "^0.0.35" }, "engines": { "vscode": "^1.73.0" } }, + "packages/open-collaboration-vscode/node_modules/@types/lodash": { + "version": "4.17.7", + "resolved": "https://registry.npmjs.org/@types/lodash/-/lodash-4.17.7.tgz", + "integrity": "sha512-8wTvZawATi/lsmNu10/j2hk1KEP0IvjubqPE3cu1Xz7xfXXt5oCq3SNUz4fMIP4XGF9Ky+Ue2tBA3hcS7LSBlA==", + "dev": true + }, "packages/open-collaboration-vscode/node_modules/@types/vscode": { "version": "1.89.0", "dev": true, diff --git a/packages/open-collaboration-protocol/src/connection.ts b/packages/open-collaboration-protocol/src/connection.ts index f97b4b5..7fb8a71 100644 --- a/packages/open-collaboration-protocol/src/connection.ts +++ b/packages/open-collaboration-protocol/src/connection.ts @@ -35,8 +35,8 @@ export interface EditorHandler { export interface FileSystemHandler { onReadFile(handler: Handler<[types.Path], types.FileData>): void; readFile(target: MessageTarget, path: types.Path): Promise; - onWriteFile(handler: Handler<[types.Path, string]>): void; - writeFile(target: MessageTarget, path: types.Path, content: string): Promise; + onWriteFile(handler: Handler<[types.Path, types.FileData]>): void; + writeFile(target: MessageTarget, path: types.Path, content: types.FileData): Promise; onStat(handler: Handler<[types.Path], types.FileSystemStat>): void; stat(target: MessageTarget, path: types.Path): Promise; onMkdir(handler: Handler<[types.Path]>): void; diff --git a/packages/open-collaboration-protocol/src/messages.ts b/packages/open-collaboration-protocol/src/messages.ts index f33f493..8a6ddba 100644 --- a/packages/open-collaboration-protocol/src/messages.ts +++ b/packages/open-collaboration-protocol/src/messages.ts @@ -40,7 +40,7 @@ export namespace Messages { export const Stat = new RequestType<[types.Path], types.FileSystemStat>('fileSystem/stat'); export const Mkdir = new RequestType<[types.Path], undefined>('fileSystem/mkdir'); export const ReadFile = new RequestType<[types.Path], types.FileData>('fileSystem/readFile'); - export const WriteFile = new RequestType<[types.Path, string], undefined>('fileSystem/writeFile'); + export const WriteFile = new RequestType<[types.Path, types.FileData], undefined>('fileSystem/writeFile'); export const ReadDir = new RequestType<[types.Path], Record>('fileSystem/readDir'); export const Delete = new RequestType<[types.Path], undefined>('fileSystem/delete'); export const Rename = new RequestType<[types.Path, types.Path], undefined>('fileSystem/rename'); diff --git a/packages/open-collaboration-vscode/package.json b/packages/open-collaboration-vscode/package.json index 4654ae1..ca0141c 100644 --- a/packages/open-collaboration-vscode/package.json +++ b/packages/open-collaboration-vscode/package.json @@ -208,16 +208,14 @@ "reflect-metadata": "^0.2.2", "open-collaboration-yjs": "0.1.0", "open-collaboration-protocol": "0.1.0", - "lodash.debounce": "^4.0.8", + "lodash": "^4.17.21", "node-fetch": "^2.0.0" }, "devDependencies": { - "@types/lodash.debounce": "^4.0.9", + "@types/lodash": "^4.17.7", "@types/node-fetch": "^2.0.0", - "@types/path-browserify": "^1.0.2", "@types/vscode": "^1.73.0", - "@vscode/l10n-dev": "^0.0.35", - "path-browserify": "1.0.1" + "@vscode/l10n-dev": "^0.0.35" }, "volta": { "node": "18.20.3", diff --git a/packages/open-collaboration-vscode/src/collaboration-file-system.ts b/packages/open-collaboration-vscode/src/collaboration-file-system.ts index 453e302..498d449 100644 --- a/packages/open-collaboration-vscode/src/collaboration-file-system.ts +++ b/packages/open-collaboration-vscode/src/collaboration-file-system.ts @@ -48,14 +48,13 @@ export class CollaborationFileSystemProvider implements vscode.FileSystemProvide const stringValue = this.yjs.getText(path); return this.encoder.encode(stringValue.toString()); } else { - // Attempt to stat the file to see if it exists on the host system - await this.stat(uri); - // Just return an empty file. It will be filled by YJS - return new Uint8Array(); + const file = await this.connection.fs.readFile(this.host.id, path); + return file.content; } } - writeFile(_uri: vscode.Uri, _content: Uint8Array, _options: { readonly create: boolean; readonly overwrite: boolean; }): void { - // Do nothing + writeFile(uri: vscode.Uri, content: Uint8Array, _options: { readonly create: boolean; readonly overwrite: boolean; }): void { + const path = this.getHostPath(uri); + this.connection.fs.writeFile(this.host.id, path, { content }); } delete(uri: vscode.Uri, _options: { readonly recursive: boolean; }): Promise { return this.connection.fs.delete(this.host.id, this.getHostPath(uri)); @@ -69,6 +68,8 @@ export class CollaborationFileSystemProvider implements vscode.FileSystemProvide } protected getHostPath(uri: vscode.Uri): string { + // When creating a URI as a guest, we always prepend it with the name of the workspace + // This just removes the workspace name from the path to get the path expected by the protocol const path = uri.path.substring(1).split('/'); return path.slice(1).join('/'); } diff --git a/packages/open-collaboration-vscode/src/collaboration-instance.ts b/packages/open-collaboration-vscode/src/collaboration-instance.ts index edac4bc..4091599 100644 --- a/packages/open-collaboration-vscode/src/collaboration-instance.ts +++ b/packages/open-collaboration-vscode/src/collaboration-instance.ts @@ -10,12 +10,13 @@ import * as Y from 'yjs'; import * as awarenessProtocol from 'y-protocols/awareness'; import * as types from 'open-collaboration-protocol'; import { CollaborationFileSystemProvider } from './collaboration-file-system'; -import paths from 'path-browserify'; import { LOCAL_ORIGIN, OpenCollaborationYjsProvider } from 'open-collaboration-yjs'; import debounce from 'lodash/debounce'; +import throttle from 'lodash/throttle'; import { inject, injectable, postConstruct } from 'inversify'; import { removeWorkspaceFolders } from './utils/workspace'; import { Mutex } from 'async-mutex'; +import { CollaborationUri } from './utils/uri'; export class DisposablePeer implements vscode.Disposable { @@ -77,12 +78,12 @@ export class DisposablePeer implements vscode.Disposable { ...selection, after: cursor }); - const beforeNameTag = this.createNameTag(colorCss, 'top: -1rem;'); - const beforeInvertedNameTag = this.createNameTag(colorCss, 'bottom: -1rem;'); + const nameTag = this.createNameTag(colorCss, 'top: -1rem;'); + const invertedNameTag = this.createNameTag(colorCss, 'bottom: -1rem;'); return new ClientTextEditorDecorationType(before, after, { - default: beforeNameTag, - inverted: beforeInvertedNameTag + default: nameTag, + inverted: invertedNameTag }, color); } @@ -191,6 +192,7 @@ export class CollaborationInstance implements vscode.Disposable { private documentDisposables = new Map(); private peers = new Map(); private throttles = new Map void>(); + private fileSystem?: CollaborationFileSystemProvider; private _following?: string; get following(): string | undefined { @@ -314,8 +316,15 @@ export class CollaborationInstance implements vscode.Disposable { this.yjsAwareness.setLocalStateField('peer', peer.id); this.identity.resolve(peer); }); + + this.registerFileEvents(); + this.registerEditorEvents(); + } + + private registerFileEvents() { + const connection = this.connection; connection.fs.onStat(async (_, path) => { - const uri = this.getResourceUri(path); + const uri = CollaborationUri.getResourceUri(path); if (uri) { const stat = await vscode.workspace.fs.stat(uri); return { @@ -329,7 +338,7 @@ export class CollaborationInstance implements vscode.Disposable { } }); connection.fs.onReaddir(async (_, path) => { - const uri = this.getResourceUri(path); + const uri = CollaborationUri.getResourceUri(path); if (uri) { const result = await vscode.workspace.fs.readDirectory(uri); return result.reduce((acc, [name, type]) => { acc[name] = type; return acc; }, {} as types.FileSystemDirectory); @@ -338,7 +347,7 @@ export class CollaborationInstance implements vscode.Disposable { } }); connection.fs.onReadFile(async (_, path) => { - const uri = this.getResourceUri(path); + const uri = CollaborationUri.getResourceUri(path); if (uri) { const content = await vscode.workspace.fs.readFile(uri); return { @@ -348,15 +357,74 @@ export class CollaborationInstance implements vscode.Disposable { throw new Error('Could not read file'); } }); - connection.editor.onOpen(async (_, path) => { - const uri = this.getResourceUri(path); + connection.fs.onDelete(async (_, path) => { + const uri = CollaborationUri.getResourceUri(path); if (uri) { - await vscode.workspace.openTextDocument(uri); + await vscode.workspace.fs.delete(uri, { recursive: true }); } else { - throw new Error('Could not open file'); + throw new Error('Could not delete file'); + } + }); + connection.fs.onRename(async (_, oldPath, newPath) => { + const oldUri = CollaborationUri.getResourceUri(oldPath); + const newUri = CollaborationUri.getResourceUri(newPath); + if (oldUri && newUri) { + await vscode.workspace.fs.rename(oldUri, newUri, { overwrite: true }); + } else { + throw new Error('Could not rename file'); + } + }); + connection.fs.onMkdir(async (_, path) => { + const uri = CollaborationUri.getResourceUri(path); + if (uri) { + await vscode.workspace.fs.createDirectory(uri); + } else { + throw new Error('Could not create directory'); + } + }); + connection.fs.onChange(async (_, changes) => { + if (this.fileSystem) { + const vscodeChanges: vscode.FileChangeEvent[] = []; + for (const change of changes.changes) { + const uri = CollaborationUri.getResourceUri(change.path); + if (uri) { + vscodeChanges.push({ + type: this.convertChangeType(change.type), + uri + }); + } + } + this.fileSystem.triggerEvent(vscodeChanges); + } + }); + connection.fs.onWriteFile(async (_, path, content) => { + const uri = CollaborationUri.getResourceUri(path); + if (uri) { + const document = vscode.workspace.textDocuments.find(e => e.uri.toString() === uri.toString()); + if (document) { + const textContent = new TextDecoder().decode(content.content); + // In case the supplied content differs from the current document content, apply the change first + if (textContent !== document.getText()) { + await vscode.workspace.applyEdit(this.createFullDocumentEdit(document, textContent)); + } + // Then save the document + await document.save(); + } else { + await vscode.workspace.fs.writeFile(uri, content.content); + } } }); - this.registerEditorEvents(); + } + + private convertChangeType(type: types.FileChangeEventType): vscode.FileChangeType { + switch (type) { + case types.FileChangeEventType.Create: + return vscode.FileChangeType.Created; + case types.FileChangeEventType.Delete: + return vscode.FileChangeType.Deleted; + case types.FileChangeEventType.Update: + return vscode.FileChangeType.Changed; + } } async leave(): Promise { @@ -389,6 +457,15 @@ export class CollaborationInstance implements vscode.Disposable { private registerEditorEvents() { + this.connection.editor.onOpen(async (_, path) => { + const uri = CollaborationUri.getResourceUri(path); + if (uri) { + await vscode.workspace.openTextDocument(uri); + } else { + throw new Error('Could not open file'); + } + }); + vscode.workspace.textDocuments.forEach(document => { if (!this.isNotebookCell(document)) { this.registerTextDocument(document); @@ -425,6 +502,11 @@ export class CollaborationInstance implements vscode.Disposable { this.updateTextSelection(event.textEditor); })); + if (this.host) { + // Only the host should create the watcher + this.createFileWatcher(); + } + const awarenessDebounce = debounce(() => { this.rerenderPresence(); }, 2000); @@ -438,6 +520,35 @@ export class CollaborationInstance implements vscode.Disposable { }); } + protected createFileWatcher(): void { + // Batch all changes and send them in one go + // We don't want to send hundreds of messages in case of multiple changes in a short time + // However, we also don't want to wait too long to send the changes. This will send the changes every 100ms + const queue: types.FileChange[] = []; + const sendChanges = throttle(() => { + const changes = queue.splice(0, queue.length); + this.connection.fs.change({ changes }); + }, 100, { + leading: false, + trailing: true + }); + const pushChange = (uri: vscode.Uri, type: types.FileChangeEventType) => { + const path = CollaborationUri.getProtocolPath(uri); + if (path) { + queue.push({ + path, + type + }); + sendChanges(); + } + }; + const watcher = vscode.workspace.createFileSystemWatcher('**/*'); + watcher.onDidChange(uri => pushChange(uri, types.FileChangeEventType.Update)); + watcher.onDidCreate(uri => pushChange(uri, types.FileChangeEventType.Create)); + watcher.onDidDelete(uri => pushChange(uri, types.FileChangeEventType.Delete)); + this.toDispose.push(watcher); + } + protected isNotebookCell(doc: vscode.TextDocument): boolean { return doc.uri.scheme === 'vscode-notebook-cell'; } @@ -466,7 +577,7 @@ export class CollaborationInstance implements vscode.Disposable { } protected async followSelection(selection: types.ClientTextSelection): Promise { - const uri = this.getResourceUri(selection.path); + const uri = CollaborationUri.getResourceUri(selection.path); if (uri && selection.visibleRanges && selection.visibleRanges.length > 0) { let editor = vscode.window.visibleTextEditors.find(e => e.document.uri.toString() === uri.toString()); if (!editor) { @@ -485,7 +596,7 @@ export class CollaborationInstance implements vscode.Disposable { return; } const uri = editor.document.uri; - const path = this.getProtocolPath(uri); + const path = CollaborationUri.getProtocolPath(uri); if (path) { const ytext = this.yjs.getText(path); const selections: types.RelativeTextSelection[] = []; @@ -524,7 +635,7 @@ export class CollaborationInstance implements vscode.Disposable { protected registerTextDocument(document: vscode.TextDocument): void { const uri = document.uri; - const path = this.getProtocolPath(uri); + const path = CollaborationUri.getProtocolPath(uri); if (path) { const text = document.getText(); const yjsText = this.yjs.getText(path); @@ -536,16 +647,10 @@ export class CollaborationInstance implements vscode.Disposable { } else { this.options.connection.editor.open(this.options.hostId, path); } - const ytextContent = yjsText.toString(); - if (text !== ytextContent) { - const edit = new vscode.WorkspaceEdit(); - edit.replace(uri, new vscode.Range(0, 0, document.lineCount, 0), ytextContent); - vscode.workspace.applyEdit(edit); - } const resyncThrottle = this.getOrCreateThrottle(path, document); const observer = (textEvent: Y.YTextEvent) => { - if (textEvent.transaction.local) { - // Ignore own events + if (textEvent.transaction.local || yjsText.toString() === document.getText()) { + // Ignore own events or if the document is already in sync return; } let index = 0; @@ -579,7 +684,7 @@ export class CollaborationInstance implements vscode.Disposable { protected updateTextDocument(event: vscode.TextDocumentChangeEvent): void { const uri = event.document.uri; - const path = this.getProtocolPath(uri); + const path = CollaborationUri.getProtocolPath(uri); if (path) { if (this.updates.has(path)) { return; @@ -605,10 +710,8 @@ export class CollaborationInstance implements vscode.Disposable { const yjsText = this.yjs.getText(path); const newContent = yjsText.toString(); if (newContent !== document.getText()) { - const edit = new vscode.WorkspaceEdit(); - edit.replace(document.uri, new vscode.Range(0, 0, document.lineCount, 0), newContent); this.updates.add(path); - await vscode.workspace.applyEdit(edit); + await vscode.workspace.applyEdit(this.createFullDocumentEdit(document, newContent)); this.updates.delete(path); } }); @@ -618,6 +721,14 @@ export class CollaborationInstance implements vscode.Disposable { return value; } + private createFullDocumentEdit(document: vscode.TextDocument, content: string): vscode.WorkspaceEdit { + const edit = new vscode.WorkspaceEdit(); + const startPosition = new vscode.Position(0, 0); + const endPosition = document.lineAt(document.lineCount - 1).range.end; + edit.replace(document.uri, new vscode.Range(startPosition, endPosition), content); + return edit; + } + protected rerenderPresence() { const states = this.yjsAwareness.getStates() as Map; for (const [clientID, state] of states.entries()) { @@ -639,7 +750,7 @@ export class CollaborationInstance implements vscode.Disposable { protected renderTextPresence(peer: DisposablePeer, selection: types.ClientTextSelection): void { const nameTagVisible = peer.lastUpdated !== undefined && Date.now() - peer.lastUpdated < 1900; const { path, textSelections } = selection; - const uri = this.getResourceUri(path); + const uri = CollaborationUri.getResourceUri(path); const editorsToRemove = new Set(vscode.window.visibleTextEditors); if (uri) { const editors = vscode.window.visibleTextEditors.filter(e => e.document.uri.toString() === uri.toString()); @@ -722,38 +833,7 @@ export class CollaborationInstance implements vscode.Disposable { for (const peer of [data.host, ...data.guests]) { this.peers.set(peer.id, new DisposablePeer(this.yjsAwareness, peer)); } - this.toDispose.push(vscode.workspace.registerFileSystemProvider('oct', new CollaborationFileSystemProvider(this.options.connection, this.yjs, data.host))); - } - - getProtocolPath(uri?: vscode.Uri): string | undefined { - if (!uri) { - return undefined; - } - const path = uri.path.toString(); - const roots = (vscode.workspace.workspaceFolders ?? []); - for (const root of roots) { - const rootUri = root.uri.path + '/'; - if (path.startsWith(rootUri)) { - return root.name + '/' + path.substring(rootUri.length); - } - } - return undefined; - } - - getResourceUri(path?: string): vscode.Uri | undefined { - if (!path) { - return undefined; - } - const parts = path.split('/'); - const root = parts[0]; - const rest = parts.slice(1); - const stat = (vscode.workspace.workspaceFolders ?? []).find(e => e.name === root); - if (stat) { - const uriPath = paths.posix.join(stat.uri.path, ...rest); - const uri = stat.uri.with({ path: uriPath }); - return uri; - } else { - return undefined; - } + this.fileSystem = new CollaborationFileSystemProvider(this.options.connection, this.yjs, data.host); + this.toDispose.push(vscode.workspace.registerFileSystemProvider('oct', this.fileSystem)); } } diff --git a/packages/open-collaboration-vscode/src/utils/uri.ts b/packages/open-collaboration-vscode/src/utils/uri.ts index 2b95d6a..52235c4 100644 --- a/packages/open-collaboration-vscode/src/utils/uri.ts +++ b/packages/open-collaboration-vscode/src/utils/uri.ts @@ -14,4 +14,53 @@ export namespace CollaborationUri { return vscode.Uri.parse(`${SCHEME}:///${workspace}${path ? '/' + path : ''}`); } + export function getProtocolPath(uri?: vscode.Uri): string | undefined { + if (!uri) { + return undefined; + } + const path = uri.path.toString(); + const roots = (vscode.workspace.workspaceFolders ?? []); + for (const root of roots) { + const rootUri = root.uri.path + '/'; + if (path.startsWith(rootUri)) { + return root.name + '/' + path.substring(rootUri.length); + } + } + return undefined; + } + + export function getResourceUri(path?: string): vscode.Uri | undefined { + if (!path) { + return undefined; + } + const parts = path.split('/'); + const root = parts[0]; + const rest = parts.slice(1); + const stat = (vscode.workspace.workspaceFolders ?? []).find(e => e.name === root); + if (stat) { + const uriPath = join(stat.uri.path, ...rest); + const uri = stat.uri.with({ path: uriPath }); + return uri; + } else { + return undefined; + } + } + + function join(...parts: string[]): string { + if (parts.length === 0) + return '.'; + let joined: string | undefined; + for (const part of parts) { + if (part.length > 0) { + if (joined === undefined) + joined = part; + else + joined += '/' + part; + } + } + if (joined === undefined) + return '.'; + return joined; + } + }