diff --git a/extensions/positron-python b/extensions/positron-python index 67df41d9088..25895a2c980 160000 --- a/extensions/positron-python +++ b/extensions/positron-python @@ -1 +1 @@ -Subproject commit 67df41d90888a86e070b05aa89ed846c930cdd0a +Subproject commit 25895a2c980a5fd42bba42d12be65a2ee59d1816 diff --git a/positron/comms/data_explorer-backend-openrpc.json b/positron/comms/data_explorer-backend-openrpc.json index 946eed0a64d..5333b578f2a 100644 --- a/positron/comms/data_explorer-backend-openrpc.json +++ b/positron/comms/data_explorer-backend-openrpc.json @@ -33,9 +33,7 @@ "name": "table_schema", "description": "The schema for a table-like object", "required": [ - "columns", - "num_rows", - "total_num_columns" + "columns" ], "properties": { "columns": { @@ -44,14 +42,6 @@ "items": { "$ref": "#/components/schemas/column_schema" } - }, - "num_rows": { - "type": "integer", - "description": "Numbers of rows in the unfiltered dataset" - }, - "total_num_columns": { - "type": "integer", - "description": "Total number of columns in the unfiltered dataset" } } } @@ -279,18 +269,38 @@ { "name": "get_state", "summary": "Get the state", - "description": "Request the current backend state (applied filters and sort columns)", + "description": "Request the current table state (applied filters and sort columns)", "params": [], "result": { "schema": { "type": "object", - "name": "backend_state", - "description": "The current backend state", + "name": "table_state", + "description": "The current backend table state", "required": [ + "table_shape", "filters", "sort_keys" ], "properties": { + "table_shape": { + "type": "object", + "name": "table_shape", + "description": "Provides number of rows and columns in table", + "required": [ + "num_rows", + "num_columns" + ], + "properties": { + "num_rows": { + "type": "integer", + "description": "Numbers of rows in the unfiltered dataset" + }, + "num_columns": { + "type": "integer", + "description": "Number of columns in the unfiltered dataset" + } + } + }, "filters": { "type": "array", "description": "The set of currently applied filters", diff --git a/positron/comms/generate-comms.ts b/positron/comms/generate-comms.ts index 706356d80f3..fda728e4683 100644 --- a/positron/comms/generate-comms.ts +++ b/positron/comms/generate-comms.ts @@ -45,7 +45,7 @@ const tsOutputDir = `${__dirname}/../../src/vs/workbench/services/languageRuntim const rustOutputDir = `${__dirname}/../../../amalthea/crates/amalthea/src/comm`; /// The directory to write the generated Python files to -const pythonOutputDir = `${__dirname}/../../extensions/positron-python/pythonFiles/positron`; +const pythonOutputDir = `${__dirname}/../../extensions/positron-python/pythonFiles/positron/positron_ipykernel`; const year = new Date().getFullYear(); diff --git a/src/vs/workbench/services/languageRuntime/common/languageRuntimeDataExplorerClient.ts b/src/vs/workbench/services/languageRuntime/common/languageRuntimeDataExplorerClient.ts index 6db8d8f5fe0..07f2da7684e 100644 --- a/src/vs/workbench/services/languageRuntime/common/languageRuntimeDataExplorerClient.ts +++ b/src/vs/workbench/services/languageRuntime/common/languageRuntimeDataExplorerClient.ts @@ -6,7 +6,14 @@ import { Emitter, Event } from 'vs/base/common/event'; import { Disposable } from 'vs/base/common/lifecycle'; import { generateUuid } from 'vs/base/common/uuid'; import { IRuntimeClientInstance } from 'vs/workbench/services/languageRuntime/common/languageRuntimeClientInstance'; -import { ColumnSortKey, PositronDataExplorerComm, SchemaUpdateEvent, TableData, TableSchema } from 'vs/workbench/services/languageRuntime/common/positronDataExplorerComm'; +import { + ColumnSortKey, + PositronDataExplorerComm, + SchemaUpdateEvent, + TableData, + TableSchema, + TableState +} from 'vs/workbench/services/languageRuntime/common/positronDataExplorerComm'; /** * A data explorer client instance. @@ -81,6 +88,14 @@ export class DataExplorerClientInstance extends Disposable { return this._positronDataExplorerComm.getSchema(startIndex, numColumns); } + /** + * Get the current active state of the data explorer backend. + * @returns A promose that resolves to the current table state. + */ + async getState(): Promise { + return this._positronDataExplorerComm.getState(); + } + /** * Get a rectangle of data values. * @param rowStartIndex The first row to fetch (inclusive). diff --git a/src/vs/workbench/services/languageRuntime/common/positronDataExplorerComm.ts b/src/vs/workbench/services/languageRuntime/common/positronDataExplorerComm.ts index b663716ab01..d5b8fefd574 100644 --- a/src/vs/workbench/services/languageRuntime/common/positronDataExplorerComm.ts +++ b/src/vs/workbench/services/languageRuntime/common/positronDataExplorerComm.ts @@ -19,16 +19,6 @@ export interface TableSchema { */ columns: Array; - /** - * Numbers of rows in the unfiltered dataset - */ - num_rows: number; - - /** - * Total number of columns in the unfiltered dataset - */ - total_num_columns: number; - } /** @@ -126,9 +116,14 @@ export interface FreqtableCounts { } /** - * The current backend state + * The current backend table state */ -export interface BackendState { +export interface TableState { + /** + * Provides number of rows and columns in table + */ + table_shape: TableShape; + /** * The set of currently applied filters */ @@ -141,6 +136,22 @@ export interface BackendState { } +/** + * Provides number of rows and columns in table + */ +export interface TableShape { + /** + * Numbers of rows in the unfiltered dataset + */ + num_rows: number; + + /** + * Number of columns in the unfiltered dataset + */ + num_columns: number; + +} + /** * Schema for a column in a table */ @@ -447,12 +458,12 @@ export class PositronDataExplorerComm extends PositronBaseComm { /** * Get the state * - * Request the current backend state (applied filters and sort columns) + * Request the current table state (applied filters and sort columns) * * - * @returns The current backend state + * @returns The current backend table state */ - getState(): Promise { + getState(): Promise { return super.performRpc('get_state', [], []); } diff --git a/src/vs/workbench/services/positronDataExplorer/browser/tableDataDataGridInstance.tsx b/src/vs/workbench/services/positronDataExplorer/browser/tableDataDataGridInstance.tsx index 198e7ea9c9d..aa0dcf020f8 100644 --- a/src/vs/workbench/services/positronDataExplorer/browser/tableDataDataGridInstance.tsx +++ b/src/vs/workbench/services/positronDataExplorer/browser/tableDataDataGridInstance.tsx @@ -33,10 +33,12 @@ export class TableDataDataGridInstance extends DataGridInstance { */ private readonly _dataExplorerClientInstance: DataExplorerClientInstance; + private tableShape?: [number, number]; + private _dataCache?: TableDataCache; private _lastFetchedData?: FetchedData; - private _schemaCache: TableSchemaCache; + private _schemaCache?: TableSchemaCache; private _lastFetchedSchema?: FetchedSchema; //#endregion Private Properties @@ -72,13 +74,6 @@ export class TableDataDataGridInstance extends DataGridInstance { cellBorder: true }); - this._schemaCache = new TableSchemaCache( - async (req: SchemaFetchRange) => { - return this._dataExplorerClientInstance.getSchema(req.startIndex, - req.endIndex - req.startIndex); - } - ); - // Set the data explorer client instance. this._dataExplorerClientInstance = dataExplorerClientInstance; @@ -92,12 +87,16 @@ export class TableDataDataGridInstance extends DataGridInstance { this._firstColumnIndex = 0; this._firstRowIndex = 0; - // Resets data schema, fetches initial schema and data + // Resets data schema, fetches table shape, initial schema, and data this.initialize(); }); this._dataExplorerClientInstance.onDidDataUpdate(async (_evt) => { this._lastFetchedData = undefined; + + const state = await this._dataExplorerClientInstance.getState(); + this.tableShape = [state.table_shape.num_rows, state.table_shape.num_columns]; + this._dataCache?.clear(); this.fetchData(); }); @@ -109,42 +108,47 @@ export class TableDataDataGridInstance extends DataGridInstance { * Gets the number of columns. */ get columns() { - return this._lastFetchedSchema ? this._lastFetchedSchema.schema.total_num_columns : 0; + return this.tableShape ? this.tableShape[1] : 0; } /** * Gets the number of rows. */ get rows() { - return this._lastFetchedSchema ? this._lastFetchedSchema.schema.num_rows : 0; + return this.tableShape ? this.tableShape[0] : 0; } /** * */ - initialize() { - this._schemaCache.clear(); - this._schemaCache.initialize().then(async (_) => { - this._lastFetchedSchema = await this._schemaCache?.fetch({ startIndex: 0, endIndex: 1000 }); - - this._dataCache = new TableDataCache( - this._schemaCache?.tableShape!, - async (req: DataFetchRange) => { - // Build the column indices to fetch. - const columnIndices: number[] = []; - for (let i = req.columnStartIndex; i < req.columnEndIndex; i++) { - columnIndices.push(i); - } - return this._dataExplorerClientInstance.getDataValues( - req.rowStartIndex, - req.rowEndIndex - req.rowStartIndex, - columnIndices - ); - }); - - // Fetch data. - this.fetchData(); - }); + async initialize() { + const state = await this._dataExplorerClientInstance.getState(); + this.tableShape = [state.table_shape.num_rows, state.table_shape.num_columns]; + this._schemaCache = new TableSchemaCache( + this.tableShape, + async (req: SchemaFetchRange) => { + return this._dataExplorerClientInstance.getSchema(req.startIndex, + req.endIndex - req.startIndex); + } + ); + this._lastFetchedSchema = await this._schemaCache?.fetch({ startIndex: 0, endIndex: 1000 }); + this._dataCache = new TableDataCache( + this.tableShape, + async (req: DataFetchRange) => { + // Build the column indices to fetch. + const columnIndices: number[] = []; + for (let i = req.columnStartIndex; i < req.columnEndIndex; i++) { + columnIndices.push(i); + } + return this._dataExplorerClientInstance.getDataValues( + req.rowStartIndex, + req.rowEndIndex - req.rowStartIndex, + columnIndices + ); + }); + + // Fetch data. + this.fetchData(); } /** diff --git a/src/vs/workbench/services/positronDataExplorer/browser/tableSummaryDataGridInstance.tsx b/src/vs/workbench/services/positronDataExplorer/browser/tableSummaryDataGridInstance.tsx index 0e937c2c2f6..d9ea76a77aa 100644 --- a/src/vs/workbench/services/positronDataExplorer/browser/tableSummaryDataGridInstance.tsx +++ b/src/vs/workbench/services/positronDataExplorer/browser/tableSummaryDataGridInstance.tsx @@ -71,7 +71,7 @@ export class TableSummaryDataGridInstance extends DataGridInstance { * Gets the number of rows. */ get rows() { - return this._tableSchema ? this._tableSchema.total_num_columns : 0; + return this._tableSchema ? this._tableSchema.columns.length : 0; } /** diff --git a/src/vs/workbench/services/positronDataExplorer/common/positronDataExplorerCache.ts b/src/vs/workbench/services/positronDataExplorer/common/positronDataExplorerCache.ts index 07a17453fe8..e4a47928721 100644 --- a/src/vs/workbench/services/positronDataExplorer/common/positronDataExplorerCache.ts +++ b/src/vs/workbench/services/positronDataExplorer/common/positronDataExplorerCache.ts @@ -154,6 +154,10 @@ export class TableDataCache extends FetchCache { this._fetchFunc = fetchFunc; } + updateShape(newShape: [number, number]) { + this._tableShape = newShape; + } + getRangeTotalSize(range: DataFetchRange): number { return ((range.columnEndIndex - range.columnStartIndex) * (range.rowEndIndex - range.rowStartIndex)); @@ -197,17 +201,16 @@ export class TableSchemaCache extends FetchCache = { * @param numRows * @param columnIndices Columns to select by index. Can be sequetial, sparse, or random */ -export function getExampleTableData(schema: TableSchema, rowStartIndex: number, +export function getExampleTableData(shape: [number, number], schema: TableSchema, rowStartIndex: number, numRows: number, columnIndices: Array): TableData { const generatedColumns = []; // Don't generate virtual data beyond the extent of the table, and if // rowStartIndex is at or after end of the table, return nothing - numRows = Math.max(Math.min(numRows, schema.num_rows - rowStartIndex), 0); + numRows = Math.max(Math.min(numRows, shape[0] - rowStartIndex), 0); for (const columnIndex of columnIndices) { const exampleValues: string[] = exampleTypeData[schema.columns[columnIndex].type_name]; diff --git a/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerInternals.test.ts b/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerInternals.test.ts index 14d462edc5c..e8cefaca2e6 100644 --- a/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerInternals.test.ts +++ b/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerInternals.test.ts @@ -24,7 +24,7 @@ class MockDataCache extends TableDataCache { for (let i = req.columnStartIndex; i < req.columnEndIndex; i++) { columnIndices.push(i); } - return mocks.getExampleTableData(this.schema, + return mocks.getExampleTableData(tableShape, this.schema, req.rowStartIndex, req.rowEndIndex - req.rowStartIndex, columnIndices @@ -38,12 +38,10 @@ class MockDataCache extends TableDataCache { class MockSchemaCache extends TableSchemaCache { private schema: TableSchema; - constructor(schema: TableSchema) { - super(async (req: SchemaFetchRange) => { + constructor(tableShape: [number, number], schema: TableSchema) { + super(tableShape, async (req: SchemaFetchRange) => { return { - columns: this.schema.columns.slice(req.startIndex, req.endIndex), - num_rows: this.schema.num_rows, - total_num_columns: this.schema.total_num_columns + columns: this.schema.columns.slice(req.startIndex, req.endIndex) }; }); this.schema = schema; @@ -124,11 +122,7 @@ suite('DataExplorerInternals', () => { const numRows = 100000; const numColumns = 1000; const schema = mocks.getTableSchema(numRows, numColumns); - const fetcher = new MockSchemaCache(schema); - - await fetcher.initialize(); - - assert.deepEqual(fetcher.tableShape, [numRows, numColumns]); + const fetcher = new MockSchemaCache([numRows, numColumns], schema); const range: SchemaFetchRange = { startIndex: 50, @@ -139,8 +133,6 @@ suite('DataExplorerInternals', () => { assert.equal(result.startIndex, 0); assert.equal(result.endIndex, 150); assert.equal(result.schema.columns.length, 150); - assert.equal(result.schema.num_rows, numRows); - assert.equal(result.schema.total_num_columns, numColumns); assert.deepEqual(result.schema.columns, schema.columns.slice(0, 150)); assert.equal(fetcher.currentCacheSize(), 150); diff --git a/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerMocks.test.ts b/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerMocks.test.ts index 9a328c96796..579b94b116b 100644 --- a/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerMocks.test.ts +++ b/src/vs/workbench/services/positronDataExplorer/test/common/positronDataExplorerMocks.test.ts @@ -3,6 +3,7 @@ *--------------------------------------------------------------------------------------------*/ import * as assert from 'assert'; +import { ensureNoDisposablesAreLeakedInTestSuite } from 'vs/base/test/common/utils'; import { ColumnFilterCompareOp, ColumnFilterFilterType, @@ -14,22 +15,26 @@ import * as mocks from "vs/workbench/services/positronDataExplorer/common/positr * Basic smoke tests for debugging the mock functions */ suite('DataExplorerMocks', () => { + ensureNoDisposablesAreLeakedInTestSuite(); + test('Test getTableSchema', () => { const schema = mocks.getTableSchema(1000, 10000); assert.equal(schema.columns.length, 10000); - assert.equal(schema.num_rows, 1000); }); test('Test getExampleTableData', () => { const schema = mocks.getTableSchema(1000, 10000); - let data = mocks.getExampleTableData(schema, 0, 100, [0, 1, 2, 3, 4]); + + const tableShape: [number, number] = [1000, 10000]; + + let data = mocks.getExampleTableData(tableShape, schema, 0, 100, [0, 1, 2, 3, 4]); assert.equal(data.columns.length, 5); assert.equal(data.columns[0].length, 100); - data = mocks.getExampleTableData(schema, 999, 100, [999]); + data = mocks.getExampleTableData(tableShape, schema, 999, 100, [999]); assert.equal(data.columns.length, 1); assert.equal(data.columns[0].length, 1); - data = mocks.getExampleTableData(schema, 1000, 100, []); + data = mocks.getExampleTableData(tableShape, schema, 1000, 100, []); assert.equal(data.columns.length, 0); });