From e76e2f3d35c4b8188391239cb9f88c5eac6c0770 Mon Sep 17 00:00:00 2001 From: Ben Schmidt Date: Wed, 15 May 2024 10:37:20 -0400 Subject: [PATCH] rename dataset to deeptable --- src/{Dataset.ts => Deeptable.ts} | 17 ++++---- src/aesthetics/Aesthetic.ts | 6 +-- src/aesthetics/AestheticSet.ts | 6 +-- src/aesthetics/ScaledAesthetic.ts | 14 ++++--- src/aesthetics/StatefulAesthetic.ts | 8 ++-- src/deepscatter.ts | 2 +- src/interaction.ts | 24 ++++++------ src/label_rendering.ts | 2 +- src/regl_rendering.ts | 28 +++++++------- src/rendering.ts | 12 +++--- src/scatterplot.ts | 60 +++++++++++++---------------- src/selection.ts | 56 +++++++++++++-------------- src/shared.d.ts | 32 +++++++-------- src/tile.ts | 50 ++++++++++++------------ src/wrap_arrow.ts | 14 +++---- 15 files changed, 163 insertions(+), 168 deletions(-) rename src/{Dataset.ts => Deeptable.ts} (98%) diff --git a/src/Dataset.ts b/src/Deeptable.ts similarity index 98% rename from src/Dataset.ts rename to src/Deeptable.ts index 8bb37d184..efc07e6ac 100644 --- a/src/Dataset.ts +++ b/src/Deeptable.ts @@ -1,4 +1,3 @@ -// A Dataset manages the production and manipulation of *tiles*. import { Tile, Rectangle, p_in_rect } from './tile'; import { max, bisectLeft, extent } from 'd3-array'; import type * as DS from './shared'; @@ -62,11 +61,11 @@ const defaultTransformations: Record = { }; /** - * A Dataset manages the production and manipulation of tiles. Each plot has a - * single dataset; the dataset handles all transformations around data through + * A Deeptable manages the production and manipulation of tiles. Each plot has a + * single deeptable; the deeptable handles all transformations around data through * batchwise operations. */ -export class Dataset { +export class Deeptable { public transformations: Record = defaultTransformations; public _plot: Scatterplot | null; @@ -113,7 +112,7 @@ export class Dataset { // optional with non-undefined defaults. rootKey = '0/0/0', tileStructure = 'quadtree', - }: DS.DatasetCreateParams) { + }: DS.DeeptableCreateParams) { this._plot = plot; this.tileProxy = tileProxy; this.tileStucture = tileStructure; @@ -274,12 +273,12 @@ export class Dataset { ); } - static from_quadfeather(url: string, plot: Scatterplot | null): Dataset { - const options: Partial = {}; + static from_quadfeather(url: string, plot: Scatterplot | null): Deeptable { + const options: Partial = {}; if (plot.tileProxy) { options['tileProxy'] = plot.tileProxy; } - return new Dataset({ + return new Deeptable({ tileProxy: plot.tileProxy ? plot.tileProxy : undefined, baseUrl: url, rootKey: '0/0/0', @@ -295,7 +294,7 @@ export class Dataset { * @param plot The Scatterplot to use. * @returns */ - static fromArrowTable(table: Table, plot: Scatterplot): Dataset { + static fromArrowTable(table: Table, plot: Scatterplot): Deeptable { return wrapArrowTable(tableToIPC(table), plot); } diff --git a/src/aesthetics/Aesthetic.ts b/src/aesthetics/Aesthetic.ts index ac780e144..6ab530ac8 100644 --- a/src/aesthetics/Aesthetic.ts +++ b/src/aesthetics/Aesthetic.ts @@ -79,8 +79,8 @@ export abstract class Aesthetic< } } - get dataset() { - return this.scatterplot.dataset; + get deeptable() { + return this.scatterplot.deeptable; } abstract apply(point: Datum): Output['rangeType']; @@ -136,7 +136,7 @@ export abstract class Aesthetic< if (this.field === null || this.field === undefined) { return (this.column = null); } - return (this.column = this.dataset.root_tile.record_batch.getChild( + return (this.column = this.deeptable.root_tile.record_batch.getChild( this.field, ) as Vector); } diff --git a/src/aesthetics/AestheticSet.ts b/src/aesthetics/AestheticSet.ts index 3d1519b5d..ebfb40a3d 100644 --- a/src/aesthetics/AestheticSet.ts +++ b/src/aesthetics/AestheticSet.ts @@ -2,7 +2,7 @@ import type { Regl, Texture2D } from 'regl'; import { dimensions } from './StatefulAesthetic'; import type { Scatterplot } from '../scatterplot'; -import type { Dataset } from '../Dataset'; +import type { Deeptable } from '../Deeptable'; import { StatefulAesthetic } from './StatefulAesthetic'; import type { Encoding } from '../shared'; import type * as DS from '../shared'; @@ -19,7 +19,7 @@ type StateList = { }; export class AestheticSet { - public tileSet: Dataset; + public tileSet: Deeptable; public scatterplot: Scatterplot; public regl: Regl; public encoding: Encoding = {}; @@ -31,7 +31,7 @@ export class AestheticSet { } = { jitter_method: { current: 'None', last: 'None' }, }; - constructor(scatterplot: Scatterplot, regl: Regl, tileSet: Dataset) { + constructor(scatterplot: Scatterplot, regl: Regl, tileSet: Deeptable) { this.scatterplot = scatterplot; this.regl = regl; this.tileSet = tileSet; diff --git a/src/aesthetics/ScaledAesthetic.ts b/src/aesthetics/ScaledAesthetic.ts index d879e8956..91d325bf5 100644 --- a/src/aesthetics/ScaledAesthetic.ts +++ b/src/aesthetics/ScaledAesthetic.ts @@ -111,7 +111,7 @@ export abstract class ScaledAesthetic< const bands = scaleBand() .domain(vals) - .range(this.dataset.extent.x) + .range(this.deeptable.extent.x) .padding(0.1) .round(true) .align(0.5); @@ -230,7 +230,7 @@ export abstract class ScaledAesthetic< return [1, 1]; } - const domain = this.dataset.domain(this.field); + const domain = this.deeptable.domain(this.field); return domain; } @@ -250,7 +250,7 @@ export abstract class ScaledAesthetic< Input['domainType'], ]; } else { - return this.scatterplot.dataset.domain(this.field); + return this.scatterplot.deeptable.domain(this.field); } } @@ -322,8 +322,12 @@ export abstract class PositionalAesthetic< return this.default_range; } get default_range(): [number, number] { - if (this.dataset.extent && this.axis && this.dataset.extent[this.axis]) { - return this.dataset.extent[this.axis]; + if ( + this.deeptable.extent && + this.axis && + this.deeptable.extent[this.axis] + ) { + return this.deeptable.extent[this.axis]; } return [-1, 1]; } diff --git a/src/aesthetics/StatefulAesthetic.ts b/src/aesthetics/StatefulAesthetic.ts index 4931c2c22..48c79eaf4 100644 --- a/src/aesthetics/StatefulAesthetic.ts +++ b/src/aesthetics/StatefulAesthetic.ts @@ -52,7 +52,7 @@ export type ConcreteScaledAesthetic = | Jitter_radius | Color; -import type { Dataset } from '../Dataset'; +import type { Deeptable } from '../Deeptable'; import type { Regl } from 'regl'; import type { TextureSet } from './AestheticSet'; @@ -67,7 +67,7 @@ export class StatefulAesthetic { * channel in a scatterplot has exactly on StatefulAesthetic that persists for the lifetime of the Plot. */ public states: [T, T]; - public dataset: Dataset; + public deeptable: Deeptable; public regl: Regl; public scatterplot: Scatterplot; public needs_transitions = false; @@ -78,7 +78,7 @@ export class StatefulAesthetic { constructor( scatterplot: Scatterplot, regl: Regl, - dataset: Dataset, + deeptable: Deeptable, aesthetic_map: TextureSet, Factory: DS.Newable, ) { @@ -87,7 +87,7 @@ export class StatefulAesthetic { } this.scatterplot = scatterplot; this.regl = regl; - this.dataset = dataset; + this.deeptable = deeptable; this.aesthetic_map = aesthetic_map; this.factory = Factory; this.ids = [Math.random().toString(), Math.random().toString()]; diff --git a/src/deepscatter.ts b/src/deepscatter.ts index f1dd90dcd..b3aa7110f 100644 --- a/src/deepscatter.ts +++ b/src/deepscatter.ts @@ -1,5 +1,5 @@ export { Scatterplot } from './scatterplot'; export { Bitmask, DataSelection } from './selection'; -export { Dataset } from './Dataset'; +export { Deeptable } from './Deeptable'; export { LabelMaker } from './label_rendering'; export { dictionaryFromArrays } from './utilityFunctions'; diff --git a/src/interaction.ts b/src/interaction.ts index 9a975b788..7031fe7ff 100644 --- a/src/interaction.ts +++ b/src/interaction.ts @@ -11,7 +11,7 @@ import type { Renderer } from './rendering'; import { ReglRenderer } from './regl_rendering'; import { StructRowProxy } from 'apache-arrow'; import { Rectangle } from './tile'; -import type { Dataset } from './Dataset'; +import type { Deeptable } from './Deeptable'; import type * as DS from './shared'; import type { Scatterplot } from './scatterplot'; import { PositionalAesthetic } from './aesthetics/ScaledAesthetic'; @@ -34,7 +34,7 @@ export class Zoom { public width: number; public height: number; public renderers: Map; - public tileSet?: Dataset; + public deeptable?: Deeptable; public _timer?: d3.Timer; public _scales?: Record>; public zoomer?: d3.ZoomBehavior; @@ -60,8 +60,8 @@ export class Zoom { this.renderers = new Map(); } - attach_tiles(tiles: Dataset) { - this.tileSet = tiles; + attach_tiles(tiles: Deeptable) { + this.deeptable = tiles; return this; } @@ -278,14 +278,14 @@ export class Zoom { return this._timer; } - data(dataset: undefined): Dataset; - data(dataset: Dataset): Zoom; + data(deeptable: undefined): Deeptable; + data(deeptable: Deeptable): Zoom; - data(dataset: Dataset | undefined) { - if (dataset === undefined) { - return this.tileSet; + data(deeptable: Deeptable | undefined) { + if (deeptable === undefined) { + return this.deeptable; } - this.tileSet = dataset; + this.deeptable = deeptable; return this as Zoom; } @@ -305,10 +305,10 @@ export class Zoom { } const { width, height } = this; - if (this.tileSet === undefined) { + if (this.deeptable === undefined) { throw new Error('Error--scales created before tileSet present.'); } - const { extent } = this.tileSet; + const { extent } = this.deeptable; const scales: Record> = {}; if (extent === undefined) { throw new Error('Error--scales created before extent present.'); diff --git a/src/label_rendering.ts b/src/label_rendering.ts index f72c40d16..93cdc5744 100644 --- a/src/label_rendering.ts +++ b/src/label_rendering.ts @@ -47,7 +47,7 @@ export class LabelMaker extends Renderer { ) { super( scatterplot.div!.node() as HTMLDivElement, - scatterplot.dataset, + scatterplot.deeptable, scatterplot, ); this.options = options; diff --git a/src/regl_rendering.ts b/src/regl_rendering.ts index 0a2aea8f7..79a63527a 100644 --- a/src/regl_rendering.ts +++ b/src/regl_rendering.ts @@ -23,7 +23,7 @@ import { rgb } from 'd3-color'; import type * as DS from './shared'; import type { Tile } from './tile'; import REGL from 'regl'; -import { Dataset } from './Dataset'; +import { Deeptable } from './Deeptable'; import { ConcreteScaledAesthetic, dimensions, @@ -52,7 +52,7 @@ export class ReglRenderer extends Renderer { public buffer_size = 1024 * 1024 * 64; // Use GPU memory in blocks of 64 MB buffers by default. private _buffers: MultipurposeBufferSet; public _initializations: Promise; - public dataset: Dataset; + public deeptable: Deeptable; public _zoom?: Zoom; public most_recent_restart?: number; public _default_webgl_scale?: number[]; @@ -72,7 +72,7 @@ export class ReglRenderer extends Renderer { constructor( selector: string | Node, - tileSet: Dataset, + tileSet: Deeptable, scatterplot: Scatterplot, ) { super(selector, tileSet, scatterplot); @@ -91,7 +91,7 @@ export class ReglRenderer extends Renderer { ], canvas: c, }); - this.dataset = tileSet; + this.deeptable = tileSet; this.aes = new AestheticSet(scatterplot, this.regl, tileSet); @@ -101,7 +101,7 @@ export class ReglRenderer extends Renderer { // Not the right way, for sure. this._initializations = Promise.all([ // some things that need to be initialized before the renderer is loaded. - this.dataset.promise.then(() => { + this.deeptable.promise.then(() => { this.remake_renderer(); this._webgl_scale_history = [ this.default_webgl_scale, @@ -119,12 +119,12 @@ export class ReglRenderer extends Renderer { return this._buffers; } - data(dataset: Dataset) { - if (dataset === undefined) { + data(deeptable: Deeptable) { + if (deeptable === undefined) { // throw - return this.dataset; + return this.deeptable; } - this.dataset = dataset; + this.deeptable = deeptable; return this; } @@ -261,12 +261,12 @@ export class ReglRenderer extends Renderer { * Actions that run on a single animation tick. */ tick() { - const { prefs, dataset, props } = this; + const { prefs, deeptable, props } = this; this.tick_num = this.tick_num || 0; this.tick_num++; // Set a download call in motion. if (this._use_scale_to_download_tiles) { - dataset.spawnDownloads( + deeptable.spawnDownloads( this.zoom.current_corners(), this.props.max_ix, 5, @@ -275,7 +275,7 @@ export class ReglRenderer extends Renderer { ); } else { // console.warn("No good rules here yet.") - dataset.spawnDownloads( + deeptable.spawnDownloads( undefined, prefs.max_points, 5, @@ -1139,14 +1139,14 @@ export class TileBufferManager { type ColumnType = Vector | Float | Bool | Int | Timestamp>; if (!tile.hasLoadedColumn(key)) { - if (tile.dataset.transformations[key] !== undefined) { + if (tile.deeptable.transformations[key] !== undefined) { throw new Error( 'Attempted to create buffer data on an unloaded transformation', ); } else { let col_names = [ ...tile.record_batch.schema.fields.map((d) => d.name), - ...Object.keys(tile.dataset.transformations), + ...Object.keys(tile.deeptable.transformations), ]; if (!key.startsWith('_')) { // Don't warn internal columns unless the user is in internal-column land. diff --git a/src/rendering.ts b/src/rendering.ts index 906df7d9e..372569dba 100644 --- a/src/rendering.ts +++ b/src/rendering.ts @@ -7,7 +7,7 @@ import type { Tile } from './tile'; import type { Zoom } from './interaction'; import type { AestheticSet } from './aesthetics/AestheticSet'; import { timer, Timer } from 'd3-timer'; -import { Dataset } from './Dataset'; +import { Deeptable } from './Deeptable'; import type * as DS from './shared.d'; import { Table } from 'apache-arrow'; import { StatefulAesthetic } from './aesthetics/StatefulAesthetic'; @@ -126,7 +126,7 @@ export class Renderer { public scatterplot: Scatterplot; public holder: d3.Selection; public canvas: HTMLCanvasElement; - public dataset: Dataset; + public deeptable: Deeptable; public width: number; public height: number; // The renderer handles periodic dispatches of calls @@ -137,7 +137,7 @@ export class Renderer { public render_props: RenderProps = new RenderProps(); constructor( selector: string | Node, - tileSet: Dataset, + tileSet: Deeptable, scatterplot: Scatterplot, ) { this.scatterplot = scatterplot; @@ -145,7 +145,7 @@ export class Renderer { this.canvas = select( this.holder!.node()!.firstElementChild, ).node() as HTMLCanvasElement; - this.dataset = tileSet; + this.deeptable = tileSet; this.width = +select(this.canvas).attr('width'); this.height = +select(this.canvas).attr('height'); this.deferred_functions = []; @@ -206,7 +206,7 @@ export class Renderer { const pixel_area = (width * height) / pixelRatio; const total_intended_points = min([ max_ix, - this.dataset.highest_known_ix || 1e10, + this.deeptable.highest_known_ix || 1e10, ]) as number; const total_points = total_intended_points * (1 - discard_share); @@ -242,7 +242,7 @@ export class Renderer { // yield the currently visible tiles based on the zoom state // and a maximum index passed manually. const { max_ix } = this; - const { dataset: tileSet } = this; + const { deeptable: tileSet } = this; // Materialize using a tileset method. if (!this.aes) throw new Error('Aesthetic missing'); diff --git a/src/scatterplot.ts b/src/scatterplot.ts index f5ac414c2..3bc3cb07e 100644 --- a/src/scatterplot.ts +++ b/src/scatterplot.ts @@ -4,7 +4,7 @@ import merge from 'lodash.merge'; import { Zoom } from './interaction'; import { neededFieldsToPlot, ReglRenderer } from './regl_rendering'; import { tableFromIPC, type StructRowProxy } from 'apache-arrow'; -import { Dataset } from './Dataset'; +import { Deeptable } from './Deeptable'; import type { FeatureCollection } from 'geojson'; import { LabelMaker } from './label_rendering'; import { Renderer } from './rendering'; @@ -43,14 +43,6 @@ const base_elements = [ // after each plotAPI update. type Hook = () => void; -// interface AdditionalProps { -// Bitmask: typeof Bitmask; -// Dataset: typeof Dataset; -// QuadtileDataset: typeof Dataset; -// DataSelection: typeof DataSelection; -// wrapArrow?: (t: Table) => Dataset; -// } - /** * The core type of the module is a single scatterplot that manages * all data and renderering. @@ -59,7 +51,7 @@ export class Scatterplot { public _renderer?: ReglRenderer; public width: number; public height: number; - public _root?: Dataset; + public _root?: Deeptable; public elements?: Selection[]; public secondary_renderers: Record = {}; public tileProxy?: DS.TileProxy; @@ -119,8 +111,8 @@ export class Scatterplot { if (options.tileProxy) { this.tileProxy = options.tileProxy; } - if (options.dataset) { - void this.load_dataset(options.dataset); + if (options.deeptable) { + void this.load_deeptable(options.deeptable); } this.prefs = { ...default_API_call } as DS.CompletePrefs; } @@ -205,7 +197,7 @@ export class Scatterplot { params: IdSelectParams | BooleanColumnParams | FunctionSelectParams, duration = this.prefs.duration, ): Promise { - const selection = await this.dataset.select_data(params); + const selection = await this.deeptable.select_data(params); if (selection === null) { throw new Error(`Invalid selection: ${JSON.stringify(params)}`); } @@ -245,7 +237,7 @@ export class Scatterplot { true_codes = codes; } - this.dataset.add_label_identifiers(true_codes, name, key_field); + this.deeptable.add_label_identifiers(true_codes, name, key_field); } async add_labels_from_url( @@ -257,7 +249,7 @@ export class Scatterplot { ): Promise { await this.ready; - await this.dataset.promise; + await this.deeptable.promise; return fetch(url) .then(async (data) => { const features = await (data.json() as Promise); @@ -274,7 +266,7 @@ export class Scatterplot { * @param features A geojson feature collection containing point labels * @param name A unique key to associate with this labelset. Labels can be enabled or disabled using this key. * @param label_key The text field in which the labels are stored in the geojson object. - * @param size_key A field in the dataset to associate with the *size* of the labels. + * @param size_key A field in the deeptable to associate with the *size* of the labels. * @param label_options Additional custom passed to the labeller. * * Usage: @@ -306,9 +298,9 @@ export class Scatterplot { /** * An alias to avoid using the underscored method directly. */ - get dataset() { + get deeptable() { if (this._root === undefined) { - throw new Error('No dataset has been loaded'); + throw new Error('No deeptable has been loaded'); } return this._root; } @@ -339,14 +331,14 @@ export class Scatterplot { ); } - async load_dataset(params: DS.DataSpec): Promise { + async load_deeptable(params: DS.DataSpec): Promise { if (params.source_url !== undefined) { - this._root = Dataset.from_quadfeather(params.source_url, this); + this._root = Deeptable.from_quadfeather(params.source_url, this); } else if (params.arrow_table !== undefined) { - this._root = Dataset.fromArrowTable(params.arrow_table, this); + this._root = Deeptable.fromArrowTable(params.arrow_table, this); } else if (params.arrow_buffer !== undefined) { const tb = tableFromIPC(params.arrow_buffer); - this._root = Dataset.fromArrowTable(tb, this); + this._root = Deeptable.fromArrowTable(tb, this); } else { throw new Error('No source_url or arrow_table specified'); } @@ -357,18 +349,18 @@ export class Scatterplot { async reinitialize() { const { prefs } = this; - await this.dataset.ready; + await this.deeptable.ready; console.log('HERE'); - await this.dataset.root_tile.get_column('x'); + await this.deeptable.root_tile.get_column('x'); console.log('HERE 2'); this._renderer = new ReglRenderer( '#container-for-webgl-canvas', - this.dataset, + this.deeptable, this, ); this._zoom = new Zoom('#deepscatter-svg', this.prefs, this); - this._zoom.attach_tiles(this.dataset); + this._zoom.attach_tiles(this.deeptable); this._zoom.attach_renderer('regl', this._renderer); this._zoom.initialize_zoom(); @@ -389,7 +381,7 @@ export class Scatterplot { ctx.fillRect(0, 0, window.innerWidth * 2, window.innerHeight * 2); void this._renderer.initialize(); - void this.dataset.promise.then(() => this.mark_ready()); + void this.deeptable.promise.then(() => this.mark_ready()); return this.ready; } @@ -441,7 +433,7 @@ export class Scatterplot { ctx.clearRect(0, 0, 10_000, 10_000); const { x_, y_ } = this._zoom.scales(); ctx.strokeStyle = '#888888'; - const tiles = this.dataset.map((t) => t); + const tiles = this.deeptable.map((t) => t); for (const i of range(20)) { setTimeout(() => { for (const tile of tiles) { @@ -647,7 +639,7 @@ export class Scatterplot { if (this._renderer) { const promises: Promise[] = []; const sine_qua_non: Promise[] = [ - this.dataset.root_tile.require_columns(needed_keys), + this.deeptable.root_tile.require_columns(needed_keys), ]; // Immediately @@ -732,13 +724,13 @@ export class Scatterplot { 'The initial API call specify exactly one of source_url, arrow_table, or arrow_buffer', ); } - await this.load_dataset(dataSpec); + await this.load_deeptable(dataSpec); } if (prefs.transformations) { for (const [k, func] of Object.entries(prefs.transformations)) { - if (!this.dataset.transformations[k]) { - this.dataset.register_transformation(k, func); + if (!this.deeptable.transformations[k]) { + this.deeptable.register_transformation(k, func); } } } @@ -819,7 +811,7 @@ export class Scatterplot { if (!this._root) { throw new Error('No dataset has been loaded'); } - return this.dataset.root_tile.record_batch; + return this.deeptable.root_tile.record_batch; } /** @@ -848,7 +840,7 @@ export class Scatterplot { sample_points(n = 10): Record[] { const vals: Record[] = []; - for (const p of this.dataset.points(this.zoom.current_corners())) { + for (const p of this.deeptable.points(this.zoom.current_corners())) { vals.push({ ...p }); if (vals.length >= n * 3) { break; diff --git a/src/selection.ts b/src/selection.ts index a25178247..f9fc582fb 100644 --- a/src/selection.ts +++ b/src/selection.ts @@ -1,4 +1,4 @@ -import { Dataset } from './Dataset'; +import { Deeptable } from './Deeptable'; import { Scatterplot } from './scatterplot'; import { Tile } from './tile'; import type * as DS from './shared.d'; @@ -267,7 +267,7 @@ export class Bitmask { } } export class DataSelection { - dataset: Dataset; + deeptable: Deeptable; plot: Scatterplot; // The match count is the number of matches **per tile**; // used to access numbers by index. @@ -288,7 +288,7 @@ export class DataSelection { */ name: string; /** - * Has the selection been applied to the dataset + * Has the selection been applied to the deeptable * (does *not* mean it has been applied to all points.) */ ready: Promise; @@ -299,7 +299,7 @@ export class DataSelection { cursor: number = 0; /** - * Has the selection run on all tiles in the dataset? + * Has the selection run on all tiles in the deeptable? */ complete: boolean = false; @@ -333,16 +333,16 @@ export class DataSelection { private events: { [key: string]: Array<(args) => void> } = {}; constructor( - dataset: Dataset, + deeptable: Deeptable, params: | IdSelectParams | BooleanColumnParams | FunctionSelectParams | CompositeSelectParams, ) { - this.dataset = dataset; - if (dataset === undefined) { - throw new Error("Can't create a selection without a dataset"); + this.deeptable = deeptable; + if (deeptable === undefined) { + throw new Error("Can't create a selection without a deeptable"); } this.name = params.name; let markReady = function () {}; @@ -394,7 +394,7 @@ export class DataSelection { /** * * Ensures that the selection has been evaluated on all - * tiles loaded in the dataset. This is useful if, for example, + * tiles loaded in the deeptable. This is useful if, for example, * your selection represents a search, and you are zoomed in on * one portion of the map; this will immediately execute the search * (subject to delays to avoid blocking the main thread) on all tiles @@ -405,8 +405,8 @@ export class DataSelection { applyToAllLoadedTiles(): Promise { return Promise.all( - this.dataset.map((tile) => { - // triggers creation of the dataset column as a side-effect. + this.deeptable.map((tile) => { + // triggers creation of the deeptable column as a side-effect. return tile.get_column(this.name); }), ).then(() => {}); @@ -414,9 +414,9 @@ export class DataSelection { /** * - * Downloads all unloaded tiles in the dataset and applies the + * Downloads all unloaded tiles in the deeptable and applies the * transformation to them. Use with care! For > 10,000,000 point - * datasets, if called from Europe this may cause the transatlantic fiber-optic internet backbone + * deeptables, if called from Europe this may cause the transatlantic fiber-optic internet backbone * cables to melt. */ @@ -430,7 +430,7 @@ export class DataSelection { * selection that is the union of the two. */ union(other: DataSelection, name: string | undefined): DataSelection { - return new DataSelection(this.dataset, { + return new DataSelection(this.deeptable, { name: name || this.name + ' union ' + other.name, composition: ['OR', this, other], }); @@ -444,7 +444,7 @@ export class DataSelection { * disjunctive normal form constructor. */ intersection(other: DataSelection, name: string | undefined): DataSelection { - return new DataSelection(this.dataset, { + return new DataSelection(this.deeptable, { name: name || this.name + ' intersection ' + other.name, composition: ['AND', this, other], }); @@ -604,7 +604,7 @@ export class DataSelection { } return mask.to_arrow(); }; - const selection = new DataSelection(this.dataset, { + const selection = new DataSelection(this.deeptable, { name: newName, tileFunction, }); @@ -636,20 +636,20 @@ export class DataSelection { } /** * - * @param name the name for the column to assign in the dataset. + * @param name the name for the column to assign in the deeptable. * @param tileFunction The transformation to apply */ async add_function_column( name: string, tileFunction: DS.BoolTransformation, ): Promise { - if (this.dataset.has_column(name)) { + if (this.deeptable.has_column(name)) { throw new Error(`Column ${name} already exists, can't create`); } - this.dataset.transformations[name] = + this.deeptable.transformations[name] = this.wrapWithSelectionMetadata(tileFunction); // Await the application to the root tile, which may be necessary - await this.dataset.root_tile.apply_transformation(name); + await this.deeptable.root_tile.apply_transformation(name); } /** @@ -686,10 +686,10 @@ export class DataSelection { /** * The total number of points in the set. At present, always a light wrapper around - * the total number of points in the dataset. + * the total number of points in the deeptable. */ get totalSetSize() { - return this.dataset.highest_known_ix; + return this.deeptable.highest_known_ix; } /** @@ -757,19 +757,19 @@ export class DataSelection { key_field: string, // options: IdentifierOptions = {}, ): Promise { - if (this.dataset.has_column(name)) { + if (this.deeptable.has_column(name)) { throw new Error(`Column ${name} already exists, can't create`); } if (typeof codes[0] === 'string') { const matcher = stringmatcher(key_field, codes as string[]); - this.dataset.transformations[name] = + this.deeptable.transformations[name] = this.wrapWithSelectionMetadata(matcher); - await this.dataset.root_tile.apply_transformation(name); + await this.deeptable.root_tile.apply_transformation(name); } else if (typeof codes[0] === 'bigint') { const matcher = bigintmatcher(key_field, codes as bigint[]); - this.dataset.transformations[name] = + this.deeptable.transformations[name] = this.wrapWithSelectionMetadata(matcher); - await this.dataset.root_tile.apply_transformation(name); + await this.deeptable.root_tile.apply_transformation(name); } else { console.error('Unable to match type', typeof codes[0]); } @@ -807,7 +807,7 @@ function bigintmatcher(field: string, matches: bigint[]) { * more to work with.) * * - * @param field The column in the deepscatter dataset to search in + * @param field The column in the deepscatter deeptable to search in * @param matches A list of strings to match in that column * @returns */ diff --git a/src/shared.d.ts b/src/shared.d.ts index e18d2c7d1..dd1de94db 100644 --- a/src/shared.d.ts +++ b/src/shared.d.ts @@ -13,7 +13,7 @@ import type { Vector, } from 'apache-arrow'; import type { Renderer } from './rendering'; -import type { Dataset } from './Dataset'; +import type { Deeptable } from './Deeptable'; import type { ConcreteAesthetic } from './aesthetics/StatefulAesthetic'; import type { Buffer } from 'regl'; import type { DataSelection } from './selection'; @@ -22,7 +22,7 @@ import { ZoomTransform } from 'd3-zoom'; import { TileBufferManager } from './regl_rendering'; import type { Tile } from './tile'; import type { Rectangle } from './tile'; -export type { Renderer, Dataset, ConcreteAesthetic }; +export type { Renderer, Deeptable, ConcreteAesthetic }; export type BufferLocation = { buffer: Buffer; @@ -52,10 +52,10 @@ export interface TileProxy { export type ScatterplotOptions = { tileProxy?: TileProxy; - dataset?: DataSpec; + deeptable?: DataSpec; }; -// The orientation of the dataset. Quadtree datasets +// The orientation of the deeptable. Quadtree deeptables // allow certain optimizations. export type TileStructure = 'quadtree' | 'other'; @@ -70,16 +70,16 @@ export type TileManifest = { }; /** - * The arguments passed to new Dataset(). + * The arguments passed to new Deeptable(). */ -export type DatasetCreateParams = { +export type DeeptableCreateParams = { // A URL for the root of the quadtree if files are not stored locally. // At the present time only // http:// and https:// urls are allowed, but other sources can be read // from the baseUrl: string; - // A Deepscatter Scatterplot object. If null, dataset operations will still be + // A Deepscatter Scatterplot object. If null, deeptable operations will still be // possible. plot: Scatterplot | null; @@ -91,7 +91,7 @@ export type DatasetCreateParams = { // may be inferred. tileStructure?: TileStructure; - // A manifest listing all the tiles in the dataset. + // A manifest listing all the tiles in the deeptable. // Currently this must be passed as a recursive structure. tileManifest?: Partial; @@ -432,7 +432,7 @@ export type Dimension = keyof Encoding; // 1. A Table object. // 2. A URL to a Quadtile source. // 3. An array buffer that contains a serialized Table. -// 4. A deepscatter Dataset. +// 4. A created deeptable. /** * A DataSpec is a record that describes how to load data into the @@ -441,7 +441,7 @@ export type Dimension = keyof Encoding; * 2. An Arrow Table object. (Use this with care! Minor differences in JS Apache Arrow builds * can cause this to fail in deeply confusing ways.) * 3. A Uint8Array containing a serialized Arrow Table. (This is safer than passing an Arrow Table.) - * 4. An already-created dataset. + * 4. An already-created deeptable. */ export type DataSpec = Record & ( @@ -449,27 +449,27 @@ export type DataSpec = Record & source_url?: never; arrow_table?: never; arrow_buffer: Uint8Array; - dataset?: never; + deeptable?: never; } | { source_url: string; arrow_table?: never; arrow_buffer?: never; - dataset?: never; + deeptable?: never; } // Pass an arrow table. This may | { source_url?: never; arrow_table: Table; arrow_buffer?: never; - dataset?: never; + deeptable?: never; } - // Pass an already instantiated dataset. + // Pass an already instantiated deeptable. | { source_url?: never; arrow_table: never; arrow_buffer?: never; - dataset: Dataset; + deeptable: Deeptable; } ); @@ -543,7 +543,7 @@ export type APICall = { // a set of functions by name from the existing data to a number. // These are used to transform the data, and can create new columns - // in your dataset. + // in your deeptable. transformations?: Record number>; encoding?: Encoding; diff --git a/src/tile.ts b/src/tile.ts index 09bd96754..a3fdf3b2e 100644 --- a/src/tile.ts +++ b/src/tile.ts @@ -5,8 +5,8 @@ import { StructRowProxy, Schema, } from 'apache-arrow'; -import { add_or_delete_column } from './Dataset'; -import type { Dataset } from './Dataset'; +import { add_or_delete_column } from './Deeptable'; +import type { Deeptable } from './Deeptable'; type MinMax = [number, number]; export type Rectangle = { @@ -40,7 +40,7 @@ export type RecordBatchCache = let tile_identifier = 0; /** - * A Tile is a collection of points in the dataset that are grouped together: + * A Tile is a collection of points in the deeptable that are grouped together: * it represents the basic unit of operation for most batched operations in * deepscatter including network packets, GPU calculations, * transformations on data, and render calls. It corresponds to a record batch @@ -57,7 +57,7 @@ export class Tile { parent: Tile | null; private _children: Array = []; public _highest_known_ix?: number; - public dataset: Dataset; + public deeptable: Deeptable; public _transformations: Record> = {}; public _deriveManifestFromTileMetadata?: Promise; //private _promiseOfChildren? = Promise; @@ -79,12 +79,12 @@ export class Tile { * @param key Either the string identifier of the tile, * OR a `TileManifest` object including an identifier. * * @param parent The parent tile -- used to navigate through the tree. - * @param dataset The full atlas dataset of which this tile is a part. + * @param deeptable The full atlas deeptable of which this tile is a part. */ constructor( key: string | (Partial & { key: string }), parent: Tile | null, - dataset: Dataset, + deeptable: Deeptable, ) { // If it's just initiated with a key, build that into a minimal manifest. let manifest: Partial & { key: string }; @@ -101,10 +101,10 @@ export class Tile { // manifest.max_ix = manifest.min_ix + 1e5; // } this.parent = parent; - this.dataset = dataset; + this.deeptable = deeptable; - if (dataset === undefined) { - throw new Error('No dataset provided'); + if (deeptable === undefined) { + throw new Error('No deeptable provided'); } // Grab the next identifier off the queue. This should be async safe with the current setup, but // the logic might fall apart in truly parallel situations. @@ -127,7 +127,7 @@ export class Tile { if (existing) { return existing; } - if (this.dataset.transformations[colname]) { + if (this.deeptable.transformations[colname]) { await this.apply_transformation(colname); const vector = this.record_batch.getChild(colname); if (vector === null) { @@ -157,7 +157,7 @@ export class Tile { if (this.transformation_holder[name] !== undefined) { return this.transformation_holder[name]; } - const transform = this.dataset.transformations[name]; + const transform = this.deeptable.transformations[name]; if (transform === undefined) { throw new Error(`Transformation ${name} is not defined`); } @@ -298,7 +298,7 @@ export class Tile { throw new Error('Attempted to set an incomplete manifest.'); } this._children = manifest.children.map((k: TileManifest | string) => { - return new Tile(k, this, this.dataset); + return new Tile(k, this, this.deeptable); }); this.highest_known_ix = manifest.max_ix; this._completeManifest = manifest; @@ -410,7 +410,7 @@ export class Tile { return existing.batch; } - let url = `${this.dataset.base_url}/${this.key}.feather`; + let url = `${this.deeptable.base_url}/${this.key}.feather`; if (suffix !== null) { // 3/4/3 // suffix: 'text' @@ -420,12 +420,12 @@ export class Tile { let bufferPromise: Promise; - if (this.dataset.tileProxy !== undefined) { + if (this.deeptable.tileProxy !== undefined) { const endpoint = new URL(url).pathname; // This method apiCall is crafted to match the // ts-nomic package, but can accept other authentication. - bufferPromise = this.dataset.tileProxy + bufferPromise = this.deeptable.tileProxy .apiCall(endpoint, 'GET', null, null, { octetStreamAsUint8: true }) .then((d) => d.buffer as ArrayBuffer); } else { @@ -493,11 +493,11 @@ export class Tile { // For every column in the root tile, // define a transformation for other children that says // 'load the main batch and pull out this column'. - const { dataset } = this; + const { deeptable } = this; for (const field of batch.schema.fields) { - if (!dataset.transformations[field.name]) { - dataset.transformations[field.name] = async (tile: Tile) => { + if (!deeptable.transformations[field.name]) { + deeptable.transformations[field.name] = async (tile: Tile) => { const batch = await tile.get_arrow(null); const vector = batch.getChild(field.name); if (vector === null) { @@ -531,11 +531,11 @@ export class Tile { // For every column in the root tile, // define a transformation for other children that says // 'load the main batch and pull out this column'. - const { dataset } = this; + const { deeptable } = this; for (const field of batch.schema.fields) { - if (!dataset.transformations[field.name]) { - dataset.transformations[field.name] = async (tile: Tile) => { + if (!deeptable.transformations[field.name]) { + deeptable.transformations[field.name] = async (tile: Tile) => { const batch = await tile.get_arrow(null); const vector = batch.getChild(field.name); if (vector === null) { @@ -629,7 +629,7 @@ export class Tile { } if (this._partialManifest?.children) { for (const child of this.manifest.children) { - const childTile = new Tile(child, this, this.dataset); + const childTile = new Tile(child, this, this.deeptable); this._children.push(childTile); } return this._children; @@ -641,11 +641,11 @@ export class Tile { // Quadtree tiles can have their limits calculated based on the structure. // There are cases where these may not be exact. get theoretical_extent(): Rectangle { - if (this.dataset.tileStucture === 'other') { + if (this.deeptable.tileStucture === 'other') { // Only three-length-keys are treated as quadtrees. - return this.dataset.extent; + return this.deeptable.extent; } - const base = this.dataset.extent; + const base = this.deeptable.extent; const [z, x, y] = this.key.split('/').map((d) => parseInt(d)); const x_step = base.x[1] - base.x[0]; diff --git a/src/wrap_arrow.ts b/src/wrap_arrow.ts index 8a0ba9b39..3b36fc97d 100644 --- a/src/wrap_arrow.ts +++ b/src/wrap_arrow.ts @@ -9,20 +9,20 @@ import { Vector, Float, } from 'apache-arrow'; -import { Dataset } from './Dataset'; -import { add_or_delete_column } from './Dataset'; +import { Deeptable } from './Deeptable'; +import { add_or_delete_column } from './Deeptable'; import type * as DS from './shared'; import { extent } from 'd3-array'; import { Rectangle } from './tile'; -// This function is used to wrap an arrow table into a deepscatter -// dataset so that record batches can be fetched asynchronously. +// This function is used to wrap an arrow table into a +// deeptable so that record batches can be fetched asynchronously. // The point display order is the same as in the original file. -// It is exposed primarily as Dataset.from +// It is exposed primarily as Deeptable.from export function wrapArrowTable( tbArray: Uint8Array, plot: Scatterplot | null, -): Dataset { +): Deeptable { let tb = tableFromIPC(tbArray); let batches = tb.batches; if (tb.getChild('ix') === null) { @@ -59,7 +59,7 @@ export function wrapArrowTable( y: extent([...(y as Iterable)]), } as Rectangle; - return new Dataset({ + return new Deeptable({ baseUrl: `feather://table`, plot, tileProxy: proxy,