From e9464f14ae6ee07559422ce673e0d47c58955190 Mon Sep 17 00:00:00 2001 From: Leo Nicolle Date: Sun, 16 Jul 2023 21:17:35 +0200 Subject: [PATCH 1/5] added webgl to the project --- client/package.json | 5 +- client/src/search-worker/gpu-worker.ts | 61 +++++ .../src/search-worker/shaders/fragment.frag | 6 + client/src/search-worker/shaders/vertex.vert | 6 + client/src/search-worker/webgl-utils.ts | 209 ++++++++++++++++++ client/vite.config.ts | 7 +- 6 files changed, 290 insertions(+), 4 deletions(-) create mode 100644 client/src/search-worker/gpu-worker.ts create mode 100644 client/src/search-worker/shaders/fragment.frag create mode 100644 client/src/search-worker/shaders/vertex.vert create mode 100644 client/src/search-worker/webgl-utils.ts diff --git a/client/package.json b/client/package.json index 1a47da8..90977f6 100644 --- a/client/package.json +++ b/client/package.json @@ -2,7 +2,7 @@ "name": "client", "version": "0.1.0", "scripts": { - "dev": "vite", + "dev": "vite -c vite.config.ts", "build": "vite build", "pretest": "rm -rf reports/ && vite build --mode testing", "update:snapshot": "UPDATE_SNAPSHOTS=true vitest run --mode testing Grid-view", @@ -28,8 +28,8 @@ "uuid": "^9.0.0", "vector2js": "^2.0.1", "vue": "^3.2.13", - "vue-router": "^4.0.3", "vue-i18n": "^9.2.2", + "vue-router": "^4.0.3", "vue3-highlightjs": "^1.0.5" }, "devDependencies": { @@ -56,6 +56,7 @@ "rimraf": "^4.4.1", "typescript": "~4.5.5", "vite": "^4.1.4", + "vite-plugin-glslify": "^2.0.2", "vitest": "^0.29.5", "vue-tsc": "^1.2.0" }, diff --git a/client/src/search-worker/gpu-worker.ts b/client/src/search-worker/gpu-worker.ts new file mode 100644 index 0000000..221ffae --- /dev/null +++ b/client/src/search-worker/gpu-worker.ts @@ -0,0 +1,61 @@ +import {bindUniforms, createAttributeLocs, createProgram, createUniforms} from './webgl-utils'; +import vert from './shaders/vertex.vert'; +import frag from './shaders/fragment.frag'; +const w = 256; +export class WebGLWorker{ + private _canvas: HTMLCanvasElement; + private _gl: WebGL2RenderingContext; + private _program: WebGLProgram; + private _attributes: Record; + private _uniforms: Record; + + constructor(canvas ? : HTMLCanvasElement){ + this._canvas = canvas || document.createElement("canvas"); + this._canvas.width = w; + this._canvas.height = w; + this._gl = this._canvas.getContext("webgl2") as WebGL2RenderingContext; + this._program = createProgram(this._gl, [vert, frag]); + const pixels = new Float32Array(w * w * 4); + // const framebuffers = [outputTexture, bodyTexture].map(({ texture }) => + // createFrameBuffer(gl, texture) + // ); + // functions computing the simulation and drawing the result on the current FBO + this._attributes = createAttributeLocs(this._gl, this._program, { + position: new Float32Array([-1, -1, 1, -1, -1, 1, -1, 1, 1, -1, 1, 1]) + }); + + this._uniforms = createUniforms(this._gl, this._program, { + + }); + this._gl.viewport(0, 0, w, w); + + } + + + frame(){ + const {_gl: gl, _program: program, _uniforms: uniforms, _attributes: attributes } = this; + // gl.bindFramebuffer(gl.FRAMEBUFFER, framebuffers[i].handle); + gl.viewport(0, 0, w, w); + // gl.viewport(0, 0, bodyTexture.width, bodyTexture.height); + gl.bindBuffer(gl.ARRAY_BUFFER, attributes.position.buffer); + gl.enableVertexAttribArray(attributes.position.location); + gl.vertexAttribPointer( + attributes.position.location, + 2, + gl.FLOAT, + false, + 0, + 0 + ); + gl.useProgram(program); + bindUniforms(gl, uniforms, { color: [1, 0, 0, 1] }); + gl.drawArrays(gl.TRIANGLES, 0, 6); + } + + + + + + + +} \ No newline at end of file diff --git a/client/src/search-worker/shaders/fragment.frag b/client/src/search-worker/shaders/fragment.frag new file mode 100644 index 0000000..0e02b10 --- /dev/null +++ b/client/src/search-worker/shaders/fragment.frag @@ -0,0 +1,6 @@ +precision mediump float; +uniform vec4 color; + +void main() { + gl_FragColor = color; +} \ No newline at end of file diff --git a/client/src/search-worker/shaders/vertex.vert b/client/src/search-worker/shaders/vertex.vert new file mode 100644 index 0000000..2e8efe5 --- /dev/null +++ b/client/src/search-worker/shaders/vertex.vert @@ -0,0 +1,6 @@ +precision mediump float; +attribute vec2 position; + +void main() { + gl_Position = vec4(position, 0, 1); +} \ No newline at end of file diff --git a/client/src/search-worker/webgl-utils.ts b/client/src/search-worker/webgl-utils.ts new file mode 100644 index 0000000..992ba66 --- /dev/null +++ b/client/src/search-worker/webgl-utils.ts @@ -0,0 +1,209 @@ +export type FBO = { + handle: WebGLFramebuffer; + texture: WebGLTexture; +}; +export type Uniform = { + type: GLint; + size: number; + value: number[] | number | WebGLTexture; + location: WebGLUniformLocation; +}; +export type Attribute = { + buffer: WebGLBuffer; + value: ArrayBuffer; + location: GLint; +}; +export type Texture = { + width: number; + height: number; + data: ArrayBufferView; + // handle for the texture + texture: WebGLTexture; + type: GLenum; + format: GLenum; +}; +const typeToBind = { + // gl.FLOAT, + 5126: 'uniform1f', + // gl.FLOAT_VEC2, + 35664: 'uniform2fv', + // gl.FLOAT_VEC3, + 35665: 'uniform3fv', + // gl.FLOAT_VEC4, + 35666: 'uniform4fv', + // gl.INT, + 5124: 'uniform1i', + // gl.INT_VEC2, + 35667: 'uniform2iv', + // gl.INT_VEC3, + 35668: 'uniform3iv', + // gl.INT_VEC4, + 35669: 'uniform4iv', + // gl.FLOAT_MAT2, + 35674: 'uniformMatrix2fv', + // gl.FLOAT_MAT3, + 35675: 'uniformMatrix3fv', + // gl.FLOAT_MAT4, + 35676: 'uniformMatrix4fv', + // gl.SAMPLER_2D, + 35678: 'uniform1i', + // gl.SAMPLER_CUBE + 35680: 'uniform1i' +}; + +export function createProgram( + gl: WebGL2RenderingContext, + shaders: [string, string] +) { + const program = gl.createProgram(); + if (!program) throw new Error('Could not create program'); + shaders.forEach((source, i) => { + const shader = gl.createShader( + i === 0 ? gl.VERTEX_SHADER : gl.FRAGMENT_SHADER + ); + if (!shader) throw new Error('Could not create shader'); + gl.shaderSource(shader, source); + gl.compileShader(shader); + + if (!gl.getShaderParameter(shader, gl.COMPILE_STATUS)) { + const info = gl.getShaderInfoLog(shader); + throw `Could not compile WebGL program. \n\n${info}`; + } + gl.attachShader(program, shader); + }); + gl.linkProgram(program); + // Check the link status + const linked = gl.getProgramParameter(program, gl.LINK_STATUS); + if (!linked) { + // something went wrong with the link + const lastError = gl.getProgramInfoLog(program); + gl.deleteProgram(program); + throw new Error('Error in program linking:' + lastError); + } + return program; +} + +export function createTexture( + gl: WebGL2RenderingContext, + { + data, + width, + height, + type, + format + }: { + data: ArrayBufferView; + width: number; + height: number; + type: GLenum; + format: GLenum; + } +): Texture { + const empty = width * height === 0; + const tex = gl.createTexture(); + if (!tex) throw new Error('Could not create texture'); + gl.bindTexture(gl.TEXTURE_2D, tex); + gl.texImage2D( + gl.TEXTURE_2D, + 0, + format, // internal format + empty ? 1 : width, + empty ? 1 : height, + 0, + format, + type, // type + empty ? new Float32Array(4) : data + ); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.NEAREST); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE); + gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE); + return { + texture: tex, + width, + height, + data, + type: gl.FLOAT, + format: gl.RGBA + }; +} + +export function createFrameBuffer( + gl: WebGL2RenderingContext, + texture: WebGLTexture +): FBO { + const handle = gl.createFramebuffer(); + if (!handle) throw new Error('Could not create framebuffer'); + gl.bindFramebuffer(gl.FRAMEBUFFER, handle); + gl.framebufferTexture2D( + gl.FRAMEBUFFER, + gl.COLOR_ATTACHMENT0, + gl.TEXTURE_2D, + texture, + 0 + ); + return { + handle, + texture + }; +} +export function createUniforms( + gl: WebGL2RenderingContext, + program: WebGLProgram, + uniforms: Record +) { + gl.useProgram(program); + const res = {} as Record; + const nbUniforms = gl.getProgramParameter(program, gl.ACTIVE_UNIFORMS); + for (let i = 0; i < nbUniforms; i++) { + const { name, size, type }: { name: string; size: number; type: GLint } = + gl.getActiveUniform(program, i) as WebGLActiveInfo; + const location = gl.getUniformLocation(program, name); + if (!location) throw new Error(`Could not get location for ${name}`); + res[name] = { + size, + value: uniforms[name], + type, + location + }; + } + return res; +} +export function bindUniforms( + gl: WebGL2RenderingContext, + uniforms: Record, + newValues: Record +): void { + let textureCount = 0; + return Object.entries(uniforms).forEach( + ([key, { location, type, value }]) => { + let newValue = newValues[key] !== undefined ? newValues[key] : value; + if (type === gl.SAMPLER_2D) { + gl.activeTexture(gl.TEXTURE0 + textureCount); + gl.bindTexture(gl.TEXTURE_2D, newValue); + newValue = textureCount; + textureCount++; + } + // @ts-ignore + gl[typeToBind[type]](location, newValue); + } + ); +} + +export function createAttributeLocs( + gl: WebGL2RenderingContext, + program: WebGLProgram, + attributes: Record +) { + return Object.entries(attributes).reduce((acc, [key, value]) => { + const buffer = gl.createBuffer() as WebGLBuffer; + gl.bindBuffer(gl.ARRAY_BUFFER, buffer); + gl.bufferData(gl.ARRAY_BUFFER, value, gl.STATIC_DRAW); + acc[key] = { + buffer, + value, + location: gl.getAttribLocation(program, key) + }; + return acc; + }, {} as Record); +} diff --git a/client/vite.config.ts b/client/vite.config.ts index 74839ad..4ba986a 100644 --- a/client/vite.config.ts +++ b/client/vite.config.ts @@ -1,9 +1,12 @@ import { defineConfig } from 'vite'; import vue from '@vitejs/plugin-vue'; +import glsl from 'vite-plugin-glslify'; // https://vitejs.dev/config/ export default defineConfig({ - plugins: [vue(), + plugins: [ + vue(), + glsl(), { configureServer: (server) => { server.middlewares.use((_req, res, next) => { @@ -14,5 +17,5 @@ export default defineConfig({ }, }], envDir: 'envs/', - + }); From 5904d228879863ab57f5f075d92cb1e03581fbdb Mon Sep 17 00:00:00 2001 From: Leo Nicolle Date: Thu, 27 Jul 2023 01:48:05 +0200 Subject: [PATCH 2/5] I think we got something here. Much faster algo, and seems reliable --- client/src/search-worker/dico.ts | 8 + client/src/search-worker/heatmap.ts | 282 +++++++++++++++++++++++++--- client/src/search-worker/tree.ts | 152 +++++++++++++++ 3 files changed, 421 insertions(+), 21 deletions(-) create mode 100644 client/src/search-worker/tree.ts diff --git a/client/src/search-worker/dico.ts b/client/src/search-worker/dico.ts index d6bfdd3..529c5d4 100644 --- a/client/src/search-worker/dico.ts +++ b/client/src/search-worker/dico.ts @@ -1,6 +1,8 @@ import { StringBS } from "./string-bs"; export const ACode = "A".charCodeAt(0); +export const ZCode = "Z".charCodeAt(0); + /** * Dico is a sigleton class that contains the dictionnary @@ -62,6 +64,12 @@ export class Dico { this.stringBS = new StringBS(this.words, this.sorted); } + findLengthInterval(length: number) { + const start = this.stringBS.byLengthStart(length, 0, this.words.length - 1); + const end = this.stringBS.byLengthEnd(length, start, this.words.length - 1); + return [start, end]; + } + findInterval(query: string) { const startL = this.stringBS.byLengthStart( query.length, diff --git a/client/src/search-worker/heatmap.ts b/client/src/search-worker/heatmap.ts index 3627c45..c99bbc2 100644 --- a/client/src/search-worker/heatmap.ts +++ b/client/src/search-worker/heatmap.ts @@ -1,11 +1,19 @@ import { Direction, Grid, CellProba, Vec, CellBest, Bounds } from "grid"; import { dico, ACode } from "./dico"; -// window ? window.dico = dico: undefined; +import { Tree, Node, intersect, mapToWords, getPossibleParents } from './tree'; + +interface Possibles { + nodes: Node[]; + isImpossible: boolean; +} +// window ? window.dico = dico: undefined; +const trees = new Map(); type Interval = [number, number]; +const cachedResult = new Map(); export type BailOptions = { sharedArray: Uint8Array }; export function initCellMap(grid: Grid) { - return grid.cells.reduce((acc, row, y) => { + const res = grid.cells.reduce((acc, row, y) => { acc.push( row.map(({ definition, text }, x) => { return { @@ -25,6 +33,7 @@ export function initCellMap(grid: Grid) { ); return acc; }, [] as CellProba[][]); + return res; } function intersection(cellMap: CellProba[][], cellBest?: CellBest[][]) { @@ -129,6 +138,226 @@ export function getCellProbasFast(grid: Grid, options?: BailOptions) { }; } +interface CellA { + previousH: CellA | null; + previousV: CellA | null; + nextH: CellA | null; + nextV: CellA | null; + x: number; + y: number; + text: string; + validH: boolean; + validV: boolean; + nodesH: Map; + nodesV: Map; + impossible: Set; + // impossibleV: Set; +} + +function filter(cell: CellA) { + const { text, validH, validV, nodesH, nodesV, impossible } = cell; + // filter out and intersect + if (text !== '*') { + const code = text.charCodeAt(0); + if (validH) { + const key = nodesH.get(code); + nodesH.clear(); + nodesH.set(code, key || []); + } + if (validV) { + const key = nodesV.get(code); + nodesV.clear(); + nodesV.set(code, key || []); + } + } else if (validH && validV) { + intersect(nodesH, nodesV); + } + //TODO: check if needed + impossible.forEach(code => { + nodesH.delete(code); + nodesV.delete(code); + }); +} +export function getCellProbasAccurate2(grid: Grid, options?: BailOptions) { + const cellsA = grid.cells.reduce((acc: CellA[][], row, y) => { + const r = row.map((cell, x) => { + const { definition, text } = cell; + return { + previousH: null, + previousV: null, + nextH: null, + nextV: null, + text: text.length ? text : '*', + x, + y, + validH: false, + validV: false, + nodesH: new Map(), + nodesV: new Map(), + impossible: new Set(), + // impossibleV: new Set(), + }; + }); + acc.push(r); + return acc; + }, [] as CellA[][]); + + + let hasBailed = false; + (["horizontal", "vertical"] as Direction[]).forEach((dir) => { + grid + .getWords(dir) + .filter(({ length }) => length > 1) + .forEach((bounds) => { + const tree = trees.get(bounds.length); + const query = bounds.cells[0].text.length ? bounds.cells[0].text : '*'; + const suffix = dir === 'horizontal' ? 'H' : 'V'; + cellsA[bounds.start.y][bounds.start.x][`nodes${suffix}`] = tree?.getIntervals(query) || new Map(); + bounds.cells.forEach(({ x, y }, i) => { + const cellA = cellsA[y][x]; + const isV = +(dir === 'vertical'); + const isH = +(dir === 'horizontal'); + cellA[`previous${suffix}`] = i === 0 ? null : cellsA[y - 1 * isV][x - 1 * isH]; + cellA[`next${suffix}`] = i === bounds.length - 1 ? null : cellsA[y + 1 * isV][x + 1 * isH]; + cellA[`index${suffix}`] = i; + cellA[`valid${suffix}`] = true; + }); + }); + }); + const sortedCells = cellsA.flat() + .filter(({ x, y }) => !grid.cells[y][x].definition) + .sort(({ x: xa, y: ya }, { x: xb, y: yb }) => { + const dx = xa - xb; + const dy = ya - yb; + return dy === 0 ? dx : dy; + }); + const lastCells = sortedCells.slice().reverse().filter(({ nextH, nextV }) => !nextH || !nextV); + sortedCells.forEach(cell => { + const mapH = new Map(); + const mapV = new Map(); + const { previousH, previousV, nodesH, nodesV, impossible } = cell; + [ + { + prev: previousH, + prevNodes: previousH && previousH.nodesH, + map: mapH, + key: 'nodesH', + }, + { + prev: previousV, + prevNodes: previousV && previousV.nodesV, + map: mapV, + key: 'nodesV', + } + ].forEach(({ prev, key, map, prevNodes }) => { + if (!prev || !prevNodes) return; + for (let i = 0; i < 26; i++) { + map.set(ACode + i, []); + } + prevNodes.forEach((nodes) => { + nodes + .forEach(node => { + node.children.forEach(child => { + map.get(child.code) && map.get(child.code).push(child); + }); + }); + }); + cell[key] = map; + }); + filter(cell); + + }); + // console.log('sortedCells', sortedCells.some(c => Array.isArray(c.nodesH) || Array.isArray(c.nodesV))) + let shouldRerun = true; + const max = 1; + let i = 0; + // console.time('firstloop'); + while (i++ < max) { + shouldRerun = false; + const Q = lastCells.slice(); + // console.time('firstloop2') + while (Q.length) { + const cell = Q.shift()!; + const { previousH, previousV, nodesH, nodesV } = cell; + [{ prev: previousH, nodes: nodesH, key: 'nodesH' }, { prev: previousV, nodes: nodesV, key: 'nodesV' }] + .forEach(({ prev, nodes, key }) => { + if (!prev) return; + Q.push(prev); + const possibleParentsCode = new Set(); + getPossibleParents(nodes, possibleParentsCode); + [...prev[key].keys()].forEach(code => { + if (!possibleParentsCode.has(code)) { + prev[key].delete(code); + } + }); + }); + } + // console.timeEnd('firstloop2') + // console.time('firstloop3') + sortedCells.forEach(cell => { + if (cell.validH && cell.validV) { + for (let i = 0; i < 26; i++) { + if (!cell.nodesH.has(ACode + i) || !cell.nodesV.has(ACode + i)) { + cell.impossible.add(ACode + i); + } + } + } else if (cell.validH) { + for (let i = 0; i < 26; i++) { + if (!cell.nodesH.has(ACode + i)) { + cell.impossible.add(ACode + i); + } + } + } else if (cell.validV) { + for (let i = 0; i < 26; i++) { + if (!cell.nodesH.has(ACode + i)) { + cell.impossible.add(ACode + i); + } + } + } + }); + // console.timeEnd('firstloop3') + // console.time('firstloop4') + sortedCells.forEach(cell => { + const { nextH, nextV, nodesH, nodesV, impossible } = cell; + if (nextV) { + const keys = [...nextV.nodesV.keys()]; + keys.forEach(code => { + nextV.nodesV.set(code, nextV.nodesV.get(code)!.filter(n => !impossible.has(n.parent.code))); + }); + } + if (nextH) { + const keys = [...nextH.nodesH.keys()]; + keys.forEach(code => { + nextH.nodesH.set(code, nextH.nodesH.get(code)!.filter(n => !impossible.has(n.parent.code))); + }); + } + }); + // console.timeEnd('firstloop4') + } + + + // (["horizontal", "vertical"] as Direction[]).forEach((dir) => { + // const scores = {}; + // grid + // .getWords(dir) + // .filter(({ length }) => length > 1) + // .forEach((bounds) => { + // const suffix = dir === 'horizontal' ? 'H' : 'V'; + + // bounds + // .cells.slice().reverse() + // .forEach(({ x, y }, i) => { + // const cellA = cellsA[y][x]; + // const isV = +(dir === 'vertical'); + // const isH = +(dir === 'horizontal'); + // }); + // }); + + console.timeEnd('firstloop'); + console.log(i, sortedCells.filter(c => c.y === 1).map(c => mapToWords(c.nodesH))); +} + + export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { const cellMap = initCellMap(grid); const cellToWordIndexH: Record = {}; @@ -178,7 +407,6 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { }); }); }); - const sortedCells = grid.cells.flat() .filter(({ x, y }) => cellMap[y][x].empty && !grid.cells[y][x].definition) .sort(({ x: xa, y: ya }, { x: xb, y: yb }) => { @@ -197,16 +425,6 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { let newStackH: Interval[] = []; const validH = cellMap[y][x].validH; const validV = cellMap[y][x].validV; - // console.log(key, { - // vTotal: (intervalsV[iV] || []) - // .reduce((acc, [start, end]) => acc + end - start + 1, 0), - // v: (intervalsV[iV] || []).length, - // hTotal: (intervalsH[iH] || []) - // .reduce((acc, [start, end]) => acc + end - start + 1, 0), - // h: (intervalsH[iH] || []).length, - - // }) - // console.time(`cell ${key}`); for (let j = 0; j < 26; j++) { let stackV = (validV && intervalsV[iV].slice()) as Interval[]; let stackH = (validH && intervalsH[iH].slice()) as Interval[]; @@ -265,9 +483,7 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { intervalsV[iV] = newStackV; newStackV = []; } - // console.timeEnd(`cell ${key}`); }); - // count how many words have each letter in each cell sortedCells.forEach(({ x, y }) => { const key = `${y}-${x}`; @@ -303,11 +519,9 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { cellMap[y][x].inter = inter; cellMap[y][x].totalInter = total; }, []); - // compute the scores for each words const bestWordsH: Record = {}; const bestWordsV: Record = {}; - (["horizontal", "vertical"] as Direction[]).forEach((dir) => { const cellToWordIndex = dir === "horizontal" ? cellToWordIndexH : cellToWordIndexV; @@ -338,9 +552,7 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { bestWords[`${y}-${x}`] = best; }); }); - }); - return { cellProbas: grid.cells.reduce((acc, row, y) => { acc.push( @@ -352,7 +564,7 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { validV, empty, bestWordsV: (bestWordsV[key] || []).map((index) => dico.words[dico.sorted[index]]), - bestWordsH: (bestWordsH[key] || []).map((index) => dico.words[dico.sorted[index]]) , + bestWordsH: (bestWordsH[key] || []).map((index) => dico.words[dico.sorted[index]]), inter, x, y, @@ -368,7 +580,28 @@ export function getCellProbasAccurate(grid: Grid, options?: BailOptions) { export function getCellProbas(grid: Grid, options: BailOptions) { + const hash = grid.serialize(); + if (cachedResult.has(hash)) { + return { hasBailed: false, cellProbas: cachedResult.get(hash)! }; + } + if (!trees.size) { + console.time('initTree'); + const min = dico.words[dico.sorted[0]].length; + const max = dico.words[dico.sorted[dico.sorted.length - 1]].length; + for (let i = min; i <= max; i++) { + trees.set(i, new Tree(i)); + } + console.timeEnd('initTree'); + } + console.log('getCellProbas') const cellMap = initCellMap(grid); + console.time('getCellProbasAccurate2') + getCellProbasAccurate2(grid, options); + console.timeEnd('getCellProbasAccurate2') + console.time('getCellProbasAccurate1') + getCellProbasAccurate(grid, options); + console.timeEnd('getCellProbasAccurate1') + return cellMap; const cellToWordIndexH: Record = {}; const cellToWordIndexV: Record = {}; const cellToBoundsH: Record = {}; @@ -415,6 +648,13 @@ export function getCellProbas(grid: Grid, options: BailOptions) { if (total > 60000) { // return getCellProbasFast(grid, options); } - return getCellProbasAccurate(grid, options); + const result = getCellProbasAccurate(grid, options); + if (cachedResult.size > 10) { + cachedResult.delete([...cachedResult.keys()][0]); + } + if (!result.hasBailed) { + cachedResult.set(hash, result.cellProbas); + } + return result; } diff --git a/client/src/search-worker/tree.ts b/client/src/search-worker/tree.ts new file mode 100644 index 0000000..de50d2c --- /dev/null +++ b/client/src/search-worker/tree.ts @@ -0,0 +1,152 @@ +import { remove } from 'fs-extra'; +import { ACode, ZCode, dico } from './dico'; + +export type Node = { + code: number; + start: number; + end: number; + parent: Node | undefined; + index: number; + children: Node[]; +}; + +function getChildren(start: number, end: number, index: number, size: number, parent?: Node): Node[] { + if (index === size) return []; + if (start === end) return [{ + code: dico.words[start].charCodeAt(index), + start, + end, + index, + parent, + children: [] + }]; + return new Array(26).fill(0).map((e, i) => i + 65) + .map(code => { + const newStart = dico.stringBS.findStartIdx( + code, + index, + start, + end + ); + const newEnd = dico.stringBS.findEndIdx( + code, + index, + newStart, + end + ); + start = newStart; + if (newStart > newEnd) return null; + const node = { + code, + start: newStart, + end: newEnd, + index, + parent + } as Node; + node.children = getChildren(newStart, newEnd, index + 1, size, node).filter(c => !isNaN(c.code) && c.code >= ACode && c.code <= ZCode); + return node; + }).filter(e => e) as Node[]; +} + +let interSet: Set; +export function intersect(a: Map, b: Map) { + // const interA: Node[] = []; + // const interB: Node[] = []; + // interSet = new Set(a.map(c => c.code)); + // for (let i = 0; i < b.length; i++) { + // if (!interSet.has(b[i].code)) { + // // removedB.push(b[i]) + // continue; + // } + // interB.push(b[i]); + // } + // interSet.clear(); + // interSet = new Set(b.map(c => c.code)); + // for (let i = 0; i < a.length; i++) { + // if (!interSet.has(a[i].code)) { + // // removedA.push(a[i]); + // continue; + // } + // interA.push(a[i]); + // } + for (let i = ACode; i < ACode + 26; i++) { + if (!a.has(i) || !b.has(i)) { + a.delete(i); + b.delete(i); + } + } +} + +export function bottomUp(callback: (node: Node) => void, node: Node) { + const Q = [node]; + while (Q.length) { + const n = Q.pop()!; + callback(n); + if (!n?.parent) break; + Q.push(n?.parent); + } +} + + +function parentHasCode(nodes: Node[], code: number, start: number, end: number, foundCodes: Set) { + while (start <= end) { + const middle = (start + end) >>> 1; + const parentCode = nodes[middle].parent!.code; + if (!parentCode) break; + foundCodes.add(parentCode) + if (parentCode < code) start = middle + 1; + else end = middle - 1; + } + return nodes[start] && nodes[start].parent!.code === code; +} +export function getPossibleParents(nodes: Map, res: Set) { + for (let i = ACode; i <= ZCode; i++) { + if (res.has(i)) continue; + for (const [_, ns] of nodes) { + if (!parentHasCode(ns, i, 0, ns.length - 1, res)) continue; + res.add(i); + } + } +} + +export function mapToWords(map: Map) { + const words: string[] = []; + for (const [_, nodes] of map) { + for (const node of nodes) { + new Array(node.end - node.start + 1).fill(0).map((e, i) => i + node.start).forEach(i => words.push(dico.words[dico.sorted[i]])); + } + } + return words; +} + +export class Tree { + private roots: Node[]; + constructor(size: number) { + const [min, max] = dico.findLengthInterval(size); + let start = min; + let end = max; + this.roots = getChildren(start, end, 0, size); + } + + getIntervals(query: string): Map { + return this.getIntervalsR(query, Array.from(this.roots.values()).flat()) + .reduce((acc, node) => { + return acc.set(node.code, acc.get(node.code) ? [...acc.get(node.code)!, node] : [node]); + }, new Map()); + } + + getIntervalsR(query: string, stack: Node[]): Node[] { + if (query.length === 1) { + if (query === "*") return stack;//.map(node => [node.start, node.end]); + return stack.filter(node => node.code === query.charCodeAt(0));//.map(node => [node.start, node.end]); + } + const code = query.charCodeAt(0); + if (query.charAt(0) !== "*") { + const newStack = stack.filter(node => node.code === code).flatMap(node => node.children); + return this.getIntervalsR(query.slice(1), newStack); + } + const newStack = stack.flatMap(node => Array.from(node.children)); + return this.getIntervalsR(query.slice(1), newStack); + } + +} \ No newline at end of file From 89e931876fb8f51d0deb10c411382bd30b485286 Mon Sep 17 00:00:00 2001 From: Leo Nicolle Date: Fri, 1 Dec 2023 18:24:11 +0100 Subject: [PATCH 3/5] gitignore --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 79105cb..576c127 100644 --- a/.gitignore +++ b/.gitignore @@ -14,4 +14,4 @@ build/ dist/ docs/.vuepress/.cache docs/.vuepress/.temp -scripts/api-key.js \ No newline at end of file +scripts/api-key.mjs \ No newline at end of file From ae24a2ea536175bb73449b5d589144657f691010 Mon Sep 17 00:00:00 2001 From: Leo Nicolle Date: Fri, 1 Dec 2023 21:06:13 +0100 Subject: [PATCH 4/5] finished faster search --- client/src/components/Suggestion.vue | 43 ++--- client/src/search-worker/heatmap.ts | 233 +++++++++++++++++---------- client/src/search-worker/index.ts | 14 +- client/src/search-worker/tree.ts | 21 +-- 4 files changed, 178 insertions(+), 133 deletions(-) diff --git a/client/src/components/Suggestion.vue b/client/src/components/Suggestion.vue index 1519f42..08f2f90 100644 --- a/client/src/components/Suggestion.vue +++ b/client/src/components/Suggestion.vue @@ -12,10 +12,7 @@ - + @@ -63,7 +51,7 @@ import { Method, Ordering } from "../types"; /** * Component to display words suggestions */ -const results = ref<{word:string}[]>([]); +const results = ref<{ word: string; }[]>([]); const totalResults = ref(0); const suggestion = ref(null); const version = ref(0); @@ -143,7 +131,7 @@ function getSuggestions( words.sort(() => Math.random() - 0.5); } totalResults.value = words.length; - results.value = words.map((word) => ({word})); + results.value = words.map((word) => ({ word })); } function orderingText() { @@ -174,19 +162,22 @@ function onMouseEvt(evt: MouseEvent, click = false) {