fix(driver): bin packing text renderer

packages/driver/src/js/renderer/renderer.ts

@@ -7,6 +7,7 @@ export type TextureBitmap = {
   id: string;
   bitmap: ImageBitmap | ImageData | OffscreenCanvas;
   flags: number;
+  updateSubImage?: () => { x: number; y: number; width: number; height: number; source: ArrayBufferView };
 };
 
 const WHITE_TEXTURE_BITMAP = (() => {
@@ -241,19 +242,19 @@ export class Renderer {
         segments.push({
           text: subtext,
           color: this.currentColor,
-          bitmap: this.textRasterizer.get(height, font, subtext),
+          render: this.textRasterizer.get(height, font, subtext),
         });
       }
     }
     if (text.length > 0) {
       segments.push({
         text,
         color: this.currentColor,
-        bitmap: this.textRasterizer.get(height, font, text),
+        render: this.textRasterizer.get(height, font, text),
       });
     }
 
-    const width = segments.reduce((width, segment) => width + segment.bitmap.width, 0);
+    const width = segments.reduce((width, segment) => width + segment.render.width, 0);
 
     let x = pos.x;
     switch (align) {
@@ -272,15 +273,15 @@ export class Renderer {
     }
 
     for (const segment of segments) {
-      if (segment.bitmap.bitmap) {
+      if (segment.render.bitmap) {
         this.backend?.drawQuad(
-          [x, pos.y, x + segment.bitmap.width, pos.y, x + segment.bitmap.width, pos.y + height, x, pos.y + height],
-          [0, 0, 1, 0, 1, 1, 0, 1],
-          segment.bitmap.bitmap,
+          [x, pos.y, x + segment.render.width, pos.y, x + segment.render.width, pos.y + height, x, pos.y + height],
+          segment.render.coords,
+          segment.render.bitmap,
           segment.color,
         );
       }
-      x += segment.bitmap.width;
+      x += segment.render.width;
     }
 
     pos.y += height;

packages/driver/src/js/renderer/text.ts

@@ -42,7 +42,7 @@ export class TextMetrics {
 
   measure(size: number, fontNum: number, text: string) {
     this.context.font = font(size, fontNum);
-    return this.context.measureText(text.replaceAll(reColorGlobal, "")).width;
+    return Math.ceil(this.context.measureText(text.replaceAll(reColorGlobal, "")).width);
   }
 
   measureCursorIndex(size: number, fontNum: number, text: string, cursorX: number, cursorY: number) {
@@ -67,6 +67,7 @@ export class TextMetrics {
 export interface TextRender {
   width: number;
   bitmap: TextureBitmap | undefined;
+  coords: number[];
 }
 
 export interface TextRasterizer {
@@ -80,8 +81,8 @@ export class SimpleTextRasterizer implements TextRasterizer {
 
   get(size: number, fontNum: number, text: string) {
     const key = `${size}:${fontNum}:${text}`;
-    let bitmap = this.cache.get(key);
-    if (!bitmap) {
+    let render = this.cache.get(key);
+    if (!render) {
       const width = this.textMetrics.measure(size, fontNum, text);
       if (width > 0) {
         const canvas = new OffscreenCanvas(width, size);
@@ -91,30 +92,173 @@ export class SimpleTextRasterizer implements TextRasterizer {
         context.fillStyle = "white";
         context.textBaseline = "bottom";
         context.fillText(text, 0, size);
-        bitmap = {
+        render = {
           width,
           bitmap: { id: key, bitmap: canvas, flags: TextureFlags.TF_NOMIPMAP | TextureFlags.TF_CLAMP },
+          coords: [0, 0, 1, 0, 1, 1, 0, 1],
         };
-        this.cache.set(key, bitmap);
+        this.cache.set(key, render);
       } else {
-        bitmap = {
+        render = {
           width: 0,
           bitmap: undefined,
+          coords: [0, 0, 1, 0, 1, 1, 0, 1],
         };
       }
     }
-    return bitmap;
+    return render;
+  }
+}
+
+type Rectangle = {
+  width: number;
+  height: number;
+  x: number;
+  y: number;
+};
+
+interface BinPack {
+  add(width: number, height: number): Rectangle | undefined;
+}
+
+class BinaryBinPack implements BinPack {
+  private readonly freeRectangles: Rectangle[];
+
+  constructor(
+    readonly width: number,
+    readonly height: number,
+  ) {
+    this.freeRectangles = [{ width: width, height: height, x: 0, y: 0 }];
+  }
+
+  private findFreeRectangle(width: number, height: number): { index: number; rect: Rectangle } | null {
+    for (let i = 0; i < this.freeRectangles.length; i++) {
+      const rect = this.freeRectangles[i];
+      if (rect.width >= width && rect.height >= height) {
+        return { index: i, rect: rect };
+      }
+    }
+    return null;
+  }
+
+  private splitFreeRectangle(freeRect: Rectangle, width: number, height: number): void {
+    const rightSplit = {
+      width: freeRect.width - width,
+      height: height,
+      x: freeRect.x + width,
+      y: freeRect.y,
+    };
+    const bottomSplit = {
+      width: freeRect.width,
+      height: freeRect.height - height,
+      x: freeRect.x,
+      y: freeRect.y + height,
+    };
+    if (rightSplit.width > 0 && rightSplit.height > 0) this.freeRectangles.push(rightSplit);
+    if (bottomSplit.width > 0 && bottomSplit.height > 0) this.freeRectangles.push(bottomSplit);
+  }
+
+  add(width: number, height: number): Rectangle | undefined {
+    const found = this.findFreeRectangle(width, height);
+    if (found) {
+      const { index, rect } = found;
+      const newRect = { width, height, x: rect.x, y: rect.y };
+
+      // Remove used rectangle and split free space
+      this.freeRectangles.splice(index, 1);
+      this.splitFreeRectangle(rect, width, height);
+      return newRect;
+    }
+    // No suitable free rectangle found
   }
 }
 
 export class BinPackingTextRasterizer {
-  private readonly maxTextureSize: number;
+  private size: { width: number; height: number };
+  // @ts-ignore
+  private canvas: OffscreenCanvas;
+  // @ts-ignore
+  private context: OffscreenCanvasRenderingContext2D;
+  // @ts-ignore
+  private packer: BinPack;
+  private cache: Map<string, TextRender> = new Map();
+  private generation = 0;
 
   constructor(readonly textMetrics: TextMetrics) {
     const canvas = new OffscreenCanvas(1, 1);
     const gl = canvas.getContext("webgl");
-    if (gl) {
-      this.maxTextureSize = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number;
+    if (!gl) throw new Error("Failed to get WebGL context");
+    const maxTextureSize = gl.getParameter(gl.MAX_TEXTURE_SIZE) as number;
+    const size = 1024;
+    this.size = { width: Math.min(size, maxTextureSize), height: Math.min(size, maxTextureSize) };
+
+    this.reset();
+  }
+
+  private reset() {
+    this.canvas = new OffscreenCanvas(this.size.width, this.size.height);
+    const context = this.canvas.getContext("2d", { willReadFrequently: true });
+    if (!context) throw new Error("Failed to get 2D context");
+    this.context = context;
+    this.context.fillStyle = "white";
+    this.context.textBaseline = "bottom";
+    this.packer = new BinaryBinPack(this.size.width, this.size.height);
+    this.cache.clear();
+    this.generation += 1;
+    console.log("BinPackingTextRasterizer: reset");
+  }
+
+  get(height: number, fontNum: number, text: string) {
+    const key = `${height}:${fontNum}:${text}`;
+    let render = this.cache.get(key);
+    if (!render) {
+      const width = this.textMetrics.measure(height, fontNum, text);
+      if (width > 0) {
+        let rect = this.packer.add(width, height);
+        if (!rect) {
+          this.reset();
+          rect = this.packer.add(width, height);
+          if (!rect) throw new Error("Failed to add text to texture");
+        }
+
+        this.context.font = font(height, fontNum);
+        this.context.fillText(text, rect.x, rect.y + rect.height);
+
+        const u1 = rect.x / this.size.width;
+        const v1 = rect.y / this.size.height;
+        const u2 = (rect.x + rect.width) / this.size.width;
+        const v2 = (rect.y + rect.height) / this.size.height;
+        const bitmap = {
+          id: `@text:${this.generation}`,
+          bitmap: this.canvas,
+          flags: TextureFlags.TF_NOMIPMAP | TextureFlags.TF_CLAMP,
+        };
+        render = {
+          width,
+          bitmap,
+          coords: [u1, v1, u2, v1, u2, v2, u1, v2],
+        };
+        this.cache.set(key, render);
+        render = {
+          ...render,
+          bitmap: {
+            ...bitmap,
+            updateSubImage: () => {
+              return {
+                ...rect,
+                source: this.context.getImageData(rect.x, rect.y, rect.width, rect.height).data,
+              };
+            },
+          },
+        };
+      } else {
+        render = {
+          width: 0,
+          bitmap: undefined,
+          coords: [0, 0, 1, 0, 1, 1, 0, 1],
+        };
+      }
     }
+    return render;
   }
 }

packages/driver/src/js/renderer/webgl_backend.ts

@@ -192,7 +192,7 @@ export class WebGL1Backend {
   private drawCount = 0;
   private readonly vbo: WebGLBuffer;
   private readonly maxTextures: number;
-  private batchTextures: Map<string, TextureBitmap & { index: number }> = new Map();
+  private batchTextures: Map<string, { index: number; texture: WebGLTexture }> = new Map();
   private batchTextureCount = 0;
   private dispatchCount = 0;
 
@@ -276,12 +276,18 @@ export class WebGL1Backend {
   }
 
   begin() {
-    this.vertices = new VertexBuffer();
-    this.drawCount = 0;
-    this.dispatchCount = 0;
+    this.resetBatch();
   }
 
   end() {
+    // if (this.textures.get("@text:1")) {
+    //   this.drawQuad(
+    //     [0, 0, 1024, 0, 1024, 1024, 0, 1024],
+    //     [0, 0, 1, 0, 1, 1, 0, 1],
+    //     { id: "@text:1", bitmap: new ImageData(1024, 1024), flags: 0 },
+    //     [1, 1, 1, 1],
+    //   );
+    // }
     this.dispatch();
     // console.log(`Draw count: ${this.drawCount}, Dispatch count: ${this.dispatchCount}`);
   }
@@ -294,9 +300,16 @@ export class WebGL1Backend {
       if (this.batchTextures.size >= this.maxTextures) {
         this.dispatch();
       }
-      t = { ...textureBitmap, index: this.batchTextureCount++ };
+      const texture = this.getTexture(textureBitmap);
+      t = { index: this.batchTextureCount++, texture };
       this.batchTextures.set(textureBitmap.id, t);
     }
+    if (textureBitmap.updateSubImage) {
+      const gl = this.gl;
+      gl.bindTexture(gl.TEXTURE_2D, t.texture);
+      const sub = textureBitmap.updateSubImage();
+      gl.texSubImage2D(gl.TEXTURE_2D, 0, sub.x, sub.y, sub.width, sub.height, gl.RGBA, gl.UNSIGNED_BYTE, sub.source);
+    }
 
     for (const i of [0, 1, 2, 0, 2, 3]) {
       this.vertices.push(i, coords, texCoords, tintColor, this.viewport, t.index);
@@ -320,7 +333,7 @@ export class WebGL1Backend {
       // Set up the texture
       for (const t of this.batchTextures.values()) {
         gl.activeTexture(gl.TEXTURE0 + t.index);
-        gl.bindTexture(gl.TEXTURE_2D, this.getTexture(t));
+        gl.bindTexture(gl.TEXTURE_2D, t.texture);
         gl.uniform1i(p.textures[t.index], t.index);
       }
 
@@ -347,6 +360,10 @@ export class WebGL1Backend {
       // gl.disableVertexAttribArray(p.texId);
       // gl.bindBuffer(gl.ARRAY_BUFFER, null);
     });
+    this.resetBatch();
+  }
+
+  private resetBatch() {
     this.vertices = new VertexBuffer();
     this.batchTextures = new Map();
     this.batchTextureCount = 0;
@@ -385,10 +402,11 @@ export class WebGL1Backend {
 
       gl.texImage2D(gl.TEXTURE_2D, 0, gl.RGBA, gl.RGBA, gl.UNSIGNED_BYTE, textureBitmap.bitmap);
       if ((textureBitmap.flags & TextureFlags.TF_NOMIPMAP) === 0) {
-        gl.generateMipmap(gl.TEXTURE_2D);
+        // gl.generateMipmap(gl.TEXTURE_2D);
       }
 
       this.textures.set(textureBitmap.id, texture);
+      // console.log("Created texture", textureBitmap.id, textureBitmap.bitmap.width, textureBitmap.bitmap.height);
     }
     return texture;
   }

packages/driver/src/js/worker.ts

@@ -6,7 +6,14 @@ import type { UIState } from "./event.ts";
 import { ImageRepository } from "./image";
 // @ts-ignore
 import { createNODEFS } from "./nodefs.js";
-import { Renderer, SimpleTextRasterizer, TextMetrics, type TextRasterizer, WebGL1Backend, loadFonts } from "./renderer";
+import {
+  BinPackingTextRasterizer,
+  Renderer,
+  TextMetrics,
+  type TextRasterizer,
+  WebGL1Backend,
+  loadFonts,
+} from "./renderer";
 
 interface DriverModule extends EmscriptenModule {
   FS: typeof FS;
@@ -83,7 +90,7 @@ export class DriverWorker {
 
     await loadFonts();
     this.textMetrics = new TextMetrics();
-    this.textRasterizer = new SimpleTextRasterizer(this.textMetrics);
+    this.textRasterizer = new BinPackingTextRasterizer(this.textMetrics);
 
     this.renderer = new Renderer(this.imageRepo, this.textRasterizer, this.screenSize);
     this.hostCallbacks = {
@@ -124,12 +131,12 @@ export class DriverWorker {
           backend: Zip,
           zipData: await rootZip.arrayBuffer(),
           name: "root.zip",
-        },
+        } as any,
         "/user": {
           name: "LocalStorage",
           backend: zenfs.Port,
           port: self as unknown as any,
-        },
+        } as any,
       },
     });