feat(tokenizer): add line info to tokenizer

packages/abell/package.json

@@ -42,10 +42,12 @@
   "homepage": "https://abelljs.org",
   "license": "MIT",
   "dependencies": {
+    "@jridgewell/sourcemap-codec": "^1.4.15",
     "commander": "^7.2.0",
     "express": "^4.17.2",
     "node-match-path": "^0.6.3",
     "postcss-selector-parser": "^6.0.10",
+    "source-map": "^0.7.4",
     "stylis": "^4.1.0",
     "vite": "4.4.9"
   },

packages/abell/src/vite-plugin-abell/compiler/generic-tokenizer.ts

@@ -2,13 +2,16 @@
  * Copied from https://gist.github.com/borgar/451393/7698c95178898c9466214867b46acb2ab2f56d68
  * Made by borgar (https://github.com/borgar)
  *
- * Saurabh has absolutely no idea how this works.
+ * Saurabh made some tiny modifications to support TS, and add position and line information
  */
 
 export type Token<T> = {
   text: string;
   type: keyof T | 'default';
   matches?: string[];
+  line?: number;
+  pos?: number;
+  col?: number;
 };
 
 /*
@@ -29,6 +32,8 @@ function tokenize<T extends Record<string, RegExp>>(
   let m;
   let r;
   let t;
+  let skippedCodeLength = 0;
+  const fullCodeString = s;
   const tokens = [];
   while (s) {
     t = null;
@@ -38,10 +43,15 @@ function tokenize<T extends Record<string, RegExp>>(
       // try to choose the best match if there are several
       // where "best" is the closest to the current starting point
       if (r && r.index < m) {
+        const pos = skippedCodeLength + r.index;
+        const codeTillHere = fullCodeString.slice(0, pos);
         t = {
           text: r[0],
           type: key,
-          matches: r.slice(1)
+          matches: r.slice(1),
+          pos,
+          line: codeTillHere.split('\n').length,
+          col: codeTillHere.slice(codeTillHere.lastIndexOf('\n')).length
         };
         m = r.index;
       }
@@ -58,7 +68,9 @@ function tokenize<T extends Record<string, RegExp>>(
       // push current token onto sequence
       tokens.push(t);
     }
-    s = s.substr(m + (t ? t.text.length : 0));
+    const currentSkippedLength = m + (t ? t.text.length : 0);
+    skippedCodeLength += currentSkippedLength;
+    s = s.substr(currentSkippedLength);
   }
   return tokens;
 }

packages/abell/src/vite-plugin-abell/compiler/syntax-blocks.ts

@@ -23,6 +23,12 @@ export const getSyntaxBlocks = (
     outText: ''
   };
 
+  // const maps = {
+  //   importTextMap: [],
+  //   declarationTextMap: [],
+  //   abellTextMap: []
+  // };
+
   const cssBlocks = [];
   const outBlocks: { text: string }[] = [];
 

packages/abell/src/vite-plugin-abell/compiler/tokenizer.spec.ts

@@ -0,0 +1,80 @@
+import { describe, expect, test } from 'vitest';
+import tokenize from './generic-tokenizer.js';
+import { tokenSchema } from './token-schema.js';
+
+describe('tokenize', () => {
+  test("should return token that matches abell's schema", () => {
+    const abellCode = `
+    <h1>{{ 2 + 3 }}</h1>
+    {{
+      /* @declaration */
+      const a = 3;
+      const b = 9;
+    }}
+    `;
+    const tokens = tokenize(abellCode, tokenSchema, 'default');
+    expect(tokens).toMatchInlineSnapshot(`
+      [
+        {
+          "text": "
+          <h1>",
+          "type": "default",
+        },
+        {
+          "col": 9,
+          "line": 2,
+          "matches": [],
+          "pos": 9,
+          "text": "{{",
+          "type": "BLOCK_START",
+        },
+        {
+          "text": " 2 + 3 ",
+          "type": "default",
+        },
+        {
+          "col": 18,
+          "line": 2,
+          "matches": [],
+          "pos": 18,
+          "text": "}}",
+          "type": "BLOCK_END",
+        },
+        {
+          "text": "</h1>
+          ",
+          "type": "default",
+        },
+        {
+          "col": 5,
+          "line": 3,
+          "matches": [],
+          "pos": 30,
+          "text": "{{",
+          "type": "BLOCK_START",
+        },
+        {
+          "text": "
+            /* @declaration */
+            const a = 3;
+            const b = 9;
+          ",
+          "type": "default",
+        },
+        {
+          "col": 5,
+          "line": 7,
+          "matches": [],
+          "pos": 100,
+          "text": "}}",
+          "type": "BLOCK_END",
+        },
+        {
+          "text": "
+          ",
+          "type": "default",
+        },
+      ]
+    `);
+  });
+});

packages/abell/src/vite-plugin-abell/vite-plugin-abell.ts

@@ -32,6 +32,7 @@ export function vitePluginAbell(abellOptions?: AbellOptions): PluginOption {
           cwd: process.cwd()
         });
         let outCode = jsCode;
+        // let outMap = undefined;
         // If loader is not defined, skip the esbuild transform
         if (abellOptions?.esbuild?.loader) {
           const esbuildOut = await transformWithEsbuild(
@@ -40,9 +41,11 @@ export function vitePluginAbell(abellOptions?: AbellOptions): PluginOption {
             abellOptions.esbuild
           );
           outCode = esbuildOut.code;
+          // outMap = esbuildOut.map;
         }
         return {
           code: outCode
+          // map: outMap
         };
       }
     }