microsoft · pelikhan · Aug 29, 2024 · Aug 29, 2024 · Aug 29, 2024 · github-actions
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -19,6 +19,7 @@
         "gptool",
         "gptools",
         "gptoolsjs",
+        "limitrows",
         "llmify",
         "llmrequest",
         "localai",
@@ -34,6 +35,7 @@
         "promptfoo",
         "stringifying",
         "sysr",
+        "tabletojson",
         "treesitter",
         "typecheck",
         "unfence",

diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts
diff --git a/genaisrc/docs_sample_generator.genai.js → genaisrc/docs-sample-generator.genai.mts b/genaisrc/docs_sample_generator.genai.js → genaisrc/docs-sample-generator.genai.mts
@@ -1,4 +1,5 @@
 script({
+    model: "openai:gpt-4",
     title: "generating tests from samples",
     system: ["system"],
     parameters: {

diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts
diff --git a/packages/cli/src/playwright.ts b/packages/cli/src/playwright.ts
@@ -86,7 +86,6 @@ export class BrowserManager {
 
         logVerbose(`browsing ${ellipseUri(url)}`)
         const browser = await this.launchBrowser(options)
-        logVerbose(`navigating...`)
         let page: Page
         if (incognito) {
             const context = await browser.newContext(rest)
@@ -96,7 +95,6 @@ export class BrowserManager {
         }
         if (timeout !== undefined) page.setDefaultTimeout(timeout)
         if (url) await page.goto(url)
-        logVerbose(`page ready`)
         return page
     }
 }
diff --git a/packages/core/package.json b/packages/core/package.json
@@ -57,6 +57,7 @@
     "sanitize-html": "^2.13.0",
     "semver": "^7.6.3",
     "serialize-error": "^11.0.3",
+    "tabletojson": "^4.1.4",
     "toml": "^3.0.0",
     "tree-sitter-wasms": "^0.1.11",
     "ts-dedent": "^2.2.0",
@@ -76,4 +77,4 @@
     "@types/turndown": "^5.0.5",
     "turndown": "^7.2.0"
   }
-}
+}
diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/globals.ts b/packages/core/src/globals.ts
@@ -8,7 +8,7 @@ import {
     updateFrontmatter,
 } from "./frontmatter"
 import { JSONLStringify, JSONLTryParse } from "./jsonl"
-import { HTMLToMarkdown, HTMLToText } from "./html"
+import { HTMLTablesToJSON, HTMLToMarkdown, HTMLToText } from "./html"
 
 export function resolveGlobal(): any {
     if (typeof window !== "undefined")
@@ -58,6 +58,7 @@ export function installGlobals() {
         },
     })
     glb.HTML = Object.freeze<HTML>({
+        convertTablesToJSON: HTMLTablesToJSON,
         convertToMarkdown: HTMLToMarkdown,
         convertToText: HTMLToText,
     })

diff --git a/packages/core/src/html.test.ts b/packages/core/src/html.test.ts
@@ -1,8 +1,26 @@
 import test, { describe } from "node:test"
-import { HTMLToMarkdown, HTMLToText } from "./html"
+import { HTMLTablesToJSON, HTMLToMarkdown, HTMLToText } from "./html"
 import assert from "node:assert/strict"
 
 describe("html", () => {
+    test("convert HTML table to JSON", () => {
+        const html = `
+            <table>
+                <tr>
+                    <th>Header 1</th>
+                    <th>Header 2</th>
+                </tr>
+                <tr>
+                    <td>Value 1</td>
+                    <td>Value 2</td>
+                </tr>
+            </table>
+        `
+        const expected = [{ "Header 1": "Value 1", "Header 2": "Value 2" }]
+        const result = HTMLTablesToJSON(html)[0]
+        console.log(JSON.stringify(result, null, 2))
+        assert.deepStrictEqual(result, expected)
+    })
     test("converts HTML to text", () => {
         const html = "<p>Hello, world!</p>"
         const expected = "Hello, world!"

diff --git a/packages/core/src/html.ts b/packages/core/src/html.ts
@@ -1,6 +1,12 @@
-import { convert } from "html-to-text"
+import { convert as convertToText } from "html-to-text"
 import { TraceOptions } from "./trace"
 import Turndown from "turndown"
+import { tabletojson } from "tabletojson"
+
+export function HTMLTablesToJSON(html: string, options?: {}): object[][] {
+    const res = tabletojson.convert(html, options)
+    return res
+}
 
 export function HTMLToText(
     html: string,
@@ -11,7 +17,7 @@
     const { trace } = options || {}
 
     try {
-        const text = convert(html, options)
+        const text = convertToText(html, options)
         return text
     } catch (e) {
         trace?.error("HTML conversion failed", e)

diff --git a/packages/core/src/promptdom.ts b/packages/core/src/promptdom.ts
@@ -255,7 +255,7 @@ export function createFileOutput(output: FileOutput): FileOutputNode {
     return { type: "fileOutput", output }
 }
 
-export function createDefDataNode(
+export function createDefData(
     name: string,
     data: object | object[],
     options?: DefDataOptions

diff --git a/packages/core/src/promptrunner.ts b/packages/core/src/promptrunner.ts
@@ -85,7 +85,7 @@
     assert(model !== undefined)
 
     try {
-        trace.itemValue("🧠 model", model ?? "??")
+        trace.heading(3, `🧠 running ${template.id} with model ${model ?? ""}`)
         if (cliInfo) traceCliArgs(trace, template, options)
 
         const vars = await resolveExpansionVars(

diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts
@@ -3,7 +3,7 @@
     appendChild,
     createAssistantNode,
     createChatParticipant,
-    createDefDataNode,
+    createDefData,
     createDefNode,
     createFileOutput,
     createFunctionNode,
@@ -114,7 +114,7 @@
             return name
         },
         defData: (name, data, defOptions) => {
-            appendChild(node, createDefDataNode(name, data, defOptions))
+            appendChild(node, createDefData(name, data, defOptions))
             return name
         },
         fence(body, options?: DefOptions) {

diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -1156,14 +1156,40 @@ interface XML {
     parse(text: string, options?: XMLParseOptions): any
 }
 
+interface HTMLTableToJSONOptions {
+    useFirstRowForHeadings?: boolean
+    headers?: HeaderRows
+    stripHtmlFromHeadings?: boolean
+    stripHtmlFromCells?: boolean
+    stripHtml?: boolean | null
+    forceIndexAsNumber?: boolean
+    countDuplicateHeadings?: boolean
+    ignoreColumns?: number[] | null
+    onlyColumns?: number[] | null
+    ignoreHiddenRows?: boolean
+    id?: string[] | null
+    headings?: string[] | null
+    containsClasses?: string[] | null
+    limitrows?: number | null
+}
+
 interface HTML {
+    /**
+     * Converts all HTML tables to JSON.
+     * @param html
+     * @param options
+     */
+    convertTablesToJSON(
+        html: string,
+        options?: HTMLTableToJSONOptions
+    ): object[][]
     /**
      * Converts HTML markup to plain text
      * @param html
      */
     convertToText(html: string): string
     /**
-     * Converts HMTL markup to markdown
+     * Converts HTML markup to markdown
      * @param html
      */
     convertToMarkdown(html: string): string

diff --git a/packages/sample/genaisrc/browse-text.genai.mts b/packages/sample/genaisrc/browse-text.genai.mts
@@ -7,8 +7,7 @@ const page = await host.browse(
 )
 const table = page.locator('table[data-testid="csv-table"]')
 const html = await table.innerHTML()
-console.log(`HTML:` + html)
-const csv = HTML.convertToText(html)
-console.log(`TEXT: ` + csv)
-def("DATA", csv)
-$`Analyze DATA.`
+const csv = HTML.convertTablesToJSON("<table>" + html + "</table>")[0]
+csv.forEach((row) => delete row[Object.keys(row)[0]]) // remove the first column
+defData("DATA", csv, { format: "csv" })
+$`Analyze DATA and provide a statistical summary.`
diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts