added JSONL parser

microsoft · Jun 10, 2024 · 19ea829 · 19ea829
1 parent 524ca79
commit 19ea829
Show file tree

Hide file tree

Showing 19 changed files with 123 additions and 2 deletions.
diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts
diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md
@@ -115,7 +115,7 @@ Options:
   -td, --test-delay <string>          delay between tests in seconds
   --no-cache                          disable LLM result cache
   -v, --verbose                       verbose output
-  -pv, --promptfoo-version [version]  promptfoo version, default is ^0.62.1
+  -pv, --promptfoo-version [version]  promptfoo version, default is ^0.63.0
   -os, --out-summary <file>           append output summary in file
   -h, --help                          display help for command
 ```

diff --git a/docs/src/content/docs/reference/scripts/parsers.md b/docs/src/content/docs/reference/scripts/parsers.md
@@ -83,6 +83,21 @@ To parse, use `parsers.TOML`. It supports both a text content or a file as input
 const res = parsers.TOML("...")
 ```
 
+## JSONL
+
+JSON**L** is a format that stores JSON objects in a line-by-line format. Each line is a valid JSON(5) object (we use the JSON5 parser to be more error resilient).
+
+```jsonl title="data.jsonl"
+{"name": "Alice"}
+{"name": "Bob"}
+```
+
+You can use `parsers.JSONL` to parse the JSONL files into an array of object (`any[]`).
+
+```js
+const res = parsers.JSONL(file)
+```
+
 ## XML
 
 The `parsers.XML` function parses for the [XML format](https://en.wikipedia.org/wiki/XML).

diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/jsonl.ts b/packages/core/src/jsonl.ts
@@ -38,7 +38,7 @@ export async function readJSONL(fn: string) {
     return res
 }
 
-export function JSONLParse(
+export function JSONLTryParse(
     text: string,
     options?: {
         repair?: boolean

diff --git a/packages/core/src/parsers.test.ts b/packages/core/src/parsers.test.ts
@@ -24,6 +24,12 @@ describe("parsers", () => {
         assert.deepStrictEqual(result, { key: "value" })
     })
 
+    test("JSONL", () => {
+        const result = parsers.JSONL('{"key": "value"}\n{"key2": "value2"}')
+        assert.deepStrictEqual(result[0], { key: "value" })
+        assert.deepStrictEqual(result[1], { key2: "value2" })
+    })
+
     test("YAML", () => {
         const result = parsers.YAML("key: value")
         assert.deepStrictEqual(result, { key: "value" })

diff --git a/packages/core/src/parsers.ts b/packages/core/src/parsers.ts
@@ -20,6 +20,7 @@ import { validateJSONWithSchema } from "./schema"
 import { XLSXTryParse } from "./xlsx"
 import { host } from "./host"
 import { unzip } from "./zip"
+import { JSONLTryParse } from "./jsonl"
 
 export function createParsers(options: {
     trace: MarkdownTrace
@@ -29,6 +30,7 @@ export function createParsers(options: {
     return Object.freeze<Parsers>({
         JSON5: (text, options) =>
             JSON5TryParse(filenameOrFileToContent(text), options?.defaultValue),
+        JSONL: (text) => JSONLTryParse(filenameOrFileToContent(text)),
         YAML: (text, options) =>
             YAMLTryParse(filenameOrFileToContent(text), options?.defaultValue),
         XML: (text, options) => {

diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -796,6 +796,13 @@ interface Parsers {
         content: string | WorkspaceFile,
         options?: { defaultValue?: any }
     ): any | undefined
+
+    /**
+     * Parses text or file as a JSONL payload. Empty lines are ignore, and JSON5 is used for parsing.
+     * @param content
+     */
+    JSONL(content: string | WorkspaceFile): any[] | undefined
+
     /**
      * Parses text as a YAML paylaod
      */

diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts
diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts
diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts
diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts
diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts
diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts
diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts
diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts
diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts
diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts