added JSONL object

microsoft · Jun 10, 2024 · a6e2b74 · a6e2b74
1 parent 19ea829
commit a6e2b74
Show file tree

Hide file tree

Showing 17 changed files with 388 additions and 16 deletions.
diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts
diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts
@@ -36,6 +36,7 @@ import { fuzzSearch } from "./fuzzsearch"
 import { parseModelIdentifier, resolveModelConnectionInfo } from "./models"
 import { renderAICI } from "./aici"
 import { MODEL_PROVIDER_AICI } from "./constants"
+import { JSONLStringify, JSONLTryParse } from "./jsonl"
 
 function stringLikeToFileName(f: string | WorkspaceFile) {
     return typeof f === "string" ? f : f?.filename
@@ -74,6 +75,10 @@ export function createPromptContext(
     const XML = Object.freeze<XML>({
         parse: XMLParse,
     })
+    const JSONL = Object.freeze<JSONL>({
+        parse: JSONLTryParse,
+        stringify: JSONLStringify,
+    })
     const AICI = Object.freeze<AICI>({
         gen: (options: AICIGenOptions) => {
             // validate options
@@ -211,6 +216,7 @@ export function createPromptContext(
         INI,
         AICI,
         XML,
+        JSONL,
         retrieval,
         host: promptHost,
         defOutputProcessor,

diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts
@@ -1021,6 +1021,19 @@ interface XML {
     parse(text: string): any
 }
 
+interface JSONL {
+    /**
+     * Parses a JSONL string to an array of objects
+     * @param text 
+     */
+    parse(text: string): any[]
+    /**
+     * Converts objects to JSONL format
+     * @param objs 
+     */
+    stringify(objs: any[]): string
+}
+
 interface INI {
     /**
      * Parses a .ini file
@@ -1538,6 +1551,7 @@ interface PromptContext extends ChatGenerationContext {
     workspace: WorkspaceFileSystem
     YAML: YAML
     XML: XML
+    JSONL: JSONL
     CSV: CSV
     INI: INI
     AICI: AICI

diff --git a/packages/core/src/types/prompt_type.d.ts b/packages/core/src/types/prompt_type.d.ts
@@ -1,5 +1,7 @@
 /// <reference path="./prompt_template.d.ts"/>
 
+import { JSONL } from "./prompt_template"
+
 // keep in sync with PromptContext!
 
 /**
@@ -116,6 +118,16 @@ declare var YAML: YAML
  */
 declare var INI: INI
 
+/**
+ * XML parsing and stringifying.
+ */
+declare var XML: XML
+
+/**
+ * JSONL parsing and stringifying.
+ */
+declare var JSONL: JSONL
+
 /**
  * AICI operations
  */

diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts
diff --git a/packages/sample/genaisrc/multi-turn.genai.js b/packages/sample/genaisrc/multi-turn.genai.js
@@ -6,9 +6,15 @@ script({
     tests: {},
 })
 
+
+def("FILE", env.files)
+$`Generate a set of questions for the files to build a FAQ.`
+
+
+// turn 2
 let turn = 0
 defChatParticipant(
-    async (_, messages) => {
+    async (ctx, messages) => {
         turn++
         if (turn <= 1) {
             const text = messages.at(-1).content
@@ -18,28 +24,27 @@ defChatParticipant(
                     .map((q) => q.trim())
                     .filter((q) => q.length > 0) || []
 
-            _.$`Here is the list of answers to the questions in the file. 
+            ctx.$`Here is the list of answers to the questions in the file. 
             
 ## Task 1:
 
 Validate the quality of the answer.
 
 ## Task 2:
 
-Write the question/answers pairs for each file in a "<filename>.qt.txt" file
-using the following format:
+Write the question/answers pairs for each file in a "<filename>.qt.jsonl" file
+using the JSONL format:
 
 \`\`\`\`markdown
-File: <filename>.qt.txt
+File: <filename>.qt.jsonl
 \`\`\`
-## <question>
-
-<answer>
+{ q: "<question1>", a: "<answer1>" }
+{ q: "<question2>", a: "<answer2>" }
 ...
 \`\`\`
 \`\`\`\`
 
-
+### Questions:
             `
 
             for (const question of questions) {
@@ -67,16 +72,13 @@ Answer the QUESTION using the contents in FILE.
                     { label: question }
                 )
 
-                _.$`
+                ctx.$`
             
 - question: ${question}`
-                _.fence(res.text)
-                _.$`\n\n`
+                ctx.fence(res.text)
+                ctx.$`\n\n`
             }
         }
     },
     { label: "answerer" }
-)
-
-def("FILE", env.files)
-$`Generate a set of questions for the files to build a FAQ. Format one line per question in text.`
+)