From a6e2b7404b1eb7b9b1dcec20a3f8749eeac59a4f Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Mon, 10 Jun 2024 21:02:03 +0000 Subject: [PATCH] added JSONL object --- docs/genaisrc/genaiscript.d.ts | 26 ++++++++++++++ genaisrc/genaiscript.d.ts | 26 ++++++++++++++ packages/core/src/genaisrc/genaiscript.d.ts | 26 ++++++++++++++ packages/core/src/promptcontext.ts | 6 ++++ packages/core/src/types/prompt_template.d.ts | 14 ++++++++ packages/core/src/types/prompt_type.d.ts | 12 +++++++ packages/sample/genaisrc/genaiscript.d.ts | 26 ++++++++++++++ packages/sample/genaisrc/multi-turn.genai.js | 34 ++++++++++--------- .../sample/genaisrc/node/genaiscript.d.ts | 26 ++++++++++++++ .../sample/genaisrc/python/genaiscript.d.ts | 26 ++++++++++++++ .../sample/genaisrc/style/genaiscript.d.ts | 26 ++++++++++++++ packages/sample/src/aici/genaiscript.d.ts | 26 ++++++++++++++ packages/sample/src/errors/genaiscript.d.ts | 26 ++++++++++++++ packages/sample/src/makecode/genaiscript.d.ts | 26 ++++++++++++++ packages/sample/src/tla/genaiscript.d.ts | 26 ++++++++++++++ packages/sample/src/vision/genaiscript.d.ts | 26 ++++++++++++++ slides/genaisrc/genaiscript.d.ts | 26 ++++++++++++++ 17 files changed, 388 insertions(+), 16 deletions(-) diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/docs/genaisrc/genaiscript.d.ts +++ b/docs/genaisrc/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/genaisrc/genaiscript.d.ts +++ b/genaisrc/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/core/src/genaisrc/genaiscript.d.ts +++ b/packages/core/src/genaisrc/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/core/src/promptcontext.ts b/packages/core/src/promptcontext.ts index 8f33130ab7..d93ff58fbf 100644 --- a/packages/core/src/promptcontext.ts +++ b/packages/core/src/promptcontext.ts @@ -36,6 +36,7 @@ import { fuzzSearch } from "./fuzzsearch" import { parseModelIdentifier, resolveModelConnectionInfo } from "./models" import { renderAICI } from "./aici" import { MODEL_PROVIDER_AICI } from "./constants" +import { JSONLStringify, JSONLTryParse } from "./jsonl" function stringLikeToFileName(f: string | WorkspaceFile) { return typeof f === "string" ? f : f?.filename @@ -74,6 +75,10 @@ export function createPromptContext( const XML = Object.freeze({ parse: XMLParse, }) + const JSONL = Object.freeze({ + parse: JSONLTryParse, + stringify: JSONLStringify, + }) const AICI = Object.freeze({ gen: (options: AICIGenOptions) => { // validate options @@ -211,6 +216,7 @@ export function createPromptContext( INI, AICI, XML, + JSONL, retrieval, host: promptHost, defOutputProcessor, diff --git a/packages/core/src/types/prompt_template.d.ts b/packages/core/src/types/prompt_template.d.ts index ee2d8c63f0..f699ae8f7d 100644 --- a/packages/core/src/types/prompt_template.d.ts +++ b/packages/core/src/types/prompt_template.d.ts @@ -1021,6 +1021,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1538,6 +1551,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI diff --git a/packages/core/src/types/prompt_type.d.ts b/packages/core/src/types/prompt_type.d.ts index 33a1ede45a..3369c5fafe 100644 --- a/packages/core/src/types/prompt_type.d.ts +++ b/packages/core/src/types/prompt_type.d.ts @@ -1,5 +1,7 @@ /// +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -116,6 +118,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/genaisrc/genaiscript.d.ts b/packages/sample/genaisrc/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/genaisrc/genaiscript.d.ts +++ b/packages/sample/genaisrc/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/genaisrc/multi-turn.genai.js b/packages/sample/genaisrc/multi-turn.genai.js index 1b826081f2..6e349a2a1e 100644 --- a/packages/sample/genaisrc/multi-turn.genai.js +++ b/packages/sample/genaisrc/multi-turn.genai.js @@ -6,9 +6,15 @@ script({ tests: {}, }) + +def("FILE", env.files) +$`Generate a set of questions for the files to build a FAQ.` + + +// turn 2 let turn = 0 defChatParticipant( - async (_, messages) => { + async (ctx, messages) => { turn++ if (turn <= 1) { const text = messages.at(-1).content @@ -18,7 +24,7 @@ defChatParticipant( .map((q) => q.trim()) .filter((q) => q.length > 0) || [] - _.$`Here is the list of answers to the questions in the file. + ctx.$`Here is the list of answers to the questions in the file. ## Task 1: @@ -26,20 +32,19 @@ Validate the quality of the answer. ## Task 2: -Write the question/answers pairs for each file in a ".qt.txt" file -using the following format: +Write the question/answers pairs for each file in a ".qt.jsonl" file +using the JSONL format: \`\`\`\`markdown -File: .qt.txt +File: .qt.jsonl \`\`\` -## - - +{ q: "", a: "" } +{ q: "", a: "" } ... \`\`\` \`\`\`\` - +### Questions: ` for (const question of questions) { @@ -67,16 +72,13 @@ Answer the QUESTION using the contents in FILE. { label: question } ) - _.$` + ctx.$` - question: ${question}` - _.fence(res.text) - _.$`\n\n` + ctx.fence(res.text) + ctx.$`\n\n` } } }, { label: "answerer" } -) - -def("FILE", env.files) -$`Generate a set of questions for the files to build a FAQ. Format one line per question in text.` +) \ No newline at end of file diff --git a/packages/sample/genaisrc/node/genaiscript.d.ts b/packages/sample/genaisrc/node/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/genaisrc/node/genaiscript.d.ts +++ b/packages/sample/genaisrc/node/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/genaisrc/python/genaiscript.d.ts b/packages/sample/genaisrc/python/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/genaisrc/python/genaiscript.d.ts +++ b/packages/sample/genaisrc/python/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/genaisrc/style/genaiscript.d.ts b/packages/sample/genaisrc/style/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/genaisrc/style/genaiscript.d.ts +++ b/packages/sample/genaisrc/style/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/src/aici/genaiscript.d.ts b/packages/sample/src/aici/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/src/aici/genaiscript.d.ts +++ b/packages/sample/src/aici/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/src/errors/genaiscript.d.ts b/packages/sample/src/errors/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/src/errors/genaiscript.d.ts +++ b/packages/sample/src/errors/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/src/makecode/genaiscript.d.ts b/packages/sample/src/makecode/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/src/makecode/genaiscript.d.ts +++ b/packages/sample/src/makecode/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/src/tla/genaiscript.d.ts b/packages/sample/src/tla/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/src/tla/genaiscript.d.ts +++ b/packages/sample/src/tla/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/packages/sample/src/vision/genaiscript.d.ts b/packages/sample/src/vision/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/packages/sample/src/vision/genaiscript.d.ts +++ b/packages/sample/src/vision/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */ diff --git a/slides/genaisrc/genaiscript.d.ts b/slides/genaisrc/genaiscript.d.ts index 5e4aa5f3e5..23d6074e84 100644 --- a/slides/genaisrc/genaiscript.d.ts +++ b/slides/genaisrc/genaiscript.d.ts @@ -1055,6 +1055,19 @@ interface XML { parse(text: string): any } +interface JSONL { + /** + * Parses a JSONL string to an array of objects + * @param text + */ + parse(text: string): any[] + /** + * Converts objects to JSONL format + * @param objs + */ + stringify(objs: any[]): string +} + interface INI { /** * Parses a .ini file @@ -1572,6 +1585,7 @@ interface PromptContext extends ChatGenerationContext { workspace: WorkspaceFileSystem YAML: YAML XML: XML + JSONL: JSONL CSV: CSV INI: INI AICI: AICI @@ -1580,6 +1594,8 @@ interface PromptContext extends ChatGenerationContext { +import { JSONL } from "./prompt_template" + // keep in sync with PromptContext! /** @@ -1696,6 +1712,16 @@ declare var YAML: YAML */ declare var INI: INI +/** + * XML parsing and stringifying. + */ +declare var XML: XML + +/** + * JSONL parsing and stringifying. + */ +declare var JSONL: JSONL + /** * AICI operations */