generate json schema (json) files as part of run

packages/cli/src/llamaindexretrieval.ts

@@ -28,6 +28,7 @@ import {
     JSON_MIME_TYPE,
     PDF_MIME_TYPE,
     DOCX_MIME_TYPE,
+    JSON_SCHEMA_MIME_TYPE,
 } from "genaiscript-core"
 import { type BaseReader, OllamaEmbedding } from "llamaindex"
 import type { GenericFileSystem } from "@llamaindex/env"
@@ -90,6 +91,7 @@ export class LlamaIndexRetrievalService
             "text/plain": new this.module.TextFileReader(),
             [JAVASCRIPT_MIME_TYPE]: new this.module.TextFileReader(),
             [JSON_MIME_TYPE]: new this.module.TextFileReader(),
+            [JSON_SCHEMA_MIME_TYPE]: new this.module.TextFileReader(),
             [PDF_MIME_TYPE]: new this.module.PDFReader(),
             "text/markdown": new this.module.MarkdownReader(),
             [DOCX_MIME_TYPE]: new this.module.DocxReader(),

packages/cli/src/run.ts

@@ -26,8 +26,9 @@ import {
     resolveModelConnectionInfo,
     CONFIGURATION_ERROR_CODE,
     parseKeyValuePairs,
-    stringifySchemaToTypeScript,
+    JSONSchemaStringifyToTypeScript,
     filePathOrUrlToWorkspaceFile,
+    JSONSchemaStringify,
 } from "genaiscript-core"
 import { capitalize } from "inflection"
 import { basename, resolve, join } from "node:path"
@@ -304,16 +305,17 @@ ${Array.from(files)
         }
         if (res.schemas) {
             for (const [sname, schema] of Object.entries(res.schemas)) {
-                const schemaFile = join(
-                    out,
-                    `${sname.toLocaleLowerCase()}.schema.d.ts`
-                )
                 await writeText(
-                    schemaFile,
-                    stringifySchemaToTypeScript(schema, {
+                    join(out, `${sname.toLocaleLowerCase()}.schema.ts`),
+                    JSONSchemaStringifyToTypeScript(schema, {
                         typeName: capitalize(sname),
+                        export: true,
                     })
                 )
+                await writeText(
+                    join(out, `${sname.toLocaleLowerCase()}.schema.json`),
+                    JSONSchemaStringify(schema)
+                )
             }
         }
         if (annotationf) {

packages/core/src/constants.ts

@@ -113,4 +113,7 @@ export const PDF_MIME_TYPE = "application/pdf"
 export const DOCX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
 export const XLSX_MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 export const JSON_MIME_TYPE = "application/json"
-export const JAVASCRIPT_MIME_TYPE = "application/javascript"
+export const JSON_SCHEMA_MIME_TYPE = "application/schema+json"
+export const JAVASCRIPT_MIME_TYPE = "application/javascript"
+
+export const JSON_META_SCHEMA_URI = "https://json-schema.org/draft/2020-12/schema"

packages/core/src/expander.ts

@@ -1,5 +1,5 @@
 import { Fragment, Project, PromptScript } from "./ast"
-import { assert, normalizeFloat, normalizeInt, normalizeString } from "./util"
+import { assert, normalizeFloat, normalizeInt } from "./util"
 import { MarkdownTrace } from "./trace"
 import { errorMessage, isCancelError } from "./error"
 import { estimateTokens } from "./tokens"
@@ -21,9 +21,8 @@ import {
     toChatCompletionUserMessage,
 } from "./chat"
 import { importPrompt } from "./importprompt"
-import { lookupMime } from "./mime"
 import { parseModelIdentifier } from "./models"
-import { stringifySchemaToTypeScript } from "./schema"
+import { JSONSchemaStringifyToTypeScript } from "./schema"
 
 const defaultTopP: number = undefined
 const defaultSeed: number = undefined
@@ -387,7 +386,7 @@ export async function expandTemplate(
     if (responseSchema) {
         responseType = "json_object"
         const typeName = "Output"
-        const schemaTs = stringifySchemaToTypeScript(responseSchema, {
+        const schemaTs = JSONSchemaStringifyToTypeScript(responseSchema, {
             typeName,
         })
         messages.unshift({

packages/core/src/parameters.ts

@@ -1,4 +1,3 @@
-import { assert } from "node:console"
 import { Project } from "./ast"
 import { NotSupportedError } from "./error"
 import { resolveSystems } from "./expander"

packages/core/src/promptdom.ts

@@ -1,7 +1,7 @@
 import { CSVToMarkdown, CSVTryParse } from "./csv"
 import { resolveFileContent } from "./file"
 import { addLineNumbers } from "./liner"
-import { stringifySchemaToTypeScript } from "./schema"
+import { JSONSchemaStringifyToTypeScript } from "./schema"
 import { estimateTokens } from "./tokens"
 import { MarkdownTrace, TraceOptions } from "./trace"
 import { assert, toStringList, trimNewlines } from "./util"
@@ -548,7 +548,7 @@ export async function renderPromptNode(
                     schemaText = YAMLStringify(schema)
                     break
                 default:
-                    schemaText = stringifySchemaToTypeScript(schema, {
+                    schemaText = JSONSchemaStringifyToTypeScript(schema, {
                         typeName: schemaName,
                     })
                     break

packages/core/src/retrieval.ts

@@ -2,6 +2,7 @@ import {
     DOCX_MIME_TYPE,
     JAVASCRIPT_MIME_TYPE,
     JSON_MIME_TYPE,
+    JSON_SCHEMA_MIME_TYPE,
     PDF_MIME_TYPE,
 } from "./constants"
 import {
@@ -16,6 +17,7 @@ const UPSERTFILE_MIME_TYPES = [
     PDF_MIME_TYPE,
     DOCX_MIME_TYPE,
     JSON_MIME_TYPE,
+    JSON_SCHEMA_MIME_TYPE,
     JAVASCRIPT_MIME_TYPE,
 ]
 

packages/core/src/schema.test.ts

@@ -1,6 +1,6 @@
 import { describe, test } from "node:test"
 import assert from "node:assert/strict"
-import { stringifySchemaToTypeScript } from "./schema"
+import { JSONSchemaStringifyToTypeScript } from "./schema"
 
 describe("schema", () => {
     test("cities", () => {
@@ -30,7 +30,7 @@ describe("schema", () => {
             },
         }
 
-        const ts = stringifySchemaToTypeScript(source, { typeName: "Foo" })
+        const ts = JSONSchemaStringifyToTypeScript(source, { typeName: "Foo" })
      //   console.log(ts)
         assert.equal(
             ts,
@@ -68,7 +68,7 @@ of the city.`,
                 required: ["name", "url"],
             }
 
-            const ts = stringifySchemaToTypeScript(source)
+            const ts = JSONSchemaStringifyToTypeScript(source)
           //  console.log(ts)
             assert.equal(
                 ts,

packages/core/src/schema.ts

@@ -10,16 +10,18 @@ export function isJSONSchema(obj: any) {
     return false
 }
 
-export function stringifySchemaToTypeScript(
+export function JSONSchemaStringifyToTypeScript(
     schema: JSONSchema,
-    options?: { typeName?: string }
+    options?: { typeName?: string; export?: boolean }
 ) {
     const { typeName = "Response" } = options || {}
     let lines: string[] = []
     let indent = 0
 
     appendJsDoc(schema.description)
-    append(`type ${typeName.replace(/\s+/g, "_")} =`)
+    append(
+        `${options?.export ? "export " : ""}type ${typeName.replace(/\s+/g, "_")} =`
+    )
     stringifyNode(schema)
     return lines.join("\n")
 
@@ -171,3 +173,15 @@ export function validateFencesWithSchema(
     }
     return frames
 }
+
+export function JSONSchemaStringify(schema: JSONSchema) {
+    return JSON.stringify(
+        {
+            $schema:
+                schema.$schema ?? "http://json-schema.org/draft-07/schema#",
+            ...schema,
+        },
+        null,
+        2
+    )
+}

packages/core/src/types/prompt_template.d.ts

@@ -617,6 +617,7 @@ interface JSONSchemaBoolean {
 }
 
 interface JSONSchemaObject {
+    $schema?: string
     type: "object"
     description?: string
     properties?: {
@@ -627,6 +628,7 @@ interface JSONSchemaObject {
 }
 
 interface JSONSchemaArray {
+    $schema?: string
     type: "array"
     description?: string
     items?: JSONSchemaType