Enhance file edit handling and update model configurations 📂✨

eval/extrism/genaisrc/extrism-tool.genai.mts

@@ -1,11 +1,9 @@
 script({
     model: "openai:gpt-4o",
     title: "Runs a extrism sample",
-    system: ["system", "system.explanations", "system.python", "system.files"],
 })
 
 const practiceDir = "python/exercises/practice"
-const { sample = "anagram" } = env.vars
 
 // create container
 const container = await host.container({
@@ -23,22 +21,32 @@ await container.exec("pip install -r requirements.txt --user", {
 await container.exec("pip install pytest --user", { cwd: "python" })
 await container.disconnect()
 
-// generate sample
-const cwd = path.join(practiceDir, sample)
-const { files: samplefiles } = JSON.parse(
-    await container.readText(path.join(cwd, ".meta/config.json"))
-)
-const { solution } = samplefiles
-const filename = path.join(cwd, solution[0])
-let instructions = ""
-for (const introname of ["introduction", "instructions", "instructions.app"]) {
-    const intro = await container.readText(
-        path.join(cwd, `.docs/${introname}.md`)
+const samples = (await container.exec("ls -1", { cwd: practiceDir })).stdout
+    .trim()
+    .split(/\n/g)
+for (const sample of samples) {
+    const cwd = path.join(practiceDir, sample)
+    console.log(cwd)
+    const { files: samplefiles } = JSON.parse(
+        await container.readText(path.join(cwd, ".meta/config.json"))
     )
-    if (intro) instructions += intro + "\n\n"
-}
+    const { solution } = samplefiles
+    const filename = path.join(cwd, solution[0])
+    let instructions = ""
+    for (const introname of [
+        "introduction",
+        "instructions",
+        "instructions.app",
+    ]) {
+        const intro = await container.readText(
+            path.join(cwd, `.docs/${introname}.md`)
+        )
+        if (intro) instructions += intro + "\n\n"
+    }
 
-$`
+    await runPrompt(
+        (ctx) => {
+            ctx.$`
 
 ## Task 1:
 
@@ -62,39 +70,53 @@ If the tests fail, update the generated code in Task 1 and re-run the tests in T
 If the tests passed, stop.
 `
 
-def("INSTRUCTIONS", instructions, { language: "markdown" })
-def("TEMPLATE", { filename }, { language: "python" })
-
-let generatedCode = ""
-let testPassed = true
-defTool(
-    "test_code",
-    "Run unit tests against generated solution",
-    {
-        code: {
-            type: "string",
-            description: "Generated Python code to solve the problem",
+            ctx.def("INSTRUCTIONS", instructions, { language: "markdown" })
+            ctx.def("TEMPLATE", { filename }, { language: "python" })
+
+            let generatedCode = ""
+            let testPassed = true
+            ctx.defTool(
+                "test_code",
+                "Run unit tests against generated solution",
+                {
+                    code: {
+                        type: "string",
+                        description:
+                            "Generated Python code to solve the problem",
+                    },
+                },
+                async ({ code }) => {
+                    generatedCode = code
+                    console.log(code)
+                    await container.writeText(filename, code)
+                    const res = await container.exec(
+                        `python3.11 -m pytest -o markers=task ${sample}_test.py`,
+                        { cwd }
+                    )
+                    if (res.exitCode) {
+                        console.log(res.stdout || "")
+                        console.log(res.stderr || "")
+                    }
+                    testPassed = true
+                    return res
+                }
+            )
+
+            ctx.defOutputProcessor(async (res) => {
+                if (!generatedCode) throw new Error("Generated code is missing")
+                if (!testPassed) throw new Error("Unit tests failed")
+                return { text: res.text + "\n\n<VALIDATED>" }
+            })
         },
-    },
-    async ({ code }) => {
-        generatedCode = code
-        console.log(code)
-        await container.writeText(filename, code)
-        const res = await container.exec(
-            `python3.11 -m pytest -o markers=task ${sample}_test.py`,
-            { cwd }
-        )
-        if (res.exitCode) {
-            console.log(res.stdout || "")
-            console.log(res.stderr || "")
+        {
+            label: sample,
+            applyEdits: true,
+            system: [
+                "system",
+                "system.explanations",
+                "system.python",
+                "system.files",
+            ],
         }
-        testPassed = true
-        return res
-    }
-)
-
-defOutputProcessor(async (res) => {
-    if (!generatedCode) throw new Error("Generated code is missing")
-    if (!testPassed) throw new Error("Unit tests failed")
-    return { text: res.text + "\n\n<VALIDATED>" }
-})
+    )
+}

packages/core/src/chat.ts

@@ -47,6 +47,7 @@ import { fenceMD, prettifyMarkdown } from "./markdown"
 import { YAMLStringify } from "./yaml"
 import { resolveTokenEncoder } from "./encoders"
 import { estimateTokens, truncateTextToTokens } from "./tokens"
+import { computeFileEdits } from "./fileedits"
 
 export function toChatCompletionUserMessage(
     expanded: string,
@@ -401,16 +402,19 @@ function assistantText(
     return text
 }
 
-function structurifyChatSession(
+async function structurifyChatSession(
     messages: ChatCompletionMessageParam[],
     schemas: Record<string, JSONSchema>,
     genVars: Record<string, string>,
+    fileOutputs: FileOutput[],
+    outputProcessors: PromptOutputProcessorHandler[],
+    fileMerges: FileMergeHandler[],
     options: GenerationOptions,
     others?: {
         resp?: ChatCompletionResponse
         err?: any
     }
-): RunPromptResult {
+): Promise<RunPromptResult> {
     const { trace, responseType, responseSchema } = options
     const { resp, err } = others || {}
     const text = assistantText(messages, responseType)
@@ -458,7 +462,7 @@ function structurifyChatSession(
     if (fences?.length)
         frames.push(...validateFencesWithSchema(fences, schemas, { trace }))
 
-    return {
+    const res = <RunPromptResult>{
         text,
         annotations,
         finishReason,
@@ -469,6 +473,14 @@ function structurifyChatSession(
         genVars,
         schemas,
     }
+    await computeFileEdits(res, {
+        trace,
+        schemas,
+        fileOutputs,
+        fileMerges,
+        outputProcessors,
+    })
+    return res
 }
 
 async function processChatMessage(
@@ -479,6 +491,9 @@ async function processChatMessage(
     chatParticipants: ChatParticipant[],
     schemas: Record<string, JSONSchema>,
     genVars: Record<string, string>,
+    fileOutputs: FileOutput[],
+    outputProcessors: PromptOutputProcessorHandler[],
+    fileMerges: FileMergeHandler[],
     options: GenerationOptions
 ): Promise<RunPromptResult> {
     const {
@@ -554,9 +569,18 @@ async function processChatMessage(
         if (needsNewTurn) return undefined
     }
 
-    return structurifyChatSession(messages, schemas, genVars, options, {
-        resp,
-    })
+    return structurifyChatSession(
+        messages,
+        schemas,
+        genVars,
+        fileOutputs,
+        outputProcessors,
+        fileMerges,
+        options,
+        {
+            resp,
+        }
+    )
 }
 
 export function mergeGenerationOptions(
@@ -585,6 +609,9 @@ export async function executeChatSession(
     messages: ChatCompletionMessageParam[],
     toolDefinitions: ToolCallback[],
     schemas: Record<string, JSONSchema>,
+    fileOutputs: FileOutput[],
+    outputProcessors: PromptOutputProcessorHandler[],
+    fileMerges: FileMergeHandler[],
     completer: ChatCompletionHandler,
     chatParticipants: ChatParticipant[],
     genOptions: GenerationOptions
@@ -683,6 +710,9 @@ export async function executeChatSession(
                     chatParticipants,
                     schemas,
                     genVars,
+                    fileOutputs,
+                    outputProcessors,
+                    fileMerges,
                     genOptions
                 )
                 if (output) return output
@@ -691,6 +721,9 @@ export async function executeChatSession(
                     messages,
                     schemas,
                     genVars,
+                    fileOutputs,
+                    outputProcessors,
+                    fileMerges,
                     genOptions,
                     { resp, err }
                 )

packages/core/src/fileedits.ts

@@ -22,11 +22,7 @@ export async function computeFileEdits(
         fileMerges?: FileMergeHandler[]
         outputProcessors?: PromptOutputProcessorHandler[]
     }
-): Promise<{
-    edits: Edits[]
-    fileEdits: Record<string, FileUpdate>
-    changelogs: string[]
-}> {
+): Promise<void> {
     const { trace, fileOutputs, fileMerges, outputProcessors, schemas } =
         options || {}
     const { fences, frames, genVars } = res
@@ -197,7 +193,11 @@ export async function computeFileEdits(
             })
         )
 
-    return { fileEdits, changelogs, edits }
+    res.text = text
+    res.fileEdits = fileEdits
+    res.changelogs = changelogs
+    res.annotations = annotations
+    res.edits = edits
 }
 
 // Validate file outputs against specified schemas and patterns

packages/core/src/promptcontext.ts

@@ -193,11 +193,6 @@ export async function createPromptContext(
         },
     }
 
-    // Default output processor for the prompt
-    const defOutputProcessor = (fn: PromptOutputProcessorHandler) => {
-        if (fn) appendPromptChild(createOutputProcessor(fn))
-    }
-
     // Define the host for executing commands, browsing, and other operations
     const promptHost: PromptHost = Object.freeze<PromptHost>({
         exec: async (
@@ -263,10 +258,6 @@ export async function createPromptContext(
         parsers,
         retrieval,
         host: promptHost,
-        defOutputProcessor,
-        defFileMerge: (fn) => {
-            appendPromptChild(createFileMerge(fn))
-        },
     }
     env.generator = ctx
     ctx.env = Object.freeze(env)