feat: add jobUrl param, optimize memory queries 🛠️

microsoft · Oct 16, 2024 · 40fc22d · 40fc22d
1 parent 25a44c6
commit 40fc22d
Show file tree

Hide file tree

Showing 8 changed files with 40 additions and 10 deletions.
diff --git a/docs/src/content/docs/reference/scripts/system.mdx b/docs/src/content/docs/reference/scripts/system.mdx
@@ -320,6 +320,7 @@ defAgent(
     `Your are a helpfull LLM agent that can query GitHub to accomplish tasks. Answer the question in QUERY.
     - Prefer diffing job logs rather downloading entire logs which can be very large.
     - Always return sha, head_sha information for runs
+    - do NOT return full job logs, they are too large and will fill the response buffer.
     `,
     {
         model,

diff --git a/packages/core/src/chat.ts b/packages/core/src/chat.ts
@@ -694,6 +694,7 @@ export async function executeChatSession(
                                 }
                               : undefined,
                 }
+                if (model === "gpt-4-32k") delete req.max_completion_tokens
                 try {
                     trace.startDetails(`📤 llm request`)
                     resp = await completer(

diff --git a/packages/core/src/genaisrc/system.agent_github.genai.mjs b/packages/core/src/genaisrc/system.agent_github.genai.mjs
@@ -10,6 +10,7 @@ defAgent(
     `Your are a helpfull LLM agent that can query GitHub to accomplish tasks. Answer the question in QUERY.
     - Prefer diffing job logs rather downloading entire logs which can be very large.
     - Always return sha, head_sha information for runs
+    - do NOT return full job logs, they are too large and will fill the response buffer.
     `,
     {
         model,

diff --git a/packages/core/src/genaisrc/system.files.genai.js b/packages/core/src/genaisrc/system.files.genai.js
@@ -25,6 +25,7 @@ $`- Make sure to use precisely \`\`\` to guard file code sections.
 - Always sure to use precisely \`\`\`\`\` to guard file markdown sections.
 - Use full path of filename in code section header.
 - do NOT use ## headers with filename
+- do NOT emit "preferred_output_format=FILE" when responding to an LLM.
 `
 if (folder !== ".")
     $`When generating new files, place files in folder "${folder}".`

diff --git a/packages/core/src/git.ts b/packages/core/src/git.ts
@@ -22,6 +22,7 @@ import { resolveTokenEncoder } from "./encoders"
 export class GitClient implements Git {
     readonly git = "git" // Git command identifier
     private _defaultBranch: string // Stores the default branch name
+    private _branch: string // Stores the current branch name
     private async resolveExcludedPaths(options?: {
         excludedPaths?: ElementOrArray<string>
     }): Promise<string[]> {
@@ -55,8 +56,11 @@ export class GitClient implements Git {
      * @returns
      */
     async branch(): Promise<string> {
-        const res = await this.exec(["branch", "--show-current"])
-        return res.trim()
+        if (!this._branch) {
+            const res = await this.exec(["branch", "--show-current"])
+            this._branch = res.trim()
+        }
+        return this._branch
     }
 
     async listBranches(): Promise<string[]> {

diff --git a/packages/core/src/promptdom.ts b/packages/core/src/promptdom.ts
@@ -246,9 +246,7 @@ function renderDefNode(def: PromptDefNode): string {
     const diffFormat =
         body.length > 500
             ? " preferred_output_format=CHANGELOG "
-            : body.length < 200
-              ? " preferred_output_format=FILE "
-              : ""
+            : ""
     const res =
         (name ? name + ":\n" : "") +
         dfence +

diff --git a/packages/core/src/runpromptcontext.ts b/packages/core/src/runpromptcontext.ts
@@ -19,7 +19,10 @@ import {
 } from "./promptdom"
 import { MarkdownTrace } from "./trace"
 import { GenerationOptions } from "./generation"
-import { promptParametersSchemaToJSONSchema } from "./parameters"
+import {
+    parametersToVars,
+    promptParametersSchemaToJSONSchema,
+} from "./parameters"
 import { consoleLogFormat } from "./logging"
 import { isGlobMatch } from "./glob"
 import { arrayify, logError, logVerbose } from "./util"
@@ -62,6 +65,7 @@ import { dedent } from "./indent"
 import { runtimeHost } from "./host"
 import { writeFileEdits } from "./fileedits"
 import { agentAddMemory, agentQueryMemory, defMemory } from "./agent"
+import { shellQuote } from "./shell"
 
 export function createChatTurnGenerationContext(
     options: GenerationOptions,
@@ -353,12 +357,22 @@ export function createChatGenerationContext(
                 required: ["query"],
             },
             async (args) => {
-                const { context, query } = args
-                infoCb?.({ text: `${agentLabel}: ${query}` })
+                // the LLM automatically adds extract arguments to the context
+                const { context, ...rest } = args
+                const { query, ...argsNoQuery } = rest
+                infoCb?.({
+                    text: `${agentLabel}: ${query} ${parametersToVars(argsNoQuery)}`,
+                })
 
+                const hasExtraArgs = Object.keys(argsNoQuery).length > 0
                 let memoryAnswer: string
                 if (memory && query && !disableMemoryQuery)
-                    memoryAnswer = await agentQueryMemory(ctx, query)
+                    memoryAnswer = await agentQueryMemory(
+                        ctx,
+                        query + hasExtraArgs
+                            ? `\n${YAML.stringify(argsNoQuery)}`
+                            : ""
+                    )
 
                 const res = await ctx.runPrompt(
                     async (_) => {
@@ -374,6 +388,11 @@ export function createChatGenerationContext(
                         if (memoryAnswer)
                             _.$`- The QUERY applied to the agent memory is in MEMORY.`
                         _.def("QUERY", query)
+                        if (Object.keys(argsNoQuery).length)
+                            _.defData("QUERY_CONTEXT", argsNoQuery, {
+                                format: "yaml",
+                            })
+
                         if (memoryAnswer) _.def("MEMORY", memoryAnswer)
                         if (memory)
                             _.defOutputProcessor(async ({ text }) => {

diff --git a/packages/sample/genaisrc/github-agent.genai.mts b/packages/sample/genaisrc/github-agent.genai.mts
@@ -7,6 +7,7 @@ script({
         "agent_docs",
     ],
     parameters: {
+        jobUrl: { type: "string" }, // URL of the job
         workflow: { type: "string" }, // Workflow name
         failure_run_id: { type: "number" }, // ID of the failed run
         branch: { type: "string" }, // Branch name
@@ -17,9 +18,13 @@ const {
     workflow = "build.yml",
     failure_run_id,
     branch = await git.branch(),
+    jobUrl,
 } = env.vars
 
-if (failure_run_id) {
+if (jobUrl) {
+    $`1. Extract the run id and job id from the  ${jobUrl}`
+    $`2. Find the last successful run before the failed run for the same workflow and branch`
+} else if (failure_run_id) {
     $`1. Find the failed run ${failure_run_id} of ${workflow} for branch ${branch}
     2. Find the last successful run before the failed run for the same workflow and branch`
 } else {