Add RangeOptions interface for line range specification (#777)

* ✨ feat: add RangeOptions interface with line range

* ✨ feat: add RangeOptions to DefOptions interface

* ✨ feat(core): add extractRange function to liner

* ✨ feat: add defrange script and tests

* feat: add fs_ask_file and improve cost render logic 💡

* 🛠️ refactor: Modify file slicing and tool handling

* ✨ refactor: set default file count & update paths

* documentation update script

* ✨ feat(docs): add caching and update file path pattern

docs/src/content/docs/reference/scripts/system.mdx

@@ -218,6 +218,7 @@ defAgent(
             "md_read_frontmatter",
             "fs_find_files",
             "fs_read_file",
+            "fs_ask_file",
         ],
         maxTokens: 5000,
     }
@@ -858,6 +859,8 @@ system({
     description: "Find files with glob and content regex.",
 })
 
+const findFilesCount = env.vars.fsFindFilesCount || 64
+
 defTool(
     "fs_find_files",
     "Finds file matching a glob pattern. Use pattern to specify a regular expression to search for in the file content. Be careful about asking too many files.",
@@ -888,7 +891,13 @@ defTool(
         required: ["glob"],
     },
     async (args) => {
-        const { glob, pattern, frontmatter, context, count = 20 } = args
+        const {
+            glob,
+            pattern,
+            frontmatter,
+            context,
+            count = findFilesCount,
+        } = args
         context.log(
             `ls ${glob} ${pattern ? `| grep ${pattern}` : ""} ${frontmatter ? "--frontmatter" : ""}`
         )
@@ -898,9 +907,10 @@ defTool(
         if (!res?.length) return "No files found."
 
         let suffix = ""
-        if (res.length > count) {
-            res = res.slice(0, count)
-            suffix = "\n...Too many files found. Showing first 20..."
+        if (res.length > findFilesCount) {
+            res = res.slice(0, findFilesCount)
+            suffix =
+                "\n<too many files found. Showing first 100. Use 'count' to specify how many and/or use 'pattern' to do a grep search>"
         }
 
         if (frontmatter) {
@@ -1883,7 +1893,7 @@ system({
     title: "Tools to help with documentation tasks",
 })
 
-const model = (env.vars.mdSummaryModel = "gpt-4o-mini")
+const model = env.vars.mdSummaryModel || "gpt-4o-mini"
 
 defTool(
     "md_find_files",

packages/core/src/fileedits.ts

@@ -98,17 +98,23 @@ export async function computeFileEdits(
             }
         } else if (/^changelog$/i.test(name) || /^changelog/i.test(language)) {
             changelogs.push(val)
-            const cls = parseChangeLogs(val)
-            for (const changelog of cls) {
-                const { filename } = changelog
-                const fn = /^[^\/]/.test(filename) // TODO
-                    ? runtimeHost.resolvePath(projFolder, filename)
-                    : filename
-                const fileEdit = await getFileEdit(fn)
-                fileEdit.after = applyChangeLog(
-                    fileEdit.after || fileEdit.before || "",
-                    changelog
-                )
+            try {
+                const cls = parseChangeLogs(val)
+                for (const changelog of cls) {
+                    const { filename } = changelog
+                    const fn = /^[^\/]/.test(filename) // TODO
+                        ? runtimeHost.resolvePath(projFolder, filename)
+                        : filename
+                    const fileEdit = await getFileEdit(fn)
+                    fileEdit.after = applyChangeLog(
+                        fileEdit.after || fileEdit.before || "",
+                        changelog
+                    )
+                }
+            } catch (e) {
+                logError(e)
+                trace.error(`error parsing changelog`, e)
+                trace.detailsFenced(`changelog`, val, "text")
             }
         }
     }

packages/core/src/genaisrc/system.agent_docs.genai.mjs

@@ -33,6 +33,7 @@ defAgent(
             "md_read_frontmatter",
             "fs_find_files",
             "fs_read_file",
+            "fs_ask_file",
         ],
         maxTokens: 5000,
     }

packages/core/src/genaisrc/system.fs_find_files.genai.mjs

@@ -3,6 +3,8 @@ system({
     description: "Find files with glob and content regex.",
 })
 
+const findFilesCount = env.vars.fsFindFilesCount || 64
+
 defTool(
     "fs_find_files",
     "Finds file matching a glob pattern. Use pattern to specify a regular expression to search for in the file content. Be careful about asking too many files.",
@@ -33,7 +35,13 @@ defTool(
         required: ["glob"],
     },
     async (args) => {
-        const { glob, pattern, frontmatter, context, count = 20 } = args
+        const {
+            glob,
+            pattern,
+            frontmatter,
+            context,
+            count = findFilesCount,
+        } = args
         context.log(
             `ls ${glob} ${pattern ? `| grep ${pattern}` : ""} ${frontmatter ? "--frontmatter" : ""}`
         )
@@ -43,9 +51,10 @@ defTool(
         if (!res?.length) return "No files found."
 
         let suffix = ""
-        if (res.length > count) {
-            res = res.slice(0, count)
-            suffix = "\n...Too many files found. Showing first 20..."
+        if (res.length > findFilesCount) {
+            res = res.slice(0, findFilesCount)
+            suffix =
+                "\n<too many files found. Showing first 100. Use 'count' to specify how many and/or use 'pattern' to do a grep search>"
         }
 
         if (frontmatter) {

packages/core/src/genaisrc/system.md_find_files.genai.mjs

@@ -2,7 +2,7 @@ system({
     title: "Tools to help with documentation tasks",
 })
 
-const model = (env.vars.mdSummaryModel = "gpt-4o-mini")
+const model = env.vars.mdSummaryModel || "gpt-4o-mini"
 
 defTool(
     "md_find_files",

packages/core/src/globals.ts

@@ -145,4 +145,8 @@ export function installGlobals() {
      * @returns Fetch result.
      */
     glb.fetchText = fetchText // Assign fetchText function to global
+
+    // these are overriden, ignored
+    glb.script = () => {}
+    glb.system = () => {}
 }

packages/core/src/liner.ts

@@ -44,3 +44,21 @@ export function removeLineNumbers(text: string) {
 
     return lines.map((line) => line.replace(rx, "")).join("\n") // Remove line numbers and join lines back
 }
+
+/**
+ * Extracts a line range from a given text using 1-based inclusive line numbers.
+ * @param text
+ * @param options
+ */
+export function extractRange(
+    text: string,
+    options?: { lineStart?: number; lineEnd?: number }
+) {
+    const { lineStart, lineEnd } = options || {}
+    if (isNaN(lineStart) && isNaN(lineEnd)) return text
+
+    const lines = text.split("\n")
+    const startLine = lineStart || 1
+    const endLine = lineEnd || lines.length
+    return lines.slice(startLine - 1, endLine).join("\n")
+}

packages/core/src/promptdom.ts

@@ -1,7 +1,7 @@
 // Importing various utility functions and constants from different modules.
 import { CSVToMarkdown, CSVTryParse } from "./csv"
 import { renderFileContent, resolveFileContent } from "./file"
-import { addLineNumbers } from "./liner"
+import { addLineNumbers, extractRange } from "./liner"
 import { JSONSchemaStringifyToTypeScript } from "./schema"
 import { estimateTokens, truncateTextToTokens } from "./tokens"
 import { MarkdownTrace, TraceOptions } from "./trace"
@@ -210,9 +210,11 @@ export function createDefDiff(
 
 // Function to render a definition node to a string.
 function renderDefNode(def: PromptDefNode): string {
-    const { name, resolved } = def
-    const file = resolved
+    const { name, resolved: file } = def
     const { language, lineNumbers, schema } = def || {}
+
+    file.content = extractRange(file.content, def)
+
     const fence =
         language === "markdown" || language === "mdx"
             ? MARKDOWN_PROMPT_FENCE

packages/core/src/types/prompt_template.d.ts

@@ -810,7 +810,22 @@ interface ContextExpansionOptions {
     ephemeral?: boolean
 }
 
-interface DefOptions extends FenceOptions, ContextExpansionOptions, DataFilter {
+interface RangeOptions {
+    /**
+     * The inclusive start of the line range, with a 1-based index
+     */
+    lineStart?: number
+    /**
+     * The inclusive end of the line range, with a 1-based index
+     */
+    lineEnd?: number
+}
+
+interface DefOptions
+    extends FenceOptions,
+        ContextExpansionOptions,
+        DataFilter,
+        RangeOptions {
     /**
      * Filename filter based on file suffix. Case insensitive.
      */

packages/core/src/usage.ts

@@ -63,6 +63,7 @@ export function estimateCost(modelId: string, usage: ChatCompletionUsage) {
  */
 export function renderCost(value: number) {
     if (isNaN(value)) return ""
+    if (value === 0) return `0$ (cached)`
     return value <= 0.01
         ? `${(value * 100).toFixed(3)}¢`
         : value <= 0.1

packages/sample/genaisrc/defrange.genai.mjs

@@ -0,0 +1,16 @@
+script({
+    model: "small",
+    files: "src/basic.prompty",
+    tests: {
+        keywords: ["CORRECT1", "CORRECT2", "CORRECT3"],
+    },
+})
+
+def("START", env.files, { lineStart: 23 })
+def("END", env.files, { lineEnd: 27 })
+
+def("RANGE", env.files, { lineStart: 23, lineEnd: 27 })
+
+$`Respond CORRECT1 if START start with "system:" otherwise INCORRECT`
+$`Respond CORRECT2 if END end with "user:" otherwise INCORRECT`
+$`Respond CORRECT3 if RANGE start with "system:" and end with "user:" otherwise INCORRECT`

packages/sample/genaisrc/docs-up.genai.mjs

@@ -0,0 +1,47 @@
+script({
+    title: "Pull Request Descriptor",
+    description: "Generate a pull request description from the git diff",
+    temperature: 0.5,
+    tools: ["fs", "md"],
+    system: ["system", "system.files"],
+    cache: "docs-up"
+})
+
+const tip = env.vars.tip
+const defaultBranch = await git.defaultBranch()
+const branch = await git.branch()
+if (branch === defaultBranch) cancel("you are already on the default branch")
+
+// compute diff
+const changes = await git.diff({
+    base: defaultBranch,
+    paths: [
+        "**/prompt_template.d.ts",
+        "**/prompt_type.d.ts",
+        "packages/sample/**",
+    ],
+})
+console.log(changes)
+
+// task
+$`You are an expert software developer and architect.
+
+## Task
+
+- Analyze and summarize the changes in the codebase described in GIT_DIFF in your own dialog and extract a list of impacted public APIs.
+- Find the list of related documentation pages of those APIs that need to be updated.
+- Update the documentation markdown files according to the changes.
+
+## Guidance
+
+${tip || ""}
+- the documentation markdown is located under docs/src/content/docs/**/*.md*
+- do NOT try to call tools within the agents
+- do NOT create new documentation pages
+`
+
+def("GIT_DIFF", changes, { maxTokens: 30000 })
+defFileOutput(
+    "docs/src/content/docs/**/*.md*",
+    "Updated documentation markdown pages"
+)