Add 'convert' command for GenAIScript processing

.github/workflows/ollama.yml

@@ -31,7 +31,10 @@ jobs:
             - name: start ollama
               run: yarn ollama:start
             - name: run summarize-ollama-phi3
-              run: yarn test:summarize --model ollama:phi3.5 --out ./temp/summarize-ollama-phi3
+              run: yarn test:summarize --model ollama:phi3.5:latest --out ./temp/summarize-ollama-phi3
               env:
-                  OLLAMA_HOST: "http://localhost:11434"                  
-
+                  OLLAMA_HOST: "http://localhost:11434"
+            - name: run convert-ollama-phi3
+              run: yarn cli convert summarize --model ollama:phi3.5:latest "packages/sample/src/rag/*.md" --cache-name sum
+              env:
+                  OLLAMA_HOST: "http://localhost:11434"

.gitignore

@@ -31,3 +31,4 @@ packages/core/*.temp.*
 packages/sample/test.txt
 packages/sample/poems/*.txt
 packages/sample/src/rag/markdown.md.txt
+*.genai.md

docs/src/content/docs/reference/cli/commands.md

@@ -46,10 +46,10 @@
   -tp, --top-p <number>                      top-p for the run
   -mt, --max-tokens <number>                 maximum completion tokens for the run
   -mdr, --max-data-repairs <number>          maximum data repairs
   -mtc, --max-tool-calls <number>            maximum tool calls for the run
   -se, --seed <number>                       seed for the run
   -em, --embeddings-model <string>           embeddings model for the run
-  --cache                                    enable LLM result cache
+  -c, --cache                                enable LLM result cache
   -cn, --cache-name <name>                   custom cache file name
   -cs, --csv-separator <string>              csv separator (default: "\t")
   -ff, --fence-format <string>               fence format (choices: "xml", "markdown", "none")
@@ -126,9 +126,43 @@
 Launch test viewer
 
 Options:
   -h, --help  display help for command
 ```
 
+## `convert`
+
+```
+Usage: genaiscript convert [options] <script> [files...]
+
+Converts file through a GenAIScript. Each file is processed separately through
+the GenAIScript and the LLM output is saved to a <filename>.genai.md (or custom
+suffix).
+
+Options:
+  -s, --suffix <string>              suffix for output files (default:
+                                     ".genai.md")
+  -ef, --excluded-files <string...>  excluded files
+  -egi, --exclude-git-ignore         exclude files that are ignored through the
+                                     .gitignore file in the workspace root
+  -m, --model <string>               'large' model alias (default)
+  -sm, --small-model <string>        'small' alias model
+  -vm, --vision-model <string>       'vision' alias model
+  -ma, --model-alias <nameid...>     model alias as name=modelid
+  -ft, --fallback-tools              Enable prompt-based tools instead of
+                                     builtin LLM tool calling builtin tool
+                                     calls
+  -o, --out <string>                 output folder. Extra markdown fields for
+                                     output and trace will also be generated
+  --vars <namevalue...>              variables, as name=value, stored in
+                                     env.vars. Use environment variables
+                                     GENAISCRIPT_VAR_name=value to pass
+                                     variable through the environment
+  -c, --cache                        enable LLM result cache
+  -cn, --cache-name <name>           custom cache file name
+  -cc, --concurrency <number>        number of concurrent conversions
+  -h, --help                         display help for command
+```
+
 ## `scripts`
 
 ```

packages/cli/src/api.ts

@@ -34,10 +34,7 @@ export async function run(
          */
         signal?: AbortSignal
     }
-): Promise<{
-    exitCode: number
-    result?: GenerationResult
-}> {
+): Promise<GenerationResult> {
     if (!scriptId) throw new Error("scriptId is required")
     if (typeof files === "string") files = [files]
 

packages/cli/src/cli.ts

@@ -37,6 +37,7 @@ import {
     OPENAI_MAX_RETRY_DELAY,
     OPENAI_RETRY_DEFAULT_DEFAULT,
     OPENAI_MAX_RETRY_COUNT,
+    GENAI_MD_EXT,
 } from "../../core/src/constants" // Core constants
 import {
     errorMessage,
@@ -47,6 +48,7 @@ import {
 import { CORE_VERSION, GITHUB_REPO } from "../../core/src/version" // Core version and repository info
 import { logVerbose } from "../../core/src/util" // Utility logging
 import { semverSatisfies } from "../../core/src/semver" // Semantic version checking
+import { convertFiles } from "./convert"
 
 /**
  * Main function to initialize and run the CLI.
@@ -177,7 +179,7 @@ export async function cli() {
             "-em, --embeddings-model <string>",
             "embeddings model for the run"
         )
-        .option("--cache", "enable LLM result cache")
+        .option("-c, --cache", "enable LLM result cache")
         .option("-cn, --cache-name <name>", "custom cache file name")
         .option("-cs, --csv-separator <string>", "csv separator", "\t")
         .addOption(
@@ -245,6 +247,43 @@ export async function cli() {
         .description("Launch test viewer")
         .action(scriptTestsView) // Action to view the tests
 
+    program
+        .command("convert")
+        .description(
+            "Converts file through a GenAIScript. Each file is processed separately through the GenAIScript and the LLM output is saved to a <filename>.genai.md (or custom suffix)."
+        )
+        .arguments("<script> [files...]")
+        .option(
+            "-s, --suffix <string>",
+            "suffix for output files",
+            GENAI_MD_EXT
+        )
+        .option("-ef, --excluded-files <string...>", "excluded files")
+        .option(
+            "-egi, --exclude-git-ignore",
+            "exclude files that are ignored through the .gitignore file in the workspace root"
+        )
+        .option("-m, --model <string>", "'large' model alias (default)")
+        .option("-sm, --small-model <string>", "'small' alias model")
+        .option("-vm, --vision-model <string>", "'vision' alias model")
+        .option("-ma, --model-alias <nameid...>", "model alias as name=modelid")
+        .option(
+            "-ft, --fallback-tools",
+            "Enable prompt-based tools instead of builtin LLM tool calling builtin tool calls"
+        )
+        .option(
+            "-o, --out <string>",
+            "output folder. Extra markdown fields for output and trace will also be generated"
+        )
+        .option(
+            "--vars <namevalue...>",
+            "variables, as name=value, stored in env.vars. Use environment variables GENAISCRIPT_VAR_name=value to pass variable through the environment"
+        )
+        .option("-c, --cache", "enable LLM result cache")
+        .option("-cn, --cache-name <name>", "custom cache file name")
+        .option("-cc, --concurrency <number>", "number of concurrent conversions")
+        .action(convertFiles)
+
     // Define 'scripts' command group for script management tasks
     const scripts = program
         .command("scripts")
@@ -281,7 +320,7 @@ export async function cli() {
 
     // Define 'cache' command for cache management
     const cache = program.command("cache").description("Cache management")
-    const clear = cache
+    cache
         .command("clear")
         .description("Clear cache")
         .argument("[name]", "Name of the cache, tests")

packages/cli/src/convert.ts

@@ -0,0 +1,129 @@
+import {
+    FILES_NOT_FOUND_ERROR_CODE,
+    GENAI_ANY_REGEX,
+    GENAI_MD_EXT,
+    HTTPS_REGEX,
+} from "../../core/src/constants"
+import { filePathOrUrlToWorkspaceFile, tryReadText } from "../../core/src/fs"
+import { host } from "../../core/src/host"
+import { MarkdownTrace, TraceOptions } from "../../core/src/trace"
+import {
+    logError,
+    logInfo,
+    logVerbose,
+    normalizeInt,
+} from "../../core/src/util"
+import { buildProject } from "./build"
+import { run } from "./api"
+import { writeText } from "../../core/src/fs"
+import { PromptScriptRunOptions } from "./main"
+import { PLimitPromiseQueue } from "../../core/src/concurrency"
+import { createPatch } from "diff"
+
+export async function convertFiles(
+    scriptId: string,
+    fileGlobs: string[],
+    options: Partial<PromptScriptRunOptions> & {
+        suffix?: string
+        concurrency?: string
+    } & TraceOptions
+): Promise<void> {
+    const {
+        trace = new MarkdownTrace(),
+        excludedFiles,
+        excludeGitIgnore,
+        suffix = GENAI_MD_EXT,
+        concurrency,
+        ...restOptions
+    } = options || {}
+    const fail = (msg: string, exitCode: number, url?: string) => {
+        throw new Error(msg)
+    }
+    const { resolve } = host.path
+
+    const toolFiles: string[] = []
+    if (GENAI_ANY_REGEX.test(scriptId)) toolFiles.push(scriptId)
+    const prj = await buildProject({
+        toolFiles,
+    })
+    const script = prj.scripts.find(
+        (t) =>
+            t.id === scriptId ||
+            (t.filename &&
+                GENAI_ANY_REGEX.test(scriptId) &&
+                resolve(t.filename) === resolve(scriptId))
+    )
+    if (!script) throw new Error(`script ${scriptId} not found`)
+
+    // resolve files
+    const resolvedFiles = new Set<string>()
+    for (let arg of fileGlobs) {
+        if (HTTPS_REGEX.test(arg)) {
+            resolvedFiles.add(arg)
+            continue
+        }
+        const stats = await host.statFile(arg)
+        if (stats?.type === "directory") arg = host.path.join(arg, "**", "*")
+        const ffs = await host.findFiles(arg, {
+            applyGitIgnore: excludeGitIgnore,
+        })
+        if (!ffs?.length) {
+            return fail(
+                `no files matching ${arg} under ${process.cwd()}`,
+                FILES_NOT_FOUND_ERROR_CODE
+            )
+        }
+        for (const file of ffs) {
+            if (file.toLocaleLowerCase().endsWith(suffix)) continue
+            resolvedFiles.add(filePathOrUrlToWorkspaceFile(file))
+        }
+    }
+    if (excludedFiles?.length) {
+        for (const arg of excludedFiles) {
+            const ffs = await host.findFiles(arg)
+            for (const f of ffs)
+                resolvedFiles.delete(filePathOrUrlToWorkspaceFile(f))
+        }
+    }
+
+    // processing
+    const files = Array.from(resolvedFiles).map(
+        (filename) => ({ filename }) as WorkspaceFile
+    )
+
+    const p = new PLimitPromiseQueue(normalizeInt(concurrency) || 1)
+    await p.mapAll(files, async (file) => {
+        const outf = file.filename + suffix
+        logInfo(`${file.filename} -> ${outf}`)
+        const fileTrace = trace.startTraceDetails(file.filename)
+        try {
+            // apply AI transformation
+            const result = await run(script.filename, file.filename, {
+                label: file.filename,
+                ...restOptions,
+            })
+            const { text, error } = result || {}
+            if (error) throw error
+            // save file
+            const existing = await tryReadText(outf)
+            if (existing !== text) {
+                const patch = createPatch(
+                    outf,
+                    existing || "",
+                    text || "",
+                    undefined,
+                    undefined,
+                    {}
+                )
+                logVerbose(patch)
+                await writeText(outf, text)
+            }
+        } catch (error) {
+            logError(error)
+            fileTrace.error(undefined, error)
+        } finally {
+            logVerbose("")
+            trace.endDetails()
+        }
+    })
+}

packages/core/src/constants.ts

@@ -26,6 +26,7 @@ export const CLI_JS = TOOL_ID + ".cjs"
 export const GENAI_SRC = "genaisrc"
 export const GENAI_JS_EXT = ".genai.js"
 export const GENAI_MJS_EXT = ".genai.mjs"
+export const GENAI_MD_EXT = ".genai.md"
 export const GENAI_ANYJS_GLOB =
     "**/*{.genai.js,.genai.mjs,.genai.ts,.genai.mts,.prompty}"
 export const GENAI_ANY_REGEX = /\.(genai\.(ts|mts|mjs|js)|prompty)$/i

packages/core/src/fs.ts

@@ -22,7 +22,7 @@ export async function writeText(fn: string, content: string) {
 
 export async function fileExists(fn: string) {
     try {
-        return await host.readFile(fn) !== undefined
+        return (await host.readFile(fn)) !== undefined
     } catch {
         return false
     }

packages/core/src/ollama.ts

@@ -51,6 +51,7 @@ const OllamaCompletion: ChatCompletionHandler = async (
                     body: JSON.stringify({ name: model, stream: false }),
                 })
                 if (!res.ok) {
+                    trace.error(`ollama: failed to pull model ${model}`)
                     throw new Error(
                         `Failed to pull model ${model}: ${res.status} ${res.statusText}`
                     )

packages/core/src/usage.ts

@@ -258,8 +258,8 @@ export class GenerationStats {
         const c = this.cost()
         if (this.model && isNaN(c) && isCosteable(this.model))
             unknowns.add(this.model)
-        if (this.resolvedModel || c) {
-            const au = this.accumulatedUsage()
+        const au = this.accumulatedUsage()
+        if (au?.total_tokens > 0 && (this.resolvedModel || c)) {
             logVerbose(
                 `${indent}${this.label ? `${this.label} (${this.resolvedModel})` : this.resolvedModel}> ${au.total_tokens} tokens (${au.prompt_tokens} -> ${au.completion_tokens}) ${renderCost(c)}`
             )