Always use node.js exec (#473)

* always call exec through node.js * formatting * python name patch * add exec test * use 3.5 model * use tools
microsoft · May 22, 2024 · 45de014 · 45de014
1 parent c898e42
commit 45de014
Show file tree

Hide file tree

Showing 28 changed files with 228 additions and 235 deletions.
diff --git a/docs/genaisrc/genaiscript.d.ts b/docs/genaisrc/genaiscript.d.ts
diff --git a/genaisrc/genaiscript.d.ts b/genaisrc/genaiscript.d.ts
diff --git a/packages/cli/src/nodehost.ts b/packages/cli/src/nodehost.ts
@@ -8,8 +8,9 @@ import {
     OAIToken,
     ReadFileOptions,
     RetrievalService,
+    SHELL_EXEC_TIMEOUT,
     ServerManager,
-    ShellCallOptions,
+    TraceOptions,
     UTF8Decoder,
     UTF8Encoder,
     createBundledParsers,
@@ -170,17 +171,48 @@ export class NodeHost implements Host {
         await remove(name)
     }
 
-    async exec(command: string, args: string[], options: ShellCallOptions) {
-        const { cwd, timeout, stdin: input } = options
-        const exec = execa
-        const { stdout, stderr, exitCode, failed } = await exec(command, args, {
-            cleanup: true,
-            input,
-            timeout,
+    async exec(
+        command: string,
+        args: string[],
+        options: ShellOptions & TraceOptions
+    ) {
+        const {
+            trace,
+            label,
             cwd,
-            preferLocal: true,
-            stripFinalNewline: true,
-        })
-        return { stdout, stderr, exitCode, failed }
+            timeout = SHELL_EXEC_TIMEOUT,
+            stdin: input,
+        } = options
+        try {
+            trace?.startDetails(label || command)
+
+            // python3 on windows -> python
+            if (command === "python3" && process.platform === "win32")
+                command = "python"
+            if (command === "python" && process.platform !== "win32")
+                command = "python3"
+
+            trace?.itemValue(`cwd`, cwd)
+            trace?.item(`\`${command}\` ${args.join(" ")}`)
+
+            const { stdout, stderr, exitCode, failed } = await execa(
+                command,
+                args,
+                {
+                    cleanup: true,
+                    input,
+                    timeout,
+                    cwd,
+                    preferLocal: true,
+                    stripFinalNewline: true,
+                }
+            )
+            trace?.itemValue(`exit code`, `${exitCode}`)
+            trace?.detailsFenced(`📩 stdout`, stdout)
+            trace?.detailsFenced(`📩 stderr`, stderr)
+            return { stdout, stderr, exitCode, failed }
+        } finally {
+            trace?.endDetails()
+        }
     }
 }
diff --git a/packages/cli/src/scripts.ts b/packages/cli/src/scripts.ts
@@ -2,7 +2,6 @@ import { buildProject } from "./build"
 import {
     copyPrompt,
     createScript as coreCreateScript,
-    exec,
     fixPromptDefinitions,
     host,
     logVerbose,
@@ -38,21 +37,20 @@ export async function compileScript() {
     await fixPromptDefinitions(project)
     for (const folder of project.folders()) {
         logVerbose(`compiling ${host.path.join(folder, "*.genai.js")}`)
-        const res = await exec(host, {
-            label: folder,
-            call: {
+        const res = await host.exec(
+            "npx",
+            [
+                "--yes",
+                "--package",
+                `typescript@${TYPESCRIPT_VERSION}`,
+                "tsc",
+                "--project",
+                host.path.resolve(folder, "jsconfig.json"),
+            ],
+            {
                 cwd: folder,
-                command: "npx",
-                args: [
-                    "--yes",
-                    "--package",
-                    `typescript@${TYPESCRIPT_VERSION}`,
-                    "tsc",
-                    "--project",
-                    host.path.resolve(folder, "jsconfig.json"),
-                ],
-            },
-        })
+            }
+        )
         logVerbose(res.output)
     }
 }
diff --git a/packages/cli/src/server.ts b/packages/cli/src/server.ts
@@ -9,6 +9,7 @@ import {
     CORE_VERSION,
     ServerResponse,
     serializeError,
+    ShellCallResponse,
 } from "genaiscript-core"
 import { runPromptScriptTests } from "./test"
 import { PROMPTFOO_VERSION } from "./version"
@@ -54,7 +55,9 @@ export async function startServer(options: { port: string }) {
                     case "retrieval.vectorClear":
                         console.log(`retrieval: clear`)
                         await host.retrieval.init()
-                        response = await host.retrieval.vectorClear(data.options)
+                        response = await host.retrieval.vectorClear(
+                            data.options
+                        )
                         break
                     case "retrieval.vectorUpsert": {
                         console.log(`retrieval: upsert ${data.filename}`)
@@ -90,10 +93,21 @@ export async function startServer(options: { port: string }) {
                             ...(data.options || {}),
                             cache: true,
                             verbose: true,
-                            promptfooVersion: PROMPTFOO_VERSION
+                            promptfooVersion: PROMPTFOO_VERSION,
                         })
                         break
                     }
+                    case "shell.call": {
+                        console.log(`exec ${data.command}`)
+                        const { command, args, options } = data
+                        const value = await host.exec(command, args, options)
+                        response = <ShellCallResponse>{
+                            value,
+                            ok: !value.failed,
+                            status: value.exitCode,
+                        }
+                        break
+                    }
                     default:
                         throw new Error(`unknown message type ${type}`)
                 }

diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts
@@ -120,4 +120,6 @@ export const JSON_MIME_TYPE = "application/json"
 export const JSON_SCHEMA_MIME_TYPE = "application/schema+json"
 export const JAVASCRIPT_MIME_TYPE = "application/javascript"
 
-export const JSON_META_SCHEMA_URI = "https://json-schema.org/draft/2020-12/schema"
+export const JSON_META_SCHEMA_URI = "https://json-schema.org/draft/2020-12/schema"
+
+export const SHELL_EXEC_TIMEOUT = 300000
diff --git a/packages/core/src/exec.ts b/packages/core/src/exec.ts
diff --git a/packages/core/src/genaisrc/genaiscript.d.ts b/packages/core/src/genaisrc/genaiscript.d.ts
diff --git a/packages/core/src/host.ts b/packages/core/src/host.ts
@@ -39,17 +39,6 @@ export interface ReadFileOptions {
     virtual?: boolean
 }
 
-export interface ShellCallOptions extends ShellOptions {
-    timeout?: number
-    stdin?: string
-    keepOnError?: boolean
-    outputdir: string
-    stdinfile: string
-    stdoutfile: string
-    stderrfile: string
-    exitcodefile: string
-}
-
 export interface RetrievalClientOptions {
     progress?: Progress
     token?: CancellationToken
@@ -148,9 +137,7 @@ export interface Host {
 
     // read a secret from the environment or a .env file
     readSecret(name: string): Promise<string | undefined>
-    getSecretToken(
-        modelId: string
-    ): Promise<OAIToken | undefined>
+    getSecretToken(modelId: string): Promise<OAIToken | undefined>
 
     log(level: LogLevel, msg: string): void
 
@@ -178,7 +165,7 @@ export interface Host {
     exec(
         command: string,
         args: string[],
-        options: ShellCallOptions
+        options: ShellOptions & TraceOptions
     ): Promise<Partial<ShellOutput>>
 }
 

diff --git a/packages/core/src/import.ts b/packages/core/src/import.ts
@@ -1,4 +1,3 @@
-import { exec } from "./exec"
 import { host } from "./host"
 import { MarkdownTrace } from "./trace"
 import { fileExists } from "./fs"
@@ -15,15 +14,8 @@ export async function installImport(
     const args = yarn
         ? ["add", mod]
         : ["install", "--no-save", "--ignore-scripts", mod]
-    const res = await exec(host, {
-        trace,
-        label: `install ${mod}`,
-        keepOnError: true,
-        call: {
-            command,
-            args,
-            cwd,
-        },
+    const res = await host.exec(command, args, {
+        cwd,
     })
     return res.exitCode === 0
 }

diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts
@@ -61,6 +61,5 @@ export * from "./parsers"
 export * from "./html"
 export * from "./parameters"
 export * from "./scripts"
-export * from "./exec"
 export * from "./math"
 export * from "./fence"