test delay (#456)

* adding test delay

* upgrading deps

* force type

.github/workflows/build-genai.yml

@@ -1,4 +1,4 @@
-name: main build genai tests
+name: build genai tests
 
 on:
     workflow_dispatch:
@@ -36,4 +36,4 @@ jobs:
             - name: download ollama docker
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run test within scripts
-              run: yarn test:scripts --out-summary $GITHUB_STEP_SUMMARY
+              run: yarn test:scripts --out-summary $GITHUB_STEP_SUMMARY --test-delay 10

docs/package.json

@@ -18,12 +18,12 @@
     "genai:alt-text": "node scripts/image-alt-text.mjs"
   },
   "dependencies": {
-    "@astrojs/check": "^0.6.0",
-    "@astrojs/starlight": "^0.22.2",
-    "astro": "^4.8.3",
+    "@astrojs/check": "^0.7.0",
+    "@astrojs/starlight": "^0.22.3",
+    "astro": "^4.8.4",
     "typescript": "5.4.5"
   },
   "devDependencies": {
-    "zx": "^8.0.2"
+    "zx": "^8.1.0"
   }
 }

package.json

@@ -76,8 +76,8 @@
         }
     },
     "devDependencies": {
-        "glob": "^10.3.12",
-        "vitest": "^1.5.0",
-        "zx": "^8.0.2"
+        "glob": "^10.3.15",
+        "vitest": "^1.6.0",
+        "zx": "^8.1.0"
     }
 }

packages/cli/package.json

@@ -24,7 +24,7 @@
   "license": "MIT",
   "dependencies": {
     "pdfjs-dist": "4.0.379",
-    "promptfoo": "^0.58.0",
+    "promptfoo": "^0.58.1",
     "tree-sitter-wasms": "^0.1.11",
     "typescript": "5.4.5",
     "web-tree-sitter": "^0.22.2"
@@ -38,7 +38,7 @@
   "devDependencies": {
     "@llamaindex/env": "^0.0.5",
     "@types/fs-extra": "^11.0.4",
-    "@types/node": "^20.12.11",
+    "@types/node": "^20.12.12",
     "@types/papaparse": "^5.3.14",
     "@types/pg": "^8.11.2",
     "@types/prompts": "^2.4.9",
@@ -47,7 +47,7 @@
     "@types/ws": "^8.5.10",
     "commander": "^12.0.0",
     "dotenv": "^16.4.5",
-    "esbuild": "^0.21.2",
+    "esbuild": "^0.21.3",
     "execa": "^9.1.0",
     "file-type": "^19.0.0",
     "fs-extra": "^11.2.0",
@@ -56,14 +56,14 @@
     "glob": "^10.3.15",
     "mammoth": "^1.7.2",
     "node-sarif-builder": "^3.1.0",
-    "openai": "^4.45.0",
+    "openai": "^4.47.1",
     "ora": "^8.0.1",
     "pretty-bytes": "^6.1.1",
     "prompts": "^2.4.2",
     "replace-ext": "^2.0.0",
     "semver": "^7.6.2",
-    "tsx": "^4.10.2",
-    "zx": "^8.0.2"
+    "tsx": "^4.10.3",
+    "zx": "^8.1.0"
   },
   "scripts": {
     "compile": "esbuild src/main.ts --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:pdfjs-dist --external:llamaindex --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo && node ../../scripts/patch-cli.mjs",

packages/cli/src/cli.ts

@@ -189,6 +189,7 @@ export async function cli() {
         .option("-rmo, --remove-out", "remove output folder if it exists")
         .option("--cli <string>", "override path to the cli")
         .option("-tp, --test-provider <string>", "test provider")
+        .option("-td, --test-delay <string>", "delay between tests in seconds")
         .option("--no-cache", "disable LLM result cache")
         .option("-v, --verbose", "verbose output")
         .option(

packages/cli/src/test.ts

@@ -22,6 +22,8 @@ import {
     EMOJI_SUCCESS,
     GENAI_JS_REGEX,
     JSON5TryParse,
+    normalizeInt,
+    delay,
 } from "genaiscript-core"
 
 import { readFile, writeFile, appendFile } from "node:fs/promises"
@@ -70,6 +72,7 @@ export async function runPromptScriptTests(
         write?: boolean
         promptfooVersion?: string
         outSummary?: string
+        testDelay?: string
     }
 ): Promise<PromptScriptTestRunResponse> {
     const prj = await buildProject()
@@ -88,6 +91,7 @@ export async function runPromptScriptTests(
     const outSummary = options.outSummary
     const provider = join(out, "provider.mjs")
     const models = options?.models
+    const testDelay = normalizeInt(options?.testDelay)
     logInfo(`writing tests to ${out}`)
 
     if (options?.removeOut) await emptyDir(out)
@@ -172,6 +176,11 @@ export async function runPromptScriptTests(
             script: script.id,
             value,
         })
+
+        if (testDelay > 0) {
+            logVerbose(`  waiting ${testDelay}s`)
+            await delay(testDelay * 1000)
+        }
     }
 
     const ok = results.every((r) => !!r.ok)
@@ -194,6 +203,7 @@ export async function scriptsTest(
         write?: boolean
         promptfooVersion?: string
         outSummary?: string
+        testDelay?: string
     }
 ) {
     const { status, value = [] } = await runPromptScriptTests(ids, options)

packages/core/package.json

@@ -21,12 +21,12 @@
     "@types/inflection": "^1.13.2",
     "@types/ini": "^4.1.0",
     "@types/mime-types": "^2.1.4",
-    "@types/node": "^20.12.11",
+    "@types/node": "^20.12.12",
     "ajv": "^8.13.0",
     "cross-fetch": "^4.0.0",
-    "csv-parse": "^5.5.5",
+    "csv-parse": "^5.5.6",
     "dotenv": "^16.4.5",
-    "esbuild": "^0.21.2",
+    "esbuild": "^0.21.3",
     "fast-xml-parser": "^4.3.6",
     "fetch-retry": "^6.0.0",
     "file-type": "^19.0.0",
@@ -36,15 +36,15 @@
     "inflection": "^3.0.0",
     "ini": "^4.1.2",
     "json5": "^2.2.3",
-    "jsonrepair": "^3.7.1",
+    "jsonrepair": "^3.8.0",
     "magic-string": "^0.30.10",
     "mammoth": "^1.7.2",
     "markdown-table": "^3.0.3",
     "mathjs": "^12.4.2",
     "mime-types": "^2.1.35",
     "minimatch": "^9.0.4",
     "minisearch": "^6.3.0",
-    "openai": "^4.45.0",
+    "openai": "^4.47.1",
     "prettier": "^3.2.5",
     "pretty-bytes": "^6.1.1",
     "serialize-error": "^11.0.3",

packages/sample/genaisrc/fetch.genai.js

@@ -4,7 +4,10 @@ script({
     },
 })
 
-const res = await fetch(
+/**
+ * @type {any}
+ */
+const res  = await fetch(
     "https://raw.githubusercontent.com/microsoft/genaiscript/main/package.json",
     { method: "GET" }
 )

packages/vscode/package.json

@@ -317,8 +317,8 @@
     "devDependencies": {
         "@types/mocha": "^10.0.6",
         "@types/vscode": "^1.86.0",
-        "@typescript-eslint/eslint-plugin": "^7.8.0",
-        "@typescript-eslint/parser": "^7.8.0",
+        "@typescript-eslint/eslint-plugin": "^7.9.0",
+        "@typescript-eslint/parser": "^7.9.0",
         "@vscode/test-web": "^0.0.53",
         "@vscode/vsce": "^2.25.0",
         "assert": "^2.1.0",
@@ -331,7 +331,7 @@
         "vscode-uri": "^3.0.8",
         "webfont": "^11.2.26",
         "websocket-polyfill": "0.0.3",
-        "zx": "^8.0.2"
+        "zx": "^8.1.0"
     },
     "dependencies": {
         "mammoth": "^1.7.2",

packages/vscode/src/state.ts

@@ -41,8 +41,6 @@ import { configureLanguageModelAccess, pickLanguageModel } from "./lmaccess"
 import { startLocalAI } from "./localai"
 import { hasOutputOrTraceOpened } from "./markdowndocumentprovider"
 
-const MAX_HISTORY_LENGTH = 500
-
 export const TOKEN_DOCUMENTATION_URL =
     "https://github.com/microsoft/genaiscript/blob/main/docs/token.md"
 export const CONTEXT_LENGTH_DOCUMENTATION_URL =

slides/package.json

@@ -11,12 +11,12 @@
     "dev": "slidev --open"
   },
   "dependencies": {
-    "@slidev/cli": "^0.49.0-beta.6",
+    "@slidev/cli": "^0.49.2",
     "@slidev/theme-default": "latest",
     "@slidev/theme-seriph": "latest",
-    "vue": "^3.4.26"
+    "vue": "^3.4.27"
   },
   "devDependencies": {
-    "zx": "^8.0.2"
+    "zx": "^8.1.0"
   }
 }