transformers.js sample (#581)

* add transformers test

* updated yarn.lock

* better logging when pull with ollama

* added guide on transformers.js

* jsconfig only applies to .js

* typo

demo/genaisrc/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

docs/genaisrc/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

docs/src/content/docs/guides/transformers-js.mdx

@@ -0,0 +1,80 @@
+---
+title: Transformer.js
+sidebar:
+    order: 20
+---
+import { Code } from "@astrojs/starlight/components"
+import sampleSrc from "../../../../../packages/sample/genaisrc/summary-with-transformers.genai?raw"
+
+
+HuggingFace [Transformers.js](https://huggingface.co/docs/transformers.js/index) is a JavaScript library
+that lets you run pretrained models locally on your machine. The library uses [onnxruntime](https://onnxruntime.ai/)
+to leverage the CPU/GPU capabilities of your hardware.
+
+In this guide, we will show how to create [summaries](https://huggingface.co/tasks/summarization) using the [Transformers.js](https://huggingface.co/docs/transformers.js/api/pipelines#module_pipelines.SummarizationPipeline) library.
+
+:::tip
+
+Transformers.js has an extensive list of tasks available. This guide will only cover one but checkout their [documentation](https://huggingface.co/docs/transformers.js/pipelines#tasks)
+for more.
+
+:::
+
+## Installation
+
+Following the [installation instructions](https://huggingface.co/docs/transformers.js/installation), 
+we add the [@xenova/transformers](https://www.npmjs.com/package/@xenova/transformers) to the current project.
+
+```bash
+npm install @xenova/transformers
+```
+
+You can also install this library globally to be able to use on any project
+
+```bash "-g"
+npm install -g @xenova/transformers
+```
+
+## Import the pipeline
+
+The snippet below imports the Transformers.js library and loads the summarizer pipeline and model.
+You can specify a model name or let the library pick the latest and greatest.
+
+```js
+import { pipeline } from "@xenova/transformers"
+const summarizer = await pipeline("summarization")
+```
+
+Allocating and loading the model can take some time, 
+so it's best to do this at the beginning of your script
+and only once.
+
+:::note[Migrate your script to `.mjs`]
+
+To use the `Transformers.js` library, you need to use the `.mjs` extension for your script (or `.mts` for TypeScript support).
+If your script is ending in `.genai.js`, rename it to `.genai.mjs`.
+
+:::
+
+## Invoke the pipeline
+
+The summarizer pipeline has a single argument, the content to summarize. It returns an array of summaries
+which we need to unpack to access the final summary text. This is what we do below and `summary_index` contains the summary text.
+
+```js
+const [summary] = await summarizer(content)
+// @ts-ignore
+const { summary_text } = summary
+```
+
+## Final code
+
+The example below generates a summary of each input file
+before letting the model generate a full summary.
+
+<Code
+    title="transformers.genai.mjs"
+    code={sampleSrc}
+    wrap={true}
+    lang="js"
+/>

docs/src/content/docs/reference/cli/run.md

@@ -36,7 +36,7 @@ See [configuration](/genaiscript/getting-started/configuration).
 
 `run` takes one or more [glob](https://en.wikipedia.org/wiki/Glob_(programming)) patterns to match files in the workspace.
 
-```npx sh
+```bash sh
 npx genaiscript run <script> "**/*.md" "**/*.ts"
 ```
 

genaisrc/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/cli/src/nodehost.ts

@@ -45,7 +45,7 @@ import {
 } from "../../core/src/models"
 import { createBundledParsers } from "../../core/src/pdf"
 import { AbortSignalOptions, TraceOptions } from "../../core/src/trace"
-import { unique } from "../../core/src/util"
+import { logVerbose, unique } from "../../core/src/util"
 
 class NodeServerManager implements ServerManager {
     async start(): Promise<void> {
@@ -73,6 +73,7 @@ class ModelManager implements ModelService {
         if (provider === MODEL_PROVIDER_OLLAMA) {
             if (this.pulled.includes(modelid)) return { ok: true }
 
+            logVerbose(`ollama: pulling ${modelid}...`)
             const conn = await this.getModelToken(modelid)
             const res = await fetch(`${conn.base}/api/pull`, {
                 method: "POST",

packages/core/bundleprompts.mjs

@@ -33,7 +33,7 @@ const promptDefs = {
                 allowJs: true,
                 skipLibCheck: true,
             },
-            include: ["*.js", "*.mjs", "./genaiscript.d.ts"],
+            include: ["*.js", "./genaiscript.d.ts"],
         },
         null,
         4

packages/core/src/genaisrc/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/core/src/importprompt.ts

@@ -2,7 +2,7 @@ import { assert } from "console"
 import { host } from "./host"
 import { logError } from "./util"
 import { TraceOptions } from "./trace"
-import { pathToFileURL } from "url"
+import { fileURLToPath, pathToFileURL } from "url"
 
 function resolveGlobal(): any {
     if (typeof window !== "undefined")
@@ -46,10 +46,10 @@ export async function importPrompt(
             import.meta.url ??
             pathToFileURL(__filename ?? host.projectFolder()).toString()
 
-        trace?.itemValue(`import`, `${modulePath}, parent: ${parentURL}`)
         const onImport = (file: string) => {
-            trace?.itemValue("📦 import", file)
+            // trace?.itemValue("📦 import", fileURLToPath(file))
         }
+        onImport(modulePath)
         const { tsImport, register } = await import("tsx/esm/api")
         unregister = register({ onImport })
         const module = await tsImport(modulePath, {

packages/core/src/runpromptcontext.ts

@@ -19,6 +19,7 @@ import { isJSONSchema } from "./schema"
 import { consoleLogFormat } from "./logging"
 import { resolveFileDataUri } from "./file"
 import { isGlobMatch } from "./glob"
+import { logVerbose } from "./util"
 
 export function createChatTurnGenerationContext(
     options: GenerationOptions,
@@ -28,7 +29,10 @@ export function createChatTurnGenerationContext(
 
     const log = (...args: any[]) => {
         const line = consoleLogFormat(...args)
-        if (line) trace.log(line)
+        if (line) {
+            trace.log(line)
+            logVerbose(line)
+        }
     }
     const console = Object.freeze<PromptGenerationConsole>({
         log,

packages/sample/genaisrc/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/genaisrc/node/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/genaisrc/python/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/genaisrc/style/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/genaisrc/summary-with-transformers.genai.mjs

@@ -0,0 +1,28 @@
+script({
+    title: "summary of summary - transformers.js",
+    model: "ollama:phi3",
+    files: ["src/rag/markdown.md"],
+    tests: {
+        files: ["src/rag/markdown.md"],
+        keywords: ["markdown"],
+    },
+})
+
+console.log("loading summarizer transformer")
+import { pipeline } from "@xenova/transformers"
+const summarizer = await pipeline("summarization")
+
+for (const file of env.files) {
+    console.log(`summarizing ${file.filename}`)
+    const [summary] = await summarizer(file.content)
+    // @ts-ignore
+    const { summary_text } = summary
+    def("FILE", {
+        filename: file.filename,
+        // @ts-ignore
+        content: summary_text,
+    })
+}
+
+console.log(`summarize all summaries`)
+$`Summarize all the contents in FILE.`

packages/sample/package.json

@@ -15,6 +15,7 @@
     "test:scripts:view": "node ../cli/built/genaiscript.cjs test view"
   },
   "devDependencies": {
-    "@tidyjs/tidy": "^2.5.2"
+    "@tidyjs/tidy": "^2.5.2",
+    "@xenova/transformers": "^2.17.2"
   }
 }

packages/sample/src/aici/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/src/errors/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/src/makecode/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/src/tla/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

packages/sample/src/vision/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }

slides/genaisrc/jsconfig.json

@@ -12,7 +12,6 @@
     },
     "include": [
         "*.js",
-        "*.mjs",
         "./genaiscript.d.ts"
     ]
 }