adding zips to parsers (#461)

* adding zips to parsers

* add fflate

* add unzip tests

* updated docs

* added zip support

docs/src/content/docs/reference/scripts/parsers.md

@@ -161,6 +161,14 @@ const res = parsers.XLSX("...filename.xlsx", {
 })
 ```
 
+## Unzip
+
+Unpacks the contents of a zip file and returns an array of files.
+
+```js
+const files = await parsers.unzip(env.files[0])
+```
+
 ## HTML to Text
 
 The `parsers.HTMLToText` converts HTML to plain text using [html-to-text](https://www.npmjs.com/package/html-to-text).

packages/core/package.json

@@ -29,6 +29,7 @@
     "esbuild": "^0.21.3",
     "fast-xml-parser": "^4.3.6",
     "fetch-retry": "^6.0.0",
+    "fflate": "^0.8.2",
     "file-type": "^19.0.0",
     "gpt-tokenizer": "^2.1.2",
     "html-escaper": "^3.0.3",

packages/core/src/parsers.test.ts

@@ -3,8 +3,24 @@ import assert from "node:assert/strict"
 import { createParsers } from "./parsers"
 import { MarkdownTrace } from "./trace"
 import { XSLXParse } from "./xslx"
-import { readFile } from "fs/promises"
+import { readFile, writeFile } from "fs/promises"
 import { resolve } from "path"
+import {
+    setHost,
+    Host,
+    AskUserOptions,
+    LogLevel,
+    ModelService,
+    OAIToken,
+    ParseService,
+    ReadFileOptions,
+    RetrievalService,
+    ServerManager,
+    ShellCallOptions,
+    UTF8Decoder,
+    UTF8Encoder,
+} from "./host"
+import { TestHost } from "./testhost"
 
 describe("parsers", () => {
     let trace: MarkdownTrace
@@ -15,6 +31,7 @@ describe("parsers", () => {
         trace = new MarkdownTrace()
         model = "test model"
         parsers = createParsers({ trace, model })
+        TestHost.install()
     })
 
     test("JSON5", () => {
@@ -43,7 +60,9 @@ describe("parsers", () => {
     })
 
     test("XSLX", async () => {
-        const result = XSLXParse(await readFile(resolve("./src/parsers.test.xlsx")))
+        const result = XSLXParse(
+            await readFile(resolve("./src/parsers.test.xlsx"))
+        )
         assert.deepStrictEqual(result, [{ key: 1, value: 2 }])
     })
 
@@ -52,6 +71,18 @@ describe("parsers", () => {
         assert.deepStrictEqual(result, { key: "value" })
     })
 
+    test("zip", async () => {
+        const result = await parsers.unzip(
+            {
+                filename: "./src/parsers.test.zip",
+                content: undefined,
+            },
+            { glob: "*.md" }
+        )
+        assert(result.find((f) => f.filename === "markdown.md"))
+        assert(!result.find((f) => f.filename === "loremipsum.pdf"))
+    })
+
     test("math", () => {
         const res = parsers.math("1 + 3")
         assert.strictEqual(res, 4)

packages/core/src/parsers.ts

@@ -19,6 +19,7 @@ import { MathTryEvaluate } from "./math"
 import { validateJSONWithSchema } from "./schema"
 import { XSLXTryParse } from "./xslx"
 import { host } from "./host"
+import { unzip } from "./zip"
 
 export function createParsers(options: {
     trace: MarkdownTrace
@@ -49,6 +50,8 @@ export function createParsers(options: {
         dotEnv: (text) => dotEnvTryParse(filenameOrFileToContent(text)),
         INI: (text, options) =>
             INITryParse(filenameOrFileToContent(text), options?.defaultValue),
+        unzip: async (file, options) =>
+            await unzip(await host.readFile(file.filename), options),
         tokens: (text) => estimateTokens(model, filenameOrFileToContent(text)),
         fences: (text) => extractFenced(filenameOrFileToContent(text)),
         annotations: (text) => parseAnnotations(filenameOrFileToContent(text)),

packages/core/src/testhost.ts

@@ -0,0 +1,96 @@
+import { readFile, writeFile } from "fs/promises"
+import {
+    setHost,
+    Host,
+    AskUserOptions,
+    LogLevel,
+    ModelService,
+    OAIToken,
+    ParseService,
+    ReadFileOptions,
+    RetrievalService,
+    ServerManager,
+    ShellCallOptions,
+    UTF8Decoder,
+    UTF8Encoder,
+} from "./host"
+import { resolve } from "path"
+
+export class TestHost implements Host {
+    userState: any
+    parser: ParseService
+    retrieval: RetrievalService
+    models: ModelService
+    server: ServerManager
+    path: Path
+    workspace: WorkspaceFileSystem
+
+    static install() {
+        setHost(new TestHost())
+    }
+
+    createUTF8Decoder(): UTF8Decoder {
+        return new TextDecoder("utf-8")
+    }
+    createUTF8Encoder(): UTF8Encoder {
+        return new TextEncoder()
+    }
+    projectFolder(): string {
+        throw new Error("Method not implemented.")
+    }
+    installFolder(): string {
+        throw new Error("Method not implemented.")
+    }
+    resolvePath(...segments: string[]): string {
+        throw new Error("Method not implemented.")
+    }
+    readSecret(name: string): Promise<string> {
+        throw new Error("Method not implemented.")
+    }
+    getSecretToken(modelId: string): Promise<OAIToken> {
+        throw new Error("Method not implemented.")
+    }
+    log(level: LogLevel, msg: string): void {
+        throw new Error("Method not implemented.")
+    }
+    async readFile(
+        name: string,
+        options?: ReadFileOptions
+    ): Promise<Uint8Array> {
+        return new Uint8Array(await readFile(resolve(name)))
+    }
+    async writeFile(name: string, content: Uint8Array): Promise<void> {
+        await writeFile(resolve(name), content)
+    }
+    deleteFile(name: string): Promise<void> {
+        throw new Error("Method not implemented.")
+    }
+    findFiles(glob: string): Promise<string[]> {
+        throw new Error("Method not implemented.")
+    }
+    clearVirtualFiles(): void {
+        throw new Error("Method not implemented.")
+    }
+    setVirtualFile(name: string, content: string): void {
+        throw new Error("Method not implemented.")
+    }
+    isVirtualFile(name: string): boolean {
+        throw new Error("Method not implemented.")
+    }
+    createDirectory(name: string): Promise<void> {
+        throw new Error("Method not implemented.")
+    }
+    deleteDirectory(name: string): Promise<void> {
+        throw new Error("Method not implemented.")
+    }
+    askUser(options: AskUserOptions): Promise<string> {
+        throw new Error("Method not implemented.")
+    }
+    exec(
+        command: string,
+        args: string[],
+        options: ShellCallOptions
+    ): Promise<Partial<ShellOutput>> {
+        throw new Error("Method not implemented.")
+    }
+}

packages/core/src/types/prompt_template.d.ts

@@ -747,6 +747,9 @@ interface ParseXLSXOptions {
     range?: string
 }
 
+interface ParseZipOptions {
+    glob?: string
+}
 
 interface Parsers {
     /**
@@ -852,6 +855,13 @@ interface Parsers {
         options?: HTMLToTextOptions
     ): string
 
+    /**
+     * Extracts the contents of a zip archive file
+     * @param file 
+     * @param options 
+     */
+    unzip(file: WorkspaceFile, options?: ParseZipOptions): Promise<WorkspaceFile[]>
+
     /**
      * Estimates the number of tokens in the content.
      * @param content content to tokenize

packages/core/src/zip.ts

@@ -0,0 +1,28 @@
+import { unzipSync } from "fflate"
+import { minimatch } from "minimatch"
+import { lookupMime } from "./mime"
+import { isBinaryMimeType } from "./parser"
+import { host } from "./host"
+
+export async function unzip(
+    data: Uint8Array,
+    options?: ParseZipOptions
+): Promise<WorkspaceFile[]> {
+    const { glob } = options || {}
+    const res = unzipSync(data, {
+        filter: (file: { name: string; originalSize: number }) => {
+            if (glob)
+                return minimatch(file.name, glob, {
+                    windowsPathsNoEscape: true,
+                })
+            return true
+        },
+    })
+    const decoder = host.createUTF8Decoder()
+    return Object.entries(res).map(([filename, data]) => {
+        const mime = lookupMime(filename)
+        if (isBinaryMimeType(mime))
+            return <WorkspaceFile>{ filename } // TODO bytes support
+        else return <WorkspaceFile>{ filename, content: decoder.decode(data) }
+    })
+}