add comment to PR with test results (#496)

* add comment to PR with test results

* also add comment on octokit

* always build ollama

* use temp folder

* run ollama on pull requests

* trigger build

* don't double build

* fix ensuredir/basename

* use require in ollama file

* trigger build

* tweak .yml and trigger build

* updated permissions

* another build

* inline trace comment

* make alerts more compatible with github parsers

.github/workflows/build-genai.yml

@@ -5,12 +5,12 @@ on:
     schedule:
         - cron: "0 7 * * *"
     push:
-      branches: [main]
-      paths:
-        - "packages/core/**"
-        - "packages/sample/**"
-        - "packages/cli/**"
-      
+        branches: [main]
+        paths:
+            - "packages/core/**"
+            - "packages/sample/**"
+            - "packages/cli/**"
+
 jobs:
     build:
         runs-on: ubuntu-latest
@@ -37,4 +37,21 @@ jobs:
             - name: download ollama docker
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run test within scripts
-              run: yarn test:scripts --out-summary $GITHUB_STEP_SUMMARY --test-delay 10
+              run: yarn test:scripts --out-summary ./summary.txt --test-delay 10
+            - name: Append summary.txt to $GITHUB_STEP_SUMMARY
+              run: |
+                  cat ./summary.txt >> $GITHUB_STEP_SUMMARY
+            - name: Add comment to PR
+              uses: actions/github-script@v5
+              if: github.event_name == 'pull_request'
+              with:
+                  github-token: ${{ secrets.GITHUB_TOKEN }}
+                  script: |
+                      import { readFile } from 'fs/promises';
+                      const comment = await readFile('./summary.txt', 'utf8');
+                      await github.rest.issues.createComment({
+                        issue_number: context.issue.number,
+                        owner: context.repo.owner,
+                        repo: context.repo.repo,
+                        body: comment
+                      })

.github/workflows/ollama.yml

@@ -1,14 +1,23 @@
 name: ollama smoke tests
 on:
     workflow_dispatch:
+    pull_request:
+        paths:
+            - yarn.lock
+            - "packages/core/**/*"
+            - "packages/cli/**/*"
+            - "packages/samples/**/*"
     push:
-        branches: [main]
+        branches:
+            - main
         paths:
             - yarn.lock
             - "packages/core/**/*"
             - "packages/cli/**/*"
             - "packages/samples/**/*"
-
+permissions:
+    issues: write
+    pull-requests: write
 jobs:
     tests:
         runs-on: ubuntu-latest
@@ -29,4 +38,18 @@ jobs:
             - name: download ollama docker
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run summarize-ollama-phi3
-              run: yarn test:summarize --model ollama:phi3
+              run: yarn test:summarize --model ollama:phi3 --out ./temp
+            - name: Add comment to PR
+              uses: actions/github-script@v5
+              if: github.event_name == 'pull_request'
+              with:
+                  github-token: ${{ secrets.GITHUB_TOKEN }}
+                  script: |
+                      const fs = require('fs/promises');
+                      const comment = await fs.readFile('./temp/res.trace.md', 'utf8');
+                      await github.rest.issues.createComment({
+                        issue_number: context.issue.number,
+                        owner: context.repo.owner,
+                        repo: context.repo.repo,
+                        body: comment
+                      })

packages/cli/src/test.ts

@@ -28,7 +28,7 @@ import {
 
 import { readFile, writeFile, appendFile } from "node:fs/promises"
 import { execa } from "execa"
-import { basename, join, resolve } from "node:path"
+import { dirname, join, resolve } from "node:path"
 import { emptyDir, ensureDir, exists } from "fs-extra"
 import type { OutputFile } from "promptfoo"
 import { PROMPTFOO_VERSION } from "./version"
@@ -44,6 +44,7 @@ function parseModelSpec(m: string): ModelOptions {
     else return { model: m }
 }
 
+// build trigger..
 async function resolveTestProvider(script: PromptScript) {
     const token = await host.getLanguageModelConfiguration(script.model)
     if (token && token.type === "azure") return token.base
@@ -89,6 +90,8 @@ export async function runPromptScriptTests(
     const cli = options.cli || resolve(__filename)
     const out = options.out || join(GENAISCRIPT_FOLDER, "tests")
     const outSummary = options.outSummary
+        ? resolve(options.outSummary)
+        : undefined
     const provider = join(out, "provider.mjs")
     const models = options?.models
     const testDelay = normalizeInt(options?.testDelay)
@@ -118,10 +121,17 @@ export async function runPromptScriptTests(
         configurations.push({ script, configuration: fn })
     }
 
-    logVerbose(`running tests with promptfoo`)
     await ensureDir(PROMPTFOO_CACHE_PATH)
     await ensureDir(PROMPTFOO_CONFIG_DIR)
-    if (outSummary) await ensureDir(basename(resolve(outSummary)))
+    if (outSummary) {
+        await ensureDir(dirname(outSummary))
+        await appendFile(
+            outSummary,
+            `## GenAIScript Test Results
+
+`
+        )
+    }
 
     const results: PromptScriptTestResult[] = []
     for (const config of configurations) {

packages/core/src/fence.ts

@@ -198,7 +198,8 @@ ${v}
 \`\`\`\`\`
 ${
     validation?.error
-        ? `> [!CAUTION] Schema ${args.schema} validation errors
+        ? `> [!CAUTION] 
+> Schema ${args.schema} validation errors
 ${validation.error.split("\n").join("\n> ")}`
         : ""
 }

packages/core/src/trace.ts

@@ -213,15 +213,18 @@ ${this.toResultIcon(success, "")}${title}
     }
 
     warn(msg: string) {
-        this.content += `\n> [!WARNING] ${msg}\n`
+        this.content += `\n> [!WARNING]
+> ${msg}\n`
     }
 
     caution(msg: string) {
-        this.content += `\n> [!CAUTION] ${msg}\n`
+        this.content += `\n> [!CAUTION]
+> ${msg}\n`
     }
 
     note(msg: string) {
-        this.content += `\n> [!NOTE] ${msg}\n`
+        this.content += `\n> [!NOTE]
+> ${msg}\n`
     }
 
     files(

packages/sample/genaisrc/summarize.genai.js

@@ -15,6 +15,6 @@ def("FILE", env.files)
 
 $`
 Summarize each FILE with one paragraph.
-
+- Do not wrap results in code section.
 - Use less than 20 words.
 `