upload annotations in github pull requests

.github/workflows/genai-pr.yml

@@ -36,16 +36,30 @@ jobs:
             - name: git stuff
               run: git fetch origin && git pull origin main:main
             - name: genaiscript pr-describe
-              run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp -prd pr-describe
+              run: node packages/cli/built/genaiscript.cjs run pr-describe --out ./temp/genai/pr-describe -prd pr-describe
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
                   OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
                   OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
             - name: genaiscript pr-review
-              run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp -prd pr-review
+              run: node packages/cli/built/genaiscript.cjs run pr-review --out ./temp/genai/pr-review -prc pr-review
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  GITHUB_COMMIT_SHA: ${{ github.event.pull_request.base.sha}}
                   OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
                   OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
                   OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+            - name: genaiscript pr-review-commit
+              run: node packages/cli/built/genaiscript.cjs run pr-review-commit --out ./temp/genai/pr-review-commit -prr
+              env:
+                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+                  GITHUB_COMMIT_SHA: ${{ github.event.pull_request.head.sha}}
+                  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+                  OPENAI_API_TYPE: ${{ secrets.OPENAI_API_TYPE }}
+                  OPENAI_API_BASE: ${{ secrets.OPENAI_API_BASE }}
+            - name: Archive genai results
+              uses: actions/upload-artifact@v4
+              with:
+                  name: genai-results
+                  path: ./temp/genai/**

.github/workflows/ollama.yml

@@ -17,8 +17,6 @@ on:
             - "packages/core/**/*"
             - "packages/cli/**/*"
             - "packages/samples/**/*"
-permissions:
-    pull-requests: write
 jobs:
     tests:
         runs-on: ubuntu-latest
@@ -39,6 +37,4 @@ jobs:
             - name: download ollama docker
               run: docker run -d -v ollama:/root/.ollama -p 11434:11434 --name ollama ollama/ollama
             - name: run summarize-ollama-phi3
-              run: yarn test:summarize --model ollama:phi3 --out ./temp -prc ollama
-              env:
-                  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+              run: yarn test:summarize --model ollama:phi3 --out ./temp

.vscode/settings.json

@@ -9,6 +9,7 @@
         "execa",
         "genai",
         "Genaiscript",
+        "genaisrc",
         "gpspec",
         "gpspecs",
         "gptool",

docs/src/content/docs/reference/cli/commands.md

@@ -24,6 +24,7 @@ Options:
   -ocl, --out-changelog <string>             output file for changelogs
   -prc, --pull-request-comment <string>      create comment on a pull request.
   -prd, --pull-request-description <string>  upsert comment on a pull request description.
+  -prr, --pull-request-reviews               create pull request reviews from annotations
   -j, --json                                 emit full JSON response to output
   -y, --yaml                                 emit full YAML response to output
   -p, --prompt                               dry run, don't execute LLM and return expanded prompt

genaisrc/pr-describe.genai.js

@@ -9,6 +9,8 @@ const { stdout: changes } = await host.exec("git", [
     "main",
     "--",
     ":!**/genaiscript.d.ts",
+    ":!genaisrc/*",
+    ":!.github/*",
     ":!.vscode/*",
     ":!yarn.lock",
 ])
@@ -19,13 +21,13 @@ $`You are an expert software developer and architect.
 
 ## Task
 
-- Describe the changes in GIT_DIFF in a way that a software engineer will understand.
+- Describe a summary of the changes in GIT_DIFF in a way that a software engineer will understand.
 
 ## Instructions
 
 - use bullet points to list the changes
 - use emojis to make the description more engaging
-- if needed inline code snippets can be used. The code snippets should be taken
-from the changes in GIT_DIFF.
+- focus on the most important changes
+- ignore comments about imports (like added, remove, changed, etc.)
 
 `

genaisrc/pr-review-commit.genai.js

@@ -0,0 +1,39 @@
+script({
+    model: "openai:gpt-4",
+    files: [],
+    title: "pull request commit review",
+    system: [
+        "system",
+        "system.typescript",
+        "system.fs_find_files",
+        "system.fs_read_file",
+        "system.annotations",
+    ],
+})
+const { stdout: changes } = await host.exec("git", [
+    "diff",
+    "HEAD^",
+    "HEAD",
+    "--",
+    "**.ts",
+])
+
+def("GIT_DIFF", changes, { maxTokens: 20000, lineNumbers: false })
+
+$`You are an expert software developer and architect. You are
+an expert in software reliability, security, scalability, and performance.
+
+## Task
+
+Review the changes in GIT_DIFF which contains the changes of the last commit in the pull request branch.
+Provide feedback to the author using annotations.
+
+Think step by step and for each annotation explain your result.
+
+- report errors only, ignore notes and warnings
+- only report issues you are absolutely certain about
+- use a friendly tone
+- use emojis
+- read the full source code of the files if you need more context
+- only report issues about code in GIT_DIFF
+`

genaisrc/pr-review.genai.js

@@ -7,14 +7,16 @@ script({
         "system.typescript",
         "system.fs_find_files",
         "system.fs_read_file",
-        "system.annotations"
     ],
 })
+
 const { stdout: changes } = await host.exec("git", [
     "diff",
     "main",
     "--",
     ":!**/genaiscript.d.ts",
+    ":!genaisrc/*",
+    ":!.github/*",
     ":!.vscode/*",
     ":!yarn.lock",
 ])
@@ -26,10 +28,10 @@ an expert in software reliability, security, scalability, and performance.
 
 ## Task
 
-Review the changes in GIT_DIFF and provide feedback to the author using annotations.
+GIT_DIFF contains the changes the pull request branch.
+
+Provide a high level review of the changes in the pull request. Do not enter into details.
+
+If the changes look good, respond LGTM (Looks Good To Me). If you have any concerns, provide a brief description of the concerns.
 
-- report errors only, ignore notes and warnings.
-- use a friendly tone
-- use emojis
-- read the full source code of the files if you need more context
 `

packages/cli/src/cli.ts

@@ -82,7 +82,7 @@ export async function cli() {
     program.on("option:quiet", () => setQuiet(true))
 
     program
-        .command("run")
+        .command("run", { isDefault: true })
         .description("Runs a GenAIScript against files.")
         .arguments("<script> [files...]")
         .option("-ef, --excluded-files <string...>", "excluded files")
@@ -109,6 +109,10 @@ export async function cli() {
             "-prd, --pull-request-description <string>",
             "upsert comment on a pull request description."
         )
+        .option(
+            "-prr, --pull-request-reviews",
+            "create pull request reviews from annotations"
+        )
         .option("-j, --json", "emit full JSON response to output")
         .option("-y, --yaml", "emit full YAML response to output")
         .option(

packages/cli/src/run.ts

@@ -32,10 +32,9 @@ import {
     CSV_REGEX,
     CLI_RUN_FILES_FOLDER,
     parseGHTokenFromEnv,
-    githubCreateIssueComment,
-    prettifyMarkdown,
-    GITHUB_COMMENT_ID_NONE,
     githubUpsetPullRequest,
+    githubCreatePullRequestReviews,
+    githubCreateIssueComment,
 } from "genaiscript-core"
 import { capitalize } from "inflection"
 import { basename, resolve, join, relative } from "node:path"
@@ -62,6 +61,7 @@ export async function runScript(
         outChangelogs: string
         pullRequestComment: string
         pullRequestDescription: string
+        pullRequestReviews: boolean
         outData: string
         label: string
         temperature: string
@@ -92,6 +92,7 @@ export async function runScript(
     const outChangelogs = options.outChangelogs
     const pullRequestComment = options.pullRequestComment
     const pullRequestDescription = options.pullRequestDescription
+    const pullRequestReviews = options.pullRequestReviews
     const outData = options.outData
     const label = options.label
     const temperature = normalizeFloat(options.temperature)
@@ -365,47 +366,33 @@ ${Array.from(files)
         }
     }
 
+    if (pullRequestReviews && res.annotations?.length) {
+        const info = parseGHTokenFromEnv(process.env)
+        if (info.repository && info.issue)
+            await githubCreatePullRequestReviews(script, info, res.annotations)
+    }
+
     if (pullRequestComment && res.text) {
         const info = parseGHTokenFromEnv(process.env)
         if (info.repository && info.issue) {
-            const ghres = await githubCreateIssueComment(
+            await githubCreateIssueComment(
+                script,
                 info,
-                prettifyMarkdown(
-                    `${res.text}\n\n> generated by genaiscript ${script.id}.`
-                ),
+                res.text,
                 pullRequestComment
             )
-            if (!ghres.created) {
-                logError(
-                    `pull request ${info.repository}/pull/${info.issue} comment failed ${ghres.statusText}`
-                )
-                process.exit(CONFIGURATION_ERROR_CODE)
-            }
-            logVerbose(
-                `pull request ${info.repository}/pull/${info.issue} comment created at ${ghres.html_url}`
-            )
         }
     }
 
     if (pullRequestDescription && res.text) {
         const info = parseGHTokenFromEnv(process.env)
         if (info.repository && info.issue) {
-            const ghres = await githubUpsetPullRequest(
+            await githubUpsetPullRequest(
+                script,
                 info,
-                prettifyMarkdown(
-                    `${res.text}\n\n> generated by genaiscript ${script.id}.`
-                ),
+                res.text,
                 pullRequestDescription
             )
-            if (!ghres.updated) {
-                logError(
-                    `pull request ${info.repository}/pull/${info.issue} update failed ${ghres.statusText}`
-                )
-                process.exit(CONFIGURATION_ERROR_CODE)
-            }
-            logVerbose(
-                `pull request ${info.repository}/pull/${info.issue} updated`
-            )
         }
     }
     // final fail

packages/core/src/annotations.ts

@@ -1,4 +1,5 @@
 import { host } from "./host"
+import { relativePath } from "./util"
 
 const GITHUB_ANNOTATIONS_RX =
     /^::(?<severity>notice|warning|error)\s*file=(?<file>[^,]+),\s*line=(?<line>\d+),\s*endLine=(?<endLine>\d+)\s*::(?<message>.*)$/gim
@@ -24,9 +25,12 @@ export function parseAnnotations(text: string): Diagnostic[] {
     text?.replace(
         GITHUB_ANNOTATIONS_RX,
         (_, severity, file, line, endLine, message) => {
-            const filename = /^[^\/]/.test(file)
-                ? host.resolvePath(projectFolder, file)
-                : file
+            const filename = relativePath(
+                projectFolder,
+                /^[^\/]/.test(file)
+                    ? host.resolvePath(projectFolder, file)
+                    : file
+            )
             const annotation: Diagnostic = {
                 severity: sevMap[severity] || severity,
                 filename,

packages/core/src/constants.ts

@@ -170,4 +170,5 @@ export const DOCKER_CONTAINER_VOLUME = "/app"
 export const CLI_RUN_FILES_FOLDER = "files"
 
 export const GITHUB_API_VERSION = "2022-11-28"
-export const GITHUB_COMMENT_ID_NONE = "none"
+export const GITHUB_COMMENT_ID_NONE = "none"
+export const GITHUB_TOKEN = "GITHUB_TOKEN"

packages/core/src/fetch.ts

@@ -24,6 +24,9 @@ export async function createFetch(
         retryDelay = FETCH_RETRY_DEFAULT_DEFAULT,
         maxDelay = FETCH_RETRY_MAX_DELAY_DEFAULT,
     } = options || {}
+
+    if (!retryOn?.length) return crossFetch
+
     const fetchRetry = await wrapFetch(crossFetch, {
         retryOn,
         retries,