Skip to content

Commit

Permalink
✨ Add XML parsing and improve output detection
Browse files Browse the repository at this point in the history
  • Loading branch information
pelikhan committed Oct 22, 2024
1 parent 4dc77e8 commit 17affcb
Show file tree
Hide file tree
Showing 5 changed files with 34 additions and 11 deletions.
27 changes: 23 additions & 4 deletions packages/core/src/chat.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@ import { MarkdownTrace } from "./trace"
import { PromptImage, renderPromptNode } from "./promptdom"
import { LanguageModelConfiguration, host } from "./host"
import { GenerationOptions } from "./generation"
import { JSON5parse, JSONLLMTryParse, isJSONObjectOrArray } from "./json5"
import {
JSON5TryParse,
JSON5parse,
JSONLLMTryParse,
isJSONObjectOrArray,
} from "./json5"
import {
CancellationOptions,
CancellationToken,
Expand Down Expand Up @@ -43,10 +48,12 @@ import {
} from "./chatrender"
import { promptParametersSchemaToJSONSchema } from "./parameters"
import { fenceMD, prettifyMarkdown } from "./markdown"
import { YAMLStringify } from "./yaml"
import { YAMLStringify, YAMLTryParse } from "./yaml"
import { resolveTokenEncoder } from "./encoders"
import { estimateTokens, truncateTextToTokens } from "./tokens"
import { computeFileEdits } from "./fileedits"
import { HTMLEscape } from "./html"
import { XMLTryParse } from "./xml"

export function toChatCompletionUserMessage(
expanded: string,
Expand Down Expand Up @@ -763,6 +770,18 @@ export async function executeChatSession(

export function tracePromptResult(trace: MarkdownTrace, resp: RunPromptResult) {
const { json, text } = resp
trace.details(`🔠 output`, prettifyMarkdown(text), { expanded: true })
if (resp.json) trace.detailsFenced("📩 JSON (parsed)", json, "json")

// try to sniff the output type
const language = JSON5TryParse(text)
? "json"
: XMLTryParse(text)
? "xml"
: /^(-|\*|#+|```)\s/im.test(text)
? "markdown"
: "text"
trace.detailsFenced(`🔠 output`, text, language)
if (language === "markdown")
trace.appendContent(
"\n\n" + HTMLEscape(prettifyMarkdown(text)) + "\n\n"
)
}
3 changes: 3 additions & 0 deletions packages/core/src/html.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// It imports necessary libraries for HTML conversion and logging purposes.

import { TraceOptions } from "./trace" // Import TraceOptions for optional logging features
import { escape as HTMLEscape_ } from "html-escaper"

/**
* Converts HTML tables to JSON objects.
Expand Down Expand Up @@ -67,3 +68,5 @@ export async function HTMLToMarkdown(
return undefined
}
}

export const HTMLEscape = HTMLEscape_
3 changes: 2 additions & 1 deletion packages/core/src/promptcontext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// The context is essential for executing prompts within a project environment.

import { host } from "./host"
import { HTMLEscape, arrayify, dotGenaiscriptPath, sha256string } from "./util"
import { arrayify, dotGenaiscriptPath, sha256string } from "./util"
import { runtimeHost } from "./host"
import { MarkdownTrace } from "./trace"
import { createParsers } from "./parsers"
Expand All @@ -29,6 +29,7 @@ import { PLimitPromiseQueue } from "./concurrency"
import { NotSupportedError } from "./error"
import { MemoryCache } from "./cache"
import { proxifyVars } from "./parameters"
import { HTMLEscape } from "./html"

/**
* Creates a prompt context for the given project, variables, trace, options, and model.
Expand Down
9 changes: 6 additions & 3 deletions packages/core/src/runpromptcontext.ts
Original file line number Diff line number Diff line change
Expand Up @@ -626,7 +626,7 @@ export function createChatGenerationContext(
const systemScripts = resolveSystems(prj, runOptions ?? {})
if (systemScripts.length)
try {
trace.startDetails("👾 systems")
runTrace.startDetails("👾 systems")
for (const systemId of systemScripts) {
checkCancelled(cancellationToken)

Expand Down Expand Up @@ -674,7 +674,7 @@ export function createChatGenerationContext(
)
}
} finally {
trace.endDetails()
runTrace.endDetails()
}
if (systemMessage.content) messages.unshift(systemMessage)

Expand Down Expand Up @@ -718,7 +718,10 @@ export function createChatGenerationContext(
)
)
tracePromptResult(runTrace, resp)
await writeFileEdits(resp.fileEdits, { applyEdits, trace })
await writeFileEdits(resp.fileEdits, {
applyEdits,
trace: runTrace,
})
return resp
} catch (e) {
runTrace.error(e)
Expand Down
3 changes: 0 additions & 3 deletions packages/core/src/util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { GENAISCRIPT_FOLDER, HTTPS_REGEX } from "./constants"
import { isCancelError, serializeError } from "./error"
import { LogLevel, host } from "./host"
import { YAMLStringify } from "./yaml"
import { escape as HTMLEscape_ } from "html-escaper"

// chunk string into chunks of size n
export function chunkString(s: string, n: number) {
Expand Down Expand Up @@ -321,8 +320,6 @@ export function renderWithPrecision(
return rs
}

export const HTMLEscape = HTMLEscape_

export function tagFilter(tags: string[], tag: string) {
if (!tags?.length || !tag) return true
const ltag = tag.toLocaleLowerCase()
Expand Down

0 comments on commit 17affcb

Please sign in to comment.