diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt index afb188c03..1df855b86 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt @@ -2,12 +2,7 @@ package com.xebia.functional.xef.evaluator import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestModel import com.xebia.functional.xef.AI -import com.xebia.functional.xef.evaluator.models.ItemResult -import com.xebia.functional.xef.evaluator.models.OutputResponse -import com.xebia.functional.xef.evaluator.models.OutputResult -import com.xebia.functional.xef.evaluator.models.SuiteResults -import com.xebia.functional.xef.evaluator.output.Html -import java.io.File +import com.xebia.functional.xef.evaluator.models.* import kotlinx.serialization.SerialName import kotlinx.serialization.Serializable import kotlinx.serialization.json.Json @@ -67,18 +62,15 @@ data class SuiteSpec( block: suspend SuiteBuilder.() -> Unit ): SuiteSpec = SuiteBuilder(description, model).apply { block() }.build() - inline fun toHtml( + inline fun toHtml(result: SuiteResults, suiteName: String): Html where + E : AI.PromptClassifier, + E : Enum = + Html.get(Json.encodeToString(SuiteResults.serializer(serializer()), result), suiteName) + + inline fun toMarkdown( result: SuiteResults, - htmlFilename: String = "index.html" - ) where E : AI.PromptClassifier, E : Enum { - val content = Json.encodeToString(SuiteResults.serializer(serializer()), result) - // Copy file inside build folder - val outputPath = System.getProperty("user.dir") + "/build/testSuite" - File(outputPath).mkdirs() - val htmlFile = File("$outputPath/$htmlFilename") - htmlFile.writeText(Html.get(content)) - println("Test suite exported to ${htmlFile.absoluteFile}") - } + suiteName: String, + ): Markdown where E : AI.PromptClassifier, E : Enum = Markdown.get(result, suiteName) } } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/output/Html.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt similarity index 95% rename from evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/output/Html.kt rename to evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt index f206c40c3..d4eecb719 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/output/Html.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt @@ -1,6 +1,7 @@ -package com.xebia.functional.xef.evaluator.output +package com.xebia.functional.xef.evaluator.models -class Html { +@JvmInline +value class Html(val value: String) { companion object { @@ -168,9 +169,10 @@ class Html { """ .trimIndent() - fun get(contentJson: String): String { + fun get(contentJson: String, suiteName: String): Html { // language=html - return """ + return Html( + """ @@ -180,6 +182,7 @@ class Html { @@ -187,7 +190,8 @@ class Html { """ - .trimIndent() + .trimIndent() + ) } } } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt new file mode 100644 index 000000000..40a275b11 --- /dev/null +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt @@ -0,0 +1,44 @@ +package com.xebia.functional.xef.evaluator.models + +import com.xebia.functional.xef.AI +import com.xebia.functional.xef.evaluator.metrics.ContextualRelevancy + +@JvmInline +value class Markdown(val value: String) { + companion object { + fun get(result: SuiteResults, suiteName: String): Markdown where + E : AI.PromptClassifier, + E : Enum { + val content = + """| + |# Suite Results: $suiteName + |- Description: ${result.description} + |- Model: ${result.model} + |- Metric: ${ContextualRelevancy::class.simpleName} + |${ + result.items.joinToString("\n") { item -> + """ + |#### Input: ${item.description} + |${ + item.items.joinToString("\n") { outputResult -> + """ + |- Description: ${outputResult.description} + |- Context: ${outputResult.contextDescription} + |- Output: + |
+ |${outputResult.output} + |
+ | + |Result: ${if (outputResult.success) "✅ Success" else "❌ Failure"} (${outputResult.result}) + """.trimMargin() + } + } + """.trimMargin() + } + } + """ + .trimMargin() + return Markdown(content) + } + } +}