diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt index 94a33497b..ff85f9e95 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt @@ -68,10 +68,9 @@ data class SuiteSpec( E : Enum = Html.get(Json.encodeToString(SuiteResults.serializer(serializer()), result), suiteName) - inline fun toMarkdown( - result: SuiteResults, - suiteName: String, - ): Markdown where E : AI.PromptClassifier, E : Enum = Markdown.get(result, suiteName) + inline fun toMarkdown(result: SuiteResults, suiteName: String): Markdown where + E : AI.PromptClassifier, + E : Enum = Markdown.get(result, suiteName) } } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt index a2b1c377c..b2514dfbb 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt @@ -58,10 +58,12 @@ value class Html(val value: String) { outputDiv.innerText = 'Output: ' + test.output; blockDiv.appendChild(outputDiv); - const usageDiv = document.createElement('pre'); - usageDiv.classList.add('output'); - usageDiv.innerText = 'Usage: \n Completion Tokens: ' + test.usage?.completionTokens + '\n Prompt Tokens: ' + test.usage?.promptTokens + '\n Total Tokens: ' + test.usage?.totalTokens; - blockDiv.appendChild(usageDiv); + if (test.usage != undefined) { + const usageDiv = document.createElement('pre'); + usageDiv.classList.add('output'); + usageDiv.innerText = 'Usage: \n Prompt Tokens: ' + test.usage?.promptTokens + ' (~' + test.usage?.estimatePricePerToken + ' ' + test.usage?.currency + ')\n Completion Tokens: ' + test.usage?.completionTokens + ' (~' + test.usage?.estimatePriceCompletionToken + ' ' + test.usage?.currency + ')\n Total Tokens: ' + test.usage?.totalTokens + '\n Total Price: ~' + test.usage?.estimatePriceTotalToken + ' ' + test.usage?.currency; + blockDiv.appendChild(usageDiv); + } const result = document.createElement('div'); result.classList.add('score', test.success ? 'score-passed' : 'score-failed'); @@ -101,6 +103,10 @@ value class Html(val value: String) { border-bottom: 1px solid #eee; padding-bottom: 20px; } + + .test-block pre { + margin-bottom: 20px; + } .test-title { font-size: 1.2em; diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt index a998f08e2..92baaaf7b 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt @@ -32,9 +32,10 @@ value class Markdown(val value: String) { |
|${outputResult.usage?.let { usage -> """ - |Completion Tokens: ${usage.completionTokens} - |Prompt Tokens: ${usage.promptTokens} + |Prompt Tokens: ${usage.promptTokens} ${usage.estimatePricePerToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" } + |Completion Tokens: ${usage.completionTokens} ${usage.estimatePriceCompletionToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" } |Total Tokens: ${usage.totalTokens} + |Total Price: ${usage.estimatePriceTotalToken?.let { "${it.to2DecimalsString()} ${usage.currency ?: ""}" } ?: "Unknown"} """.trimMargin() } ?: "No usage information available"} |
@@ -50,5 +51,7 @@ value class Markdown(val value: String) { .trimMargin() return Markdown(content) } + + private fun Double.to2DecimalsString() = String.format("%.6f", this) } } diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt new file mode 100644 index 000000000..124148c84 --- /dev/null +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt @@ -0,0 +1,44 @@ +package com.xebia.functional.xef.evaluator.models + +data class ModelsPricing( + val modelName: String, + val currency: String, + val input: ModelsPricingItem, + val output: ModelsPricingItem +) { + + companion object { + + const val oneMillion = 1_000_000 + val oneThousand = 1_000 + + // The pricing for the models was updated the May 2st, 2024 + // Be sure to update the pricing for each model + + val gpt4Turbo = + ModelsPricing( + modelName = "gpt-4-turbo", + currency = "USD", + input = ModelsPricingItem(10.0, oneMillion), + output = ModelsPricingItem(30.0, oneMillion) + ) + + val gpt4 = + ModelsPricing( + modelName = "gpt-4-turbo", + currency = "USD", + input = ModelsPricingItem(30.0, oneMillion), + output = ModelsPricingItem(60.0, oneMillion) + ) + + val gpt3_5Turbo = + ModelsPricing( + modelName = "gpt-3.5-turbo", + currency = "USD", + input = ModelsPricingItem(0.5, oneMillion), + output = ModelsPricingItem(1.5, oneMillion) + ) + } +} + +data class ModelsPricingItem(val price: Double, val perTokens: Int) diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt index ca9160674..ccdcff017 100644 --- a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt +++ b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt @@ -17,10 +17,15 @@ data class OutputResponse( @JvmSynthetic suspend operator fun invoke( description: OutputDescription, + price: ModelsPricing?, block: suspend () -> MessageWithUsage ): OutputResponse { val response = block() - return OutputResponse(description, response.usage?.let { OutputTokens(it) }, response.message) + return OutputResponse( + description, + response.usage?.let { OutputTokens(it, price) }, + response.message + ) } } } @@ -28,12 +33,32 @@ data class OutputResponse( @Serializable data class OutputTokens( val promptTokens: Int? = null, + val estimatePricePerToken: Double? = null, val completionTokens: Int? = null, - val totalTokens: Int? = null + val estimatePriceCompletionToken: Double? = null, + val totalTokens: Int? = null, + val estimatePriceTotalToken: Double? = null, + val currency: String? ) { companion object { @JvmSynthetic - operator fun invoke(usage: MessagesUsage): OutputTokens = - OutputTokens(usage.promptTokens, usage.completionTokens, usage.totalTokens) + operator fun invoke(usage: MessagesUsage, price: ModelsPricing?): OutputTokens { + val estimateInputPrice = + price?.let { usage.promptTokens.let { (it * price.input.price) / price.input.perTokens } } + val estimateOutputPrice = + price?.let { + usage.completionTokens.let { (it * price.output.price) / price.output.perTokens } + } + val estimateTotalPrice = estimateInputPrice?.plus(estimateOutputPrice ?: 0.0) + return OutputTokens( + usage.promptTokens, + estimateInputPrice, + usage.completionTokens, + estimateOutputPrice, + usage.totalTokens, + estimateTotalPrice, + price?.currency + ) + } } } diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt index d51a6710d..999a6b8da 100644 --- a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt +++ b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt @@ -5,6 +5,7 @@ import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestMo import com.xebia.functional.xef.OpenAI import com.xebia.functional.xef.conversation.Conversation import com.xebia.functional.xef.evaluator.metrics.AnswerAccuracy +import com.xebia.functional.xef.evaluator.models.ModelsPricing import com.xebia.functional.xef.evaluator.models.OutputDescription import com.xebia.functional.xef.evaluator.models.OutputResponse import com.xebia.functional.xef.llm.promptMessageAndUsage @@ -31,7 +32,7 @@ object TestExample { input = "Please provide a movie title, genre and director", context = "Contains information about a movie" ) { - +OutputResponse(gpt35Description) { + +OutputResponse(gpt35Description, ModelsPricing.gpt3_5Turbo) { Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) } } @@ -42,7 +43,7 @@ object TestExample { input = "Recipe for a chocolate cake", context = "Contains instructions for making a cake" ) { - +OutputResponse(gpt35Description) { + +OutputResponse(gpt35Description, ModelsPricing.gpt3_5Turbo) { Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) } }