xebia-functional · raulraja · May 2, 2024 · May 2, 2024
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/SuiteBuilder.kt
@@ -68,10 +68,9 @@ data class SuiteSpec(
     E : Enum<E> =
       Html.get(Json.encodeToString(SuiteResults.serializer(serializer<E>()), result), suiteName)
 
-    inline fun <reified E> toMarkdown(
-      result: SuiteResults<E>,
-      suiteName: String,
-    ): Markdown where E : AI.PromptClassifier, E : Enum<E> = Markdown.get(result, suiteName)
+    inline fun <reified E> toMarkdown(result: SuiteResults<E>, suiteName: String): Markdown where
+    E : AI.PromptClassifier,
+    E : Enum<E> = Markdown.get(result, suiteName)
   }
 }
 

diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Html.kt
@@ -58,10 +58,12 @@ value class Html(val value: String) {
                     outputDiv.innerText = 'Output: ' + test.output;
                     blockDiv.appendChild(outputDiv);
 
-                    const usageDiv = document.createElement('pre');
-                    usageDiv.classList.add('output');
-                    usageDiv.innerText = 'Usage: \n Completion Tokens: ' + test.usage?.completionTokens + '\n Prompt Tokens: ' + test.usage?.promptTokens + '\n Total Tokens: ' + test.usage?.totalTokens;
-                    blockDiv.appendChild(usageDiv);
+                    if (test.usage != undefined) {
+                        const usageDiv = document.createElement('pre');
+                        usageDiv.classList.add('output');
+                        usageDiv.innerText = 'Usage: \n Prompt Tokens: ' + test.usage?.promptTokens + ' (~' + test.usage?.estimatePricePerToken + ' ' + test.usage?.currency + ')\n Completion Tokens: ' + test.usage?.completionTokens + ' (~' + test.usage?.estimatePriceCompletionToken + ' ' + test.usage?.currency + ')\n Total Tokens: ' + test.usage?.totalTokens + '\n Total Price: ~' + test.usage?.estimatePriceTotalToken + ' ' + test.usage?.currency;
+                        blockDiv.appendChild(usageDiv);
+                    }
 
                     const result = document.createElement('div');
                     result.classList.add('score', test.success ? 'score-passed' : 'score-failed');
@@ -101,6 +103,10 @@ value class Html(val value: String) {
               border-bottom: 1px solid #eee;
               padding-bottom: 20px;
           }
+
+          .test-block pre {
+              margin-bottom: 20px;
+          }
 
           .test-title {
               font-size: 1.2em;

diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/Markdown.kt
@@ -32,9 +32,10 @@ value class Markdown(val value: String) {
                 |<blockquote>
                 |${outputResult.usage?.let { usage ->
                   """
-                  |Completion Tokens: ${usage.completionTokens}
-                  |Prompt Tokens: ${usage.promptTokens}
+                  |Prompt Tokens: ${usage.promptTokens} ${usage.estimatePricePerToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" }
+                  |Completion Tokens: ${usage.completionTokens} ${usage.estimatePriceCompletionToken?.let { "(~ ${it.to2DecimalsString()} ${usage.currency ?: ""})" } ?: "" }
                   |Total Tokens: ${usage.totalTokens}
+                  |Total Price: ${usage.estimatePriceTotalToken?.let { "${it.to2DecimalsString()} ${usage.currency ?: ""}" } ?: "Unknown"}
                   """.trimMargin()
                 } ?: "No usage information available"}
                 |</blockquote>
@@ -50,5 +51,7 @@ value class Markdown(val value: String) {
           .trimMargin()
       return Markdown(content)
     }
+
+    private fun Double.to2DecimalsString() = String.format("%.6f", this)
   }
 }
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/ModelsPricing.kt
@@ -0,0 +1,44 @@
+package com.xebia.functional.xef.evaluator.models
+
+data class ModelsPricing(
+  val modelName: String,
+  val currency: String,
+  val input: ModelsPricingItem,
+  val output: ModelsPricingItem
+) {
+
+  companion object {
+
+    const val oneMillion = 1_000_000
+    val oneThousand = 1_000
+
+    // The pricing for the models was updated the May 2st, 2024
+    // Be sure to update the pricing for each model
+
+    val gpt4Turbo =
+      ModelsPricing(
+        modelName = "gpt-4-turbo",
+        currency = "USD",
+        input = ModelsPricingItem(10.0, oneMillion),
+        output = ModelsPricingItem(30.0, oneMillion)
+      )
+
+    val gpt4 =
+      ModelsPricing(
+        modelName = "gpt-4-turbo",
+        currency = "USD",
+        input = ModelsPricingItem(30.0, oneMillion),
+        output = ModelsPricingItem(60.0, oneMillion)
+      )
+
+    val gpt3_5Turbo =
+      ModelsPricing(
+        modelName = "gpt-3.5-turbo",
+        currency = "USD",
+        input = ModelsPricingItem(0.5, oneMillion),
+        output = ModelsPricingItem(1.5, oneMillion)
+      )
+  }
+}
+
+data class ModelsPricingItem(val price: Double, val perTokens: Int)
diff --git a/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt b/evaluator/src/main/kotlin/com/xebia/functional/xef/evaluator/models/TestModels.kt
@@ -17,23 +17,48 @@ data class OutputResponse(
     @JvmSynthetic
     suspend operator fun invoke(
       description: OutputDescription,
+      price: ModelsPricing?,
       block: suspend () -> MessageWithUsage
     ): OutputResponse {
       val response = block()
-      return OutputResponse(description, response.usage?.let { OutputTokens(it) }, response.message)
+      return OutputResponse(
+        description,
+        response.usage?.let { OutputTokens(it, price) },
+        response.message
+      )
     }
   }
 }
 
 @Serializable
 data class OutputTokens(
   val promptTokens: Int? = null,
+  val estimatePricePerToken: Double? = null,
   val completionTokens: Int? = null,
-  val totalTokens: Int? = null
+  val estimatePriceCompletionToken: Double? = null,
+  val totalTokens: Int? = null,
+  val estimatePriceTotalToken: Double? = null,
+  val currency: String?
 ) {
   companion object {
     @JvmSynthetic
-    operator fun invoke(usage: MessagesUsage): OutputTokens =
-      OutputTokens(usage.promptTokens, usage.completionTokens, usage.totalTokens)
+    operator fun invoke(usage: MessagesUsage, price: ModelsPricing?): OutputTokens {
+      val estimateInputPrice =
+        price?.let { usage.promptTokens.let { (it * price.input.price) / price.input.perTokens } }
+      val estimateOutputPrice =
+        price?.let {
+          usage.completionTokens.let { (it * price.output.price) / price.output.perTokens }
+        }
+      val estimateTotalPrice = estimateInputPrice?.plus(estimateOutputPrice ?: 0.0)
+      return OutputTokens(
+        usage.promptTokens,
+        estimateInputPrice,
+        usage.completionTokens,
+        estimateOutputPrice,
+        usage.totalTokens,
+        estimateTotalPrice,
+        price?.currency
+      )
+    }
   }
 }
diff --git a/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt b/examples/src/main/kotlin/com/xebia/functional/xef/evaluator/TestExample.kt
@@ -5,6 +5,7 @@ import com.xebia.functional.openai.generated.model.CreateChatCompletionRequestMo
 import com.xebia.functional.xef.OpenAI
 import com.xebia.functional.xef.conversation.Conversation
 import com.xebia.functional.xef.evaluator.metrics.AnswerAccuracy
+import com.xebia.functional.xef.evaluator.models.ModelsPricing
 import com.xebia.functional.xef.evaluator.models.OutputDescription
 import com.xebia.functional.xef.evaluator.models.OutputResponse
 import com.xebia.functional.xef.llm.promptMessageAndUsage
@@ -31,7 +32,7 @@ object TestExample {
           input = "Please provide a movie title, genre and director",
           context = "Contains information about a movie"
         ) {
-          +OutputResponse(gpt35Description) {
+          +OutputResponse(gpt35Description, ModelsPricing.gpt3_5Turbo) {
             Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) }
           }
 
@@ -42,7 +43,7 @@ object TestExample {
           input = "Recipe for a chocolate cake",
           context = "Contains instructions for making a cake"
         ) {
-          +OutputResponse(gpt35Description) {
+          +OutputResponse(gpt35Description, ModelsPricing.gpt3_5Turbo) {
             Conversation { chat.promptMessageAndUsage(Prompt(model) { +user(input) }) }
           }