better success / failure reasoning

yukinagae · Sep 26, 2024 · 3cc78a0 · 3cc78a0
1 parent 7d2902b
commit 3cc78a0
Show file tree

Hide file tree

Showing 3 changed files with 9 additions and 6 deletions.
diff --git a/examples/package.json b/examples/package.json
@@ -24,7 +24,7 @@
     "@genkit-ai/googleai": "^0.5.13",
     "@genkit-ai/vertexai": "^0.5.13",
     "express": "^4.21.0",
-    "genkitx-promptfoo": "^0.1.8",
+    "genkitx-promptfoo": "^0.1.11",
     "zod": "^3.23.8"
   },
   "devDependencies": {

diff --git a/src/index.ts b/src/index.ts
@@ -108,19 +108,23 @@ export async function promptfooScore(
   });
   const score = summary.results[0].score;
   if (summary.results[0].error) {
+    const errorReason = summary.results[0].error;
     return {
       score: score,
-      error: 'Assertion failed',
       details: {
-        reasoning: summary.results[0].error, // error reason
+        reasoning: `Assertion failed: ${errorReason}`, // error reason
       },
     };
   }
+  const successReason =
+    summary.results[0].gradingResult?.componentResults?.[0]?.reason;
   return {
     score: score,
     details: {
       reasoning:
-        summary.results[0].gradingResult?.componentResults?.[0]?.reason, // success reason
+        successReason === 'Assertion passed'
+          ? successReason
+          : `Assertion passed: ${successReason}`, // success reason
     },
   };
 }

diff --git a/tests/index_test.ts b/tests/index_test.ts
@@ -46,9 +46,8 @@ describe('promptfooScore', () => {
       },
       expected: {
         score: 0,
-        error: 'Assertion failed',
         details: {
-          reasoning: 'Expected output to contain "Hello"',
+          reasoning: 'Assertion failed: Expected output to contain "Hello"',
         },
       },
     },