Make eval_checker consistent with main branch by merging (#496)

berkeley-function-call-leaderboard/bfcl/eval_checker/checker.py

@@ -1,5 +1,3 @@
-from js_type_converter import js_type_converter
-from java_type_converter import java_type_converter
 from model_handler.constant import (
     UNDERSCORE_TO_DOT,
     JAVA_TYPE_CONVERSION,
@@ -12,6 +10,11 @@
 import time
 import json
 
+# We switch to conditional import for the following two imports to avoid unnecessary installations.
+# User doesn't need to setup the tree-sitter packages if they are not running the test for that language.
+# from js_type_converter import js_type_converter
+# from java_type_converter import java_type_converter
+
 PYTHON_TYPE_MAPPING = {
     "string": str,
     "integer": int,
@@ -362,9 +365,19 @@ def simple_function_checker(
         nested_type_converted = None
 
         if language == "Java":
+            from java_type_converter import java_type_converter
+
             expected_type_converted = JAVA_TYPE_CONVERSION[expected_type_description]
 
             if expected_type_description in JAVA_TYPE_CONVERSION:
+                if type(value) != str:
+                    result["valid"] = False
+                    result["error"].append(
+                        f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}."
+                    )
+                    result["error_type"] = "type_error:java"
+                    return result
+
                 if expected_type_description in NESTED_CONVERSION_TYPE_LIST:
                     nested_type = param_details[param]["items"]["type"]
                     nested_type_converted = JAVA_TYPE_CONVERSION[nested_type]
@@ -375,9 +388,19 @@ def simple_function_checker(
                     value = java_type_converter(value, expected_type_description)
 
         elif language == "JavaScript":
+            from js_type_converter import js_type_converter
+
             expected_type_converted = JS_TYPE_CONVERSION[expected_type_description]
 
             if expected_type_description in JS_TYPE_CONVERSION:
+                if type(value) != str:
+                    result["valid"] = False
+                    result["error"].append(
+                        f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}."
+                    )
+                    result["error_type"] = "type_error:js"
+                    return result
+
                 if expected_type_description in NESTED_CONVERSION_TYPE_LIST:
                     nested_type = param_details[param]["items"]["type"]
                     nested_type_converted = JS_TYPE_CONVERSION[nested_type]
@@ -945,4 +968,4 @@ def exec_checker(decoded_result: list, func_description: dict, test_category: st
             func_description["execution_result"][0],
             func_description["execution_result_type"][0],
             False,
-        )
+        )

berkeley-function-call-leaderboard/bfcl/eval_checker/custom_exception.py

@@ -1,10 +1,10 @@
 class NoAPIKeyError(Exception):
     def __init__(self):
-        self.message = "Please fill in the API keys in the function_credential_config.json file. If you do not provide the API keys, the executable test category results will be inaccurate."
+        self.message = "❗️Please fill in the API keys in the function_credential_config.json file. If you do not provide the API keys, the executable test category results will be inaccurate."
         super().__init__(self.message)
 
 
 class BadAPIStatusError(Exception):
-    def __init__(self, message):
-        self.message = message
-        super().__init__(self.message)
+    def __init__(self, errors, error_rate):
+        self.errors = errors
+        self.error_rate = error_rate