From 2734ae1bf714f6016c97b83778a08fac4a6ed620 Mon Sep 17 00:00:00 2001 From: Xu Hwang Date: Mon, 2 Sep 2024 12:46:48 +0800 Subject: [PATCH 1/2] add ToolACE handler --- .../bfcl/eval_checker/eval_runner_helper.py | 9 +- .../bfcl/model_handler/handler_map.py | 4 +- .../bfcl/model_handler/toolace_handler.py | 181 ++++++++++++++++++ 3 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py diff --git a/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py index 62a67203c..7ec651ed0 100644 --- a/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py +++ b/berkeley-function-call-leaderboard/bfcl/eval_checker/eval_runner_helper.py @@ -532,6 +532,12 @@ "https://huggingface.co/Salesforce/xLAM-7b-fc-r", "Salesforce", "cc-by-nc-4.0", + ], + "Team-ACE/ToolACE-8B": [ + "ToolACE-8B (FC)", + "https://huggingface.co/Team-ACE/ToolACE-8B", + "Huawei Noah & USTC", + "Apache-2.0" ] } @@ -673,7 +679,8 @@ "ibm-granite/granite-20b-functioncalling", "THUDM/glm-4-9b-chat", "Salesforce/xLAM-1b-fc-r", - "Salesforce/xLAM-7b-fc-r" + "Salesforce/xLAM-7b-fc-r", + "Team-ACE/ToolACE-8B" ] # Price got from AZure, 22.032 per hour for 8 V100, Pay As You Go Total Price diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py index 48668107e..0cff2fb5f 100644 --- a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py @@ -20,6 +20,7 @@ from bfcl.model_handler.glm_handler import GLMHandler from bfcl.model_handler.yi_handler import YiHandler from bfcl.model_handler.xlam_handler import xLAMHandler +from bfcl.model_handler.toolace_handler import ToolACEHandler handler_map = { "gorilla-openfunctions-v0": GorillaHandler, @@ -96,5 +97,6 @@ "THUDM/glm-4-9b-chat": GLMHandler, "yi-large-fc": YiHandler, "Salesforce/xLAM-1b-fc-r": xLAMHandler, - "Salesforce/xLAM-7b-fc-r": xLAMHandler + "Salesforce/xLAM-7b-fc-r": xLAMHandler, + "Team-ACE/ToolACE-8B": ToolACEHandler } diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py new file mode 100644 index 000000000..6cb2a5e5e --- /dev/null +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py @@ -0,0 +1,181 @@ +import json +from bfcl.model_handler.oss_handler import OSSHandler +from bfcl.model_handler.utils import ast_parse +from bfcl.model_handler.utils import ( + ast_parse, + _get_language_specific_hint +) + + +SYSTEM_PROMPT = """You are an expert in composing functions. You are given a question and a set of possible functions. +Based on the question, you will need to make one or more function/tool calls to achieve the purpose. +If none of the function can be used, point it out. If the given question lacks the parameters required by the function, +also point it out. +""" + +TOOL_PROMPT = """Here is a list of functions in JSON format that you can invoke:\n{functions}. +Should you decide to return the function call(s). \nPut it in the format of [func1(params_name=params_value, params_name2=params_value2...), func2(params)]\n +NO other text MUST be included. +""" + +class ToolACEHandler(OSSHandler): + def __init__(self, model_name, temperature=0.001, top_p=1, max_tokens=1024) -> None: + super().__init__(model_name, temperature, top_p, max_tokens) + + def _format_prompt(prompts, function, test_category): + + formatted_prompt = "<|begin_of_text|>" + + for prompt in prompts: + formatted_prompt += f"<|start_header_id|>{prompt['role']}<|end_header_id|>\n\n{prompt['content']}<|eot_id|>" + + formatted_prompt += f"<|start_header_id|>assistant<|end_header_id|>\n\n" + + return formatted_prompt + + + @staticmethod + def system_prompt_pre_processing(prompts, system_prompt_template, function_docs): + assert type(prompts) == list + + system_prompt = system_prompt_template.format(functions=function_docs) + + # System prompt must be in the first position + # If the question comes with a system prompt, append its content at the end of the chat template. + if prompts[0]["role"] == "system": + prompts[0]["content"] = system_prompt + prompts[0]["content"] + # Otherwise, use the system prompt template to create a new system prompt. + else: + prompts.insert( + 0, + {"role": "system", "content": system_prompt}, + ) + + return prompts + + + @staticmethod + def func_doc_language_specific_pre_processing(function, test_category): + if len(function) == 0: + return function + + assert type(function) == list + for item in function: + # Add language specific hints to the function description + item["description"] = item["description"] + # Process the parameters + properties = item["parameters"]["properties"] + if test_category == "java": + for key, value in properties.items(): + if value["type"] == "any": + properties[key]["description"] += ( + " This parameter can be of any type of Java object in string representation." + ) + else: + value["description"] += ( + f" This is Java {value['type']} type parameter in string representation." + ) + if value["type"] == "ArrayList" or value["type"] == "Array": + value["description"] += ( + f" The list elements are of type {value['items']['type']}; they are not in string representation." + ) + del value["items"] + + value["type"] = "string" + + elif test_category == "javascript": + for key, value in properties.items(): + if value["type"] == "any": + properties[key]["description"] += ( + " This parameter can be of any type of JavaScript object in string representation." + ) + else: + value["description"] += ( + f" This is JavaScript {value['type']} type parameter in string representation." + ) + if value["type"] == "array": + value["description"] += ( + f" The list elements are of type {value['items']['type']}; they are not in string representation." + ) + del value["items"] + + if value["type"] == "dict": + if "properties" in value: # not every dict has properties + value["description"] += ( + f" The dictionary entries have the following schema; they are not in string representation. {json.dumps(value['properties'])}" + ) + del value["properties"] + + value["type"] = "string" + + return function + + @staticmethod + def process_input( + test_question, + format_prompt_func, + include_system_prompt=True + ): + prompts = [] + for question in test_question: + test_category = question["id"].rsplit("_", 1)[0] + functions = ToolACEHandler.func_doc_language_specific_pre_processing( + question["function"], test_category + ) + # prompt here is a list of dictionaries, one representing a role and content + if include_system_prompt: + question["question"] = ToolACEHandler.system_prompt_pre_processing( + question["question"], + f"{SYSTEM_PROMPT}\n{TOOL_PROMPT}", + function_docs=functions + ) + + language_hint = _get_language_specific_hint(test_category) + question["question"][-1]["content"] += f"\n{language_hint}" + + prompts.append(format_prompt_func(question["question"], functions, test_category)) + return prompts + + def inference( + self, + test_question, + num_gpus, + gpu_memory_utilization, + format_prompt_func=_format_prompt, + max_model_len=None, + include_system_prompt=True + ): + return super().inference( + test_question, + num_gpus, + gpu_memory_utilization=0.7, + format_prompt_func=format_prompt_func, + include_system_prompt=include_system_prompt, + max_model_len=max_model_len + ) + + def decode_ast(self, result, language="Python"): + func = result + func = func.replace("\n", "") # remove new line characters + if not func.startswith("["): + func = "[" + func + if not func.endswith("]"): + func = func + "]" + decoded_output = ast_parse(func, language) + return decoded_output + + def decode_execute(self, result): + func = result + func = func.replace("\n", "") # remove new line characters + if not func.startswith("["): + func = "[" + func + if not func.endswith("]"): + func = func + "]" + decode_output = ast_parse(func) + execution_list = [] + for function_call in decode_output: + for key, value in function_call.items(): + execution_list.append( + f"{key}({','.join([f'{k}={repr(v)}' for k, v in value.items()])})" + ) + return execution_list From 429e8cd1903e7623dbb562b1417d6abb50cfbf15 Mon Sep 17 00:00:00 2001 From: Xu Hwang Date: Sat, 14 Sep 2024 15:01:29 +0800 Subject: [PATCH 2/2] set toolace handler as llama handler --- .../bfcl/model_handler/handler_map.py | 3 +- .../bfcl/model_handler/toolace_handler.py | 181 ------------------ 2 files changed, 1 insertion(+), 183 deletions(-) delete mode 100644 berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py index 0cff2fb5f..fb4a32ada 100644 --- a/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py +++ b/berkeley-function-call-leaderboard/bfcl/model_handler/handler_map.py @@ -20,7 +20,6 @@ from bfcl.model_handler.glm_handler import GLMHandler from bfcl.model_handler.yi_handler import YiHandler from bfcl.model_handler.xlam_handler import xLAMHandler -from bfcl.model_handler.toolace_handler import ToolACEHandler handler_map = { "gorilla-openfunctions-v0": GorillaHandler, @@ -98,5 +97,5 @@ "yi-large-fc": YiHandler, "Salesforce/xLAM-1b-fc-r": xLAMHandler, "Salesforce/xLAM-7b-fc-r": xLAMHandler, - "Team-ACE/ToolACE-8B": ToolACEHandler + "Team-ACE/ToolACE-8B": LlamaHandler, } diff --git a/berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py b/berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py deleted file mode 100644 index 6cb2a5e5e..000000000 --- a/berkeley-function-call-leaderboard/bfcl/model_handler/toolace_handler.py +++ /dev/null @@ -1,181 +0,0 @@ -import json -from bfcl.model_handler.oss_handler import OSSHandler -from bfcl.model_handler.utils import ast_parse -from bfcl.model_handler.utils import ( - ast_parse, - _get_language_specific_hint -) - - -SYSTEM_PROMPT = """You are an expert in composing functions. You are given a question and a set of possible functions. -Based on the question, you will need to make one or more function/tool calls to achieve the purpose. -If none of the function can be used, point it out. If the given question lacks the parameters required by the function, -also point it out. -""" - -TOOL_PROMPT = """Here is a list of functions in JSON format that you can invoke:\n{functions}. -Should you decide to return the function call(s). \nPut it in the format of [func1(params_name=params_value, params_name2=params_value2...), func2(params)]\n -NO other text MUST be included. -""" - -class ToolACEHandler(OSSHandler): - def __init__(self, model_name, temperature=0.001, top_p=1, max_tokens=1024) -> None: - super().__init__(model_name, temperature, top_p, max_tokens) - - def _format_prompt(prompts, function, test_category): - - formatted_prompt = "<|begin_of_text|>" - - for prompt in prompts: - formatted_prompt += f"<|start_header_id|>{prompt['role']}<|end_header_id|>\n\n{prompt['content']}<|eot_id|>" - - formatted_prompt += f"<|start_header_id|>assistant<|end_header_id|>\n\n" - - return formatted_prompt - - - @staticmethod - def system_prompt_pre_processing(prompts, system_prompt_template, function_docs): - assert type(prompts) == list - - system_prompt = system_prompt_template.format(functions=function_docs) - - # System prompt must be in the first position - # If the question comes with a system prompt, append its content at the end of the chat template. - if prompts[0]["role"] == "system": - prompts[0]["content"] = system_prompt + prompts[0]["content"] - # Otherwise, use the system prompt template to create a new system prompt. - else: - prompts.insert( - 0, - {"role": "system", "content": system_prompt}, - ) - - return prompts - - - @staticmethod - def func_doc_language_specific_pre_processing(function, test_category): - if len(function) == 0: - return function - - assert type(function) == list - for item in function: - # Add language specific hints to the function description - item["description"] = item["description"] - # Process the parameters - properties = item["parameters"]["properties"] - if test_category == "java": - for key, value in properties.items(): - if value["type"] == "any": - properties[key]["description"] += ( - " This parameter can be of any type of Java object in string representation." - ) - else: - value["description"] += ( - f" This is Java {value['type']} type parameter in string representation." - ) - if value["type"] == "ArrayList" or value["type"] == "Array": - value["description"] += ( - f" The list elements are of type {value['items']['type']}; they are not in string representation." - ) - del value["items"] - - value["type"] = "string" - - elif test_category == "javascript": - for key, value in properties.items(): - if value["type"] == "any": - properties[key]["description"] += ( - " This parameter can be of any type of JavaScript object in string representation." - ) - else: - value["description"] += ( - f" This is JavaScript {value['type']} type parameter in string representation." - ) - if value["type"] == "array": - value["description"] += ( - f" The list elements are of type {value['items']['type']}; they are not in string representation." - ) - del value["items"] - - if value["type"] == "dict": - if "properties" in value: # not every dict has properties - value["description"] += ( - f" The dictionary entries have the following schema; they are not in string representation. {json.dumps(value['properties'])}" - ) - del value["properties"] - - value["type"] = "string" - - return function - - @staticmethod - def process_input( - test_question, - format_prompt_func, - include_system_prompt=True - ): - prompts = [] - for question in test_question: - test_category = question["id"].rsplit("_", 1)[0] - functions = ToolACEHandler.func_doc_language_specific_pre_processing( - question["function"], test_category - ) - # prompt here is a list of dictionaries, one representing a role and content - if include_system_prompt: - question["question"] = ToolACEHandler.system_prompt_pre_processing( - question["question"], - f"{SYSTEM_PROMPT}\n{TOOL_PROMPT}", - function_docs=functions - ) - - language_hint = _get_language_specific_hint(test_category) - question["question"][-1]["content"] += f"\n{language_hint}" - - prompts.append(format_prompt_func(question["question"], functions, test_category)) - return prompts - - def inference( - self, - test_question, - num_gpus, - gpu_memory_utilization, - format_prompt_func=_format_prompt, - max_model_len=None, - include_system_prompt=True - ): - return super().inference( - test_question, - num_gpus, - gpu_memory_utilization=0.7, - format_prompt_func=format_prompt_func, - include_system_prompt=include_system_prompt, - max_model_len=max_model_len - ) - - def decode_ast(self, result, language="Python"): - func = result - func = func.replace("\n", "") # remove new line characters - if not func.startswith("["): - func = "[" + func - if not func.endswith("]"): - func = func + "]" - decoded_output = ast_parse(func, language) - return decoded_output - - def decode_execute(self, result): - func = result - func = func.replace("\n", "") # remove new line characters - if not func.startswith("["): - func = "[" + func - if not func.endswith("]"): - func = func + "]" - decode_output = ast_parse(func) - execution_list = [] - for function_call in decode_output: - for key, value in function_call.items(): - execution_list.append( - f"{key}({','.join([f'{k}={repr(v)}' for k, v in value.items()])})" - ) - return execution_list