From 57b02d7b265176269e64c6b9c5ee3891273ecf95 Mon Sep 17 00:00:00 2001 From: longchao1916 Date: Mon, 18 Mar 2024 20:04:55 +0800 Subject: [PATCH 1/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E4=E4=B8=AAFC=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=BB=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- appbuilder/__init__.py | 6 +- appbuilder/core/components/asr/component.py | 71 ++++++++++++++++++- .../core/components/general_ocr/component.py | 62 +++++++++++++++- .../components/object_recognize/component.py | 70 +++++++++++++++++- .../core/components/translate/component.py | 58 ++++++++++++++- appbuilder/tests/test_asr.py | 13 ++++ appbuilder/tests/test_general_ocr.py | 17 +++++ appbuilder/tests/test_object_recognize.py | 17 +++++ appbuilder/tests/test_translate.py | 13 ++++ 9 files changed, 319 insertions(+), 8 deletions(-) diff --git a/appbuilder/__init__.py b/appbuilder/__init__.py index eb22829f..08a37b98 100644 --- a/appbuilder/__init__.py +++ b/appbuilder/__init__.py @@ -61,8 +61,8 @@ def check_version(self): from .core.components.extract_table.component import ExtractTableFromDoc from .core.components.doc_parser.doc_parser import DocParser, ParserConfig from .core.components.doc_splitter.doc_splitter import DocSplitter -from .core.components.retriever.bes_retriever import BESRetriever -from .core.components.retriever.bes_retriever import BESVectorStoreIndex +from .core.components.retriever.bes.bes_retriever import BESRetriever +from .core.components.retriever.bes.bes_retriever import BESVectorStoreIndex from .core.components.dish_recognize.component import DishRecognition from .core.components.translate.component import Translation from .core.components.animal_recognize.component import AnimalRecognition @@ -81,7 +81,6 @@ def check_version(self): from .core.components.image_understand.component import ImageUnderstand from .core.components.mix_card_ocr.component import MixCardOCR - from appbuilder.core.message import Message from appbuilder.core.agent import AgentRuntime from appbuilder.core.user_session import UserSession @@ -90,7 +89,6 @@ def check_version(self): from appbuilder.core.utils import get_model_list - from .core._exception import ( BadRequestException, ForbiddenException, diff --git a/appbuilder/core/components/asr/component.py b/appbuilder/core/components/asr/component.py index dc60cd8e..94ea8d70 100644 --- a/appbuilder/core/components/asr/component.py +++ b/appbuilder/core/components/asr/component.py @@ -17,9 +17,13 @@ import uuid import json +import proto +import requests + +from appbuilder.core import utils from appbuilder.core.component import Component from appbuilder.core.message import Message -from appbuilder.core._exception import AppBuilderServerException +from appbuilder.core._exception import AppBuilderServerException, InvalidRequestArgumentError from appbuilder.core._client import HTTPClient from appbuilder.core.components.asr.model import ShortSpeechRecognitionRequest, ShortSpeechRecognitionResponse, \ ASRInMsg, ASROutMsg @@ -44,6 +48,32 @@ class ASR(Component): out = asr.run(msg) print(out.content) # eg: {"result": ["北京科技馆。"]} """ + name = "asr" + version = "v1" + + manifests = [ + { + "name": "asr", + "description": "对于输入的语音文件进行识别，输出语音识别结果。", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "输入语音文件的url,根据url获取到语音文件" + }, + "file_name": { + "type": "string", + "description": "待识别语音文件名,用于生成获取语音的url" + } + }, + "required": [ + "url" + ] + } + } + ] + @HTTPClient.check_param def run(self, message: Message, audio_format: str = "pcm", rate: int = 16000, timeout: float = None, retry: int = 0) -> Message: @@ -120,3 +150,42 @@ def _check_service_error(request_id: str, data: dict): service_err_code=data["err_no"], service_err_message=data["err_msg"] ) + + def tool_eval(self, name: str, streaming: bool, **kwargs): + """ + asr for function call + """ + url_key = kwargs.get("url", None) + file_urls = kwargs.get("file_urls", {}) + if not url_key: + url_key = kwargs.get("file_name", None) + if utils.is_url(url_key): + url = url_key + else: + url = file_urls.get(url_key, None) + if not url: + raise InvalidRequestArgumentError(f"file {url_key} url does not exist") + req = ShortSpeechRecognitionRequest() + req.cuid = str(uuid.uuid4()) + req.dev_pid = "80001" + req.speech = requests.get(url).content + req.format = "pcm" + req.rate = 16000 + result = proto.Message.to_dict(self._recognize(req)) + results = { + "识别结果": " \n".join(item for item in result["result"]) + } + res = json.dumps(results, ensure_ascii=False, indent=4) + if streaming: + yield { + "type": "text", + "text": res, + "visible_scope": 'llm', + } + yield { + "type": "text", + "text": "", + "visible_scope": 'user', + } + else: + return res diff --git a/appbuilder/core/components/general_ocr/component.py b/appbuilder/core/components/general_ocr/component.py index 30103811..606ce148 100644 --- a/appbuilder/core/components/general_ocr/component.py +++ b/appbuilder/core/components/general_ocr/component.py @@ -14,10 +14,11 @@ import base64 import json +from appbuilder.core import utils from appbuilder.core._client import HTTPClient -from appbuilder.core._exception import AppBuilderServerException +from appbuilder.core._exception import AppBuilderServerException, InvalidRequestArgumentError from appbuilder.core.component import Component from appbuilder.core.components.general_ocr.model import * from appbuilder.core.message import Message @@ -44,6 +45,31 @@ class GeneralOCR(Component): print(out.content) """ + name = "general_ocr" + version = "v1" + + manifests = [ + { + "name": "general_ocr", + "description": "提供更高精度的通用文字识别能力，能够识别图片中的文字", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "待识别图片的url,根据该url能够获取图片" + }, + "file_name": { + "type": "string", + "description": "待识别图片的文件名,用于生成图片url" + } + }, + "required": [ + "url" + ] + } + } + ] @HTTPClient.check_param def run(self, message: Message, timeout: float = None, retry: int = 0) -> Message: @@ -110,3 +136,37 @@ def _check_service_error(request_id: str, data: dict): service_err_code=data.get("error_code"), service_err_message=data.get("error_msg") ) + + def tool_eval(self, name: str, streaming: bool, **kwargs): + """ + general_ocr for function call + """ + url_key = kwargs.get("url", None) + file_urls = kwargs.get("file_urls", {}) + if not url_key: + url_key = kwargs.get("file_name", None) + if utils.is_url(url_key): + url = url_key + else: + url = file_urls.get(url_key, None) + if not url: + raise InvalidRequestArgumentError(f"file {url_key} url does not exist") + req = GeneralOCRRequest(url=url) + result = proto.Message.to_dict(self._recognize(req)) + results = { + "识别结果": " \n".join(item["words"] for item in result["words_result"]) + } + res = json.dumps(results, ensure_ascii=False, indent=4) + if streaming: + yield { + "type": "text", + "text": res, + "visible_scope": 'llm', + } + yield { + "type": "text", + "text": "", + "visible_scope": 'user', + } + else: + return res \ No newline at end of file diff --git a/appbuilder/core/components/object_recognize/component.py b/appbuilder/core/components/object_recognize/component.py index 5f5b58ba..16c6ae5e 100644 --- a/appbuilder/core/components/object_recognize/component.py +++ b/appbuilder/core/components/object_recognize/component.py @@ -15,10 +15,11 @@ import base64 import json +from appbuilder.core import utils from appbuilder.core._client import HTTPClient from appbuilder.core.component import Component from appbuilder.core.message import Message -from appbuilder.core._exception import AppBuilderServerException +from appbuilder.core._exception import AppBuilderServerException, InvalidRequestArgumentError from appbuilder.core.components.object_recognize.model import * @@ -41,6 +42,31 @@ class ObjectRecognition(Component): print(out.content) """ + name = "object_recognition" + version = "v1" + + manifests = [ + { + "name": "object_recognition", + "description": "提供通用物体及场景识别能力，即对于输入的一张图片，输出图片中的多个物体及场景标签。", + "parameters": { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "待识别图片的url,根据该url能够获取图片" + }, + "file_name": { + "type": "string", + "description": "待识别图片的文件名,用于生成图片url" + } + }, + "required": [ + "url" + ] + } + } + ] @HTTPClient.check_param def run(self, message: Message, timeout: float = None, retry: int = 0) -> Message: @@ -108,3 +134,45 @@ def _check_service_error(request_id: str, data: dict): service_err_code=data.get("error_code"), service_err_message=data.get("error_msg") ) + + def tool_eval(self, name: str, streaming: bool, **kwargs): + """ + object_recognize for function call + """ + url_key = kwargs.get("url", None) + file_urls = kwargs.get("file_urls", {}) + if not url_key: + url_key = kwargs.get("file_name", None) + if utils.is_url(url_key): + url = url_key + else: + url = file_urls.get(url_key, None) + if not url: + raise InvalidRequestArgumentError(f"file {url_key} url does not exist") + score_threshold = kwargs.get("score_threshold", 0.5) + req = ObjectRecognitionRequest(url=url) + result = proto.Message.to_dict(self._recognize(req)) + results = [] + for item in result["result"]: + if item["score"] < score_threshold and len(results) > 0: + continue + res = { + "物品名称": item["keyword"], + "置信度": item["score"], + "所属类别": item["root"], + } + results.append(res) + res = json.dumps(results, ensure_ascii=False, indent=4) + if streaming: + yield { + "type": "text", + "text": res, + "visible_scope": 'llm', + } + yield { + "type": "text", + "text": "", + "visible_scope": 'user', + } + else: + return res \ No newline at end of file diff --git a/appbuilder/core/components/translate/component.py b/appbuilder/core/components/translate/component.py index ddd60508..cb675959 100644 --- a/appbuilder/core/components/translate/component.py +++ b/appbuilder/core/components/translate/component.py @@ -22,7 +22,7 @@ from appbuilder.core.message import Message from appbuilder.core.component import Component from appbuilder.core._client import HTTPClient -from appbuilder.core._exception import AppBuilderServerException +from appbuilder.core._exception import AppBuilderServerException, InvalidRequestArgumentError from appbuilder.core.components.translate.model import * @@ -50,6 +50,30 @@ class Translation(Component): name = "translate" version = "v1" + manifests = [ + { + "name": "translation", + "description": "文本翻译通用版工具，会根据指定的目标语言对文本进行翻译，并返回翻译后的文本。", + "parameters": { + "type": "object", + "properties": { + "q": { + "type": "string", + "description": "需要翻译的源文本，文本翻译工具会将该文本翻译成对应的目标语言" + }, + "to_lang": { + "type": "string", + "description": "翻译的目标语言类型，'en'表示将原文本翻译成英文, 'zh'表示将原文本翻译成中文，默认为'en'", + "enum": ["en", "zh"] + } + }, + "required": [ + "q" + ] + } + } + ] + @HTTPClient.check_param def run(self, message: Message, from_lang: str = "auto", to_lang: str = "en", timeout: float = None, retry: int = 0) -> Message: @@ -113,3 +137,35 @@ def _translate(self, request: TranslateRequest, timeout: float = None, json_str = json.dumps(data) return TranslateResponse(TranslateResponse.from_json(json_str)) + + def tool_eval(self, name: str, streaming: bool, **kwargs): + """ + translate for function call + """ + req = TranslateRequest() + text = kwargs.get("q", None) + if not text: + raise InvalidRequestArgumentError("param `q` must be set") + req.q = text + to_lang = kwargs.get("to_lang", "en") + req.to_lang = to_lang + results = proto.Message.to_dict(self._translate(req))["result"] + trans_result = results["trans_result"] + res = { + "原文本": "\n ".join(item["src"] for item in trans_result), + "翻译结果": "\n ".join(item["dst"] for item in trans_result) + } + res = json.dumps(res, ensure_ascii=False, indent=4) + if streaming: + yield { + "type": "text", + "text": res, + "visible_scope": 'llm', + } + yield { + "type": "text", + "text": "", + "visible_scope": 'user', + } + else: + return res diff --git a/appbuilder/tests/test_asr.py b/appbuilder/tests/test_asr.py index 02988865..744d6bf4 100644 --- a/appbuilder/tests/test_asr.py +++ b/appbuilder/tests/test_asr.py @@ -4,6 +4,7 @@ import requests import appbuilder +from appbuilder.core._exception import InvalidRequestArgumentError from appbuilder.core.components.asr.model import ShortSpeechRecognitionRequest, ShortSpeechRecognitionResponse @@ -129,6 +130,18 @@ def test_check_service_error(self): data = {'err_msg': 'No Error', 'err_no': 0} self.assertIsNone(self.asr._check_service_error("", data)) + def test_tool_eval_valid(self): + """测试 tool 方法对有效请求的处理。""" + result = self.asr.tool_eval(name="asr", streaming=True, url=self.audio_file_url) + res = [item for item in result] + self.assertNotEqual(len(res), 0) + + def test_tool_eval_invalid(self): + """测试 tool 方法对无效请求的处理。""" + with self.assertRaises(InvalidRequestArgumentError): + result = self.asr.tool_eval(name="asr", streaming=True) + next(result) + if __name__ == '__main__': unittest.main() diff --git a/appbuilder/tests/test_general_ocr.py b/appbuilder/tests/test_general_ocr.py index 3ec8779b..147f2271 100644 --- a/appbuilder/tests/test_general_ocr.py +++ b/appbuilder/tests/test_general_ocr.py @@ -15,6 +15,7 @@ import unittest import requests import appbuilder +from appbuilder.core._exception import InvalidRequestArgumentError class TestGeneralOCR(unittest.TestCase): @@ -131,6 +132,22 @@ def test_run_without_image_and_url(self): with self.assertRaises(ValueError): self.general_ocr.run(message=message) + def test_tool_eval_valid(self): + """测试 tool 方法对有效请求的处理。""" + image_url = "https://bj.bcebos.com/v1/appbuilder/general_ocr_test.png?" \ + "authorization=bce-auth-v1%2FALTAKGa8m4qCUasgoljdEDAzLm%2F2024-01-" \ + "11T10%3A59%3A17Z%2F-1%2Fhost%2F081bf7bcccbda5207c82a4de074628b04ae" \ + "857a27513734d765495f89ffa5f73" + result = self.general_ocr.tool_eval(name="general_ocr", streaming=True, url=image_url) + res = [item for item in result] + self.assertNotEqual(len(res), 0) + + def test_tool_eval_invalid(self): + """测试 tool 方法对无效请求的处理。""" + with self.assertRaises(InvalidRequestArgumentError): + result = self.general_ocr.tool_eval(name="general_ocr", streaming=True) + next(result) + if __name__ == '__main__': unittest.main() diff --git a/appbuilder/tests/test_object_recognize.py b/appbuilder/tests/test_object_recognize.py index 68f1e10f..1962bb93 100644 --- a/appbuilder/tests/test_object_recognize.py +++ b/appbuilder/tests/test_object_recognize.py @@ -15,6 +15,7 @@ import unittest import requests import appbuilder +from appbuilder.core._exception import InvalidRequestArgumentError class TestObjectRecognize(unittest.TestCase): @@ -131,6 +132,22 @@ def test_run_without_image_and_url(self): with self.assertRaises(ValueError): self.object_recognition.run(message=message) + def test_tool_eval_valid(self): + """测试 tool 方法对有效请求的处理。""" + image_url = "https://bj.bcebos.com/v1/appbuilder/object_recognize_test.png?" \ + "authorization=bce-auth-v1%2FALTAKGa8m4qCUasgoljdEDAzLm%2F2024-01-" \ + "11T11%3A00%3A19Z%2F-1%2Fhost%2F2c31bf29205f61e58df661dc80af31a1dc" \ + "1ba1de0a8f072bc5a87102bd32f9e3" + result = self.object_recognition.tool_eval(name="object_recognition", streaming=True, url=image_url) + res = [item for item in result] + self.assertNotEqual(len(res), 0) + + def test_tool_eval_invalid(self): + """测试 tool 方法对无效请求的处理。""" + with self.assertRaises(InvalidRequestArgumentError): + result = self.object_recognition.tool_eval(name="object_recognition", streaming=True) + next(result) + if __name__ == '__main__': unittest.main() diff --git a/appbuilder/tests/test_translate.py b/appbuilder/tests/test_translate.py index 3dd63f46..461367c3 100644 --- a/appbuilder/tests/test_translate.py +++ b/appbuilder/tests/test_translate.py @@ -1,5 +1,6 @@ import unittest import appbuilder +from appbuilder.core._exception import InvalidRequestArgumentError class TestTranslationComponent(unittest.TestCase): @@ -19,6 +20,18 @@ def test_run_invalid_request(self): with self.assertRaises(ValueError): _ = self.translation(msg) + def test_tool_eval_valid(self): + """测试 tool 方法对有效请求的处理。""" + result = self.translation.tool_eval(name="translation", streaming=True, q="你好", to_lang="en") + res = [item for item in result] + self.assertNotEqual(len(res), 0) + + def test_tool_eval_invalid(self): + """测试 tool 方法对无效请求的处理。""" + with self.assertRaises(InvalidRequestArgumentError): + result = self.translation.tool_eval(name="translation", streaming=True, to_lang="en") + next(result) + if __name__ == '__main__': unittest.main() From 6d43d40aaec209a41493ff3ef173b28fd95db37b Mon Sep 17 00:00:00 2001 From: longchao1916 Date: Mon, 18 Mar 2024 21:16:58 +0800 Subject: [PATCH 2/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E4=E4=B8=AAFC=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=BB=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- appbuilder/core/components/asr/component.py | 38 ++++++++++------- .../core/components/general_ocr/component.py | 42 ++++++++++++------- .../components/object_recognize/component.py | 40 +++++++++++------- appbuilder/tests/test_asr.py | 2 +- appbuilder/tests/test_general_ocr.py | 2 +- appbuilder/tests/test_object_recognize.py | 2 +- appbuilder/tests/test_translate.py | 2 +- 7 files changed, 79 insertions(+), 49 deletions(-) diff --git a/appbuilder/core/components/asr/component.py b/appbuilder/core/components/asr/component.py index 94ea8d70..e5ce8507 100644 --- a/appbuilder/core/components/asr/component.py +++ b/appbuilder/core/components/asr/component.py @@ -14,6 +14,7 @@ r"""ASR component. """ +import os import uuid import json @@ -58,7 +59,7 @@ class ASR(Component): "parameters": { "type": "object", "properties": { - "url": { + "file_url": { "type": "string", "description": "输入语音文件的url,根据url获取到语音文件" }, @@ -67,8 +68,17 @@ class ASR(Component): "description": "待识别语音文件名,用于生成获取语音的url" } }, - "required": [ - "url" + "anyOf": [ + { + "required": [ + "file_url" + ] + }, + { + "required": [ + "file_name" + ] + } ] } } @@ -155,20 +165,20 @@ def tool_eval(self, name: str, streaming: bool, **kwargs): """ asr for function call """ - url_key = kwargs.get("url", None) - file_urls = kwargs.get("file_urls", {}) - if not url_key: - url_key = kwargs.get("file_name", None) - if utils.is_url(url_key): - url = url_key - else: - url = file_urls.get(url_key, None) - if not url: - raise InvalidRequestArgumentError(f"file {url_key} url does not exist") + file_url = kwargs.get("file_url", None) + if not file_url: + file_urls = kwargs.get("file_urls", {}) + file_path = kwargs.get("file_name", None) + if not file_path: + raise InvalidRequestArgumentError("file name is not set") + file_name = os.path.basename(file_path) + file_url = file_urls.get(file_name, None) + if not file_url: + raise InvalidRequestArgumentError(f"file {file_url} url does not exist") req = ShortSpeechRecognitionRequest() req.cuid = str(uuid.uuid4()) req.dev_pid = "80001" - req.speech = requests.get(url).content + req.speech = requests.get(file_url).content req.format = "pcm" req.rate = 16000 result = proto.Message.to_dict(self._recognize(req)) diff --git a/appbuilder/core/components/general_ocr/component.py b/appbuilder/core/components/general_ocr/component.py index 606ce148..82a7be5d 100644 --- a/appbuilder/core/components/general_ocr/component.py +++ b/appbuilder/core/components/general_ocr/component.py @@ -13,6 +13,7 @@ r"""general ocr component.""" import base64 import json +import os.path from appbuilder.core import utils from appbuilder.core._client import HTTPClient @@ -55,17 +56,26 @@ class GeneralOCR(Component): "parameters": { "type": "object", "properties": { - "url": { + "img_url": { "type": "string", "description": "待识别图片的url,根据该url能够获取图片" }, - "file_name": { + "img_name": { "type": "string", "description": "待识别图片的文件名,用于生成图片url" - } + }, }, - "required": [ - "url" + "anyOf": [ + { + "required": [ + "img_url" + ] + }, + { + "required": [ + "img_name" + ] + } ] } } @@ -141,17 +151,17 @@ def tool_eval(self, name: str, streaming: bool, **kwargs): """ general_ocr for function call """ - url_key = kwargs.get("url", None) - file_urls = kwargs.get("file_urls", {}) - if not url_key: - url_key = kwargs.get("file_name", None) - if utils.is_url(url_key): - url = url_key - else: - url = file_urls.get(url_key, None) - if not url: - raise InvalidRequestArgumentError(f"file {url_key} url does not exist") - req = GeneralOCRRequest(url=url) + img_url = kwargs.get("img_url", None) + if not img_url: + file_urls = kwargs.get("file_urls", {}) + img_path = kwargs.get("img_name", None) + if not img_path: + raise InvalidRequestArgumentError("file name is not set") + img_name = os.path.basename(img_path) + img_url = file_urls.get(img_name, None) + if not img_url: + raise InvalidRequestArgumentError(f"file {img_name} url does not exist") + req = GeneralOCRRequest(url=img_url) result = proto.Message.to_dict(self._recognize(req)) results = { "识别结果": " \n".join(item["words"] for item in result["words_result"]) diff --git a/appbuilder/core/components/object_recognize/component.py b/appbuilder/core/components/object_recognize/component.py index 16c6ae5e..405fafb7 100644 --- a/appbuilder/core/components/object_recognize/component.py +++ b/appbuilder/core/components/object_recognize/component.py @@ -14,6 +14,7 @@ import base64 import json +import os from appbuilder.core import utils from appbuilder.core._client import HTTPClient @@ -52,17 +53,26 @@ class ObjectRecognition(Component): "parameters": { "type": "object", "properties": { - "url": { + "img_url": { "type": "string", "description": "待识别图片的url,根据该url能够获取图片" }, - "file_name": { + "img_name": { "type": "string", "description": "待识别图片的文件名,用于生成图片url" } }, - "required": [ - "url" + "anyOf": [ + { + "required": [ + "img_url" + ] + }, + { + "required": [ + "img_name" + ] + } ] } } @@ -139,18 +149,18 @@ def tool_eval(self, name: str, streaming: bool, **kwargs): """ object_recognize for function call """ - url_key = kwargs.get("url", None) - file_urls = kwargs.get("file_urls", {}) - if not url_key: - url_key = kwargs.get("file_name", None) - if utils.is_url(url_key): - url = url_key - else: - url = file_urls.get(url_key, None) - if not url: - raise InvalidRequestArgumentError(f"file {url_key} url does not exist") + img_url = kwargs.get("img_url", None) + if not img_url: + file_urls = kwargs.get("file_urls", {}) + img_path = kwargs.get("img_name", None) + if not img_path: + raise InvalidRequestArgumentError("file name is not set") + img_name = os.path.basename(img_path) + img_url = file_urls.get(img_name, None) + if not img_url: + raise InvalidRequestArgumentError(f"file {img_name} url does not exist") score_threshold = kwargs.get("score_threshold", 0.5) - req = ObjectRecognitionRequest(url=url) + req = ObjectRecognitionRequest(url=img_url) result = proto.Message.to_dict(self._recognize(req)) results = [] for item in result["result"]: diff --git a/appbuilder/tests/test_asr.py b/appbuilder/tests/test_asr.py index 744d6bf4..a6ab77eb 100644 --- a/appbuilder/tests/test_asr.py +++ b/appbuilder/tests/test_asr.py @@ -132,7 +132,7 @@ def test_check_service_error(self): def test_tool_eval_valid(self): """测试 tool 方法对有效请求的处理。""" - result = self.asr.tool_eval(name="asr", streaming=True, url=self.audio_file_url) + result = self.asr.tool_eval(name="asr", streaming=True, file_url=self.audio_file_url) res = [item for item in result] self.assertNotEqual(len(res), 0) diff --git a/appbuilder/tests/test_general_ocr.py b/appbuilder/tests/test_general_ocr.py index 147f2271..b0e822ec 100644 --- a/appbuilder/tests/test_general_ocr.py +++ b/appbuilder/tests/test_general_ocr.py @@ -138,7 +138,7 @@ def test_tool_eval_valid(self): "authorization=bce-auth-v1%2FALTAKGa8m4qCUasgoljdEDAzLm%2F2024-01-" \ "11T10%3A59%3A17Z%2F-1%2Fhost%2F081bf7bcccbda5207c82a4de074628b04ae" \ "857a27513734d765495f89ffa5f73" - result = self.general_ocr.tool_eval(name="general_ocr", streaming=True, url=image_url) + result = self.general_ocr.tool_eval(name="general_ocr", streaming=True, img_url=image_url) res = [item for item in result] self.assertNotEqual(len(res), 0) diff --git a/appbuilder/tests/test_object_recognize.py b/appbuilder/tests/test_object_recognize.py index 1962bb93..e022de08 100644 --- a/appbuilder/tests/test_object_recognize.py +++ b/appbuilder/tests/test_object_recognize.py @@ -138,7 +138,7 @@ def test_tool_eval_valid(self): "authorization=bce-auth-v1%2FALTAKGa8m4qCUasgoljdEDAzLm%2F2024-01-" \ "11T11%3A00%3A19Z%2F-1%2Fhost%2F2c31bf29205f61e58df661dc80af31a1dc" \ "1ba1de0a8f072bc5a87102bd32f9e3" - result = self.object_recognition.tool_eval(name="object_recognition", streaming=True, url=image_url) + result = self.object_recognition.tool_eval(name="object_recognition", streaming=True, img_url=image_url) res = [item for item in result] self.assertNotEqual(len(res), 0) diff --git a/appbuilder/tests/test_translate.py b/appbuilder/tests/test_translate.py index 461367c3..5e643b6c 100644 --- a/appbuilder/tests/test_translate.py +++ b/appbuilder/tests/test_translate.py @@ -22,7 +22,7 @@ def test_run_invalid_request(self): def test_tool_eval_valid(self): """测试 tool 方法对有效请求的处理。""" - result = self.translation.tool_eval(name="translation", streaming=True, q="你好", to_lang="en") + result = self.translation.tool_eval(name="translation", streaming=True, q="你好\n中国", to_lang="en") res = [item for item in result] self.assertNotEqual(len(res), 0) From 918c043804c53720a5fc012341e7d939c7c71c88 Mon Sep 17 00:00:00 2001 From: longchao1916 Date: Tue, 19 Mar 2024 10:40:50 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E6=96=B0=E5=A2=9E4=E4=B8=AAFC=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=BB=84=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/components/retriever/baidu_vdb/baiduvdb_retriever.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/appbuilder/core/components/retriever/baidu_vdb/baiduvdb_retriever.py b/appbuilder/core/components/retriever/baidu_vdb/baiduvdb_retriever.py index d716f05f..811d9e22 100644 --- a/appbuilder/core/components/retriever/baidu_vdb/baiduvdb_retriever.py +++ b/appbuilder/core/components/retriever/baidu_vdb/baiduvdb_retriever.py @@ -136,7 +136,8 @@ def _init_client(self, instance_id, account, api_key): from pymochow.auth.bce_credentials import AppBuilderCredentials gateway = os.getenv("GATEWAY_URL") if os.getenv("GATEWAY_URL") else GATEWAY_URL - + appbuilder_token = os.getenv("APPBUILDER_TOKEN") + config = Configuration( credentials=AppBuilderCredentials(account, api_key, appbuilder_token), endpoint=gateway, From b711231728969dced9862b8948fcb4603dd1f5cd Mon Sep 17 00:00:00 2001 From: longchao1916 Date: Tue, 19 Mar 2024 15:49:43 +0800 Subject: [PATCH 4/4] =?UTF-8?q?BML-APP-9343=20[=E4=BB=BB=E5=8A=A1]function?= =?UTF-8?q?call=20ASR=E5=A2=9E=E5=8A=A0file=5Ftype=E5=8F=82=E6=95=B0&trans?= =?UTF-8?q?late=E5=A2=9E=E5=8A=A0=E6=97=A5=E9=9F=A9=E6=B3=95=E5=BE=B7?= =?UTF-8?q?=E8=AF=AD=E8=A8=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- appbuilder/core/components/asr/component.py | 14 +++++++++++--- appbuilder/core/components/translate/component.py | 5 +++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/appbuilder/core/components/asr/component.py b/appbuilder/core/components/asr/component.py index e5ce8507..754274c4 100644 --- a/appbuilder/core/components/asr/component.py +++ b/appbuilder/core/components/asr/component.py @@ -66,6 +66,11 @@ class ASR(Component): "file_name": { "type": "string", "description": "待识别语音文件名,用于生成获取语音的url" + }, + "file_type": { + "type": "string", + "description": "语音文件类型,支持pcm/wav/amr/m4a", + "enum": ["pcm", "wav", "amr", "m4a"] } }, "anyOf": [ @@ -139,7 +144,7 @@ def _recognize(self, request: ShortSpeechRecognitionRequest, timeout: float = No data = response.json() self.http_client.check_response_json(data) request_id = self.http_client.response_request_id(response) - self.__class__._check_service_error(request_id,data) + self.__class__._check_service_error(request_id, data) response = ShortSpeechRecognitionResponse.from_json(payload=json.dumps(data)) response.request_id = request_id return response @@ -175,11 +180,14 @@ def tool_eval(self, name: str, streaming: bool, **kwargs): file_url = file_urls.get(file_name, None) if not file_url: raise InvalidRequestArgumentError(f"file {file_url} url does not exist") + file_type = kwargs.get("file_type", "wav") + if file_type not in ["pcm", "wav", "amr", "m4a"]: + file_type = "wav" req = ShortSpeechRecognitionRequest() + req.speech = requests.get(file_url).content + req.format = file_type req.cuid = str(uuid.uuid4()) req.dev_pid = "80001" - req.speech = requests.get(file_url).content - req.format = "pcm" req.rate = 16000 result = proto.Message.to_dict(self._recognize(req)) results = { diff --git a/appbuilder/core/components/translate/component.py b/appbuilder/core/components/translate/component.py index cb675959..e512f1b2 100644 --- a/appbuilder/core/components/translate/component.py +++ b/appbuilder/core/components/translate/component.py @@ -63,8 +63,9 @@ class Translation(Component): }, "to_lang": { "type": "string", - "description": "翻译的目标语言类型，'en'表示将原文本翻译成英文, 'zh'表示将原文本翻译成中文，默认为'en'", - "enum": ["en", "zh"] + "description": "翻译的目标语言类型，'en'表示翻译成英语, 'zh'表示翻译成中文，'jp'表示翻译成日语，" + "'kor'表示翻译成韩语，'fra'表示翻译成法语，'de'表示翻译成德语，默认为'en'", + "enum": ["en", "zh", "jp", "kor", "fra", "de"] } }, "required": [