From a348d88bb931eacbb765397ca8334ce04e441ad5 Mon Sep 17 00:00:00 2001 From: raullese Date: Wed, 16 Oct 2024 17:07:21 +0800 Subject: [PATCH 1/8] add document understanding module --- cookbooks/components/asr.ipynb | 2 +- cookbooks/components/general_ocr.ipynb | 2 +- cookbooks/components/object_recognize.ipynb | 2 +- cookbooks/components/translate.ipynb | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cookbooks/components/asr.ipynb b/cookbooks/components/asr.ipynb index 46cabf04..3afe858c 100644 --- a/cookbooks/components/asr.ipynb +++ b/cookbooks/components/asr.ipynb @@ -148,4 +148,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/cookbooks/components/general_ocr.ipynb b/cookbooks/components/general_ocr.ipynb index 42ad0435..1a1868b2 100644 --- a/cookbooks/components/general_ocr.ipynb +++ b/cookbooks/components/general_ocr.ipynb @@ -179,4 +179,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/cookbooks/components/object_recognize.ipynb b/cookbooks/components/object_recognize.ipynb index 1f5a6b2c..381abf77 100644 --- a/cookbooks/components/object_recognize.ipynb +++ b/cookbooks/components/object_recognize.ipynb @@ -178,4 +178,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} diff --git a/cookbooks/components/translate.ipynb b/cookbooks/components/translate.ipynb index d8c27557..d70eeccb 100644 --- a/cookbooks/components/translate.ipynb +++ b/cookbooks/components/translate.ipynb @@ -362,4 +362,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} From fe6781a99e81e8b0575e2cda6824b27803bdd021 Mon Sep 17 00:00:00 2001 From: raullese Date: Wed, 16 Oct 2024 17:58:49 +0800 Subject: [PATCH 2/8] add document understanding module --- appbuilder/__init__.py | 3 +- .../document_understanding/README.md | 100 ++++++++ .../document_understanding/__init__.py | 0 .../components/document_understanding/base.py | 60 +++++ .../document_understanding/component.py | 215 ++++++++++++++++++ .../tests/test_document_understanding.py | 73 ++++++ 6 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 appbuilder/core/components/document_understanding/README.md create mode 100644 appbuilder/core/components/document_understanding/__init__.py create mode 100644 appbuilder/core/components/document_understanding/base.py create mode 100644 appbuilder/core/components/document_understanding/component.py create mode 100644 appbuilder/tests/test_document_understanding.py diff --git a/appbuilder/__init__.py b/appbuilder/__init__.py index 7162b192..b1843180 100644 --- a/appbuilder/__init__.py +++ b/appbuilder/__init__.py @@ -113,7 +113,7 @@ def get_default_header(): from .core.components.handwrite_ocr.component import HandwriteOCR from .core.components.image_understand.component import ImageUnderstand from .core.components.mix_card_ocr.component import MixCardOCR - +from .core.components.document_understanding.component import DocumentUnderstanding __COMPONENTS__ = [ "RagWithBaiduSearchPro", "RAGWithBaiduSearch", @@ -163,6 +163,7 @@ def get_default_header(): "HandwriteOCR", "ImageUnderstand", "MixCardOCR", + "DocumentUnderstanding", ] # NOQA from appbuilder.core.message import Message diff --git a/appbuilder/core/components/document_understanding/README.md b/appbuilder/core/components/document_understanding/README.md new file mode 100644 index 00000000..d706aa73 --- /dev/null +++ b/appbuilder/core/components/document_understanding/README.md @@ -0,0 +1,100 @@ +# 文件生成PPT（PPTGenerationFromFile） + +## 简介 +长文档内容理解组件（DocumentUnderstanding）支持对图片以及文档内容进行理解，并基于图片以及文档内容对用户的提问进行回答， +包括但不限于文档内容问答、总结摘要、内容分析。 +### 功能介绍 +根据用户上传的文档（支持txt、docx、pdf、xlsx、png、jpg、jpeg等多种格式）、query、指令生成大模型答案 +### 特色优势 +处理长上下文的大模型内容理解任务 +### 应用场景 +长上下文的文档问答 + +## 基本用法 +### 快速开始 + +```python + +import uuid +import os +import appbuilder + +# 请前往千帆AppBuilder官网创建密钥，流程详见：https://cloud.baidu.com/doc/AppBuilder/s/Olq6grrt6#1%E3%80%81%E5%88%9B%E5%BB%BA%E5%AF%86%E9%92%A5 +APPBUILDER_TOKEN = "YOUR-TOKEN" +os.environ["APPBUILDER_TOKEN"] = APPBUILDER_TOKEN +uid = str(uuid.uuid4()) +trace_id = str(uuid.uuid4()) +conversation_id = str(uuid.uuid4()) ## 注意你的conversation_id不能和之前的请求重复，不然会直接返回之前已有的conversation_id的答案 +du = appbuilder.DocumentUnderstanding() +query = appbuilder.Message("这篇文档讲了什么") +instruction = "请根据文档内容回答问题" +addition_instruction = "请你用一句话简短概括" ##用户增强指令，可选填，该内容会进一步增强大模型的指令跟随能力，将你最需要增强效果的指令填于此，内容可以与上述的"instruction"基础指令有重复，注意：该字段内容过多会一定程度影响大模型内容严谨度，请注意控制该字段的指令字数 +file_path = "YOUR-FILE-PATH" ##填写你的本地待分析文件路径 +stream = False ##是否开启流式输出功能 +response_ = du.run(query, + file_path, + instruction=instruction, + addition_instruction=addition_instruction, + uid=uid, + trace_id=trace_id, + conversation_id=conversation_id, + stream=stream) + +for result in response_: + print(result) ##打印输出的大模型答案 +``` + + +## 参数说明 +### 鉴权说明 +使用组件之前，请首先申请并设置鉴权参数，可参考[组件使用流程](https://cloud.baidu.com/doc/AppBuilder/s/Olq6grrt6#1%E3%80%81%E5%88%9B%E5%BB%BA%E5%AF%86%E9%92%A5)。 +```python +# 设置环境中的TOKEN，以下示例略 +import os +os.environ['APPBUILDER_TOKEN'] = 'bce-YOURTOKEN' +``` + + +### 初始化参数 + +| 参数名称 | 参数类型 | 是否必须 | 描述 | 示例值 | +| ------- | ------- | -------- | -------- | -------- | +| `secret_key` | str | 否 | 用户鉴权token，默认从环境变量中获取: `os.getenv("APPBUILDER_TOKEN", "")` | bce-v3/XXX | +| `gateway` | str | 否 | 后端网关服务地址，默认从环境变量中获取: `os.getenv("GATEWAY_URL", "")` | https://appbuilder.baidu.com | +| `lazy_certification` | bool | 否 | 延迟认证，为True时在第一次运行时认证。默认为False。 | False | + + +### 调用参数 + +| 参数名称 | 参数类型 | 是否必须 | 描述 | 示例值 | +|------------------------|------|------|--------------------------------------------------------------------------|-----------------| +| `message` | obj | 是 | 输入消息，用户输入query。 | Message(content=input_data) | +| `file_path` | str | 是 | 用户需要分析的文档 | "test.pdf" | +| `instruction` | str | 否 | 用户指令 | "你的回答要严谨客观" | +| `addition_instruction` | str | 否 | 用户增强指令，可选填，该内容会进一步增强大模型的指令跟随能力，将你最需要增强效果的指令填于此，注意：该字段内容过多会一定程度影响大模型内容严谨度 | "你的答案需要分点阐述" | + + +### 响应参数 +| 参数名称 | 参数类型 | 描述 | 示例值 | +| ------- |------| -------- | -------- | +| `result` | str | 模型运行后的输出结果 | "" | + +### 响应示例 +``` +您好，请问您是想询问关于残疾人办理什么证件的问题吗？如果是，我可以为您提供一些信息。 + +首先，如果您是首次申请办理残疾人证，需要携带身份证、户口簿和三张两寸近期免冠白底彩色照片到县残联办证窗口提出申请。如果您因身体原因无法亲自前往，可以联系村（社区）工作人员代办申请。 + +其次，如果您是指残疾类型等级证明，您需要携带相关材料到指定医院或医生进行评级，并由医生签名盖章。 + +最后，如果您是指残疾人享受低保或残疾人贫困证的一级肢体、视力、智力、精神、多重及60周岁以上的一级听力、语言的重度残疾人可以享受重度残疾人生活补助，那么您需要携带身份证、户口本和残疾证申请表到县、市、区级残联进行办理。 + +希望这些信息对您有所帮助。如果您还有其他问题，欢迎随时提问。 +``` + +## 高级用法 + +## 更新记录和贡献 +### 2024.10. 15 +#### [Added] +- 第一版 \ No newline at end of file diff --git a/appbuilder/core/components/document_understanding/__init__.py b/appbuilder/core/components/document_understanding/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/appbuilder/core/components/document_understanding/base.py b/appbuilder/core/components/document_understanding/base.py new file mode 100644 index 00000000..64d40cd7 --- /dev/null +++ b/appbuilder/core/components/document_understanding/base.py @@ -0,0 +1,60 @@ +""" +Copyright (c) 2023 Baidu, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +from pydantic import Field +from appbuilder.core.message import Message +import base64 +from appbuilder.core.component import ComponentArguments + + + +class DocumentUnderstandingArgs(ComponentArguments): + '''长文档问答配置''' + message: Message = Field(..., + variable_name="query", + description="用户输入query") + file_path: str = Field(..., + variable_name="file_path", + description="用户上传的文件路径") + instruction: str = Field(default="", + variable_name='instruction', + description='用户指令') + addition_instruction: str = Field(default="", + variable_name='addition_instruction', + description='用户增强指令') + + def file_to_base64(self, file_path): + """ + 读取指定路径的文件并将其内容转换为Base64编码的字符串。 + + :param file_path: 文件的本地路径 + :return: Base64编码的字符串 + """ + try: + # 以二进制模式读取文件内容 + with open(file_path, "rb") as file: + file_data = file.read() + + # 将文件内容编码为Base64 + base64_encoded_data = base64.b64encode(file_data).decode('utf-8') + + return base64_encoded_data + + except Exception as e: + print(f"读取文件或编码过程中出错: {e}") + return None + diff --git a/appbuilder/core/components/document_understanding/component.py b/appbuilder/core/components/document_understanding/component.py new file mode 100644 index 00000000..69b30c43 --- /dev/null +++ b/appbuilder/core/components/document_understanding/component.py @@ -0,0 +1,215 @@ +""" +Copyright (c) 2023 Baidu, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" +import os + +import requests + +import json +from typing import Optional + +from appbuilder.core.components.document_understanding.base import DocumentUnderstandingArgs + +from appbuilder.core.message import Message +from appbuilder.core.component import Component +import base64 + +class DocumentUnderstanding(Component): + """ + DocumentUnderstanding + """ + name = "document_understanding" + version = "v1" + meta = DocumentUnderstandingArgs + manifests = [{ + "name": "document_understanding", + "description": "该工具支持对图片以及文档内容进行理解，并基于图片以及文档内容对用户的提问进行回答，包括但不限于文档内容问答、" + "总结摘要、内容分析。", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "用户输入的query" + }, + "file_path": { + "type": "string", + "description": "用户上传的文档的文件路径" + }, + "instruction": { + "type": "string", + "description": "用户指令" + }, + "addition_instruction": { + "type": "string", + "description": "用户增强指令" + }, + }, + "required": ["query", "file_path", "instruction", "addition_instruction"] + } + }] + + def __init__( + self, + secret_key: Optional[str] = None, + gateway: str = "", + lazy_certification: bool = False, + instruction: Optional[Message] = None, + addition_instruction: Optional[Message] = None, + file_path: Optional[str] = None, + + ): + """初始化DocumentUnderstanding组件。 + + Args: + secret_key (str, 可选): 用户鉴权token, 默认从环境变量中获取: os.getenv("APPBUILDER_TOKEN", ""). + gateway (str, 可选): 后端网关服务地址，默认从环境变量中获取: os.getenv("GATEWAY_URL", "") + lazy_certification (bool, 可选): 延迟认证，为True时在第一次运行时认证. Defaults to False. + + Returns: + None + """ + super().__init__(DocumentUnderstandingArgs, + secret_key=secret_key, + gateway=gateway, + lazy_certification=lazy_certification) + self.url = "http://copilot-test.now.baidu-int.com/dte/api/v2/component/tool_eval" + self.instruction = instruction, + self.addition_instruction = addition_instruction + self.file_path = file_path + + + def get_addition_instruction(self, addition_instruction: str): + """拼接addition_instruction""" + return "，指令：" + addition_instruction + + def file_to_base64(self, file_path: str): + try: + # 打开文件并读取内容 + with open(file_path, "rb") as file: + file_data = file.read() + + # 将文件数据转换为base64格式 + base64_encoded_data = base64.b64encode(file_data) + + # 将base64字节数据转换为字符串 + base64_message = base64_encoded_data.decode('utf-8') + + return base64_message + + except FileNotFoundError: + return "文件未找到，请检查文件路径是否正确。" + except Exception as e: + return f"发生错误: {str(e)}" + + def run(self, + message: Message, + file_path, + instruction="", + addition_instruction="", + uid=None, + trace_id=None, + conversation_id=None, + stream=False): + ''' + run方法，用于执行长文档理解任务 + Args: + message: 用户输入query + file_path: 用户输入的文件路径 + instruction: 用户输入的人设指令 + addition_instruction: 用户输入的增强版指令(如有) + + Returns: + result (Message): 模型运行后的输出消息。 + + ''' + file_data = self.file_to_base64(file_path) + file_name = file_path.split("/")[-1] + file_type = file_name.split(".")[-1].lower() + + support_file_type = ["pdf", "docx", "xlsx", "png", "jpg", "jpeg", "txt"] + if file_type not in support_file_type: + raise Exception(f"不支持解析{file_type}类型的文件，当前仅支持解析以下几种文件类型：{support_file_type}") + payload = json.dumps({ + "component": "DocumentUnderstanding", + "stream": stream, + "component_init_args": { + "instruction": instruction, + "addition_instruction": self.get_addition_instruction(addition_instruction), + "file_data": file_data, + "file_name": file_name, + }, + "system": { + "uid": uid, + "traceid": trace_id, + "conversation_id": conversation_id, + "appbuilder_token": f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" + }, + "user": { + "query": message.content + } + }) + headers = { + 'content-type': 'application/json', + 'user-agent': 'vscode-restclient', + 'x-appbuilder-authorization': f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}", + 'x-appbuilder-from': 'sdk' + } + + # try: + response = requests.request("POST", self.url, headers=headers, data=payload, stream=True) + if response.status_code == 200: + if stream: + # 处理流式响应，逐行生成数据 + for line in response.iter_lines(): + if line: + decoded_line = line.decode('utf-8') + yield decoded_line # 使用yield逐行输出结果 + else: + result = response.json() + if result["code"] == 0: + yield result["result"].get("text") + else: + raise Exception(f"服务请求失败: {result['message']}") + else: + response.raise_for_status() + + +if __name__ == '__main__': + import uuid + APPBUILDER_TOKEN = "YOUR-TOKEN" + os.environ["APPBUILDER_TOKEN"] = APPBUILDER_TOKEN + import appbuilder + uid = str(uuid.uuid4()) + trace_id = str(uuid.uuid4()) + conversation_id = str(uuid.uuid4()) + du = DocumentUnderstanding() + query = appbuilder.Message("这篇文档讲了什么") + instruction = "请根据文档内容回答问题" + addition_instruction = "请你用一句话简短概括" + file_path = "test.docx" + stream = True + print(trace_id) + response_ = du.run(query, + file_path, + instruction=instruction, + addition_instruction=addition_instruction, + uid=uid, + trace_id=trace_id, + conversation_id=conversation_id, + stream=stream) + + for result in response_: + print(result) diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py new file mode 100644 index 00000000..c0deebc6 --- /dev/null +++ b/appbuilder/tests/test_document_understanding.py @@ -0,0 +1,73 @@ +""" +Copyright (c) 2023 Baidu, Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + + +import os +import unittest +import appbuilder +import uuid + +# 生产环境 +os.environ['APPBUILDER_TOKEN'] = "YOUR-TOKEN" + + +TEST_INPUT = { + "query": appbuilder.Message("这篇文档讲了什么"), + "instruction": "请根据文档内容回答问题", + "addition_instruction": "请你用一句话简短概括", + "file_path": "title_splitter.docx", + "stream": True, + "conversation_id": str(uuid.uuid4()), + "trace_id": str(uuid.uuid4()), + "uid": str(uuid.uuid4()), + "APPBUILDER_TOKEN": os.getenv("APPBUILDER_TOKEN", None), +} + + +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") +class TestDocumentUnderstandingComponent(unittest.TestCase): + def setUp(self): + """ + 设置环境变量。 + + Args: + 无参数，默认值为空。 + + Returns: + 无返回值，方法中执行了环境变量的赋值操作。 + """ + os.environ['APPBUILDER_TOKEN'] = TEST_INPUT.get("APPBUILDER_TOKEN") + self.du = appbuilder.DocumentUnderstanding() + + def test_run_with_default_params(self): + """测试 run 方法使用默认参数 + """ + query = TEST_INPUT.get("query") + results = self.du.run(query, + TEST_INPUT.get("file_path"), + instruction=TEST_INPUT.get("instruction"), + addition_instruction=TEST_INPUT.get("addition_instruction"), + stream=TEST_INPUT.get("stream"), + conversation_id=TEST_INPUT.get("conversation_id"), + trace_id=TEST_INPUT.get("trace_id"), + uid=TEST_INPUT.get("uid")) + + for result in results: + self.assertIsNotNone(result) + print(f'\n[result]\n{result}\n') + +if __name__ == '__main__': + unittest.main() From 24c51a559118ffefd8fe46abdd45d4d73ef4909e Mon Sep 17 00:00:00 2001 From: raullese Date: Wed, 16 Oct 2024 20:11:28 +0800 Subject: [PATCH 3/8] add document understanding modules --- .../components/document_understanding/base.py | 25 +------- .../document_understanding/component.py | 58 +------------------ .../tests/test_document_understanding.py | 39 ++++++++++++- 3 files changed, 39 insertions(+), 83 deletions(-) diff --git a/appbuilder/core/components/document_understanding/base.py b/appbuilder/core/components/document_understanding/base.py index 64d40cd7..ea191eeb 100644 --- a/appbuilder/core/components/document_understanding/base.py +++ b/appbuilder/core/components/document_understanding/base.py @@ -17,7 +17,6 @@ from pydantic import Field from appbuilder.core.message import Message -import base64 from appbuilder.core.component import ComponentArguments @@ -35,26 +34,4 @@ class DocumentUnderstandingArgs(ComponentArguments): description='用户指令') addition_instruction: str = Field(default="", variable_name='addition_instruction', - description='用户增强指令') - - def file_to_base64(self, file_path): - """ - 读取指定路径的文件并将其内容转换为Base64编码的字符串。 - - :param file_path: 文件的本地路径 - :return: Base64编码的字符串 - """ - try: - # 以二进制模式读取文件内容 - with open(file_path, "rb") as file: - file_data = file.read() - - # 将文件内容编码为Base64 - base64_encoded_data = base64.b64encode(file_data).decode('utf-8') - - return base64_encoded_data - - except Exception as e: - print(f"读取文件或编码过程中出错: {e}") - return None - + description='用户增强指令') \ No newline at end of file diff --git a/appbuilder/core/components/document_understanding/component.py b/appbuilder/core/components/document_understanding/component.py index 69b30c43..b5090a40 100644 --- a/appbuilder/core/components/document_understanding/component.py +++ b/appbuilder/core/components/document_understanding/component.py @@ -33,34 +33,6 @@ class DocumentUnderstanding(Component): name = "document_understanding" version = "v1" meta = DocumentUnderstandingArgs - manifests = [{ - "name": "document_understanding", - "description": "该工具支持对图片以及文档内容进行理解，并基于图片以及文档内容对用户的提问进行回答，包括但不限于文档内容问答、" - "总结摘要、内容分析。", - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "用户输入的query" - }, - "file_path": { - "type": "string", - "description": "用户上传的文档的文件路径" - }, - "instruction": { - "type": "string", - "description": "用户指令" - }, - "addition_instruction": { - "type": "string", - "description": "用户增强指令" - }, - }, - "required": ["query", "file_path", "instruction", "addition_instruction"] - } - }] - def __init__( self, secret_key: Optional[str] = None, @@ -184,32 +156,4 @@ def run(self, else: raise Exception(f"服务请求失败: {result['message']}") else: - response.raise_for_status() - - -if __name__ == '__main__': - import uuid - APPBUILDER_TOKEN = "YOUR-TOKEN" - os.environ["APPBUILDER_TOKEN"] = APPBUILDER_TOKEN - import appbuilder - uid = str(uuid.uuid4()) - trace_id = str(uuid.uuid4()) - conversation_id = str(uuid.uuid4()) - du = DocumentUnderstanding() - query = appbuilder.Message("这篇文档讲了什么") - instruction = "请根据文档内容回答问题" - addition_instruction = "请你用一句话简短概括" - file_path = "test.docx" - stream = True - print(trace_id) - response_ = du.run(query, - file_path, - instruction=instruction, - addition_instruction=addition_instruction, - uid=uid, - trace_id=trace_id, - conversation_id=conversation_id, - stream=stream) - - for result in response_: - print(result) + response.raise_for_status() \ No newline at end of file diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py index c0deebc6..0b03db32 100644 --- a/appbuilder/tests/test_document_understanding.py +++ b/appbuilder/tests/test_document_understanding.py @@ -52,8 +52,8 @@ def setUp(self): os.environ['APPBUILDER_TOKEN'] = TEST_INPUT.get("APPBUILDER_TOKEN") self.du = appbuilder.DocumentUnderstanding() - def test_run_with_default_params(self): - """测试 run 方法使用默认参数 + def test_run_with_stream(self): + """测试 run 方法流式输出 """ query = TEST_INPUT.get("query") results = self.du.run(query, @@ -69,5 +69,40 @@ def test_run_with_default_params(self): self.assertIsNotNone(result) print(f'\n[result]\n{result}\n') + def test_run_with_nostream(self): + """测试 run 方法非流式输出 + """ + query = TEST_INPUT.get("query") + results = self.du.run(query, + TEST_INPUT.get("file_path"), + instruction=TEST_INPUT.get("instruction"), + addition_instruction=TEST_INPUT.get("addition_instruction"), + stream=False, + conversation_id=TEST_INPUT.get("conversation_id"), + trace_id=TEST_INPUT.get("trace_id"), + uid=TEST_INPUT.get("uid")) + + for result in results: + self.assertIsNotNone(result) + print(f'\n[result]\n{result}\n') + + + def test_run_with_errortype(self): + """测试 run 方法上传非法文件类型 + """ + query = TEST_INPUT.get("query") + results = self.du.run(query, + TEST_INPUT.get("test_utils_logging_util.py"), + instruction=TEST_INPUT.get("instruction"), + addition_instruction=TEST_INPUT.get("addition_instruction"), + stream=False, + conversation_id=TEST_INPUT.get("conversation_id"), + trace_id=TEST_INPUT.get("trace_id"), + uid=TEST_INPUT.get("uid")) + + for result in results: + self.assertIsNotNone(result) + print(f'\n[result]\n{result}\n') + if __name__ == '__main__': unittest.main() From 24531c4fc403f012af0640c4b0d9ae2baceed557 Mon Sep 17 00:00:00 2001 From: raullese Date: Wed, 16 Oct 2024 20:36:32 +0800 Subject: [PATCH 4/8] document understanding fix unittest1 --- .../tests/test_document_understanding.py | 21 +++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py index 0b03db32..d173585a 100644 --- a/appbuilder/tests/test_document_understanding.py +++ b/appbuilder/tests/test_document_understanding.py @@ -20,10 +20,6 @@ import appbuilder import uuid -# 生产环境 -os.environ['APPBUILDER_TOKEN'] = "YOUR-TOKEN" - - TEST_INPUT = { "query": appbuilder.Message("这篇文档讲了什么"), "instruction": "请根据文档内容回答问题", @@ -104,5 +100,22 @@ def test_run_with_errortype(self): self.assertIsNotNone(result) print(f'\n[result]\n{result}\n') + def test_run_with_nofile(self): + """测试 run 方法上传无效文件 + """ + query = TEST_INPUT.get("query") + results = self.du.run(query, + TEST_INPUT.get("tt.txt"), + instruction=TEST_INPUT.get("instruction"), + addition_instruction=TEST_INPUT.get("addition_instruction"), + stream=False, + conversation_id=TEST_INPUT.get("conversation_id"), + trace_id=TEST_INPUT.get("trace_id"), + uid=TEST_INPUT.get("uid")) + + for result in results: + self.assertIsNotNone(result) + print(f'\n[result]\n{result}\n') + if __name__ == '__main__': unittest.main() From 4bca89bf61cacd3f72866dbd5a3411915755a8cb Mon Sep 17 00:00:00 2001 From: raullese Date: Wed, 16 Oct 2024 21:01:20 +0800 Subject: [PATCH 5/8] document understanding fix unittest2 --- .../core/components/document_understanding/component.py | 7 ++----- appbuilder/tests/test_document_understanding.py | 7 ++++--- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/appbuilder/core/components/document_understanding/component.py b/appbuilder/core/components/document_understanding/component.py index b5090a40..674ad39e 100644 --- a/appbuilder/core/components/document_understanding/component.py +++ b/appbuilder/core/components/document_understanding/component.py @@ -80,11 +80,8 @@ def file_to_base64(self, file_path: str): base64_message = base64_encoded_data.decode('utf-8') return base64_message - - except FileNotFoundError: - return "文件未找到，请检查文件路径是否正确。" - except Exception as e: - return f"发生错误: {str(e)}" + except: + return f"文件未找到，请检查文件路径是否正确。" def run(self, message: Message, diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py index d173585a..51fdd1f5 100644 --- a/appbuilder/tests/test_document_understanding.py +++ b/appbuilder/tests/test_document_understanding.py @@ -20,6 +20,8 @@ import appbuilder import uuid +os.environ['APPBUILDER_TOKEN'] = "bce-v3/ALTAK-n5AYUIUJMarF7F7iFXVeK/1bf65eed7c8c7efef9b11388524fa1087f90ea58" + TEST_INPUT = { "query": appbuilder.Message("这篇文档讲了什么"), "instruction": "请根据文档内容回答问题", @@ -28,8 +30,7 @@ "stream": True, "conversation_id": str(uuid.uuid4()), "trace_id": str(uuid.uuid4()), - "uid": str(uuid.uuid4()), - "APPBUILDER_TOKEN": os.getenv("APPBUILDER_TOKEN", None), + "uid": str(uuid.uuid4()) } @@ -45,7 +46,7 @@ def setUp(self): Returns: 无返回值，方法中执行了环境变量的赋值操作。 """ - os.environ['APPBUILDER_TOKEN'] = TEST_INPUT.get("APPBUILDER_TOKEN") + os.environ['APPBUILDER_TOKEN'] = "bce-v3/ALTAK-n5AYUIUJMarF7F7iFXVeK/1bf65eed7c8c7efef9b11388524fa1087f90ea58" self.du = appbuilder.DocumentUnderstanding() def test_run_with_stream(self): From 9075a058f2c2b29d1bce26d48a78339a3794bb4c Mon Sep 17 00:00:00 2001 From: raullese Date: Thu, 17 Oct 2024 17:57:19 +0800 Subject: [PATCH 6/8] document understanding module add tool eval& fix url --- .../document_understanding/component.py | 74 +++++++++++++++---- .../tests/test_document_understanding.py | 18 ++++- 2 files changed, 76 insertions(+), 16 deletions(-) diff --git a/appbuilder/core/components/document_understanding/component.py b/appbuilder/core/components/document_understanding/component.py index 674ad39e..845a775c 100644 --- a/appbuilder/core/components/document_understanding/component.py +++ b/appbuilder/core/components/document_understanding/component.py @@ -19,7 +19,6 @@ import json from typing import Optional - from appbuilder.core.components.document_understanding.base import DocumentUnderstandingArgs from appbuilder.core.message import Message @@ -33,6 +32,33 @@ class DocumentUnderstanding(Component): name = "document_understanding" version = "v1" meta = DocumentUnderstandingArgs + manifests = [{ + "name": "document_understanding", + "description": "该工具支持对图片以及文档内容进行理解，并基于图片以及文档内容对用户的提问进行回答，包括但不限于文档内容问答、" + "总结摘要、内容分析。", + "parameters": { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "用户输入的query" + }, + "file_path": { + "type": "string", + "description": "用户上传的文档的文件路径" + }, + "instruction": { + "type": "string", + "description": "用户指令" + }, + "addition_instruction": { + "type": "string", + "description": "用户增强指令" + }, + }, + "required": ["query", "file_path", "instruction", "addition_instruction"] + } + }] def __init__( self, secret_key: Optional[str] = None, @@ -57,7 +83,6 @@ def __init__( secret_key=secret_key, gateway=gateway, lazy_certification=lazy_certification) - self.url = "http://copilot-test.now.baidu-int.com/dte/api/v2/component/tool_eval" self.instruction = instruction, self.addition_instruction = addition_instruction self.file_path = file_path @@ -91,7 +116,8 @@ def run(self, uid=None, trace_id=None, conversation_id=None, - stream=False): + stream=False, + timeout=None): ''' run方法，用于执行长文档理解任务 Args: @@ -130,15 +156,14 @@ def run(self, "query": message.content } }) - headers = { - 'content-type': 'application/json', - 'user-agent': 'vscode-restclient', - 'x-appbuilder-authorization': f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}", - 'x-appbuilder-from': 'sdk' - } - - # try: - response = requests.request("POST", self.url, headers=headers, data=payload, stream=True) + headers = self.http_client.auth_header() + headers['content-type'] = 'application/json' + headers['user-agent'] = 'vscode-restclient' + headers['x-appbuilder-from'] = 'sdk' + headers['x-appbuilder-authorization'] = f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" + url = "http://copilot-test.now.baidu-int.com/dte/api/v2/component/tool_eval" + response = self.http_client.session.post(url, headers=headers, data=payload, timeout=timeout, stream=stream) + self.http_client.check_response_header(response) if response.status_code == 200: if stream: # 处理流式响应，逐行生成数据 @@ -153,4 +178,27 @@ def run(self, else: raise Exception(f"服务请求失败: {result['message']}") else: - response.raise_for_status() \ No newline at end of file + response.raise_for_status() + + def tool_eval(self, + message: Message, + file_path: str, + stream: bool = False, + **kwargs): + """用于function call + """ + instruction = kwargs.get("instruction", "") + addition_instruction = kwargs.get("addition_instruction", "") + uid = kwargs.get("uid", "") + trace_id = kwargs.get("trace_id", "") + conversation_id = kwargs.get("conversation_id", "") + + result = self.run(message, + file_path, + instruction=instruction, + addition_instruction=addition_instruction, + uid=uid, + trace_id=trace_id, + conversation_id=conversation_id, + stream=stream) + return result diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py index 51fdd1f5..314f1410 100644 --- a/appbuilder/tests/test_document_understanding.py +++ b/appbuilder/tests/test_document_understanding.py @@ -20,8 +20,6 @@ import appbuilder import uuid -os.environ['APPBUILDER_TOKEN'] = "bce-v3/ALTAK-n5AYUIUJMarF7F7iFXVeK/1bf65eed7c8c7efef9b11388524fa1087f90ea58" - TEST_INPUT = { "query": appbuilder.Message("这篇文档讲了什么"), "instruction": "请根据文档内容回答问题", @@ -46,7 +44,6 @@ def setUp(self): Returns: 无返回值，方法中执行了环境变量的赋值操作。 """ - os.environ['APPBUILDER_TOKEN'] = "bce-v3/ALTAK-n5AYUIUJMarF7F7iFXVeK/1bf65eed7c8c7efef9b11388524fa1087f90ea58" self.du = appbuilder.DocumentUnderstanding() def test_run_with_stream(self): @@ -118,5 +115,20 @@ def test_run_with_nofile(self): self.assertIsNotNone(result) print(f'\n[result]\n{result}\n') + def test_tool_eval(self): + '''测试tool_eval方法''' + INPUT_JON_ = { + "instruction": "请根据文档内容回答问题", + "addition_instruction": "请你用一句话简短概括", + "uid": str(uuid.uuid4()), + "trace_id": str(uuid.uuid4()), + "conversation_id": str(uuid.uuid4()), + } + query = appbuilder.Message("这篇文档讲了什么") + results = self.du.tool_eval(query, file_path="治安管理处罚条例.docx", stream=False, **INPUT_JON_) + for result in results: + print(result) + + if __name__ == '__main__': unittest.main() From ecc248b688cb3b3f7b08603ab7d5cc1e10718ab4 Mon Sep 17 00:00:00 2001 From: raullese Date: Sat, 26 Oct 2024 00:38:44 +0800 Subject: [PATCH 7/8] Add DocumentUnderstanding module 1025_1 --- .../document_understanding/README.md | 46 ++++----- .../document_understanding/component.py | 95 ++++++++++++------- .../tests/test_document_understanding.py | 37 +++----- 3 files changed, 94 insertions(+), 84 deletions(-) diff --git a/appbuilder/core/components/document_understanding/README.md b/appbuilder/core/components/document_understanding/README.md index d706aa73..1bd9333f 100644 --- a/appbuilder/core/components/document_understanding/README.md +++ b/appbuilder/core/components/document_understanding/README.md @@ -1,4 +1,4 @@ -# 文件生成PPT（PPTGenerationFromFile） +# 长文档内容理解（DocumentUnderstanding） ## 简介 长文档内容理解组件（DocumentUnderstanding）支持对图片以及文档内容进行理解，并基于图片以及文档内容对用户的提问进行回答， @@ -15,29 +15,24 @@ ```python -import uuid import os import appbuilder # 请前往千帆AppBuilder官网创建密钥，流程详见：https://cloud.baidu.com/doc/AppBuilder/s/Olq6grrt6#1%E3%80%81%E5%88%9B%E5%BB%BA%E5%AF%86%E9%92%A5 APPBUILDER_TOKEN = "YOUR-TOKEN" os.environ["APPBUILDER_TOKEN"] = APPBUILDER_TOKEN -uid = str(uuid.uuid4()) -trace_id = str(uuid.uuid4()) -conversation_id = str(uuid.uuid4()) ## 注意你的conversation_id不能和之前的请求重复，不然会直接返回之前已有的conversation_id的答案 du = appbuilder.DocumentUnderstanding() query = appbuilder.Message("这篇文档讲了什么") -instruction = "请根据文档内容回答问题" -addition_instruction = "请你用一句话简短概括" ##用户增强指令，可选填，该内容会进一步增强大模型的指令跟随能力，将你最需要增强效果的指令填于此，内容可以与上述的"instruction"基础指令有重复，注意：该字段内容过多会一定程度影响大模型内容严谨度，请注意控制该字段的指令字数 +instruction = "请根据文档内容回答问题，用一句话简短概括" +addition_instruction = "用一句话简短概括" ##用户增强指令，可选填，该内容会进一步增强大模型的指令跟随能力，将你最需要增强效果的指令填于此，内容可以与上述的"instruction"基础指令有重复，注意：该字段内容过多会一定程度影响大模型内容严谨度，请注意控制该字段的指令字数 +app_id = "YOUR-APP-ID" ##你需要在系统上自己的账号下（https://qianfan.cloud.baidu.com/appbuilder）创建任意空Agent，并获取该Agent的app_id（即界面上的应用ID，在首页->个人空间->应用 里面即会显示应用ID），这里任意空Agent就可以，无需任何配置信息，这个agent的作用只是为了获取app_id信息 file_path = "YOUR-FILE-PATH" ##填写你的本地待分析文件路径 stream = False ##是否开启流式输出功能 response_ = du.run(query, file_path, instruction=instruction, addition_instruction=addition_instruction, - uid=uid, - trace_id=trace_id, - conversation_id=conversation_id, + app_id=app_id, stream=stream) for result in response_: @@ -66,30 +61,31 @@ os.environ['APPBUILDER_TOKEN'] = 'bce-YOURTOKEN' ### 调用参数 -| 参数名称 | 参数类型 | 是否必须 | 描述 | 示例值 | -|------------------------|------|------|--------------------------------------------------------------------------|-----------------| +| 参数名称 | 参数类型 | 是否必须 | 描述 | 示例值 | +|------------------------|------|------|--------------------------------------------------------------------------|-----------------------------| | `message` | obj | 是 | 输入消息，用户输入query。 | Message(content=input_data) | -| `file_path` | str | 是 | 用户需要分析的文档 | "test.pdf" | -| `instruction` | str | 否 | 用户指令 | "你的回答要严谨客观" | -| `addition_instruction` | str | 否 | 用户增强指令，可选填，该内容会进一步增强大模型的指令跟随能力，将你最需要增强效果的指令填于此，注意：该字段内容过多会一定程度影响大模型内容严谨度 | "你的答案需要分点阐述" | - +| `file_path` | str | 是 | 用户需要分析的文档 | "test.pdf" | +| `app_id` | str | 是 | 你需要在系统上自己的账号下（https://qianfan.cloud.baidu.com/appbuilder）创建任意空Agent，并获取该Agent的app_id（即界面上的应用ID，在首页->个人空间->应用 里面即会显示应用ID），这里任意空Agent就可以，无需任何配置信息，这个agent的作用只是为了获取app_id信息 | "YOUR-APP-ID" | +| `instruction` | str | 否 | 用户指令 | "你的回答要严谨客观，且答案一定要分点阐述" | +| `addition_instruction` | str | 否 | 用户增强指令，可选填，该内容会进一步增强大模型的指令跟随能力，将你最需要增强效果的指令填于此，注意：该字段内容过多会一定程度影响大模型内容严谨度 | "你的答案需要分点阐述" | ### 响应参数 | 参数名称 | 参数类型 | 描述 | 示例值 | | ------- |------| -------- | -------- | | `result` | str | 模型运行后的输出结果 | "" | -### 响应示例 +### 响应示例-流式输出 +``` +data: {"type": "text", "text": "文件解析完成, 耗时13485.63ms\n\n"} request_id: f99a7230-649f-4170-ade7-62d8368a18e6 +data: {"type": "text", "text": "**Human", "event_status": "running"} request_id: f99a7230-649f-4170-ade7-62d8368a18e6 +data: {"type": "text", "text": "-Timescale Adaptation in an Open-Ended Task Space** 文档详细介绍了DeepMind团队开发的自适应代理（Adaptive Agent，简称", "event_status": "running"} request_id: f99a7230-649f-4170-ade7-62d8368a18e6 +data: {"type": "text", "text": "AdA）在开放任务空间中的快速适应能力。", "event_status": "running"} request_id: f99a7230-649f-4170-ade7-62d8368a18e6 +data: {"type": "text", "text": "", "event_status": "done"} request_id: f99a7230-649f-4170-ade7-62d8368a18e6 ``` -您好，请问您是想询问关于残疾人办理什么证件的问题吗？如果是，我可以为您提供一些信息。 - -首先，如果您是首次申请办理残疾人证，需要携带身份证、户口簿和三张两寸近期免冠白底彩色照片到县残联办证窗口提出申请。如果您因身体原因无法亲自前往，可以联系村（社区）工作人员代办申请。 - -其次，如果您是指残疾类型等级证明，您需要携带相关材料到指定医院或医生进行评级，并由医生签名盖章。 - -最后，如果您是指残疾人享受低保或残疾人贫困证的一级肢体、视力、智力、精神、多重及60周岁以上的一级听力、语言的重度残疾人可以享受重度残疾人生活补助，那么您需要携带身份证、户口本和残疾证申请表到县、市、区级残联进行办理。 -希望这些信息对您有所帮助。如果您还有其他问题，欢迎随时提问。 +### 响应示例-非流式输出 +``` +{'code': 0, 'message': '', 'result': {'text': '文件解析完成, 耗时14572.57ms\n\n**Human-Timescale Adaptation in an Open-Ended Task Space** 文档详细介绍了DeepMind团队开发的自适应代理（Adaptive Agent，简称AdA）在开放任务空间中的快速适应能力。以下是文档的主要内容和贡献点：\n\n1. **引言**：\n - 强调了快速适应能力对于人工智能的重要性，特别是在现实世界中的应用和与人类互动的场景中。\n - 提出了通过元强化学习（meta-RL）和自动课程学习（auto-curriculum learning）等方法，训练能够在未见过的环境中快速适应的代理。\n\n2. **自适应代理（AdA）**：\n - 介绍了AdA的设计和训练方法，包括其在开放任务空间中的适应行为、记忆架构、以及如何通过自动课程学习来优化训练过程。\n - 展示了AdA能够在几分钟内解决复杂的3D任务，且不需要进一步的代理训练，显示了其快速适应的能力。\n\n3. **实验与结果**：\n - 在多个方面评估了AdA的性能，包括其在单代理和多代理设置下的适应能力、不同架构和课程学习方法的影响、以及模型大小和记忆长度对性能的影响。\n - 通过与人类玩家的比较，证明了AdA在适应速度上与人类相当。\n\n4. **相关工作**：\n - 回顾了与本工作相关的领域，包括程序化环境生成、开放任务学习、适应性和强化学习中的Transformer应用等。\n\n5. **结论**：\n - 总结了AdA的贡献，强调了其在开放任务空间中快速适应的能力，以及通过元强化学习和自动课程学习等方法训练大型模型的可能性。\n\n6. **作者和贡献**：\n - 列出了主要贡献者和部分贡献者，以及项目的赞助商和认可。\n\n**主要贡献点**：\n- 提出了AdA，一个能够在开放任务空间中快速适应的代理，其适应速度与人类相当。\n- 通过元强化学习和自动课程学习等方法，训练了大型Transformer模型，展示了其在开放任务空间中的快速适应能力。\n- 分析了不同架构、课程学习方法、模型大小和记忆长度对AdA性能的影响，提供了详细的实验结果和比较。\n- 通过与人类玩家的比较，证明了AdA在适应速度上的优势。'}, 'request_id': '687642b0-b877-49ed-9ad9-65d76de0ea58'} ``` ## 高级用法 diff --git a/appbuilder/core/components/document_understanding/component.py b/appbuilder/core/components/document_understanding/component.py index 845a775c..27086ce0 100644 --- a/appbuilder/core/components/document_understanding/component.py +++ b/appbuilder/core/components/document_understanding/component.py @@ -24,6 +24,7 @@ from appbuilder.core.message import Message from appbuilder.core.component import Component import base64 +import uuid class DocumentUnderstanding(Component): """ @@ -55,8 +56,12 @@ class DocumentUnderstanding(Component): "type": "string", "description": "用户增强指令" }, + "app_id": { + "type": "string", + "description": "系统应用ID" + }, }, - "required": ["query", "file_path", "instruction", "addition_instruction"] + "required": ["query", "file_path", "instruction", "addition_instruction", "app_id"] } }] def __init__( @@ -67,6 +72,7 @@ def __init__( instruction: Optional[Message] = None, addition_instruction: Optional[Message] = None, file_path: Optional[str] = None, + app_id: Optional[str] = None, ): """初始化DocumentUnderstanding组件。 @@ -86,6 +92,7 @@ def __init__( self.instruction = instruction, self.addition_instruction = addition_instruction self.file_path = file_path + self.app_id = app_id def get_addition_instruction(self, addition_instruction: str): @@ -108,14 +115,42 @@ def file_to_base64(self, file_path: str): except: return f"文件未找到，请检查文件路径是否正确。" + def get_conversation_id(self, app_id: str): + url = "https://qianfan.baidubce.com/v2/app/conversation" + payload = json.dumps({ + "app_id": app_id + }) + headers = { + 'Content-Type': 'application/json', + 'X-Appbuilder-Authorization': f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" + } + response = self.http_client.session.post(url, headers=headers, data=payload) + self.http_client.check_response_header(response) + response = requests.request("POST", url, headers=headers, data=payload) + return json.loads(response.text).get("conversation_id", None) + + def get_file_id(self, conversation_id: str, app_id: str, file_path: str): + url = "https://qianfan.baidubce.com/v2/app/conversation/file/upload" + payload = { + 'app_id': app_id, + 'conversation_id': conversation_id + } + files = [ + ('file', (file_path, + open(file_path, 'rb'), 'application/{}'.format(file_path.split("."[-1])))) + ] + headers = { + 'X-Appbuilder-Authorization': f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" + } + response = requests.request("POST", url, headers=headers, data=payload, files=files) + return json.loads(response.text).get("id", None) + def run(self, message: Message, file_path, instruction="", addition_instruction="", - uid=None, - trace_id=None, - conversation_id=None, + app_id="", stream=False, timeout=None): ''' @@ -125,43 +160,36 @@ def run(self, file_path: 用户输入的文件路径 instruction: 用户输入的人设指令 addition_instruction: 用户输入的增强版指令(如有) + app_id: 用户输入的app_id Returns: result (Message): 模型运行后的输出消息。 ''' - file_data = self.file_to_base64(file_path) file_name = file_path.split("/")[-1] file_type = file_name.split(".")[-1].lower() - + request_id = str(uuid.uuid4()) support_file_type = ["pdf", "docx", "xlsx", "png", "jpg", "jpeg", "txt"] if file_type not in support_file_type: raise Exception(f"不支持解析{file_type}类型的文件，当前仅支持解析以下几种文件类型：{support_file_type}") payload = json.dumps({ - "component": "DocumentUnderstanding", "stream": stream, - "component_init_args": { - "instruction": instruction, - "addition_instruction": self.get_addition_instruction(addition_instruction), - "file_data": file_data, - "file_name": file_name, - }, - "system": { - "uid": uid, - "traceid": trace_id, - "conversation_id": conversation_id, - "appbuilder_token": f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" - }, - "user": { - "query": message.content + "batch": False, + "arguments": { + "query": message.content, + "file_ids":[self.get_file_id(self.get_conversation_id(app_id=app_id), app_id, file_path)], + "files": [], + "file_urls": {}, + "instruction": instruction, + "addition_instruction": self.get_addition_instruction(addition_instruction), } }) headers = self.http_client.auth_header() - headers['content-type'] = 'application/json' - headers['user-agent'] = 'vscode-restclient' - headers['x-appbuilder-from'] = 'sdk' - headers['x-appbuilder-authorization'] = f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" - url = "http://copilot-test.now.baidu-int.com/dte/api/v2/component/tool_eval" + headers['Content-Type'] = 'application/json' + headers['Authorization'] = f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" + headers['X-Appbuilder-Request-Id'] = request_id + url = self.http_client.service_url_v2("/components/document_understanding/version/preview") + print("url: ", url) response = self.http_client.session.post(url, headers=headers, data=payload, timeout=timeout, stream=stream) self.http_client.check_response_header(response) if response.status_code == 200: @@ -170,11 +198,14 @@ def run(self, for line in response.iter_lines(): if line: decoded_line = line.decode('utf-8') + decoded_line = f"{decoded_line} request_id: {request_id}" yield decoded_line # 使用yield逐行输出结果 else: result = response.json() + result["request_id"] = request_id if result["code"] == 0: - yield result["result"].get("text") + yield result + else: raise Exception(f"服务请求失败: {result['message']}") else: @@ -189,16 +220,12 @@ def tool_eval(self, """ instruction = kwargs.get("instruction", "") addition_instruction = kwargs.get("addition_instruction", "") - uid = kwargs.get("uid", "") - trace_id = kwargs.get("trace_id", "") - conversation_id = kwargs.get("conversation_id", "") + app_id = kwargs.get("app_id", "") result = self.run(message, file_path, instruction=instruction, addition_instruction=addition_instruction, - uid=uid, - trace_id=trace_id, - conversation_id=conversation_id, + app_id=app_id, stream=stream) - return result + return result \ No newline at end of file diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py index 314f1410..3f88324f 100644 --- a/appbuilder/tests/test_document_understanding.py +++ b/appbuilder/tests/test_document_understanding.py @@ -18,7 +18,6 @@ import os import unittest import appbuilder -import uuid TEST_INPUT = { "query": appbuilder.Message("这篇文档讲了什么"), @@ -26,9 +25,7 @@ "addition_instruction": "请你用一句话简短概括", "file_path": "title_splitter.docx", "stream": True, - "conversation_id": str(uuid.uuid4()), - "trace_id": str(uuid.uuid4()), - "uid": str(uuid.uuid4()) + "app_id": "87187054-78f0-4ef3-b710-fdcf2bfba7f2" } @@ -55,9 +52,7 @@ def test_run_with_stream(self): instruction=TEST_INPUT.get("instruction"), addition_instruction=TEST_INPUT.get("addition_instruction"), stream=TEST_INPUT.get("stream"), - conversation_id=TEST_INPUT.get("conversation_id"), - trace_id=TEST_INPUT.get("trace_id"), - uid=TEST_INPUT.get("uid")) + app_id=TEST_INPUT.get("app_id")) for result in results: self.assertIsNotNone(result) @@ -72,9 +67,7 @@ def test_run_with_nostream(self): instruction=TEST_INPUT.get("instruction"), addition_instruction=TEST_INPUT.get("addition_instruction"), stream=False, - conversation_id=TEST_INPUT.get("conversation_id"), - trace_id=TEST_INPUT.get("trace_id"), - uid=TEST_INPUT.get("uid")) + app_id=TEST_INPUT.get("app_id")) for result in results: self.assertIsNotNone(result) @@ -86,13 +79,11 @@ def test_run_with_errortype(self): """ query = TEST_INPUT.get("query") results = self.du.run(query, - TEST_INPUT.get("test_utils_logging_util.py"), + "test_utils_logging_util.py", instruction=TEST_INPUT.get("instruction"), addition_instruction=TEST_INPUT.get("addition_instruction"), stream=False, - conversation_id=TEST_INPUT.get("conversation_id"), - trace_id=TEST_INPUT.get("trace_id"), - uid=TEST_INPUT.get("uid")) + app_id=TEST_INPUT.get("app_id")) for result in results: self.assertIsNotNone(result) @@ -103,13 +94,11 @@ def test_run_with_nofile(self): """ query = TEST_INPUT.get("query") results = self.du.run(query, - TEST_INPUT.get("tt.txt"), + "tt.txt", instruction=TEST_INPUT.get("instruction"), addition_instruction=TEST_INPUT.get("addition_instruction"), stream=False, - conversation_id=TEST_INPUT.get("conversation_id"), - trace_id=TEST_INPUT.get("trace_id"), - uid=TEST_INPUT.get("uid")) + app_id=TEST_INPUT.get("app_id")) for result in results: self.assertIsNotNone(result) @@ -118,14 +107,12 @@ def test_run_with_nofile(self): def test_tool_eval(self): '''测试tool_eval方法''' INPUT_JON_ = { - "instruction": "请根据文档内容回答问题", - "addition_instruction": "请你用一句话简短概括", - "uid": str(uuid.uuid4()), - "trace_id": str(uuid.uuid4()), - "conversation_id": str(uuid.uuid4()), + "instruction": TEST_INPUT.get("instruction", ""), + "addition_instruction": TEST_INPUT.get("addition_instruction", ""), + "app_id": TEST_INPUT.get("app_id") } - query = appbuilder.Message("这篇文档讲了什么") - results = self.du.tool_eval(query, file_path="治安管理处罚条例.docx", stream=False, **INPUT_JON_) + query = TEST_INPUT.get("query", "") + results = self.du.tool_eval(query, file_path=TEST_INPUT.get("file_path", ""), stream=False, **INPUT_JON_) for result in results: print(result) From 9640c6275cefbab5a19c239c054637db54d175e8 Mon Sep 17 00:00:00 2001 From: raullese Date: Sat, 26 Oct 2024 01:24:21 +0800 Subject: [PATCH 8/8] Add DocumentUnderstanding module 1025_2 --- .../document_understanding/component.py | 16 ------- appbuilder/tests/component_collector.py | 3 +- .../tests/test_document_understanding.py | 42 +++++++------------ 3 files changed, 16 insertions(+), 45 deletions(-) diff --git a/appbuilder/core/components/document_understanding/component.py b/appbuilder/core/components/document_understanding/component.py index 27086ce0..e53053e1 100644 --- a/appbuilder/core/components/document_understanding/component.py +++ b/appbuilder/core/components/document_understanding/component.py @@ -99,21 +99,6 @@ def get_addition_instruction(self, addition_instruction: str): """拼接addition_instruction""" return "，指令：" + addition_instruction - def file_to_base64(self, file_path: str): - try: - # 打开文件并读取内容 - with open(file_path, "rb") as file: - file_data = file.read() - - # 将文件数据转换为base64格式 - base64_encoded_data = base64.b64encode(file_data) - - # 将base64字节数据转换为字符串 - base64_message = base64_encoded_data.decode('utf-8') - - return base64_message - except: - return f"文件未找到，请检查文件路径是否正确。" def get_conversation_id(self, app_id: str): url = "https://qianfan.baidubce.com/v2/app/conversation" @@ -189,7 +174,6 @@ def run(self, headers['Authorization'] = f"Bearer {os.getenv('APPBUILDER_TOKEN', '')}" headers['X-Appbuilder-Request-Id'] = request_id url = self.http_client.service_url_v2("/components/document_understanding/version/preview") - print("url: ", url) response = self.http_client.session.post(url, headers=headers, data=payload, timeout=timeout, stream=stream) self.http_client.check_response_header(response) if response.status_code == 200: diff --git a/appbuilder/tests/component_collector.py b/appbuilder/tests/component_collector.py index 00c015db..0d7381b7 100644 --- a/appbuilder/tests/component_collector.py +++ b/appbuilder/tests/component_collector.py @@ -67,7 +67,8 @@ "PlantRecognition", "HandwriteOCR", "ImageUnderstand", - "MixCardOCR", + "MixCardOCR", + "DocumentUnderstanding", ] diff --git a/appbuilder/tests/test_document_understanding.py b/appbuilder/tests/test_document_understanding.py index 3f88324f..c6a04517 100644 --- a/appbuilder/tests/test_document_understanding.py +++ b/appbuilder/tests/test_document_understanding.py @@ -29,7 +29,7 @@ } -@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") +# @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestDocumentUnderstandingComponent(unittest.TestCase): def setUp(self): """ @@ -72,37 +72,23 @@ def test_run_with_nostream(self): for result in results: self.assertIsNotNone(result) print(f'\n[result]\n{result}\n') - - - def test_run_with_errortype(self): - """测试 run 方法上传非法文件类型 - """ - query = TEST_INPUT.get("query") - results = self.du.run(query, - "test_utils_logging_util.py", - instruction=TEST_INPUT.get("instruction"), - addition_instruction=TEST_INPUT.get("addition_instruction"), - stream=False, - app_id=TEST_INPUT.get("app_id")) - - for result in results: - self.assertIsNotNone(result) - print(f'\n[result]\n{result}\n') - def test_run_with_nofile(self): """测试 run 方法上传无效文件 """ query = TEST_INPUT.get("query") - results = self.du.run(query, - "tt.txt", - instruction=TEST_INPUT.get("instruction"), - addition_instruction=TEST_INPUT.get("addition_instruction"), - stream=False, - app_id=TEST_INPUT.get("app_id")) - - for result in results: - self.assertIsNotNone(result) - print(f'\n[result]\n{result}\n') + # 使用 assertRaises 捕获预期异常 + with self.assertRaises(FileNotFoundError): # 假设无效文件抛出 FileNotFoundError 异常 + results = self.du.run(query, + "invalid_file.txt", # 使用无效文件 + instruction=TEST_INPUT.get("instruction"), + addition_instruction=TEST_INPUT.get("addition_instruction"), + stream=False, + app_id=TEST_INPUT.get("app_id")) + + # 如果 run 方法抛出异常，以下代码将不会执行 + for result in results: + self.assertIsNotNone(result) + print(f'\n[result]\n{result}\n') def test_tool_eval(self): '''测试tool_eval方法'''