Merge pull request #80 from gomate-community/pipeline

Rename Gommate to TrustRAG
gomate-community · Nov 28, 2024 · 1334c44 · 1334c44
2 parents d969659 + b4bdd5a
commit 1334c44
Show file tree

Hide file tree

Showing 158 changed files with 213 additions and 209 deletions.
diff --git a/README.md b/README.md
@@ -284,11 +284,10 @@ python app.py
 ```
 
 浏览器访问：[127.0.0.1:7860](127.0.0.1:7860)
-![demo.png](resources%2Fdemo.png)
+![trustrag_demo.png](resources%2Ftrustrag_demo.png)
 
 app后台日志：
-
-![app_logging.png](resources%2Fapp_logging.png)
+![app_logging3.png](resources%2Fapp_logging3.png)
 
 ## ⭐️ Star History
 
@@ -308,3 +307,7 @@ app后台日志：
 群满或者合作交流可以联系：
 
 <img src="https://raw.githubusercontent.com/yanqiangmiffy/Chinese-LangChain/master/images/personal.jpg" width="180px">
+
+## 致谢
+- 文档解析：[infiniflow/ragflow](https://github.com/infiniflow/ragflow/blob/main/deepdoc/README.md)
+- PDF文件解析[opendatalab/MinerU](https://github.com/opendatalab/MinerU)
diff --git a/api/apps/core/citation/views.py b/api/apps/core/citation/views.py
@@ -14,8 +14,8 @@
 
 from api.apps.core.citation.bodys import CitationBody
 from api.apps.handle.response.json_response import ApiResponse
-from gomate.modules.citation.match_citation import MatchCitation
-from gomate.modules.citation.source_citation import SourceCitation
+from trustrag.modules.citation.match_citation import MatchCitation
+from trustrag.modules.citation.source_citation import SourceCitation
 mc = MatchCitation()
 sc = SourceCitation()
 citation_router = APIRouter()

diff --git a/api/apps/core/judge/views.py b/api/apps/core/judge/views.py
@@ -14,8 +14,8 @@
 
 from api.apps.core.judge.bodys import JudgeBody
 from api.apps.handle.response.json_response import ApiResponse
-from gomate.modules.judger.bge_judger import BgeJudger, BgeJudgerConfig
-from gomate.modules.judger.chatgpt_judger import OpenaiJudger, OpenaiJudgerConfig
+from trustrag.modules.judger.bge_judger import BgeJudger, BgeJudgerConfig
+from trustrag.modules.judger.chatgpt_judger import OpenaiJudger, OpenaiJudgerConfig
 
 judge_router = APIRouter()
 

diff --git a/api/apps/core/parser/views.py b/api/apps/core/parser/views.py
@@ -17,14 +17,14 @@
 from fastapi import File, UploadFile, HTTPException
 from fastapi.responses import JSONResponse
 
-from gomate.modules.document.chunk import TextChunker
-from gomate.modules.document.docx_parser import DocxParser
-from gomate.modules.document.excel_parser import ExcelParser
-from gomate.modules.document.html_parser import HtmlParser
-from gomate.modules.document.json_parser import JsonParser
-from gomate.modules.document.pdf_parser_fast import PdfSimParser
-from gomate.modules.document.ppt_parser import PptParser
-from gomate.modules.document.txt_parser import TextParser
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.document.docx_parser import DocxParser
+from trustrag.modules.document.excel_parser import ExcelParser
+from trustrag.modules.document.html_parser import HtmlParser
+from trustrag.modules.document.json_parser import JsonParser
+from trustrag.modules.document.pdf_parser_fast import PdfSimParser
+from trustrag.modules.document.ppt_parser import PptParser
+from trustrag.modules.document.txt_parser import TextParser
 
 tc = TextChunker()
 parse_router = APIRouter()

diff --git a/api/apps/core/refiner/views.py b/api/apps/core/refiner/views.py
@@ -14,7 +14,7 @@
 
 from api.apps.core.refiner.bodys import RefinerBody
 from api.apps.handle.response.json_response import ApiResponse
-from gomate.modules.refiner.compressor import LLMCompressApi
+from trustrag.modules.refiner.compressor import LLMCompressApi
 
 refiner_router = APIRouter()
 compressor = LLMCompressApi()

diff --git a/api/apps/core/rerank/views.py b/api/apps/core/rerank/views.py
@@ -15,7 +15,7 @@
 from api.apps.core.rerank.bodys import RerankBody
 from api.apps.core.rerank.models import Application
 from api.apps.handle.response.json_response import UserNotFoundResponse, ApiResponse
-from gomate.modules.reranker.bge_reranker import BgeReranker, BgeRerankerConfig
+from trustrag.modules.reranker.bge_reranker import BgeReranker, BgeRerankerConfig
 
 # from apps.handle.exception.exception import MallException
 # from apps.core.config.models import LLMModel

diff --git a/api/apps/core/rewrite/views.py b/api/apps/core/rewrite/views.py
@@ -14,7 +14,7 @@
 
 from api.apps.core.rewrite.bodys import RewriteBody
 from api.apps.handle.response.json_response import ApiResponse
-from gomate.modules.rewriter.openai_rewrite import OpenaiRewriter,OpenaiRewriterConfig
+from trustrag.modules.rewriter.openai_rewrite import OpenaiRewriter,OpenaiRewriterConfig
 
 
 rewriter_router = APIRouter()

diff --git a/app.py b/app.py
@@ -13,19 +13,19 @@
 import gradio as gr
 import loguru
 
-from gomate.applications.rag import RagApplication, ApplicationConfig
-from gomate.modules.reranker.bge_reranker import BgeRerankerConfig
-from gomate.modules.retrieval.dense_retriever import DenseRetrieverConfig
+from trustrag.applications.rag import RagApplication, ApplicationConfig
+from trustrag.modules.reranker.bge_reranker import BgeRerankerConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig
 
 # 修改成自己的配置！！！
 app_config = ApplicationConfig()
-app_config.docs_path = "/data/users/searchgpt/yq/GoMate_dev/data/docs/"
+app_config.docs_path = "/data/users/searchgpt/yq/TrustRAG/data/docs/"
 app_config.llm_model_path = "/data/users/searchgpt/pretrained_models/glm-4-9b-chat"
 
 retriever_config = DenseRetrieverConfig(
     model_name_or_path="/data/users/searchgpt/pretrained_models/bge-large-zh-v1.5",
     dim=1024,
-    index_dir='/data/users/searchgpt/yq/GoMate/examples/retrievers/dense_cache'
+    index_path='/data/users/searchgpt/yq/TrustRAG/examples/retrievers/dense_cache'
 )
 rerank_config = BgeRerankerConfig(
     model_name_or_path="/data/users/searchgpt/pretrained_models/bge-reranker-large"
@@ -115,7 +115,7 @@ def predict(input,
 
 
 with gr.Blocks(theme="soft") as demo:
-    gr.Markdown("""<h1><center>Gomate Application</center></h1>
+    gr.Markdown("""<h1><center>TrustRAG Application</center></h1>
         <center><font size=3>
         </center></font>
         """)
@@ -177,16 +177,16 @@ def predict(input,
             # )
         with gr.Column(scale=4):
             with gr.Row():
-                chatbot = gr.Chatbot(label='Gomate Application').style(height=650)
+                chatbot = gr.Chatbot(label='TrustRAG Application').style(height=650)
             with gr.Row():
                 message = gr.Textbox(label='请输入问题')
             with gr.Row():
                 clear_history = gr.Button("🧹 清除历史对话")
                 send = gr.Button("🚀 发送")
             with gr.Row():
                 gr.Markdown("""提醒：<br>
-                                        [Gomate Application](https://github.com/gomate-community/GoMate) <br>
-                                        有任何使用问题[Github Issue区](https://github.com/gomate-community/GoMate)进行反馈. 
+                                        [TrustRAG Application](https://github.com/TrustRAG-community/TrustRAG) <br>
+                                        有任何使用问题[Github Issue区](https://github.com/TrustRAG-community/TrustRAG)进行反馈. 
                                         <br>
                                         """)
         with gr.Column(scale=2):

diff --git a/docs/quickstart.md b/docs/quickstart.md
@@ -57,14 +57,14 @@ import pickle
 import pandas as pd
 from tqdm import tqdm
 
-from gomate.modules.document.chunk import TextChunker
-from gomate.modules.document.txt_parser import TextParser
-from gomate.modules.document.utils import PROJECT_BASE
-from gomate.modules.generator.llm import GLM4Chat
-from gomate.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
-from gomate.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
-from gomate.modules.retrieval.dense_retriever import DenseRetrieverConfig
-from gomate.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.document.txt_parser import TextParser
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.generator.llm import GLM4Chat
+from trustrag.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
+from trustrag.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig
+from trustrag.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig
 ```
 
 

diff --git a/docs/retrieval.md b/docs/retrieval.md
@@ -10,9 +10,9 @@
 - `index_path`：向量维度
 
 ```python
-from gomate.modules.document.common_parser import CommonParser
-from gomate.modules.document.utils import PROJECT_BASE
-from gomate.modules.retrieval.bm25s_retriever import BM25RetrieverConfig, BM25Retriever
+from trustrag.modules.document.common_parser import CommonParser
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.retrieval.bm25s_retriever import BM25RetrieverConfig, BM25Retriever
 
 if __name__ == '__main__':
 
@@ -55,7 +55,7 @@ if __name__ == '__main__':
 import pandas as pd
 from tqdm import tqdm
 
-from gomate.modules.retrieval.dense_retriever import DenseRetriever, DenseRetrieverConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetriever, DenseRetrieverConfig
 
 if __name__ == '__main__':
     retriever_config = DenseRetrieverConfig(
@@ -91,10 +91,10 @@ if __name__ == '__main__':
 > 混合检索，将Bm25检索以及Dense检索的结果进行合并
 
 ```python
-from gomate.modules.document.common_parser import CommonParser
-from gomate.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
-from gomate.modules.retrieval.dense_retriever import DenseRetrieverConfig
-from gomate.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig
+from trustrag.modules.document.common_parser import CommonParser
+from trustrag.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig
+from trustrag.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig
 
 if __name__ == '__main__':
     # BM25 and Dense Retriever configurations

diff --git a/docs/rewriter.md b/docs/rewriter.md
@@ -1,15 +1,16 @@
 ## HyDE
+
 ```python
 import os
 
 import pandas as pd
 from tqdm import tqdm
 
-from gomate.modules.document.common_parser import CommonParser
-from gomate.modules.generator.llm import GLMChat
-from gomate.modules.retrieval.dense_retriever import DenseRetriever, DenseRetrieverConfig
-from gomate.modules.rewriter.hyde_rewriter import HydeRewriter
-from gomate.modules.rewriter.promptor import Promptor
+from trustrag.modules.document.common_parser import CommonParser
+from trustrag.modules.generator.llm import GLMChat
+from trustrag.modules.retrieval.dense_retriever import DenseRetriever, DenseRetrieverConfig
+from trustrag.modules.rewriter.hyde_rewriter import HydeRewriter
+from trustrag.modules.rewriter.promptor import Promptor
 
 if __name__ == '__main__':
     promptor = Promptor(task="WEB_SEARCH", language="zh")

diff --git a/examples/citations/match_example.py b/examples/citations/match_example.py
@@ -1,4 +1,4 @@
-from gomate.modules.citation.match_citation import MatchCitation
+from trustrag.modules.citation.match_citation import MatchCitation
 import json
 
 mc = MatchCitation()

diff --git a/examples/docs/convert2json_parse.py b/examples/docs/convert2json_parse.py
@@ -2,7 +2,7 @@
 import os
 import re
 
-from gomate.modules.document.utils import PROJECT_BASE
+from trustrag.modules.document.utils import PROJECT_BASE
 
 
 def process_markdown(markdown_content):

diff --git a/examples/judger/bgejudger_exmpale.py b/examples/judger/bgejudger_exmpale.py
@@ -10,7 +10,7 @@
 @description: coding..
 """
 import pandas as pd
-from gomate.modules.judger.bge_judger import BgeJudger, BgeJudgerConfig
+from trustrag.modules.judger.bge_judger import BgeJudger, BgeJudgerConfig
 
 if __name__ == '__main__':
     judger_config = BgeJudgerConfig(

diff --git a/examples/judger/chatgpt_judger.py b/examples/judger/chatgpt_judger.py
@@ -2,7 +2,7 @@
 
 import pandas as pd
 
-from gomate.modules.judger.chatgpt_judger import OpenaiJudger, OpenaiJudgerConfig
+from trustrag.modules.judger.chatgpt_judger import OpenaiJudger, OpenaiJudgerConfig
 
 if __name__ == '__main__':
 

diff --git a/examples/parsers/common_parser.py b/examples/parsers/common_parser.py
@@ -1,4 +1,4 @@
-from gomate.modules.document.common_parser import CommonParser
+from trustrag.modules.document.common_parser import CommonParser
 
 
 # cp=CommonParser()

diff --git a/examples/parsers/markdown_parser.py b/examples/parsers/markdown_parser.py
@@ -9,7 +9,7 @@
 @software: PyCharm
 @description: coding..
 """
-from gomate.modules.document.markdown_parser import MarkdownParser
+from trustrag.modules.document.markdown_parser import MarkdownParser
 
 
 if __name__ == '__main__':

diff --git a/examples/parsers/parser_examples.py b/examples/parsers/parser_examples.py
@@ -1,4 +1,4 @@
-from gomate.modules.document.common_parser import CommonParser
+from trustrag.modules.document.common_parser import CommonParser
 
 if __name__ == '__main__':
     parser = CommonParser()

diff --git a/examples/parsers/pdfparser_example.py b/examples/parsers/pdfparser_example.py
@@ -9,7 +9,7 @@
 @software: PyCharm
 @description: coding..
 """
-from gomate.modules.document.pdf_parser_fast import PdfParserUsingPyMuPDF
+from trustrag.modules.document.pdf_parser_fast import PdfParserUsingPyMuPDF
 
 
 if __name__ == '__main__':

diff --git a/examples/parsers/pdfparser_mineru.py b/examples/parsers/pdfparser_mineru.py
@@ -1,6 +1,6 @@
 import os
-from gomate.modules.document.pdf_mineru_parser import  PdfParserWithMinerU
-from gomate.modules.document.utils import PROJECT_BASE
+from trustrag.modules.document.pdf_mineru_parser import  PdfParserWithMinerU
+from trustrag.modules.document.utils import PROJECT_BASE
 from tqdm import tqdm
 if __name__ == '__main__':
     pdf_parser=PdfParserWithMinerU(url='http://localhost:8888/pdf_parse')

diff --git a/examples/parsers/textparser_exmaple.py b/examples/parsers/textparser_exmaple.py
@@ -9,7 +9,7 @@
 @software: PyCharm
 @description: coding..
 """
-from gomate.modules.document.text_parser import TextParser
+from trustrag.modules.document.text_parser import TextParser
 
 
 

diff --git a/examples/pipelines/insurance_qa_example.py b/examples/pipelines/insurance_qa_example.py
@@ -12,9 +12,9 @@
 import json
 import pandas as pd
 from transformers import AutoTokenizer
-from gomate.modules.retrieval.embedding import SBertEmbeddingModel
-from gomate.modules.retrieval.faiss_retriever import FaissRetriever
-from gomate.modules.retrieval.faiss_retriever import FaissRetrieverConfig
+from trustrag.modules.retrieval.embedding import SBertEmbeddingModel
+from trustrag.modules.retrieval.faiss_retriever import FaissRetriever
+from trustrag.modules.retrieval.faiss_retriever import FaissRetrieverConfig
 
 ## step1 build faiss index
 embedding_model_path = "/home/test/pretrained_models/bge-large-zh-v1.5"

diff --git a/examples/rag/app_weibo.py b/examples/rag/app_weibo.py
@@ -12,10 +12,10 @@
 
 import gradio as gr
 
-from gomate.applications.rag_weibo import WeiboRagApplication, ApplicationConfig
-from gomate.modules.document.utils import PROJECT_BASE
-from gomate.modules.reranker.bge_reranker import BgeRerankerConfig
-from gomate.modules.retrieval.dense_retriever import DenseRetrieverConfig
+from trustrag.applications.rag_weibo import WeiboRagApplication, ApplicationConfig
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.reranker.bge_reranker import BgeRerankerConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig
 
 # 修改成自己的配置！！！
 app_config = ApplicationConfig()

diff --git a/examples/rag/xunfei_rag.py b/examples/rag/xunfei_rag.py
@@ -3,14 +3,14 @@
 import pandas as pd
 from tqdm import tqdm
 
-from gomate.modules.document.chunk import TextChunker
-from gomate.modules.document.txt_parser import TextParser
-from gomate.modules.document.utils import PROJECT_BASE
-from gomate.modules.generator.llm import GLM4Chat
-from gomate.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
-from gomate.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
-from gomate.modules.retrieval.dense_retriever import DenseRetrieverConfig
-from gomate.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.document.txt_parser import TextParser
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.generator.llm import GLM4Chat
+from trustrag.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
+from trustrag.modules.retrieval.bm25s_retriever import BM25RetrieverConfig
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig
+from trustrag.modules.retrieval.hybrid_retriever import HybridRetriever, HybridRetrieverConfig
 
 
 def generate_chunks():

diff --git a/examples/rag/xunfei_rag_bm25.py b/examples/rag/xunfei_rag_bm25.py
@@ -3,12 +3,12 @@
 import pandas as pd
 from tqdm import tqdm
 
-from gomate.modules.document.chunk import TextChunker
-from gomate.modules.document.txt_parser import TextParser
-from gomate.modules.document.utils import PROJECT_BASE
-from gomate.modules.generator.llm import GLM4Chat
-from gomate.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
-from gomate.modules.retrieval.bm25s_retriever import BM25RetrieverConfig, BM25Retriever
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.document.txt_parser import TextParser
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.generator.llm import GLM4Chat
+from trustrag.modules.reranker.bge_reranker import BgeRerankerConfig, BgeReranker
+from trustrag.modules.retrieval.bm25s_retriever import BM25RetrieverConfig, BM25Retriever
 
 
 def generate_chunks():

diff --git a/examples/rag/xunfei_rag_dense.py b/examples/rag/xunfei_rag_dense.py
@@ -3,11 +3,11 @@
 import pandas as pd
 from tqdm import tqdm
 
-from gomate.modules.document.chunk import TextChunker
-from gomate.modules.document.txt_parser import TextParser
-from gomate.modules.document.utils import PROJECT_BASE
-from gomate.modules.generator.llm import QwenChat
-from gomate.modules.retrieval.dense_retriever import DenseRetrieverConfig,DenseRetriever
+from trustrag.modules.document.chunk import TextChunker
+from trustrag.modules.document.txt_parser import TextParser
+from trustrag.modules.document.utils import PROJECT_BASE
+from trustrag.modules.generator.llm import QwenChat
+from trustrag.modules.retrieval.dense_retriever import DenseRetrieverConfig,DenseRetriever
 
 
 def generate_chunks():