From a41a629a568900038f8c84c542d098224a664b80 Mon Sep 17 00:00:00 2001 From: SeeknnDestroy Date: Mon, 16 Oct 2023 16:41:18 +0300 Subject: [PATCH] update readme, add todo to document_reading module --- README.md | 11 ++++++++--- .../{markdown_processing.py => document_reading.py} | 3 ++- 2 files changed, 10 insertions(+), 4 deletions(-) rename autollm/utils/{markdown_processing.py => document_reading.py} (95%) diff --git a/README.md b/README.md index 6493223e..cf957f61 100644 --- a/README.md +++ b/README.md @@ -101,8 +101,8 @@ Create robust query engine pipelines with automatic cost logging. Supports fine- ```python from autollm import AutoQueryEngine -# Initialize a query engine with existing vector store and service context -vector_store = AutoVectorStore.from_defaults( +# Initialize a query engine with existing vector store index and service context +vector_store_index = AutoVectorStoreIndex.from_defaults( vector_store_type="VectorStoreIndex", documents=documents ) service_context = AutoServiceContext.from_defaults(enable_cost_calculator=True) @@ -136,7 +136,10 @@ query_engine = AutoQueryEngine.from_parameters( query_wrapper_prompt="Your Query Wrapper Prompt", enable_cost_calculator=True, llm_params={"model": "gpt-3.5-turbo"}, - vector_store_params={"vector_store_type": "qdrant", "index_name": "quickstart"}, + vector_store_params={"vector_store_type": "QdrantVectorStore", "client": qdrant_client.QdrantClient( + url="http://:" + api_key="", +), "collection_name": "quickstart"}, service_context_params={"chunk_size": 1024}, query_engine_params={"similarity_top_k": 10}, ) @@ -144,7 +147,9 @@ query_engine = AutoQueryEngine.from_parameters( response = query_engine.query("Why is SafeVideo AI awesome?") print(response.response) +``` +``` >> Because they redefine the movie experience by AI! ``` diff --git a/autollm/utils/markdown_processing.py b/autollm/utils/document_reading.py similarity index 95% rename from autollm/utils/markdown_processing.py rename to autollm/utils/document_reading.py index 70e3f9dd..6dfca92f 100644 --- a/autollm/utils/markdown_processing.py +++ b/autollm/utils/document_reading.py @@ -9,7 +9,8 @@ logger = logging.getLogger(__name__) -def process_and_get_documents( +# TODO: add all files supports beside md, use SimpleDirReader +def read_files_as_documents( path_or_files: Union[Path, List[Path]], read_as_single_doc: bool = False, extra_info: Optional[Dict] = None) -> List[Document]: