From b4ece2ed8290aae44c4b07c97fed92bd7000af62 Mon Sep 17 00:00:00 2001 From: Kamyab Nazari Date: Mon, 3 Jul 2023 07:44:36 +0200 Subject: [PATCH 1/5] Adding deployed functions --- backend/chat_bot_function.py | 2 +- backend/ingest.py | 13 ++++++++----- backend/main.py | 7 +++++++ 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/backend/chat_bot_function.py b/backend/chat_bot_function.py index a721536..6299f5e 100644 --- a/backend/chat_bot_function.py +++ b/backend/chat_bot_function.py @@ -18,7 +18,7 @@ def make_chain(documentId:str): embeddings = OpenAIEmbeddings() - client = QdrantClient(os.getenv('PUBLIC_QDRANT_URL')) + client = QdrantClient(url=os.getenv('PUBLIC_QDRANT_URL'), prefer_grpc=True, api_key=os.getenv('QDRANT__SERVICE_API_KEY')) qdrant = Qdrant(client, documentId, embeddings) return ConversationalRetrievalChain.from_llm( diff --git a/backend/ingest.py b/backend/ingest.py index 4c84306..d6a7136 100644 --- a/backend/ingest.py +++ b/backend/ingest.py @@ -127,12 +127,15 @@ def create_embeddings_from_pdf_file(file_path: str, documentId: str): # Step 3 + 4: Generate embeddings and store them in DB embeddings = OpenAIEmbeddings() - client = QdrantClient(url=os.getenv('PUBLIC_QDRANT_URL'), api_key=os.getenv('QDRANT_API_KEY'), https=True) - qdrant = Qdrant(client, documentId, embeddings) - - qdrant.from_documents( + url = os.getenv('PUBLIC_QDRANT_URL') + api_key = os.getenv('QDRANT__SERVICE_API_KEY') + + qdrant = Qdrant.from_documents( document_chunks, - embeddings, + embedding=embeddings, + url=url, + prefer_grpc=True, + api_key=api_key, collection_name=documentId, timeout=120 ) diff --git a/backend/main.py b/backend/main.py index d2b30ca..3646f3d 100644 --- a/backend/main.py +++ b/backend/main.py @@ -2,6 +2,7 @@ import pdfkit import io import os +from qdrant_client import QdrantClient import requests # Python package for PDF parsing @@ -80,6 +81,12 @@ async def read_favicon(): async def read_api_root(): return {"message": "Welcome to the EE API!"} +@app.get("/api/documents/{document_id}/delete_vector_file") +async def delete_api_vector_file(document_id: str): + client = QdrantClient(url=os.getenv('PUBLIC_QDRANT_URL'), prefer_grpc=True, api_key=os.getenv('QDRANT__SERVICE_API_KEY')) + client.delete_collection(document_id) + return {"message": "Vector file deleted!"} + @app.post("/api/documents/{document_id}/send_new_message/{user_id}") async def read_api_documents_send_new_message(document_id: str, user_id: str, request: Request): # Accessing Request body and converting it to a dictionary From 23369d07d4c5a2d999352fcfe4167c235a6e7c4f Mon Sep 17 00:00:00 2001 From: Kamyab Nazari Date: Mon, 3 Jul 2023 07:53:48 +0200 Subject: [PATCH 2/5] delete collection method frontend --- backend/main.py | 2 +- frontend/src/lib/components/FileTable.svelte | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/main.py b/backend/main.py index 3646f3d..26a2a90 100644 --- a/backend/main.py +++ b/backend/main.py @@ -81,7 +81,7 @@ async def read_favicon(): async def read_api_root(): return {"message": "Welcome to the EE API!"} -@app.get("/api/documents/{document_id}/delete_vector_file") +@app.post("/api/documents/{document_id}/delete_vector_file") async def delete_api_vector_file(document_id: str): client = QdrantClient(url=os.getenv('PUBLIC_QDRANT_URL'), prefer_grpc=True, api_key=os.getenv('QDRANT__SERVICE_API_KEY')) client.delete_collection(document_id) diff --git a/frontend/src/lib/components/FileTable.svelte b/frontend/src/lib/components/FileTable.svelte index 2cab2d5..82695f7 100644 --- a/frontend/src/lib/components/FileTable.svelte +++ b/frontend/src/lib/components/FileTable.svelte @@ -34,8 +34,8 @@ try { await pb.collection('documents').delete(documentID); await axios({ - url: `${env.PUBLIC_QDRANT_URL}/collections/${documentID}`, - method: 'delete', + url: `${env.PUBLIC_BACKEND_URL}/api/documents/${documentID}/delete_vector_file`, + method: 'post', headers: { 'Content-Type': 'application/json' } }); documentList = documentList.filter((document) => document.id !== documentID); From dbd087ec39f996bf17b1380d0a449464b47a8f5d Mon Sep 17 00:00:00 2001 From: Kamyab Nazari Date: Mon, 3 Jul 2023 09:24:37 +0200 Subject: [PATCH 3/5] Adding catch to pdfviewer import --- frontend/src/lib/components/PDFViewer.svelte | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/frontend/src/lib/components/PDFViewer.svelte b/frontend/src/lib/components/PDFViewer.svelte index 4ea72e6..da35367 100644 --- a/frontend/src/lib/components/PDFViewer.svelte +++ b/frontend/src/lib/components/PDFViewer.svelte @@ -21,11 +21,15 @@ let isLoading = true; - let pdfjsWorkerPromise = import('pdfjs-dist/build/pdf.worker.entry').then((worker) => { - let pdfjsWorker = URL.createObjectURL(new Blob([worker], { type: 'application/javascript' })); - GlobalWorkerOptions.workerSrc = pdfjsWorker; - return pdfjsWorker; - }); + let pdfjsWorkerPromise = import('pdfjs-dist/build/pdf.worker.entry') + .then((worker) => { + let pdfjsWorker = URL.createObjectURL(new Blob([worker], { type: 'application/javascript' })); + GlobalWorkerOptions.workerSrc = pdfjsWorker; + return pdfjsWorker; + }) + .catch((error) => { + console.error("Error importing 'pdfjs-dist/build/pdf.worker.entry':", error); + }); afterUpdate(() => { previousDocumentUrl = generatedDocumentURL; From b5cf75b5b3c45a91fa2b93087808d2a57a9c0b85 Mon Sep 17 00:00:00 2001 From: Kamyab Nazari Date: Mon, 3 Jul 2023 11:09:56 +0200 Subject: [PATCH 4/5] Fixing client creation to work on both loc and dp --- backend/README.md | 2 +- backend/chat_bot_function.py | 9 ++++++++- backend/ingest.py | 31 +++++++++++++++++++++---------- backend/main.py | 10 +++++++++- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/backend/README.md b/backend/README.md index e1bb6af..7083118 100644 --- a/backend/README.md +++ b/backend/README.md @@ -8,12 +8,12 @@ Create a `.env` file in the root of the backend directory with the following var ``` OPENAI_API_KEY={your openai api key} -QDRANT_API_KEY={your qdrant api key} POCKETBASE_ADMIN_EMAIL={your pocketbase admin email} POCKETBASE_ADMIN_PASSWORD={your pocketbase admin password} PUBLIC_POCKETBASE_URL=http://localhost:8090 PUBLIC_FRONTEND_URL=http://localhost:5173 PUBLIC_QDRANT_URL=http://localhost:6333 +QDRANT__SERVICE_API_KEY={Only for Deployed Qdrant} ``` ### Running the backend diff --git a/backend/chat_bot_function.py b/backend/chat_bot_function.py index 6299f5e..3b0a900 100644 --- a/backend/chat_bot_function.py +++ b/backend/chat_bot_function.py @@ -18,7 +18,14 @@ def make_chain(documentId:str): embeddings = OpenAIEmbeddings() - client = QdrantClient(url=os.getenv('PUBLIC_QDRANT_URL'), prefer_grpc=True, api_key=os.getenv('QDRANT__SERVICE_API_KEY')) + url = os.getenv('PUBLIC_QDRANT_URL') + api_key = os.getenv('QDRANT__SERVICE_API_KEY') + + if api_key: + client = QdrantClient(url=url, prefer_grpc=True, api_key=api_key) + else: + client = QdrantClient(url=url) + qdrant = Qdrant(client, documentId, embeddings) return ConversationalRetrievalChain.from_llm( diff --git a/backend/ingest.py b/backend/ingest.py index 9b6f770..759f9d8 100644 --- a/backend/ingest.py +++ b/backend/ingest.py @@ -124,16 +124,27 @@ def create_embeddings_from_pdf_file(file_path: str, documentId: str): # Step 3 + 4: Generate embeddings and store them in DB embeddings = OpenAIEmbeddings() - + url = os.getenv('PUBLIC_QDRANT_URL') api_key = os.getenv('QDRANT__SERVICE_API_KEY') - qdrant = Qdrant.from_documents( - document_chunks, - embedding=embeddings, - url=url, - prefer_grpc=True, - api_key=api_key, - collection_name=documentId, - timeout=120 - ) + if api_key: + Qdrant.from_documents( + document_chunks, + embedding=embeddings, + url=url, + prefer_grpc=True, + api_key=api_key, + collection_name=documentId, + timeout=120 + ) + else: + Qdrant.from_documents( + document_chunks, + embedding=embeddings, + url=url, + prefer_grpc=False, + collection_name=documentId, + timeout=120 + ) + diff --git a/backend/main.py b/backend/main.py index 26a2a90..71ee8a3 100644 --- a/backend/main.py +++ b/backend/main.py @@ -83,7 +83,15 @@ async def read_api_root(): @app.post("/api/documents/{document_id}/delete_vector_file") async def delete_api_vector_file(document_id: str): - client = QdrantClient(url=os.getenv('PUBLIC_QDRANT_URL'), prefer_grpc=True, api_key=os.getenv('QDRANT__SERVICE_API_KEY')) + + url = os.getenv('PUBLIC_QDRANT_URL') + api_key = os.getenv('QDRANT__SERVICE_API_KEY') + + if api_key: + client = QdrantClient(url=url, prefer_grpc=True, api_key=api_key) + else: + client = QdrantClient(url=url) + client.delete_collection(document_id) return {"message": "Vector file deleted!"} From 8bd4d65d582042193adcc2ac54cae7d51f8661ee Mon Sep 17 00:00:00 2001 From: Kamyab Nazari Date: Mon, 3 Jul 2023 11:13:20 +0200 Subject: [PATCH 5/5] Fixing remove account endpoint --- frontend/src/routes/(auth)/profile/+page.server.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/routes/(auth)/profile/+page.server.ts b/frontend/src/routes/(auth)/profile/+page.server.ts index 937d3b2..9508722 100644 --- a/frontend/src/routes/(auth)/profile/+page.server.ts +++ b/frontend/src/routes/(auth)/profile/+page.server.ts @@ -39,8 +39,8 @@ export const actions: Actions = { documentList.forEach(async (document: Record) => { // delete all user embeddings from Qdrant await axios({ - url: `${env.PUBLIC_QDRANT_URL}/collections/${document.id}`, - method: 'delete', + url: `${env.PUBLIC_BACKEND_URL}/api/documents/${document.id}/delete_vector_file`, + method: 'post', headers: { 'Content-Type': 'application/json' }, httpAgent: new http.Agent({ family: 4 }), // Force IPv4 httpsAgent: new https.Agent({ family: 4 }) // Force IPv4