From 87f3e4375540a8f33364ac3fc7a432c7219ffc1f Mon Sep 17 00:00:00 2001
From: yym68686 <yym68686@outlook.com>
Date: Wed, 20 Dec 2023 16:15:36 +0800
Subject: [PATCH] Add support for gpt-4-vision-preview stream mode

---
 bot.py               |  5 +++--
 utils/agent.py       | 13 +++++++++++++
 utils/chatgpt2api.py |  2 +-
 3 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/bot.py b/bot.py
index b6bdd295..d9577399 100644
--- a/bot.py
+++ b/bot.py
@@ -8,7 +8,7 @@
 from utils.chatgpt2api import Chatbot as GPT
 from utils.chatgpt2api import claudebot
 from telegram.constants import ChatAction
-from utils.agent import docQA, get_doc_from_local, Document_extract, pdfQA
+from utils.agent import docQA, get_doc_from_local, Document_extract, pdfQA, get_encode_image
 from telegram import BotCommand, InlineKeyboardButton, InlineKeyboardMarkup, InlineQueryResultArticle, InputTextMessageContent
 from telegram.ext import CommandHandler, MessageHandler, ApplicationBuilder, filters, CallbackQueryHandler, Application, AIORateLimiter, InlineQueryHandler
 from config import WEB_HOOK, PORT, BOT_TOKEN
@@ -79,11 +79,12 @@ async def command_bot(update, context, language=None, prompt=translator_prompt,
                 title = "`🤖️ gpt-4-vision-preview`\n\n"
             message = [{"type": "text", "text": message}]
             if (image_url and config.GPT_ENGINE == "gpt-4-vision-preview") or (image_url and robot == config.GPT4visionbot):
+                base64_image = get_encode_image(image_url)
                 message.append(
                     {
                         "type": "image_url",
                         "image_url": {
-                            "url": image_url
+                            "url": base64_image
                         }
                     }
                 )
diff --git a/utils/agent.py b/utils/agent.py
index ca980b3d..d7b8976b 100644
--- a/utils/agent.py
+++ b/utils/agent.py
@@ -1,6 +1,7 @@
 import os
 import re
 import json
+import base64
 
 import sys
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -540,6 +541,18 @@ def get_version_info():
     output = result.stdout.decode()
     return output
 
+def encode_image(image_path):
+  with open(image_path, "rb") as image_file:
+    return base64.b64encode(image_file.read()).decode('utf-8')
+
+def get_encode_image(image_url):
+    filename = get_doc_from_url(image_url)
+    image_path = os.getcwd() + "/" + filename
+    base64_image = encode_image(image_path)
+    prompt = f"data:image/jpeg;base64,{base64_image}"
+    os.remove(image_path)
+    return prompt
+
 if __name__ == "__main__":
     os.system("clear")
     print(get_date_time_weekday())
diff --git a/utils/chatgpt2api.py b/utils/chatgpt2api.py
index de34e555..9bed94db 100644
--- a/utils/chatgpt2api.py
+++ b/utils/chatgpt2api.py
@@ -501,9 +501,9 @@ def get_post_body(
             "model": os.environ.get("MODEL_NAME") or model or self.engine,
             "messages": self.conversation[convo_id] if pass_history else [{"role": "system","content": self.system_prompt},{"role": role, "content": prompt}],
             "max_tokens": 5000,
+            "stream": True,
         }
         body = {
-            "stream": True,
             # kwargs
             "temperature": kwargs.get("temperature", self.temperature),
             "top_p": kwargs.get("top_p", self.top_p),