Chatbot timeout increasing (#2428)

CVS-153834
openvinotoolkit · Oct 1, 2024 · c4cf789 · c4cf789
1 parent ab60d5a
commit c4cf789
Show file tree

Hide file tree

Showing 4 changed files with 4 additions and 4 deletions.
diff --git a/notebooks/llm-chatbot/llm-chatbot.ipynb b/notebooks/llm-chatbot/llm-chatbot.ipynb
@@ -1182,7 +1182,7 @@
     "    if input_ids.shape[1] > 2000:\n",
     "        history = [history[-1]]\n",
     "        input_ids = convert_history_to_token(history)\n",
-    "    streamer = TextIteratorStreamer(tok, timeout=30.0, skip_prompt=True, skip_special_tokens=True)\n",
+    "    streamer = TextIteratorStreamer(tok, timeout=3600.0, skip_prompt=True, skip_special_tokens=True)\n",
     "    generate_kwargs = dict(\n",
     "        input_ids=input_ids,\n",
     "        max_new_tokens=max_new_tokens,\n",

diff --git a/notebooks/llm-rag-langchain/llm-rag-langchain.ipynb b/notebooks/llm-rag-langchain/llm-rag-langchain.ipynb
@@ -1653,7 +1653,7 @@
     "    \"\"\"\n",
     "    streamer = TextIteratorStreamer(\n",
     "        llm.pipeline.tokenizer,\n",
-    "        timeout=60.0,\n",
+    "        timeout=3600.0,\n",
     "        skip_prompt=hide_full_prompt,\n",
     "        skip_special_tokens=True,\n",
     "    )\n",

diff --git a/notebooks/qwen2-vl/gradio_helper.py b/notebooks/qwen2-vl/gradio_helper.py
@@ -78,7 +78,7 @@ def call_local_model(model, processor, messages):
         inputs = processor(text=[text], images=image_inputs, videos=video_inputs, padding=True, return_tensors="pt").to(model.device)
 
         tokenizer = processor.tokenizer
-        streamer = TextIteratorStreamer(tokenizer, timeout=20.0, skip_prompt=True, skip_special_tokens=True)
+        streamer = TextIteratorStreamer(tokenizer, timeout=3600.0, skip_prompt=True, skip_special_tokens=True)
 
         gen_kwargs = {"max_new_tokens": 512, "streamer": streamer, **inputs}
 

diff --git a/supplementary_materials/qwen2/chat.py b/supplementary_materials/qwen2/chat.py
@@ -42,7 +42,7 @@ def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor, **kwa
         trust_remote_code=True,
     )
 
-    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
+    streamer = TextIteratorStreamer(tokenizer, timeout=3600.0, skip_prompt=True, skip_special_tokens=True)
     stop_tokens = [151643, 151645]
     stop_tokens = [StopOnTokens(stop_tokens)]