cleaned examples and added vllm example

B0-B · Oct 9, 2024 · 343280e · 343280e
1 parent b8543ab
commit 343280e
Show file tree

Hide file tree

Showing 12 changed files with 9 additions and 209 deletions.
diff --git a/examples/assistant.py b/examples/assistant.py
diff --git a/examples/bidenObamaTrump.py b/examples/bidenObamaTrump.py
diff --git a/examples/chatGPT.py b/examples/chatGPT.py
diff --git a/examples/gpt-llama-3-8B-cpu.py b/examples/gpt-llama-3-8B-cpu.py
diff --git a/examples/gpt-llama-3-8B-gpu.py b/examples/gpt-llama-3-8B-gpu.py
diff --git a/examples/llama3_GPT_CPU_console.py → examples/llama3_8B_gpt_cpu_console.py b/examples/llama3_GPT_CPU_console.py → examples/llama3_8B_gpt_cpu_console.py
diff --git a/examples/llama3_GPT_GPU_console.py → examples/llama3_8B_gpt_gpu_console.py b/examples/llama3_GPT_GPU_console.py → examples/llama3_8B_gpt_gpu_console.py
diff --git a/examples/vllm_gpt.py → examples/llama3_8B_gpt_vllm_console.py b/examples/vllm_gpt.py → examples/llama3_8B_gpt_vllm_console.py
@@ -1,7 +1,12 @@
+#!/usr/bin/env python3
+
+'''
+A GPT-like example of LLama-3-8B chat accelerated with vLLM's flash-attn. backend.
+'''
 from blowtorch import client, console
 
 chat_bot = client(hugging_face_path='MaziyarPanahi/Meta-Llama-3-8B-Instruct-GPTQ',
-                  attention=True,
+                  attention=True, # setting attention to true
                   name='llama-bot',
                   chat_format='llama-3',
                   device='gpu')

diff --git a/examples/yoda.py → ...ples/llama3_8B_yoda_scenario_cpu_webui.py b/examples/yoda.py → ...ples/llama3_8B_yoda_scenario_cpu_webui.py
@@ -12,8 +12,8 @@
 
 from blowtorch import client, console, webUI
 
-cl = client('llama-2-7b-chat.Q2_K.gguf', 
-            'TheBloke/Llama-2-7B-Chat-GGUF', 
+cl = client('Meta-Llama-3-8B-Instruct.Q2_K.gguf', 
+            'MaziyarPanahi/Meta-Llama-3-8B-Instruct-GGUF', 
             name='Yoda',
             device='cpu', 
             model_type="llama",
@@ -29,5 +29,4 @@
     top_k=60,
 )
 
-webUI(cl)
-# console(cl)
+webUI(cl)
diff --git a/examples/mistral.py → examples/mistral_7B_gpt_cpu_webui.py b/examples/mistral.py → examples/mistral_7B_gpt_cpu_webui.py
diff --git a/examples/scenario.py b/examples/scenario.py
diff --git a/examples/vicuna7B.py b/examples/vicuna7B.py