From f0812b5c7d6ec2b83512bfda176d9ddee59ed26a Mon Sep 17 00:00:00 2001 From: Nils Durner Date: Wed, 23 Oct 2024 00:36:30 +0200 Subject: [PATCH] fix pixtral image prompt order for doc VQA (#99) * fix pixtral image prompt order for doc VQA * fix code style --- mlx_vlm/prompt_utils.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mlx_vlm/prompt_utils.py b/mlx_vlm/prompt_utils.py index 3885c4f..7be150b 100644 --- a/mlx_vlm/prompt_utils.py +++ b/mlx_vlm/prompt_utils.py @@ -24,7 +24,12 @@ def add_image_tokens(message, token_format): return message if role == "user" and not skip_image_token: if isinstance(message["content"], list): - message["content"].extend([{"type": "image"}] * num_images) + if model_name == "pixtral": + message["content"] = [{"type": "image"}] * num_images + message[ + "content" + ] + else: + message["content"].extend([{"type": "image"}] * num_images) else: if model_name == "phi3_v": message["content"] = f"{token_format}{message['content']}"