From 3356f2d686c035ba42feda393678c4c5ae77a6f2 Mon Sep 17 00:00:00 2001
From: Thomas <wxy_000000@qq.com>
Date: Thu, 21 Nov 2024 08:05:22 +0800
Subject: [PATCH] chore: add more open source models

---
 tests/run_test.py | 27 ++++++++++++++++++++++-----
 1 file changed, 22 insertions(+), 5 deletions(-)

diff --git a/tests/run_test.py b/tests/run_test.py
index 670d9422..45d78f7d 100644
--- a/tests/run_test.py
+++ b/tests/run_test.py
@@ -19,7 +19,10 @@
     {"name": "llama3.1", "function_call_support": True},
     {"name": "llama3.2", "function_call_support": True},
     {"name": "mistral-nemo", "function_call_support": True},
-    # {"name": "darkmoon/olmo:7B-instruct-q6-k", "function_call_support": False}
+    {"name": "olmo", "function_call_support": False},
+    {"name": "gemma", "function_call_support": False},
+    {"name": "llava", "function_call_support": False},
+    {"name": "deepseek-coder-v2", "function_call_support": False}
 ]
 
 
@@ -68,7 +71,11 @@ def bool_to_emoji(value):
                 'score': proprietary_results[model['name']],
                 'function_call_support': bool_to_emoji(model['function_call_support'])
             })
-    proprietary_models.sort(key=lambda x: x['score'], reverse=True)
+    # Sort models, putting '-' scores at the end
+    proprietary_models.sort(
+        key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
+        reverse=True
+    )
 
     open_source_models = []
     for model in OPENSOURCE_MODELS:
@@ -78,7 +85,11 @@ def bool_to_emoji(value):
                 'score': opensource_results[model['name']],
                 'function_call_support': bool_to_emoji(model['function_call_support'])
             })
-    open_source_models.sort(key=lambda x: x['score'], reverse=True)
+    # Sort models, putting '-' scores at the end
+    open_source_models.sort(
+        key=lambda x: float('-inf') if x['score'] == '-' else x['score'], 
+        reverse=True
+    )
 
     # Set up template environment
     env = Environment(loader=FileSystemLoader('templates'))
@@ -110,11 +121,17 @@ def run_all_tests():
 
     # Test proprietary models
     for model in PROPRIETARY_MODELS:
-        proprietary_results[model['name']] = run_model_tests(model['name'])
+        if model['function_call_support']:
+            proprietary_results[model['name']] = run_model_tests(model['name'])
+        else:
+            proprietary_results[model['name']] = '-'
 
     # Test open source models
     for model in OPENSOURCE_MODELS:
-        opensource_results[model['name']] = run_model_tests(model['name'])
+        if model['function_call_support']:
+            opensource_results[model['name']] = run_model_tests(model['name'])
+        else:
+            opensource_results[model['name']] = '-'
 
     # Generate report
     generate_model_report(proprietary_results, opensource_results)