Merge pull request #41 from naaive/main

Generate benchmark report

.github/workflows/run_test_and_gen_report.yml

@@ -6,6 +6,10 @@ on:
       - 'v*.*.*'
   workflow_dispatch:
 
+# Add permissions configuration
+permissions:
+  contents: write
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -76,10 +80,31 @@ jobs:
         cd tests
         poetry run python run_test.py
 
-    - name: Commit test results
-      uses: stefanzweifel/git-auto-commit-action@v4
-      with:
-        commit_message: Add test results
+    - name: Commit and push report
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        # Switch to main branch
+        git fetch origin main
+        git checkout main
+        
+        git config --global user.name 'github-actions[bot]'
+        git config --global user.email 'github-actions[bot]@users.noreply.github.com'
+        
+        # Check if file exists
+        ls -la tests/compatible-models.mdx || echo "Report file not found!"
+        
+        # Add all changes (including new files)
+        git add -A
+        
+        # Show pending changes
+        git status
+        
+        # Create commit with timestamp
+        git commit -m "docs: update compatibility test report" || echo "No changes to commit"
+        
+        # Push changes to main branch
+        git push origin main
 
 
 

openagent/conf/llm_provider.py

@@ -11,7 +11,7 @@
 
 from openagent.conf.env import settings
 
-SUPPORTED_MODELS = {
+SUPPORTED_OLLAMA_MODELS = {
     "llama3.2": {"name": "llama3.2", "supports_tools": True},
     "mistral-nemo": {"name": "mistral-nemo", "supports_tools": True},
     "darkmoon/olmo:7B-instruct-q6-k": {"name": "olmo", "supports_tools": False},
@@ -35,12 +35,12 @@ def get_available_ollama_providers() -> List[str]:
         for model in ollama_list["models"]:
             full_name = model["name"]
             # check if the full model name is in SUPPORTED_MODELS
-            if full_name in SUPPORTED_MODELS:
+            if full_name in SUPPORTED_OLLAMA_MODELS:
                 available_models.append(full_name)
             else:
                 # try to check the base name (without version tag)
                 base_name = full_name.split(":")[0]
-                if base_name in SUPPORTED_MODELS:
+                if base_name in SUPPORTED_OLLAMA_MODELS:
                     available_models.append(base_name)
         return available_models
     except Exception as e:

openagent/ui/app.py

@@ -8,10 +8,11 @@
 from langchain.schema.runnable.config import RunnableConfig
 from langchain_core.language_models import BaseChatModel
 from langchain_core.messages import HumanMessage
+from langchain_ollama import ChatOllama
 from loguru import logger
 
 from openagent.conf.env import settings
-from openagent.conf.llm_provider import SUPPORTED_MODELS, get_available_providers
+from openagent.conf.llm_provider import SUPPORTED_OLLAMA_MODELS, get_available_providers
 from openagent.ui.profile import profile_name_to_provider_key, provider_to_profile
 from openagent.workflows.member import members
 from openagent.workflows.workflow import build_workflow
@@ -108,9 +109,9 @@ async def on_message(message: cl.Message):  # noqa
     msg = cl.Message(content="")
     agent_names = [member["name"] for member in members]
 
-    if hasattr(llm, "model"):
+    if hasattr(llm, "model") and isinstance(llm,ChatOllama):
         model_name = llm.model
-        supports_tools = SUPPORTED_MODELS.get(model_name, {}).get("supports_tools", False)
+        supports_tools = SUPPORTED_OLLAMA_MODELS.get(model_name, {}).get("supports_tools", False)
     else:
         supports_tools = True
 

openagent/ui/profile.py

@@ -1,6 +1,6 @@
 import chainlit as cl
 
-from openagent.conf.llm_provider import MODELS_ICONS, SUPPORTED_MODELS
+from openagent.conf.llm_provider import MODELS_ICONS, SUPPORTED_OLLAMA_MODELS
 
 provider_key_to_profile_info = {
     "gpt-4o": {
@@ -25,7 +25,7 @@
     },
 }
 
-for model_key, model_info in SUPPORTED_MODELS.items():
+for model_key, model_info in SUPPORTED_OLLAMA_MODELS.items():
     icon = MODELS_ICONS.get(model_info["name"], "/public/ollama.png")  # type: ignore
     provider_key_to_profile_info[model_key] = {
         "name": model_info["name"],  # type: ignore

openagent/workflows/workflow.py

@@ -3,6 +3,7 @@
 
 from langchain_core.language_models import BaseChatModel
 from langchain_core.messages import BaseMessage, HumanMessage
+from langchain_ollama import ChatOllama
 from langgraph.graph import END, StateGraph
 from loguru import logger
 
@@ -11,7 +12,7 @@
 from openagent.agents.fallback import build_fallback_agent
 from openagent.agents.feed_explore import build_feed_explorer_agent
 from openagent.agents.research_analyst import build_research_analyst_agent
-from openagent.conf.llm_provider import SUPPORTED_MODELS
+from openagent.conf.llm_provider import SUPPORTED_OLLAMA_MODELS
 
 
 class AgentState(TypedDict):
@@ -29,14 +30,15 @@ async def run(state):
 
 
 def build_workflow(llm: BaseChatModel):
-    if hasattr(llm, "model"):
+    is_ollama = isinstance(llm, ChatOllama)
+    if hasattr(llm, "model") and is_ollama:
         model_name = llm.model
     else:
         return build_tool_workflow(llm)
 
-    supports_tools = SUPPORTED_MODELS.get(model_name, {}).get("supports_tools", False)
+    supports_tools = SUPPORTED_OLLAMA_MODELS.get(model_name, {}).get("supports_tools", False)
 
-    if not supports_tools:
+    if not supports_tools and is_ollama:
         return build_simple_workflow(llm)
     else:
         return build_tool_workflow(llm)

pyproject.toml

@@ -29,7 +29,6 @@ sqlalchemy-utils = "^0.41.2"
 retrying = "^1.3.4"
 langchain-postgres = "0.0.9"
 psycopg2-binary = "^2.9.9"
-langchain-google-genai = "^1.0.4"
 psycopg-binary = "^3.1.19"
 langchain-core = ">=0.2.9,<0.3"
 langgraph="0.1.1"
@@ -50,6 +49,8 @@ moralis = "^0.1.49"
 pytest-asyncio = "^0.23.8"
 feedparser = "^6.0.11"
 jinja2 = "^3.1.4"
+langchain-google-genai = "<2.0.4"
+pytest = "^8.3.3"
 
 [tool.poetry.group.dev.dependencies]
 ruff = "^0.4.1"

tests/compatible-models.mdx

@@ -30,13 +30,17 @@ Learn more:
 
 <LinkCard id="ollama" />
 
-| Name | Score (out of 100) |
-|------|-------------------|
-
+| Name | Score (out of 100) | Function Call Support |
+|------|-------------------|---------------------|
+| mistral-nemo | 88 | ✅ |
+| llama3.2 | 80 | ✅ |
+| darkmoon/olmo:7B-instruct-q6-k | 0 | ❌ |
 
 ### Proprietary Models
 
-| Name | Score (out of 100) |
-|------|-------------------|
-| gpt-4o-mini | 100 |
-| gpt-4o | 100 | 
+| Name | Score (out of 100) | Function Call Support |
+|------|-------------------|---------------------|
+| gpt-4o-mini | 100 | ✅ |
+| gemini-1.5-flash | 100 | ✅ |
+| gpt-4o | 96 | ✅ |
+| gemini-1.5-pro | 96 | ✅ |