fix: simplify tool instructions to ~300 tokens (was 40k)

Reduce tool instructions from 40k tokens to ~300 tokens:
- List only 3 main tools (read, write, bash) with brief descriptions
- Single concise JSON format example
- Remove verbose formatting and multiple examples
- Only add instructions on first request (no assistant response yet)

This makes tool usage feasible for 8K-32K context models,
especially important for home setups with limited VRAM.
This commit is contained in:
2026-02-24 14:26:16 +01:00
parent 12eaac0d27
commit bad8732b7b
+28 -2
View File
@@ -59,8 +59,34 @@ def format_messages_with_tools(messages: list, tools: Optional[list] = None) ->
"""
formatted = []
# Tools are accepted but model responds normally
# Server-side tool execution parses the response and executes tools
# Check if there are already tool results in the conversation
has_tool_results = any(msg.role == "tool" for msg in messages)
has_assistant_response = any(msg.role == "assistant" for msg in messages)
# Add brief tool instructions if tools are present and no assistant has responded yet
if tools and not has_tool_results and not has_assistant_response:
tool_instructions = """You have access to these tools:
read: Read a file (filePath)
write: Write to a file (filePath, content)
bash: Run a shell command (command)
When you need to use a tool, respond with ONLY this JSON format:
{"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "TOOL_NAME", "arguments": "{\\"param\\": \\"value\\"}"}}]}
Do not explain - just output the JSON when using tools."""
# Add to system message or create one
has_system = False
for msg in messages:
if msg.role == "system":
msg.content = tool_instructions + "\n\n" + (msg.content or "")
has_system = True
break
if not has_system:
from api.models import ChatMessage
messages.insert(0, ChatMessage(role="system", content=tool_instructions))
for msg in messages:
role = msg.role