fix(planner): allow rationale/thinking by using tool_choice=auto
Tests / Test passed: 855

Forced tool_choice ("any"/"required") makes both Anthropic and OpenAI
skip any text/thinking block before the tool call, which silently made
rationale and thinking always None despite the planner-reflection-history
change's capture code being correct. Switch the primary call to
tool_choice="auto" (Anthropic: type=auto, disable_parallel_tool_use=true;
OpenAI: "auto") so the model can emit its reflection text, and add a
one-time forced retry (Anthropic "any", OpenAI "required", thinking
disabled) if the model responds without a tool call, guaranteeing a step
never stalls. Also add OpenAI text_output capture from message.content,
which was never extracted before (Anthropic-only gap).

Update planner-reflection-history design.md/tasks.md to document the bug
found during the pending manual smoke test (task 8.5) and the fix (new
section 9).
This commit is contained in:
2026-07-15 13:43:39 +08:00
parent 24992fc9fb
commit 367fd0d412
4 changed files with 250 additions and 16 deletions
+164 -9
View File
@@ -21,9 +21,14 @@ from tests.fakes import PNG_10X20
class FakeMessages:
def __init__(
self, *, response: object | None = None, error: Exception | None = None
self,
*,
response: object | None = None,
responses: list[object] | None = None,
error: Exception | None = None,
) -> None:
self.response = response
self._responses = list(responses) if responses is not None else None
self.error = error
self.calls: list[dict[str, Any]] = []
@@ -31,6 +36,8 @@ class FakeMessages:
self.calls.append(kwargs)
if self.error:
raise self.error
if self._responses is not None:
return self._responses.pop(0)
return self.response
@@ -41,9 +48,14 @@ class FakeTransport:
class FakeCompletions:
def __init__(
self, *, response: object | None = None, error: Exception | None = None
self,
*,
response: object | None = None,
responses: list[object] | None = None,
error: Exception | None = None,
) -> None:
self.response = response
self._responses = list(responses) if responses is not None else None
self.error = error
self.calls: list[dict[str, Any]] = []
@@ -51,6 +63,8 @@ class FakeCompletions:
self.calls.append(kwargs)
if self.error:
raise self.error
if self._responses is not None:
return self._responses.pop(0)
return self.response
@@ -67,7 +81,7 @@ class FakeOpenAITransport:
# --- Anthropic ---------------------------------------------------------
def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -> None:
def test_anthropic_tool_calling_client_sends_auto_tool_choice_request() -> None:
messages = FakeMessages(
response={
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
@@ -95,7 +109,7 @@ def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -
call = messages.calls[0]
assert call["model"] == "test-model"
assert call["timeout"] == 2.5
assert call["tool_choice"] == {"type": "any", "disable_parallel_tool_use": True}
assert call["tool_choice"] == {"type": "auto", "disable_parallel_tool_use": True}
assert call["tools"] == [
{
"name": "tap",
@@ -224,10 +238,72 @@ def test_anthropic_tool_calling_client_wraps_malformed_responses(
)
def test_anthropic_client_retries_with_forced_tool_choice_when_model_omits_tool_call() -> (
None
):
messages = FakeMessages(
responses=[
{"content": [{"type": "text", "text": "just thinking out loud"}]},
{
"content": [
{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}
]
},
]
)
client = AnthropicToolCallingClient(
model="test-model", transport=FakeTransport(messages)
)
decision = client.decide(
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
)
assert decision.tool_name == "tap"
assert len(messages.calls) == 2
assert messages.calls[0]["tool_choice"] == {
"type": "auto",
"disable_parallel_tool_use": True,
}
assert messages.calls[1]["tool_choice"] == {
"type": "any",
"disable_parallel_tool_use": True,
}
def test_anthropic_client_retry_drops_thinking_param_since_incompatible_with_forced_tool_choice() -> (
None
):
messages = FakeMessages(
responses=[
{"content": [{"type": "thinking", "thinking": "hmm, no tool yet"}]},
{
"content": [
{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}
]
},
]
)
client = AnthropicToolCallingClient(
model="test-model",
transport=FakeTransport(messages),
thinking_budget_tokens=1024,
)
client.decide(
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
)
assert "thinking" in messages.calls[0]
assert "betas" in messages.calls[0]
assert "thinking" not in messages.calls[1]
assert "betas" not in messages.calls[1]
# --- OpenAI --------------------------------------------------------------
def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> None:
def test_openai_tool_calling_client_sends_auto_tool_choice_request() -> None:
completions = FakeCompletions(
response={
"choices": [
@@ -270,7 +346,7 @@ def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> N
assert call["timeout"] == 2.5
assert call["max_completion_tokens"] == 1024
assert "max_tokens" not in call
assert call["tool_choice"] == "required"
assert call["tool_choice"] == "auto"
assert call["parallel_tool_calls"] is False
assert call["tools"] == [
{
@@ -419,6 +495,44 @@ def test_openai_tool_calling_client_wraps_malformed_responses(response: object)
)
def test_openai_client_retries_with_forced_tool_choice_when_model_omits_tool_call() -> (
None
):
completions = FakeCompletions(
responses=[
{"choices": [{"message": {"content": "just chatting, no tool"}}]},
{
"choices": [
{
"message": {
"tool_calls": [
{
"function": {
"name": "tap",
"arguments": '{"x": 1, "y": 2}',
}
}
]
}
}
]
},
]
)
client = OpenAIToolCallingClient(
model="test-model", transport=FakeOpenAITransport(completions)
)
decision = client.decide(
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
)
assert decision.tool_name == "tap"
assert len(completions.calls) == 2
assert completions.calls[0]["tool_choice"] == "auto"
assert completions.calls[1]["tool_choice"] == "required"
# --- build_client ----------------------------------------------------------
@@ -498,7 +612,9 @@ def test_anthropic_client_captures_both_thinking_and_text_output() -> None:
assert decision.text_output == "Step succeeded. Tapping next."
def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() -> None:
def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() -> (
None
):
messages = FakeMessages(
response={
"content": [
@@ -524,7 +640,12 @@ def test_openai_client_captures_reasoning_content() -> None:
"message": {
"reasoning_content": "I reasoned about this step.",
"tool_calls": [
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
{
"function": {
"name": "tap",
"arguments": '{"x": 1, "y": 2}',
}
}
],
}
}
@@ -541,6 +662,35 @@ def test_openai_client_captures_reasoning_content() -> None:
assert decision.text_output is None
def test_openai_client_captures_message_content_as_text_output() -> None:
completions = FakeCompletions(
response={
"choices": [
{
"message": {
"content": "Previous step succeeded. Now tapping login.",
"tool_calls": [
{
"function": {
"name": "tap",
"arguments": '{"x": 5, "y": 5}',
}
}
],
}
}
]
}
)
client = OpenAIToolCallingClient(
model="test-model", transport=FakeOpenAITransport(completions)
)
decision = client.decide(
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
)
assert decision.text_output == "Previous step succeeded. Now tapping login."
def test_openai_client_no_reasoning_content_gives_none_thinking() -> None:
completions = FakeCompletions(
response={
@@ -548,7 +698,12 @@ def test_openai_client_no_reasoning_content_gives_none_thinking() -> None:
{
"message": {
"tool_calls": [
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
{
"function": {
"name": "tap",
"arguments": '{"x": 1, "y": 2}',
}
}
]
}
}