Forced tool_choice ("any"/"required") makes both Anthropic and OpenAI
skip any text/thinking block before the tool call, which silently made
rationale and thinking always None despite the planner-reflection-history
change's capture code being correct. Switch the primary call to
tool_choice="auto" (Anthropic: type=auto, disable_parallel_tool_use=true;
OpenAI: "auto") so the model can emit its reflection text, and add a
one-time forced retry (Anthropic "any", OpenAI "required", thinking
disabled) if the model responds without a tool call, guaranteeing a step
never stalls. Also add OpenAI text_output capture from message.content,
which was never extracted before (Anthropic-only gap).
Update planner-reflection-history design.md/tasks.md to document the bug
found during the pending manual smoke test (task 8.5) and the fix (new
section 9).
This commit is contained in:
@@ -21,9 +21,14 @@ from tests.fakes import PNG_10X20
|
||||
|
||||
class FakeMessages:
|
||||
def __init__(
|
||||
self, *, response: object | None = None, error: Exception | None = None
|
||||
self,
|
||||
*,
|
||||
response: object | None = None,
|
||||
responses: list[object] | None = None,
|
||||
error: Exception | None = None,
|
||||
) -> None:
|
||||
self.response = response
|
||||
self._responses = list(responses) if responses is not None else None
|
||||
self.error = error
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
@@ -31,6 +36,8 @@ class FakeMessages:
|
||||
self.calls.append(kwargs)
|
||||
if self.error:
|
||||
raise self.error
|
||||
if self._responses is not None:
|
||||
return self._responses.pop(0)
|
||||
return self.response
|
||||
|
||||
|
||||
@@ -41,9 +48,14 @@ class FakeTransport:
|
||||
|
||||
class FakeCompletions:
|
||||
def __init__(
|
||||
self, *, response: object | None = None, error: Exception | None = None
|
||||
self,
|
||||
*,
|
||||
response: object | None = None,
|
||||
responses: list[object] | None = None,
|
||||
error: Exception | None = None,
|
||||
) -> None:
|
||||
self.response = response
|
||||
self._responses = list(responses) if responses is not None else None
|
||||
self.error = error
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
@@ -51,6 +63,8 @@ class FakeCompletions:
|
||||
self.calls.append(kwargs)
|
||||
if self.error:
|
||||
raise self.error
|
||||
if self._responses is not None:
|
||||
return self._responses.pop(0)
|
||||
return self.response
|
||||
|
||||
|
||||
@@ -67,7 +81,7 @@ class FakeOpenAITransport:
|
||||
# --- Anthropic ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -> None:
|
||||
def test_anthropic_tool_calling_client_sends_auto_tool_choice_request() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
@@ -95,7 +109,7 @@ def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -
|
||||
call = messages.calls[0]
|
||||
assert call["model"] == "test-model"
|
||||
assert call["timeout"] == 2.5
|
||||
assert call["tool_choice"] == {"type": "any", "disable_parallel_tool_use": True}
|
||||
assert call["tool_choice"] == {"type": "auto", "disable_parallel_tool_use": True}
|
||||
assert call["tools"] == [
|
||||
{
|
||||
"name": "tap",
|
||||
@@ -224,10 +238,72 @@ def test_anthropic_tool_calling_client_wraps_malformed_responses(
|
||||
)
|
||||
|
||||
|
||||
def test_anthropic_client_retries_with_forced_tool_choice_when_model_omits_tool_call() -> (
|
||||
None
|
||||
):
|
||||
messages = FakeMessages(
|
||||
responses=[
|
||||
{"content": [{"type": "text", "text": "just thinking out loud"}]},
|
||||
{
|
||||
"content": [
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}
|
||||
]
|
||||
},
|
||||
]
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
|
||||
assert decision.tool_name == "tap"
|
||||
assert len(messages.calls) == 2
|
||||
assert messages.calls[0]["tool_choice"] == {
|
||||
"type": "auto",
|
||||
"disable_parallel_tool_use": True,
|
||||
}
|
||||
assert messages.calls[1]["tool_choice"] == {
|
||||
"type": "any",
|
||||
"disable_parallel_tool_use": True,
|
||||
}
|
||||
|
||||
|
||||
def test_anthropic_client_retry_drops_thinking_param_since_incompatible_with_forced_tool_choice() -> (
|
||||
None
|
||||
):
|
||||
messages = FakeMessages(
|
||||
responses=[
|
||||
{"content": [{"type": "thinking", "thinking": "hmm, no tool yet"}]},
|
||||
{
|
||||
"content": [
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}
|
||||
]
|
||||
},
|
||||
]
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model",
|
||||
transport=FakeTransport(messages),
|
||||
thinking_budget_tokens=1024,
|
||||
)
|
||||
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
|
||||
assert "thinking" in messages.calls[0]
|
||||
assert "betas" in messages.calls[0]
|
||||
assert "thinking" not in messages.calls[1]
|
||||
assert "betas" not in messages.calls[1]
|
||||
|
||||
|
||||
# --- OpenAI --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> None:
|
||||
def test_openai_tool_calling_client_sends_auto_tool_choice_request() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
@@ -270,7 +346,7 @@ def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> N
|
||||
assert call["timeout"] == 2.5
|
||||
assert call["max_completion_tokens"] == 1024
|
||||
assert "max_tokens" not in call
|
||||
assert call["tool_choice"] == "required"
|
||||
assert call["tool_choice"] == "auto"
|
||||
assert call["parallel_tool_calls"] is False
|
||||
assert call["tools"] == [
|
||||
{
|
||||
@@ -419,6 +495,44 @@ def test_openai_tool_calling_client_wraps_malformed_responses(response: object)
|
||||
)
|
||||
|
||||
|
||||
def test_openai_client_retries_with_forced_tool_choice_when_model_omits_tool_call() -> (
|
||||
None
|
||||
):
|
||||
completions = FakeCompletions(
|
||||
responses=[
|
||||
{"choices": [{"message": {"content": "just chatting, no tool"}}]},
|
||||
{
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "tap",
|
||||
"arguments": '{"x": 1, "y": 2}',
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
]
|
||||
)
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
|
||||
assert decision.tool_name == "tap"
|
||||
assert len(completions.calls) == 2
|
||||
assert completions.calls[0]["tool_choice"] == "auto"
|
||||
assert completions.calls[1]["tool_choice"] == "required"
|
||||
|
||||
|
||||
# --- build_client ----------------------------------------------------------
|
||||
|
||||
|
||||
@@ -498,7 +612,9 @@ def test_anthropic_client_captures_both_thinking_and_text_output() -> None:
|
||||
assert decision.text_output == "Step succeeded. Tapping next."
|
||||
|
||||
|
||||
def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() -> None:
|
||||
def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() -> (
|
||||
None
|
||||
):
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
@@ -524,7 +640,12 @@ def test_openai_client_captures_reasoning_content() -> None:
|
||||
"message": {
|
||||
"reasoning_content": "I reasoned about this step.",
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
|
||||
{
|
||||
"function": {
|
||||
"name": "tap",
|
||||
"arguments": '{"x": 1, "y": 2}',
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
@@ -541,6 +662,35 @@ def test_openai_client_captures_reasoning_content() -> None:
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_openai_client_captures_message_content_as_text_output() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"content": "Previous step succeeded. Now tapping login.",
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "tap",
|
||||
"arguments": '{"x": 5, "y": 5}',
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.text_output == "Previous step succeeded. Now tapping login."
|
||||
|
||||
|
||||
def test_openai_client_no_reasoning_content_gives_none_thinking() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
@@ -548,7 +698,12 @@ def test_openai_client_no_reasoning_content_gives_none_thinking() -> None:
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
|
||||
{
|
||||
"function": {
|
||||
"name": "tap",
|
||||
"arguments": '{"x": 1, "y": 2}',
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user