feat(runtime): add planner reflection history with rationale and thinking
Tests / Test failed: 2, passed: 849
Tests / Test failed: 2, passed: 849
- ToolCallDecision captures thinking blocks and pre-tool text output
- AnthropicToolCallingClient supports optional extended thinking (budget_tokens + beta header)
- PlannedStep carries rationale and thinking from each LLM decision
- WorldEvent replaces scene_summary with rationale/thinking/page fields (backward-compatible)
- AI planner system prompt instructs reflection before each tool call
- _history_summary() emits compact {page, rationale, action, success} dicts
- Cloud DB migration 0011 adds nullable rationale/thinking columns to planner_decision_log
- OpenAI client extracts reasoning_content into thinking field
This commit is contained in:
@@ -435,3 +435,187 @@ def test_build_client_selects_provider_and_resolves_default_model() -> None:
|
||||
def test_build_client_honors_explicit_model_override() -> None:
|
||||
client = build_client(PlannerConfig(provider="openai", model="gpt-5.6-custom"))
|
||||
assert client.model == "gpt-5.6-custom"
|
||||
|
||||
|
||||
# --- thinking / text_output capture -----------------------------------------
|
||||
|
||||
|
||||
def test_anthropic_client_captures_thinking_block() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "I should tap the button."},
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 10, "y": 20}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking == "I should tap the button."
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_anthropic_client_captures_text_block_as_text_output() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "text", "text": "Previous step succeeded. Now tapping login."},
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 5, "y": 5}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.text_output == "Previous step succeeded. Now tapping login."
|
||||
assert decision.thinking is None
|
||||
|
||||
|
||||
def test_anthropic_client_captures_both_thinking_and_text_output() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "Deep thought."},
|
||||
{"type": "text", "text": "Step succeeded. Tapping next."},
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 1}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking == "Deep thought."
|
||||
assert decision.text_output == "Step succeeded. Tapping next."
|
||||
|
||||
|
||||
def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 0, "y": 0}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking is None
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_openai_client_captures_reasoning_content() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"reasoning_content": "I reasoned about this step.",
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
|
||||
],
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking == "I reasoned about this step."
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_openai_client_no_reasoning_content_gives_none_thinking() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking is None
|
||||
|
||||
|
||||
def test_anthropic_client_sends_thinking_param_when_budget_set() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model",
|
||||
transport=FakeTransport(messages),
|
||||
thinking_budget_tokens=2048,
|
||||
)
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
call = messages.calls[0]
|
||||
assert call["thinking"] == {"type": "enabled", "budget_tokens": 2048}
|
||||
assert "interleaved-thinking-2025-05-14" in call["betas"]
|
||||
# max_tokens must be >= budget + 1
|
||||
assert call["max_tokens"] >= 2049
|
||||
|
||||
|
||||
def test_anthropic_client_enforces_max_tokens_floor_for_thinking() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
# max_tokens=1024, budget=4096 → max_tokens should be raised to 4097
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model",
|
||||
transport=FakeTransport(messages),
|
||||
max_tokens=1024,
|
||||
thinking_budget_tokens=4096,
|
||||
)
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert messages.calls[0]["max_tokens"] == 4097
|
||||
|
||||
|
||||
def test_anthropic_client_no_thinking_param_when_budget_not_set() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
call = messages.calls[0]
|
||||
assert "thinking" not in call
|
||||
assert "betas" not in call
|
||||
|
||||
Reference in New Issue
Block a user