feat(runtime): add planner reflection history with rationale and thinking
Tests / Test failed: 2, passed: 849
Tests / Test failed: 2, passed: 849
- ToolCallDecision captures thinking blocks and pre-tool text output
- AnthropicToolCallingClient supports optional extended thinking (budget_tokens + beta header)
- PlannedStep carries rationale and thinking from each LLM decision
- WorldEvent replaces scene_summary with rationale/thinking/page fields (backward-compatible)
- AI planner system prompt instructs reflection before each tool call
- _history_summary() emits compact {page, rationale, action, success} dicts
- Cloud DB migration 0011 adds nullable rationale/thinking columns to planner_decision_log
- OpenAI client extracts reasoning_content into thinking field
This commit is contained in:
@@ -195,3 +195,62 @@ def test_ai_planner_step_prompt_reflects_scene_changes() -> None:
|
||||
assert steps_a[0].prompt != steps_b[0].prompt
|
||||
assert "Alpha" in steps_a[0].prompt
|
||||
assert "Beta" in steps_b[0].prompt
|
||||
|
||||
|
||||
def test_ai_planner_propagates_rationale_and_thinking_to_planned_step() -> None:
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(
|
||||
tool_name="tap",
|
||||
arguments={"x": 1, "y": 2},
|
||||
text_output="Previous step opened settings. Now tapping account.",
|
||||
thinking="I need to navigate deeper.",
|
||||
)
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
steps = planner.plan(goal="open account", scene=_scene(), context=_context())
|
||||
|
||||
assert steps[0].rationale == "Previous step opened settings. Now tapping account."
|
||||
assert steps[0].thinking == "I need to navigate deeper."
|
||||
|
||||
|
||||
def test_history_summary_returns_compact_format() -> None:
|
||||
from collections import deque
|
||||
from runtime.ai_planner import _history_summary
|
||||
from world.models import WorldEvent, WorldState
|
||||
|
||||
state = WorldState(
|
||||
history=deque(
|
||||
[
|
||||
WorldEvent(
|
||||
action="tap",
|
||||
success=True,
|
||||
rationale="Opened settings.",
|
||||
page="Home",
|
||||
),
|
||||
WorldEvent(
|
||||
action="swipe",
|
||||
success=False,
|
||||
rationale=None,
|
||||
page="Settings",
|
||||
),
|
||||
]
|
||||
)
|
||||
)
|
||||
|
||||
summary = _history_summary(state)
|
||||
|
||||
assert summary == [
|
||||
{"page": "Home", "action": "tap", "rationale": "Opened settings.", "success": True},
|
||||
{"page": "Settings", "action": "swipe", "rationale": None, "success": False},
|
||||
]
|
||||
# Must not contain scene element data
|
||||
for entry in summary:
|
||||
assert "scene_summary" not in entry
|
||||
assert "elements" not in entry
|
||||
|
||||
|
||||
def test_history_summary_returns_empty_for_none_world() -> None:
|
||||
from runtime.ai_planner import _history_summary
|
||||
|
||||
assert _history_summary(None) == []
|
||||
|
||||
@@ -1751,3 +1751,62 @@ def test_prune_planner_decision_log_deletes_only_old_terminal_tasks(
|
||||
)
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_record_planner_decision_stores_rationale_and_thinking(
|
||||
database_url: str,
|
||||
) -> None:
|
||||
database = CloudDatabase(database_url)
|
||||
task_id = _unique_id("reflect-task")
|
||||
host_id = _unique_id("reflect-host")
|
||||
now = datetime(2026, 7, 15, 0, 0, tzinfo=UTC)
|
||||
|
||||
try:
|
||||
database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
tool_name="tap",
|
||||
arguments_json='{"x": 1}',
|
||||
now=now,
|
||||
rationale="Previous step opened settings. Now tapping account.",
|
||||
thinking="I need to navigate to account settings.",
|
||||
)
|
||||
|
||||
decisions = database.repository.list_planner_decisions(task_id=task_id, attempt=1)
|
||||
assert len(decisions) == 1
|
||||
assert decisions[0].rationale == "Previous step opened settings. Now tapping account."
|
||||
assert decisions[0].thinking == "I need to navigate to account settings."
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
|
||||
def test_record_planner_decision_stores_null_rationale_and_thinking(
|
||||
database_url: str,
|
||||
) -> None:
|
||||
database = CloudDatabase(database_url)
|
||||
task_id = _unique_id("reflect-null-task")
|
||||
host_id = _unique_id("reflect-null-host")
|
||||
now = datetime(2026, 7, 15, 0, 0, tzinfo=UTC)
|
||||
|
||||
try:
|
||||
database.repository.record_planner_decision(
|
||||
host_id=host_id,
|
||||
task_id=task_id,
|
||||
attempt=1,
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
tool_name="tap",
|
||||
arguments_json="{}",
|
||||
now=now,
|
||||
# rationale and thinking omitted (default None)
|
||||
)
|
||||
|
||||
decisions = database.repository.list_planner_decisions(task_id=task_id, attempt=1)
|
||||
assert len(decisions) == 1
|
||||
assert decisions[0].rationale is None
|
||||
assert decisions[0].thinking is None
|
||||
finally:
|
||||
database.close()
|
||||
|
||||
@@ -70,3 +70,21 @@ def test_load_config_falls_back_to_default_timeout_when_invalid_or_non_positive(
|
||||
for value in ["not-a-number", "0", "-5"]:
|
||||
config = load_config({"AI_PLANNER_TIMEOUT_SECONDS": value, **_NO_RELEVANT_VARS})
|
||||
assert config.timeout == DEFAULT_TIMEOUT_SECONDS
|
||||
|
||||
|
||||
def test_load_config_parses_thinking_budget_tokens() -> None:
|
||||
config = load_config({"AI_PLANNER_THINKING_BUDGET_TOKENS": "4096"})
|
||||
|
||||
assert config.thinking_budget_tokens == 4096
|
||||
|
||||
|
||||
def test_load_config_thinking_budget_tokens_unset_defaults_to_none() -> None:
|
||||
config = load_config(_NO_RELEVANT_VARS)
|
||||
|
||||
assert config.thinking_budget_tokens is None
|
||||
|
||||
|
||||
def test_load_config_thinking_budget_tokens_invalid_or_non_positive_gives_none() -> None:
|
||||
for value in ["not-a-number", "0", "-1"]:
|
||||
config = load_config({"AI_PLANNER_THINKING_BUDGET_TOKENS": value, **_NO_RELEVANT_VARS})
|
||||
assert config.thinking_budget_tokens is None
|
||||
|
||||
@@ -435,3 +435,187 @@ def test_build_client_selects_provider_and_resolves_default_model() -> None:
|
||||
def test_build_client_honors_explicit_model_override() -> None:
|
||||
client = build_client(PlannerConfig(provider="openai", model="gpt-5.6-custom"))
|
||||
assert client.model == "gpt-5.6-custom"
|
||||
|
||||
|
||||
# --- thinking / text_output capture -----------------------------------------
|
||||
|
||||
|
||||
def test_anthropic_client_captures_thinking_block() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "I should tap the button."},
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 10, "y": 20}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking == "I should tap the button."
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_anthropic_client_captures_text_block_as_text_output() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "text", "text": "Previous step succeeded. Now tapping login."},
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 5, "y": 5}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.text_output == "Previous step succeeded. Now tapping login."
|
||||
assert decision.thinking is None
|
||||
|
||||
|
||||
def test_anthropic_client_captures_both_thinking_and_text_output() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "thinking", "thinking": "Deep thought."},
|
||||
{"type": "text", "text": "Step succeeded. Tapping next."},
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 1}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking == "Deep thought."
|
||||
assert decision.text_output == "Step succeeded. Tapping next."
|
||||
|
||||
|
||||
def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "tool_use", "name": "tap", "input": {"x": 0, "y": 0}},
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking is None
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_openai_client_captures_reasoning_content() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"reasoning_content": "I reasoned about this step.",
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
|
||||
],
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking == "I reasoned about this step."
|
||||
assert decision.text_output is None
|
||||
|
||||
|
||||
def test_openai_client_no_reasoning_content_gives_none_thinking() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(
|
||||
model="test-model", transport=FakeOpenAITransport(completions)
|
||||
)
|
||||
decision = client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert decision.thinking is None
|
||||
|
||||
|
||||
def test_anthropic_client_sends_thinking_param_when_budget_set() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model",
|
||||
transport=FakeTransport(messages),
|
||||
thinking_budget_tokens=2048,
|
||||
)
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
call = messages.calls[0]
|
||||
assert call["thinking"] == {"type": "enabled", "budget_tokens": 2048}
|
||||
assert "interleaved-thinking-2025-05-14" in call["betas"]
|
||||
# max_tokens must be >= budget + 1
|
||||
assert call["max_tokens"] >= 2049
|
||||
|
||||
|
||||
def test_anthropic_client_enforces_max_tokens_floor_for_thinking() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
# max_tokens=1024, budget=4096 → max_tokens should be raised to 4097
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model",
|
||||
transport=FakeTransport(messages),
|
||||
max_tokens=1024,
|
||||
thinking_budget_tokens=4096,
|
||||
)
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
assert messages.calls[0]["max_tokens"] == 4097
|
||||
|
||||
|
||||
def test_anthropic_client_no_thinking_param_when_budget_not_set() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(
|
||||
model="test-model", transport=FakeTransport(messages)
|
||||
)
|
||||
client.decide(
|
||||
system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1
|
||||
)
|
||||
call = messages.calls[0]
|
||||
assert "thinking" not in call
|
||||
assert "betas" not in call
|
||||
|
||||
@@ -27,9 +27,9 @@ def test_world_event_and_state_to_dict() -> None:
|
||||
widgets=[SemanticWidget(element_id="send", purpose="send message")],
|
||||
)
|
||||
event = WorldEvent(
|
||||
scene_summary=semantic_scene,
|
||||
action="tap",
|
||||
success=True,
|
||||
scene_summary=semantic_scene,
|
||||
)
|
||||
state = WorldState.with_history_bound(2)
|
||||
state.current_app = "com.example.chat"
|
||||
@@ -51,10 +51,38 @@ def test_world_event_and_state_to_dict() -> None:
|
||||
def test_world_state_history_evicts_oldest_entry_at_bound() -> None:
|
||||
state = WorldState.with_history_bound(2)
|
||||
|
||||
state.history.append(WorldEvent(_scene(), "first", True))
|
||||
state.history.append(WorldEvent(_scene(), "second", True))
|
||||
state.history.append(WorldEvent(_scene(), "third", True))
|
||||
state.history.append(WorldEvent(action="first", success=True))
|
||||
state.history.append(WorldEvent(action="second", success=True))
|
||||
state.history.append(WorldEvent(action="third", success=True))
|
||||
|
||||
assert len(state.history) == 2
|
||||
assert [event.action for event in state.history] == ["second", "third"]
|
||||
assert state.history.maxlen == 2
|
||||
|
||||
|
||||
def test_world_event_with_rationale_thinking_page() -> None:
|
||||
event = WorldEvent(
|
||||
action="tap",
|
||||
success=True,
|
||||
rationale="Previous step opened settings. Now tapping account.",
|
||||
thinking="I should navigate to account settings.",
|
||||
page="Settings",
|
||||
)
|
||||
data = event.to_dict()
|
||||
|
||||
assert data["rationale"] == "Previous step opened settings. Now tapping account."
|
||||
assert data["thinking"] == "I should navigate to account settings."
|
||||
assert data["page"] == "Settings"
|
||||
assert data["scene_summary"] is None
|
||||
|
||||
|
||||
def test_world_event_all_optional_fields_none() -> None:
|
||||
event = WorldEvent(action="swipe", success=False)
|
||||
data = event.to_dict()
|
||||
|
||||
assert data["rationale"] is None
|
||||
assert data["thinking"] is None
|
||||
assert data["page"] is None
|
||||
assert data["scene_summary"] is None
|
||||
assert data["action"] == "swipe"
|
||||
assert data["success"] is False
|
||||
|
||||
Reference in New Issue
Block a user