q792602257andClaude Sonnet 5
61ff3b425d
Replaces the stub Planner's fixed describe_screen/[] behavior with a real decision-maker: AIPlanner uses native tool/function calling (Anthropic or OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded action per turn, with an explicit finish_task(success, reason) tool for completion/failure instead of an ambiguous "no tool call" signal. Default disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to the existing stub Planner unchanged when disabled. Amends CONSTITUTION.md's Perception Boundary with one narrow exception: only the AI Planner may receive the current step's raw screenshot bytes alongside Scene, for vision-grounded coordinate grounding. Also fixes a latent gap in TaskRunner.run(): observe/plan exceptions are now caught per iteration and turned into a failed task with a failure_reason, instead of propagating uncaught. openspec change: ai-planner-runtime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
117 lines
3.8 KiB
Python
117 lines
3.8 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from core.errors import TaskFailedError
|
|
from core.models import Bounds, Scene, SceneElement
|
|
from runtime.ai_planner import AIPlanner
|
|
from runtime.context import TaskContext
|
|
from runtime.planner import PlannedStep
|
|
from runtime.planner_config import PlannerConfig
|
|
from runtime.tool_calling_client import ToolCallDecision
|
|
from runtime.tool_specs import ALL_TOOL_SPECS
|
|
|
|
|
|
class FakeToolCallingClient:
|
|
def __init__(self, decision: ToolCallDecision) -> None:
|
|
self.decision = decision
|
|
self.calls: list[dict[str, Any]] = []
|
|
|
|
def decide(
|
|
self,
|
|
*,
|
|
system_prompt: str,
|
|
user_prompt: str,
|
|
screenshot: bytes | None,
|
|
tools: list[Any],
|
|
timeout: float,
|
|
) -> ToolCallDecision:
|
|
self.calls.append(
|
|
{
|
|
"system_prompt": system_prompt,
|
|
"user_prompt": user_prompt,
|
|
"screenshot": screenshot,
|
|
"tools": tools,
|
|
"timeout": timeout,
|
|
}
|
|
)
|
|
return self.decision
|
|
|
|
|
|
def _scene() -> Scene:
|
|
return Scene(
|
|
width=10,
|
|
height=20,
|
|
elements=[SceneElement(id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4))],
|
|
)
|
|
|
|
|
|
def _context() -> TaskContext:
|
|
return TaskContext(task_id="task-1", goal="send a message")
|
|
|
|
|
|
def test_ai_planner_returns_single_planned_step_for_action_decision() -> None:
|
|
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
|
planner = AIPlanner(client=client)
|
|
|
|
steps = planner.plan(goal="send a message", scene=_scene(), context=_context())
|
|
|
|
assert steps == [
|
|
PlannedStep(
|
|
action="tap",
|
|
description="AI planner: tap({'x': 1, 'y': 2})",
|
|
args={"x": 1, "y": 2},
|
|
)
|
|
]
|
|
|
|
|
|
def test_ai_planner_finish_task_success_returns_empty_plan() -> None:
|
|
client = FakeToolCallingClient(
|
|
ToolCallDecision(tool_name="finish_task", arguments={"success": True, "reason": "done"})
|
|
)
|
|
planner = AIPlanner(client=client)
|
|
|
|
steps = planner.plan(goal="send a message", scene=_scene(), context=_context())
|
|
|
|
assert steps == []
|
|
|
|
|
|
def test_ai_planner_finish_task_failure_raises_task_failed_error_with_reason() -> None:
|
|
client = FakeToolCallingClient(
|
|
ToolCallDecision(tool_name="finish_task", arguments={"success": False, "reason": "stuck on login"})
|
|
)
|
|
planner = AIPlanner(client=client)
|
|
|
|
with pytest.raises(TaskFailedError, match="stuck on login"):
|
|
planner.plan(goal="send a message", scene=_scene(), context=_context())
|
|
|
|
|
|
def test_ai_planner_finish_task_failure_without_reason_uses_default_message() -> None:
|
|
client = FakeToolCallingClient(ToolCallDecision(tool_name="finish_task", arguments={"success": False}))
|
|
planner = AIPlanner(client=client)
|
|
|
|
with pytest.raises(TaskFailedError, match="task failed"):
|
|
planner.plan(goal="send a message", scene=_scene(), context=_context())
|
|
|
|
|
|
def test_ai_planner_goal_reached_is_always_false() -> None:
|
|
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
|
planner = AIPlanner(client=client)
|
|
|
|
assert planner.goal_reached(goal="anything", scene=_scene(), context=_context()) is False
|
|
|
|
|
|
def test_ai_planner_forwards_tools_screenshot_and_timeout_to_client() -> None:
|
|
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
|
planner = AIPlanner(client=client, config=PlannerConfig(timeout=12.5))
|
|
|
|
planner.plan(goal="send a message", scene=_scene(), context=_context(), screenshot=b"fake-bytes")
|
|
|
|
call = client.calls[0]
|
|
assert call["tools"] == ALL_TOOL_SPECS
|
|
assert call["screenshot"] == b"fake-bytes"
|
|
assert call["timeout"] == 12.5
|
|
assert "send a message" in call["user_prompt"]
|