feat(agent-runtime): add LLM-driven AI Planner with dual-provider tool calling
Replaces the stub Planner's fixed describe_screen/[] behavior with a real decision-maker: AIPlanner uses native tool/function calling (Anthropic or OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded action per turn, with an explicit finish_task(success, reason) tool for completion/failure instead of an ambiguous "no tool call" signal. Default disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to the existing stub Planner unchanged when disabled. Amends CONSTITUTION.md's Perception Boundary with one narrow exception: only the AI Planner may receive the current step's raw screenshot bytes alongside Scene, for vision-grounded coordinate grounding. Also fixes a latent gap in TaskRunner.run(): observe/plan exceptions are now caught per iteration and turned into a failed task with a failure_reason, instead of propagating uncaught. openspec change: ai-planner-runtime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,116 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from core.errors import TaskFailedError
|
||||
from core.models import Bounds, Scene, SceneElement
|
||||
from runtime.ai_planner import AIPlanner
|
||||
from runtime.context import TaskContext
|
||||
from runtime.planner import PlannedStep
|
||||
from runtime.planner_config import PlannerConfig
|
||||
from runtime.tool_calling_client import ToolCallDecision
|
||||
from runtime.tool_specs import ALL_TOOL_SPECS
|
||||
|
||||
|
||||
class FakeToolCallingClient:
|
||||
def __init__(self, decision: ToolCallDecision) -> None:
|
||||
self.decision = decision
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def decide(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[Any],
|
||||
timeout: float,
|
||||
) -> ToolCallDecision:
|
||||
self.calls.append(
|
||||
{
|
||||
"system_prompt": system_prompt,
|
||||
"user_prompt": user_prompt,
|
||||
"screenshot": screenshot,
|
||||
"tools": tools,
|
||||
"timeout": timeout,
|
||||
}
|
||||
)
|
||||
return self.decision
|
||||
|
||||
|
||||
def _scene() -> Scene:
|
||||
return Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[SceneElement(id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4))],
|
||||
)
|
||||
|
||||
|
||||
def _context() -> TaskContext:
|
||||
return TaskContext(task_id="task-1", goal="send a message")
|
||||
|
||||
|
||||
def test_ai_planner_returns_single_planned_step_for_action_decision() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
steps = planner.plan(goal="send a message", scene=_scene(), context=_context())
|
||||
|
||||
assert steps == [
|
||||
PlannedStep(
|
||||
action="tap",
|
||||
description="AI planner: tap({'x': 1, 'y': 2})",
|
||||
args={"x": 1, "y": 2},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def test_ai_planner_finish_task_success_returns_empty_plan() -> None:
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="finish_task", arguments={"success": True, "reason": "done"})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
steps = planner.plan(goal="send a message", scene=_scene(), context=_context())
|
||||
|
||||
assert steps == []
|
||||
|
||||
|
||||
def test_ai_planner_finish_task_failure_raises_task_failed_error_with_reason() -> None:
|
||||
client = FakeToolCallingClient(
|
||||
ToolCallDecision(tool_name="finish_task", arguments={"success": False, "reason": "stuck on login"})
|
||||
)
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
with pytest.raises(TaskFailedError, match="stuck on login"):
|
||||
planner.plan(goal="send a message", scene=_scene(), context=_context())
|
||||
|
||||
|
||||
def test_ai_planner_finish_task_failure_without_reason_uses_default_message() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="finish_task", arguments={"success": False}))
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
with pytest.raises(TaskFailedError, match="task failed"):
|
||||
planner.plan(goal="send a message", scene=_scene(), context=_context())
|
||||
|
||||
|
||||
def test_ai_planner_goal_reached_is_always_false() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
||||
planner = AIPlanner(client=client)
|
||||
|
||||
assert planner.goal_reached(goal="anything", scene=_scene(), context=_context()) is False
|
||||
|
||||
|
||||
def test_ai_planner_forwards_tools_screenshot_and_timeout_to_client() -> None:
|
||||
client = FakeToolCallingClient(ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}))
|
||||
planner = AIPlanner(client=client, config=PlannerConfig(timeout=12.5))
|
||||
|
||||
planner.plan(goal="send a message", scene=_scene(), context=_context(), screenshot=b"fake-bytes")
|
||||
|
||||
call = client.calls[0]
|
||||
assert call["tools"] == ALL_TOOL_SPECS
|
||||
assert call["screenshot"] == b"fake-bytes"
|
||||
assert call["timeout"] == 12.5
|
||||
assert "send a message" in call["user_prompt"]
|
||||
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from core.models import Bounds, Scene, SceneElement
|
||||
from runtime.ai_planner import AIPlanner
|
||||
from runtime.context import TaskContext
|
||||
from runtime.planner_config import PlannerConfig
|
||||
|
||||
|
||||
def _scene() -> Scene:
|
||||
return Scene(
|
||||
width=390,
|
||||
height=844,
|
||||
elements=[
|
||||
SceneElement(id="send", type="button", text="Send", bounds=Bounds(300, 800, 60, 30)),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_real_anthropic_ai_planner_selects_a_tool() -> None:
|
||||
if not os.environ.get("ANTHROPIC_API_KEY"):
|
||||
pytest.skip("ANTHROPIC_API_KEY is required for AI planner integration test")
|
||||
try:
|
||||
import anthropic # noqa: F401
|
||||
except ImportError:
|
||||
pytest.skip("anthropic SDK is not installed")
|
||||
|
||||
planner = AIPlanner(config=PlannerConfig(enabled=True, provider="anthropic", timeout=15.0))
|
||||
context = TaskContext(task_id="task", goal="tap the send button")
|
||||
|
||||
steps = planner.plan(goal="tap the send button", scene=_scene(), context=context)
|
||||
|
||||
assert isinstance(steps, list)
|
||||
assert len(steps) <= 1
|
||||
|
||||
|
||||
@pytest.mark.integration
|
||||
def test_real_openai_ai_planner_selects_a_tool() -> None:
|
||||
if not os.environ.get("OPENAI_API_KEY"):
|
||||
pytest.skip("OPENAI_API_KEY is required for AI planner integration test")
|
||||
try:
|
||||
import openai # noqa: F401
|
||||
except ImportError:
|
||||
pytest.skip("openai SDK is not installed")
|
||||
|
||||
planner = AIPlanner(config=PlannerConfig(enabled=True, provider="openai", timeout=15.0))
|
||||
context = TaskContext(task_id="task", goal="tap the send button")
|
||||
|
||||
steps = planner.plan(goal="tap the send button", scene=_scene(), context=context)
|
||||
|
||||
assert isinstance(steps, list)
|
||||
assert len(steps) <= 1
|
||||
@@ -0,0 +1,132 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from core.models import Bounds, Scene, SceneElement, Task
|
||||
from runtime.ai_planner import AIPlanner
|
||||
from runtime.executor import Executor, ExecutorConfig
|
||||
from runtime.planner import PlannedStep, Planner
|
||||
from runtime.planner_config import PlannerConfig
|
||||
from runtime.task import TaskRunner, TaskRunnerConfig
|
||||
from tests.fakes import PNG_10X20
|
||||
|
||||
|
||||
class RaisingPlanner(Planner):
|
||||
def __init__(self, error: Exception) -> None:
|
||||
self.error = error
|
||||
self.calls = 0
|
||||
|
||||
def plan(self, *, goal, scene, context):
|
||||
self.calls += 1
|
||||
raise self.error
|
||||
|
||||
def goal_reached(self, *, goal, scene, context):
|
||||
return False
|
||||
|
||||
|
||||
class NarrowSignaturePlanner(Planner):
|
||||
"""Predates the `screenshot` parameter added to the base Planner.plan()."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls = 0
|
||||
|
||||
def plan(self, *, goal, scene, context):
|
||||
self.calls += 1
|
||||
if context.step_results:
|
||||
return []
|
||||
return [PlannedStep(action="tap", description="tap")]
|
||||
|
||||
def goal_reached(self, *, goal, scene, context):
|
||||
return bool(context.step_results)
|
||||
|
||||
|
||||
class ScreenshotRecordingPlanner(Planner):
|
||||
def __init__(self) -> None:
|
||||
self.screenshots: list[bytes | None] = []
|
||||
|
||||
def plan(self, *, goal, scene, context, screenshot=None):
|
||||
self.screenshots.append(screenshot)
|
||||
if context.step_results:
|
||||
return []
|
||||
return [PlannedStep(action="tap", description="tap")]
|
||||
|
||||
def goal_reached(self, *, goal, scene, context):
|
||||
return bool(context.step_results)
|
||||
|
||||
|
||||
def _scene() -> Scene:
|
||||
return Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[SceneElement(id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4))],
|
||||
)
|
||||
|
||||
|
||||
def _runner(*, planner=None, planner_config=None, observer=None) -> TaskRunner:
|
||||
return TaskRunner(
|
||||
planner=planner,
|
||||
planner_config=planner_config,
|
||||
executor=Executor(
|
||||
tools={"tap": lambda **kwargs: {"ok": True}},
|
||||
config=ExecutorConfig(max_retries=1, backoff_seconds=0),
|
||||
),
|
||||
config=TaskRunnerConfig(max_steps=5),
|
||||
observer=observer or (lambda device_id: _scene()),
|
||||
screenshot_provider=lambda device_id: PNG_10X20,
|
||||
)
|
||||
|
||||
|
||||
def test_task_runner_marks_task_failed_when_planner_raises() -> None:
|
||||
planner = RaisingPlanner(RuntimeError("boom"))
|
||||
runner = _runner(planner=planner)
|
||||
|
||||
result = runner.run(Task(goal="inspect", device_id="phone"))
|
||||
|
||||
assert result.status == "failed"
|
||||
assert result.failure_reason == "RuntimeError: boom"
|
||||
assert planner.calls == 1
|
||||
|
||||
|
||||
def test_task_runner_marks_task_failed_when_observer_raises() -> None:
|
||||
def failing_observer(device_id: str) -> Scene:
|
||||
raise RuntimeError("no device")
|
||||
|
||||
runner = _runner(planner=Planner(), observer=failing_observer)
|
||||
|
||||
result = runner.run(Task(goal="inspect", device_id="phone"))
|
||||
|
||||
assert result.status == "failed"
|
||||
assert result.failure_reason == "RuntimeError: no device"
|
||||
|
||||
|
||||
def test_task_runner_omits_screenshot_kwarg_for_narrow_signature_planner() -> None:
|
||||
planner = NarrowSignaturePlanner()
|
||||
runner = _runner(planner=planner)
|
||||
|
||||
result = runner.run(Task(goal="inspect", device_id="phone"))
|
||||
|
||||
assert result.status == "completed"
|
||||
assert planner.calls == 2
|
||||
|
||||
|
||||
def test_task_runner_passes_screenshot_to_planner_that_declares_it() -> None:
|
||||
planner = ScreenshotRecordingPlanner()
|
||||
runner = _runner(planner=planner)
|
||||
|
||||
result = runner.run(Task(goal="inspect", device_id="phone"))
|
||||
|
||||
assert result.status == "completed"
|
||||
assert planner.screenshots == [PNG_10X20, PNG_10X20]
|
||||
|
||||
|
||||
def test_task_runner_default_planner_is_stub_when_ai_planner_disabled() -> None:
|
||||
runner = _runner(planner=None, planner_config=PlannerConfig(enabled=False))
|
||||
|
||||
assert type(runner.planner) is Planner
|
||||
|
||||
|
||||
def test_task_runner_default_planner_is_ai_planner_when_enabled() -> None:
|
||||
runner = _runner(
|
||||
planner=None,
|
||||
planner_config=PlannerConfig(enabled=True, provider="anthropic", model="test-model"),
|
||||
)
|
||||
|
||||
assert isinstance(runner.planner, AIPlanner)
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from runtime.planner_config import (
|
||||
DEFAULT_MODEL_BY_PROVIDER,
|
||||
DEFAULT_PROVIDER,
|
||||
DEFAULT_TIMEOUT_SECONDS,
|
||||
PlannerConfig,
|
||||
load_config,
|
||||
)
|
||||
|
||||
_NO_RELEVANT_VARS = {"UNRELATED": "1"}
|
||||
|
||||
|
||||
def test_load_config_defaults_when_unset() -> None:
|
||||
config = load_config(_NO_RELEVANT_VARS)
|
||||
|
||||
assert config == PlannerConfig(
|
||||
enabled=False,
|
||||
provider=DEFAULT_PROVIDER,
|
||||
model="",
|
||||
timeout=DEFAULT_TIMEOUT_SECONDS,
|
||||
)
|
||||
assert config.resolved_model() == DEFAULT_MODEL_BY_PROVIDER[DEFAULT_PROVIDER]
|
||||
|
||||
|
||||
def test_load_config_parses_enabled_truthy_values() -> None:
|
||||
for value in ["1", "true", "True", "yes", "on", "enabled"]:
|
||||
assert load_config({"AI_PLANNER_ENABLED": value}).enabled is True
|
||||
|
||||
|
||||
def test_load_config_parses_enabled_falsy_values() -> None:
|
||||
for value in ["0", "false", "no", "off", ""]:
|
||||
assert load_config({"AI_PLANNER_ENABLED": value}).enabled is False
|
||||
|
||||
|
||||
def test_load_config_selects_provider_and_resolves_default_model() -> None:
|
||||
config = load_config({"AI_PLANNER_PROVIDER": "openai"})
|
||||
|
||||
assert config.provider == "openai"
|
||||
assert config.resolved_model() == "gpt-5.6"
|
||||
|
||||
|
||||
def test_load_config_anthropic_default_model() -> None:
|
||||
config = load_config({"AI_PLANNER_PROVIDER": "anthropic"})
|
||||
|
||||
assert config.resolved_model() == "claude-sonnet-5"
|
||||
|
||||
|
||||
def test_load_config_falls_back_to_default_provider_when_unsupported() -> None:
|
||||
config = load_config({"AI_PLANNER_PROVIDER": "not-a-real-provider"})
|
||||
|
||||
assert config.provider == DEFAULT_PROVIDER
|
||||
|
||||
|
||||
def test_load_config_model_override_wins_regardless_of_provider() -> None:
|
||||
config = load_config(
|
||||
{"AI_PLANNER_PROVIDER": "openai", "AI_PLANNER_MODEL": "custom-model"}
|
||||
)
|
||||
|
||||
assert config.resolved_model() == "custom-model"
|
||||
|
||||
|
||||
def test_load_config_parses_valid_timeout() -> None:
|
||||
config = load_config({"AI_PLANNER_TIMEOUT_SECONDS": "12.5"})
|
||||
|
||||
assert config.timeout == 12.5
|
||||
|
||||
|
||||
def test_load_config_falls_back_to_default_timeout_when_invalid_or_non_positive() -> None:
|
||||
for value in ["not-a-number", "0", "-5"]:
|
||||
config = load_config({"AI_PLANNER_TIMEOUT_SECONDS": value, **_NO_RELEVANT_VARS})
|
||||
assert config.timeout == DEFAULT_TIMEOUT_SECONDS
|
||||
@@ -0,0 +1,296 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from runtime.planner_config import PlannerConfig
|
||||
from runtime.tool_calling_client import (
|
||||
AnthropicToolCallingClient,
|
||||
OpenAIToolCallingClient,
|
||||
ToolCallDecision,
|
||||
ToolCallUnavailable,
|
||||
build_client,
|
||||
)
|
||||
from runtime.tool_specs import FINISH_TASK_SPEC, TAP_SPEC
|
||||
from tests.fakes import PNG_10X20
|
||||
|
||||
|
||||
class FakeMessages:
|
||||
def __init__(self, *, response: object | None = None, error: Exception | None = None) -> None:
|
||||
self.response = response
|
||||
self.error = error
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def create(self, **kwargs: Any) -> object:
|
||||
self.calls.append(kwargs)
|
||||
if self.error:
|
||||
raise self.error
|
||||
return self.response
|
||||
|
||||
|
||||
class FakeTransport:
|
||||
def __init__(self, messages: FakeMessages) -> None:
|
||||
self.messages = messages
|
||||
|
||||
|
||||
class FakeCompletions:
|
||||
def __init__(self, *, response: object | None = None, error: Exception | None = None) -> None:
|
||||
self.response = response
|
||||
self.error = error
|
||||
self.calls: list[dict[str, Any]] = []
|
||||
|
||||
def create(self, **kwargs: Any) -> object:
|
||||
self.calls.append(kwargs)
|
||||
if self.error:
|
||||
raise self.error
|
||||
return self.response
|
||||
|
||||
|
||||
class FakeChat:
|
||||
def __init__(self, completions: FakeCompletions) -> None:
|
||||
self.completions = completions
|
||||
|
||||
|
||||
class FakeOpenAITransport:
|
||||
def __init__(self, completions: FakeCompletions) -> None:
|
||||
self.chat = FakeChat(completions)
|
||||
|
||||
|
||||
# --- Anthropic ---------------------------------------------------------
|
||||
|
||||
|
||||
def test_anthropic_tool_calling_client_sends_forced_single_tool_call_request() -> None:
|
||||
messages = FakeMessages(
|
||||
response={"content": [{"type": "tool_use", "name": "tap", "input": {"x": 1, "y": 2}}]}
|
||||
)
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
|
||||
decision = client.decide(
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
screenshot=None,
|
||||
tools=[TAP_SPEC, FINISH_TASK_SPEC],
|
||||
timeout=2.5,
|
||||
)
|
||||
|
||||
assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
assert len(messages.calls) == 1
|
||||
call = messages.calls[0]
|
||||
assert call["model"] == "test-model"
|
||||
assert call["timeout"] == 2.5
|
||||
assert call["tool_choice"] == {"type": "any", "disable_parallel_tool_use": True}
|
||||
assert call["tools"] == [
|
||||
{"name": "tap", "description": TAP_SPEC.description, "input_schema": TAP_SPEC.parameters},
|
||||
{
|
||||
"name": "finish_task",
|
||||
"description": FINISH_TASK_SPEC.description,
|
||||
"input_schema": FINISH_TASK_SPEC.parameters,
|
||||
},
|
||||
]
|
||||
assert call["system"][0]["text"] == "system"
|
||||
assert call["system"][0]["cache_control"] == {"type": "ephemeral"}
|
||||
assert call["messages"] == [{"role": "user", "content": [{"type": "text", "text": "user"}]}]
|
||||
|
||||
|
||||
def test_anthropic_tool_calling_client_includes_image_block_when_screenshot_present() -> None:
|
||||
messages = FakeMessages(
|
||||
response={
|
||||
"content": [
|
||||
{"type": "tool_use", "name": "finish_task", "input": {"success": True, "reason": "done"}}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
|
||||
client.decide(
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
screenshot=PNG_10X20,
|
||||
tools=[FINISH_TASK_SPEC],
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
content = messages.calls[0]["messages"][0]["content"]
|
||||
assert content[0] == {
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": base64.b64encode(PNG_10X20).decode("ascii"),
|
||||
},
|
||||
}
|
||||
assert content[1] == {"type": "text", "text": "user"}
|
||||
|
||||
|
||||
def test_anthropic_tool_calling_client_wraps_transport_errors() -> None:
|
||||
messages = FakeMessages(error=TimeoutError("timed out"))
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
[
|
||||
{"content": []},
|
||||
{"content": [{"type": "text", "text": "no tool call"}]},
|
||||
{"content": [{"type": "tool_use", "name": "tap", "input": "not-a-dict"}]},
|
||||
],
|
||||
)
|
||||
def test_anthropic_tool_calling_client_wraps_malformed_responses(response: object) -> None:
|
||||
messages = FakeMessages(response=response)
|
||||
client = AnthropicToolCallingClient(model="test-model", transport=FakeTransport(messages))
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
|
||||
|
||||
# --- OpenAI --------------------------------------------------------------
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_sends_forced_single_tool_call_request() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{"message": {"tool_calls": [{"function": {"name": "tap", "arguments": '{"x": 1, "y": 2}'}}]}}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
|
||||
decision = client.decide(
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
screenshot=None,
|
||||
tools=[TAP_SPEC, FINISH_TASK_SPEC],
|
||||
timeout=2.5,
|
||||
)
|
||||
|
||||
assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
assert len(completions.calls) == 1
|
||||
call = completions.calls[0]
|
||||
assert call["model"] == "test-model"
|
||||
assert call["timeout"] == 2.5
|
||||
assert call["max_completion_tokens"] == 1024
|
||||
assert "max_tokens" not in call
|
||||
assert call["tool_choice"] == "required"
|
||||
assert call["parallel_tool_calls"] is False
|
||||
assert call["tools"] == [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "tap",
|
||||
"description": TAP_SPEC.description,
|
||||
"parameters": TAP_SPEC.parameters,
|
||||
},
|
||||
},
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "finish_task",
|
||||
"description": FINISH_TASK_SPEC.description,
|
||||
"parameters": FINISH_TASK_SPEC.parameters,
|
||||
},
|
||||
},
|
||||
]
|
||||
assert call["messages"] == [
|
||||
{"role": "system", "content": "system"},
|
||||
{"role": "user", "content": "user"},
|
||||
]
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_includes_image_block_when_screenshot_present() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [
|
||||
{
|
||||
"message": {
|
||||
"tool_calls": [
|
||||
{
|
||||
"function": {
|
||||
"name": "finish_task",
|
||||
"arguments": '{"success": true, "reason": "done"}',
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
|
||||
client.decide(
|
||||
system_prompt="system",
|
||||
user_prompt="user",
|
||||
screenshot=PNG_10X20,
|
||||
tools=[FINISH_TASK_SPEC],
|
||||
timeout=1,
|
||||
)
|
||||
|
||||
user_message = completions.calls[0]["messages"][1]
|
||||
assert user_message["role"] == "user"
|
||||
assert user_message["content"][0] == {"type": "text", "text": "user"}
|
||||
encoded = base64.b64encode(PNG_10X20).decode("ascii")
|
||||
assert user_message["content"][1] == {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{encoded}"},
|
||||
}
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_accepts_arguments_already_as_dict() -> None:
|
||||
completions = FakeCompletions(
|
||||
response={
|
||||
"choices": [{"message": {"tool_calls": [{"function": {"name": "tap", "arguments": {"x": 1, "y": 2}}}]}}]
|
||||
}
|
||||
)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
|
||||
decision = client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
|
||||
assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
|
||||
|
||||
|
||||
def test_openai_tool_calling_client_wraps_transport_errors() -> None:
|
||||
completions = FakeCompletions(error=TimeoutError("timed out"))
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response",
|
||||
[
|
||||
{"choices": []},
|
||||
{"choices": [{"message": {"tool_calls": []}}]},
|
||||
{"choices": [{"message": {"tool_calls": [{"function": {"name": "tap", "arguments": "not-json"}}]}}]},
|
||||
],
|
||||
)
|
||||
def test_openai_tool_calling_client_wraps_malformed_responses(response: object) -> None:
|
||||
completions = FakeCompletions(response=response)
|
||||
client = OpenAIToolCallingClient(model="test-model", transport=FakeOpenAITransport(completions))
|
||||
|
||||
with pytest.raises(ToolCallUnavailable):
|
||||
client.decide(system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1)
|
||||
|
||||
|
||||
# --- build_client ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_build_client_selects_provider_and_resolves_default_model() -> None:
|
||||
anthropic_client = build_client(PlannerConfig(provider="anthropic", model=""))
|
||||
assert isinstance(anthropic_client, AnthropicToolCallingClient)
|
||||
assert anthropic_client.model == "claude-sonnet-5"
|
||||
|
||||
openai_client = build_client(PlannerConfig(provider="openai", model=""))
|
||||
assert isinstance(openai_client, OpenAIToolCallingClient)
|
||||
assert openai_client.model == "gpt-5.6"
|
||||
|
||||
|
||||
def test_build_client_honors_explicit_model_override() -> None:
|
||||
client = build_client(PlannerConfig(provider="openai", model="gpt-5.6-custom"))
|
||||
assert client.model == "gpt-5.6-custom"
|
||||
@@ -0,0 +1,72 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from runtime.tool_specs import (
|
||||
ACTION_TOOL_SPECS,
|
||||
ALL_TOOL_SPECS,
|
||||
FINISH_TASK_SPEC,
|
||||
INPUT_TEXT_SPEC,
|
||||
LAUNCH_APP_SPEC,
|
||||
SWIPE_SPEC,
|
||||
TAP_SPEC,
|
||||
TERMINATE_APP_SPEC,
|
||||
ToolSpec,
|
||||
)
|
||||
|
||||
|
||||
def test_action_tool_specs_has_five_entries_and_all_tool_specs_adds_finish_task() -> None:
|
||||
assert len(ACTION_TOOL_SPECS) == 5
|
||||
assert len(ALL_TOOL_SPECS) == 6
|
||||
assert ALL_TOOL_SPECS == [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]
|
||||
assert FINISH_TASK_SPEC not in ACTION_TOOL_SPECS
|
||||
|
||||
|
||||
def test_all_tool_spec_names_are_unique() -> None:
|
||||
names = [spec.name for spec in ALL_TOOL_SPECS]
|
||||
assert len(names) == len(set(names))
|
||||
|
||||
|
||||
def test_every_tool_spec_schema_forbids_additional_properties() -> None:
|
||||
for spec in ALL_TOOL_SPECS:
|
||||
assert isinstance(spec, ToolSpec)
|
||||
assert spec.parameters["type"] == "object"
|
||||
assert spec.parameters["additionalProperties"] is False
|
||||
|
||||
|
||||
def test_tap_spec_requires_x_and_y() -> None:
|
||||
assert TAP_SPEC.parameters["required"] == ["x", "y"]
|
||||
assert set(TAP_SPEC.parameters["properties"]) == {"x", "y"}
|
||||
|
||||
|
||||
def test_swipe_spec_requires_coordinates_and_makes_duration_optional() -> None:
|
||||
assert SWIPE_SPEC.parameters["required"] == ["start_x", "start_y", "end_x", "end_y"]
|
||||
assert set(SWIPE_SPEC.parameters["properties"]) == {
|
||||
"start_x",
|
||||
"start_y",
|
||||
"end_x",
|
||||
"end_y",
|
||||
"duration_ms",
|
||||
}
|
||||
assert "duration_ms" not in SWIPE_SPEC.parameters["required"]
|
||||
assert SWIPE_SPEC.parameters["properties"]["duration_ms"]["default"] == 500
|
||||
|
||||
|
||||
def test_input_text_spec_requires_text() -> None:
|
||||
assert INPUT_TEXT_SPEC.parameters["required"] == ["text"]
|
||||
assert set(INPUT_TEXT_SPEC.parameters["properties"]) == {"text"}
|
||||
|
||||
|
||||
def test_launch_and_terminate_app_specs_require_app_id() -> None:
|
||||
for spec in (LAUNCH_APP_SPEC, TERMINATE_APP_SPEC):
|
||||
assert spec.parameters["required"] == ["app_id"]
|
||||
assert set(spec.parameters["properties"]) == {"app_id"}
|
||||
|
||||
|
||||
def test_finish_task_spec_requires_success_and_reason() -> None:
|
||||
assert FINISH_TASK_SPEC.parameters["required"] == ["success", "reason"]
|
||||
assert set(FINISH_TASK_SPEC.parameters["properties"]) == {"success", "reason"}
|
||||
assert FINISH_TASK_SPEC.parameters["properties"]["success"]["type"] == "boolean"
|
||||
|
||||
|
||||
def test_no_tool_spec_declares_device_id() -> None:
|
||||
for spec in ALL_TOOL_SPECS:
|
||||
assert "device_id" not in spec.parameters["properties"]
|
||||
Reference in New Issue
Block a user