feat(agent-runtime): add LLM-driven AI Planner with dual-provider tool calling
Replaces the stub Planner's fixed describe_screen/[] behavior with a real decision-maker: AIPlanner uses native tool/function calling (Anthropic or OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded action per turn, with an explicit finish_task(success, reason) tool for completion/failure instead of an ambiguous "no tool call" signal. Default disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to the existing stub Planner unchanged when disabled. Amends CONSTITUTION.md's Perception Boundary with one narrow exception: only the AI Planner may receive the current step's raw screenshot bytes alongside Scene, for vision-grounded coordinate grounding. Also fixes a latent gap in TaskRunner.run(): observe/plan exceptions are now caught per iteration and turned into a failed task with a failure_reason, instead of propagating uncaught. openspec change: ai-planner-runtime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
Claude Sonnet 5
@@ -0,0 +1,298 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Protocol
|
||||
|
||||
from runtime.planner_config import PlannerConfig
|
||||
from runtime.tool_specs import ToolSpec
|
||||
|
||||
|
||||
class ToolCallUnavailable(Exception):
|
||||
"""Internal signal for expected tool-calling transport/response failures."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ToolCallDecision:
|
||||
tool_name: str
|
||||
arguments: dict[str, Any]
|
||||
|
||||
|
||||
class ToolCallingClient(Protocol):
|
||||
def decide(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[ToolSpec],
|
||||
timeout: float,
|
||||
) -> ToolCallDecision: ...
|
||||
|
||||
|
||||
class AnthropicToolCallingClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
transport: Any | None = None,
|
||||
max_tokens: int = 1024,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self._transport = transport
|
||||
self.max_tokens = max_tokens
|
||||
|
||||
def decide(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[ToolSpec],
|
||||
timeout: float,
|
||||
) -> ToolCallDecision:
|
||||
try:
|
||||
response = self._create_message(
|
||||
system_prompt,
|
||||
user_prompt,
|
||||
screenshot,
|
||||
tools,
|
||||
timeout=timeout,
|
||||
)
|
||||
return _decision_from_anthropic_response(response)
|
||||
except ToolCallUnavailable:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ToolCallUnavailable(str(exc)) from exc
|
||||
|
||||
def _create_message(
|
||||
self,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[ToolSpec],
|
||||
*,
|
||||
timeout: float,
|
||||
) -> Any:
|
||||
client = self._client()
|
||||
kwargs = {
|
||||
"model": self.model,
|
||||
"max_tokens": self.max_tokens,
|
||||
"timeout": timeout,
|
||||
"system": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": system_prompt,
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
}
|
||||
],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": _anthropic_content(user_prompt, screenshot),
|
||||
}
|
||||
],
|
||||
"tools": [_anthropic_tool(spec) for spec in tools],
|
||||
"tool_choice": {"type": "any", "disable_parallel_tool_use": True},
|
||||
}
|
||||
messages = getattr(client, "messages", None)
|
||||
if messages is not None:
|
||||
return messages.create(**kwargs)
|
||||
return client.create(**kwargs)
|
||||
|
||||
def _client(self) -> Any:
|
||||
if self._transport is not None:
|
||||
return self._transport
|
||||
|
||||
try:
|
||||
import anthropic
|
||||
except Exception as exc:
|
||||
raise ToolCallUnavailable("anthropic SDK is unavailable") from exc
|
||||
|
||||
self._transport = anthropic.Anthropic()
|
||||
return self._transport
|
||||
|
||||
|
||||
class OpenAIToolCallingClient:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
model: str,
|
||||
transport: Any | None = None,
|
||||
max_tokens: int = 1024,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self._transport = transport
|
||||
self.max_tokens = max_tokens
|
||||
|
||||
def decide(
|
||||
self,
|
||||
*,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[ToolSpec],
|
||||
timeout: float,
|
||||
) -> ToolCallDecision:
|
||||
try:
|
||||
response = self._create_completion(
|
||||
system_prompt,
|
||||
user_prompt,
|
||||
screenshot,
|
||||
tools,
|
||||
timeout=timeout,
|
||||
)
|
||||
return _decision_from_openai_response(response)
|
||||
except ToolCallUnavailable:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise ToolCallUnavailable(str(exc)) from exc
|
||||
|
||||
def _create_completion(
|
||||
self,
|
||||
system_prompt: str,
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
tools: list[ToolSpec],
|
||||
*,
|
||||
timeout: float,
|
||||
) -> Any:
|
||||
client = self._client()
|
||||
kwargs = {
|
||||
"model": self.model,
|
||||
"max_completion_tokens": self.max_tokens,
|
||||
"timeout": timeout,
|
||||
"messages": [
|
||||
{"role": "system", "content": system_prompt},
|
||||
{"role": "user", "content": _openai_content(user_prompt, screenshot)},
|
||||
],
|
||||
"tools": [_openai_tool(spec) for spec in tools],
|
||||
"tool_choice": "required",
|
||||
"parallel_tool_calls": False,
|
||||
}
|
||||
chat = getattr(client, "chat", None)
|
||||
if chat is not None:
|
||||
return chat.completions.create(**kwargs)
|
||||
return client.create(**kwargs)
|
||||
|
||||
def _client(self) -> Any:
|
||||
if self._transport is not None:
|
||||
return self._transport
|
||||
|
||||
try:
|
||||
from openai import OpenAI
|
||||
except Exception as exc:
|
||||
raise ToolCallUnavailable("openai SDK is unavailable") from exc
|
||||
|
||||
self._transport = OpenAI()
|
||||
return self._transport
|
||||
|
||||
|
||||
def build_client(config: PlannerConfig) -> ToolCallingClient:
|
||||
model = config.resolved_model()
|
||||
if config.provider == "openai":
|
||||
return OpenAIToolCallingClient(model=model)
|
||||
return AnthropicToolCallingClient(model=model)
|
||||
|
||||
|
||||
def _anthropic_content(
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
) -> list[dict[str, Any]]:
|
||||
content: list[dict[str, Any]] = []
|
||||
if screenshot is not None:
|
||||
content.append(
|
||||
{
|
||||
"type": "image",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "image/png",
|
||||
"data": base64.b64encode(screenshot).decode("ascii"),
|
||||
},
|
||||
}
|
||||
)
|
||||
content.append({"type": "text", "text": user_prompt})
|
||||
return content
|
||||
|
||||
|
||||
def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
|
||||
return {
|
||||
"name": spec.name,
|
||||
"description": spec.description,
|
||||
"input_schema": spec.parameters,
|
||||
}
|
||||
|
||||
|
||||
def _decision_from_anthropic_response(response: Any) -> ToolCallDecision:
|
||||
content = _value(response, "content")
|
||||
if not isinstance(content, list):
|
||||
raise ValueError("anthropic tool-call response missing content list")
|
||||
for block in content:
|
||||
if _value(block, "type") != "tool_use":
|
||||
continue
|
||||
name = _value(block, "name")
|
||||
arguments = _value(block, "input")
|
||||
if isinstance(name, str) and isinstance(arguments, dict):
|
||||
return ToolCallDecision(tool_name=name, arguments=arguments)
|
||||
raise ValueError("anthropic response did not include a tool_use block")
|
||||
|
||||
|
||||
def _openai_content(
|
||||
user_prompt: str,
|
||||
screenshot: bytes | None,
|
||||
) -> str | list[dict[str, Any]]:
|
||||
if screenshot is None:
|
||||
return user_prompt
|
||||
encoded = base64.b64encode(screenshot).decode("ascii")
|
||||
return [
|
||||
{"type": "text", "text": user_prompt},
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": f"data:image/png;base64,{encoded}"},
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
|
||||
return {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": spec.name,
|
||||
"description": spec.description,
|
||||
"parameters": spec.parameters,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _decision_from_openai_response(response: Any) -> ToolCallDecision:
|
||||
choices = _value(response, "choices")
|
||||
if not isinstance(choices, list) or not choices:
|
||||
raise ValueError("openai tool-call response missing choices")
|
||||
message = _value(choices[0], "message")
|
||||
tool_calls = _value(message, "tool_calls")
|
||||
if not isinstance(tool_calls, list) or not tool_calls:
|
||||
raise ValueError("openai response did not include a tool call")
|
||||
function = _value(tool_calls[0], "function")
|
||||
name = _value(function, "name")
|
||||
if not isinstance(name, str):
|
||||
raise ValueError("openai tool call missing a function name")
|
||||
arguments = _decode_openai_arguments(_value(function, "arguments"))
|
||||
return ToolCallDecision(tool_name=name, arguments=arguments)
|
||||
|
||||
|
||||
def _decode_openai_arguments(raw_arguments: Any) -> dict[str, Any]:
|
||||
if isinstance(raw_arguments, dict):
|
||||
return raw_arguments
|
||||
if isinstance(raw_arguments, str):
|
||||
decoded = json.loads(raw_arguments)
|
||||
if isinstance(decoded, dict):
|
||||
return decoded
|
||||
raise ValueError("openai tool call arguments must decode to a JSON object")
|
||||
|
||||
|
||||
def _value(source: Any, key: str) -> Any:
|
||||
if isinstance(source, dict):
|
||||
return source.get(key)
|
||||
value = getattr(source, key, None)
|
||||
return None if callable(value) else value
|
||||
Reference in New Issue
Block a user