feat(agent-runtime): add LLM-driven AI Planner with dual-provider tool calling

Replaces the stub Planner's fixed describe_screen/[] behavior with a real
decision-maker: AIPlanner uses native tool/function calling (Anthropic or
OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded
action per turn, with an explicit finish_task(success, reason) tool for
completion/failure instead of an ambiguous "no tool call" signal. Default
disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to
the existing stub Planner unchanged when disabled.

Amends CONSTITUTION.md's Perception Boundary with one narrow exception:
only the AI Planner may receive the current step's raw screenshot bytes
alongside Scene, for vision-grounded coordinate grounding. Also fixes a
latent gap in TaskRunner.run(): observe/plan exceptions are now caught per
iteration and turned into a failed task with a failure_reason, instead of
propagating uncaught.

openspec change: ai-planner-runtime.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

runtime/ai_planner.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from core.errors import TaskFailedError
+from core.models import Scene
+from runtime.context import TaskContext
+from runtime.planner import PlannedStep, Planner
+from runtime.planner_config import PlannerConfig, load_config
+from runtime.planner_prompts import PLANNER_SYSTEM_PROMPT, planner_user_prompt
+from runtime.tool_calling_client import ToolCallingClient, build_client
+from runtime.tool_specs import ALL_TOOL_SPECS
+
+if TYPE_CHECKING:
+    from world.models import WorldState
+
+FINISH_TASK_TOOL = "finish_task"
+
+
+class AIPlanner(Planner):
+    def __init__(
+        self,
+        *,
+        client: ToolCallingClient | None = None,
+        config: PlannerConfig | None = None,
+    ) -> None:
+        self.config = config or load_config()
+        self.client = client or build_client(self.config)
+
+    def plan(
+        self,
+        *,
+        goal: str,
+        scene: Scene,
+        context: TaskContext,
+        world: "WorldState | None" = None,
+        screenshot: bytes | None = None,
+    ) -> list[PlannedStep]:
+        decision = self.client.decide(
+            system_prompt=PLANNER_SYSTEM_PROMPT,
+            user_prompt=planner_user_prompt(
+                goal=goal,
+                scene_json=scene.to_dict(),
+                history_summary=_history_summary(world),
+            ),
+            screenshot=screenshot,
+            tools=ALL_TOOL_SPECS,
+            timeout=self.config.timeout,
+        )
+
+        if decision.tool_name == FINISH_TASK_TOOL:
+            if decision.arguments.get("success"):
+                return []
+            raise TaskFailedError(decision.arguments.get("reason") or "task failed")
+
+        return [
+            PlannedStep(
+                action=decision.tool_name,
+                description=f"AI planner: {decision.tool_name}({decision.arguments})",
+                args=dict(decision.arguments),
+            )
+        ]
+
+    def goal_reached(self, *, goal: str, scene: Scene, context: TaskContext) -> bool:
+        # Completion is signaled exclusively via the finish_task tool call
+        # (mapped to an empty plan above), never via this hook.
+        return False
+
+
+def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]:
+    if world is None:
+        return []
+    return [event.to_dict() for event in world.history]

runtime/planner.py

@@ -26,6 +26,7 @@ class Planner:
         scene: Scene,
         context: TaskContext,
         world: "WorldState | None" = None,
+        screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
         if context.step_results:
             return []

runtime/planner_config.py

@@ -0,0 +1,63 @@
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from dataclasses import dataclass
+
+DEFAULT_PROVIDER = "anthropic"
+DEFAULT_MODEL_BY_PROVIDER = {
+    "anthropic": "claude-sonnet-5",
+    "openai": "gpt-5.6",
+}
+DEFAULT_TIMEOUT_SECONDS = 30.0
+
+ENABLED_ENV = "AI_PLANNER_ENABLED"
+PROVIDER_ENV = "AI_PLANNER_PROVIDER"
+MODEL_ENV = "AI_PLANNER_MODEL"
+TIMEOUT_ENV = "AI_PLANNER_TIMEOUT_SECONDS"
+
+SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODEL_BY_PROVIDER)
+
+
+@dataclass(frozen=True)
+class PlannerConfig:
+    enabled: bool = False
+    provider: str = DEFAULT_PROVIDER
+    model: str = ""
+    timeout: float = DEFAULT_TIMEOUT_SECONDS
+
+    def resolved_model(self) -> str:
+        return self.model or DEFAULT_MODEL_BY_PROVIDER[self.provider]
+
+
+def load_config(env: Mapping[str, str] | None = None) -> PlannerConfig:
+    values = env or os.environ
+    return PlannerConfig(
+        enabled=_parse_bool(values.get(ENABLED_ENV), default=False),
+        provider=_parse_provider(values.get(PROVIDER_ENV)),
+        model=values.get(MODEL_ENV) or "",
+        timeout=_parse_timeout(values.get(TIMEOUT_ENV)),
+    )
+
+
+def _parse_bool(value: str | None, *, default: bool) -> bool:
+    if value is None:
+        return default
+    return value.strip().lower() in {"1", "true", "yes", "on", "enabled"}
+
+
+def _parse_provider(value: str | None) -> str:
+    if value is None:
+        return DEFAULT_PROVIDER
+    provider = value.strip().lower()
+    return provider if provider in SUPPORTED_PROVIDERS else DEFAULT_PROVIDER
+
+
+def _parse_timeout(value: str | None) -> float:
+    if value is None:
+        return DEFAULT_TIMEOUT_SECONDS
+    try:
+        timeout = float(value)
+    except ValueError:
+        return DEFAULT_TIMEOUT_SECONDS
+    return timeout if timeout > 0 else DEFAULT_TIMEOUT_SECONDS

runtime/planner_prompts.py

@@ -0,0 +1,42 @@
+from __future__ import annotations
+
+import json
+from typing import Any
+
+PLANNER_SYSTEM_PROMPT = """You are the planning brain of a mobile device automation agent.
+
+Each turn you are given a goal, the current screen as a structured Scene (a
+list of UI elements with id, type, text, and pixel bounds), and — when
+available — a screenshot of the same screen and a short history of recent
+actions and their outcomes.
+
+You must call exactly one tool per turn:
+- One of `tap`, `swipe`, `input_text`, `launch_app`, `terminate_app` to make
+  progress toward the goal.
+- `finish_task` when the goal has been reached, or when it cannot be reached
+  and no further action would help.
+
+Ground every coordinate you choose in the Scene element bounds (and the
+screenshot, if provided) for the current turn only — never reuse coordinates
+from history, since the screen may have changed. Only call `finish_task` with
+`success=True` when the current Scene shows the goal has actually been
+reached. Call it with `success=False` and a clear `reason` if you are stuck,
+repeating the same action without progress, or the goal is not achievable.
+"""
+
+
+def planner_user_prompt(
+    *,
+    goal: str,
+    scene_json: dict[str, Any],
+    history_summary: list[dict[str, Any]],
+) -> str:
+    return (
+        "Goal:\n"
+        f"{goal}\n\n"
+        "Current Scene (JSON):\n"
+        f"{json.dumps(scene_json, ensure_ascii=False, sort_keys=True)}\n\n"
+        "Recent history, oldest first (JSON):\n"
+        f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n"
+        "Call exactly one tool for this turn."
+    )

runtime/task.py

@@ -6,9 +6,11 @@ from dataclasses import dataclass, replace
 from inspect import Parameter, signature
 
 from core.models import Scene, Task, utc_now
+from runtime.ai_planner import AIPlanner
 from runtime.context import TaskContext
 from runtime.executor import Executor
 from runtime.planner import PlannedStep, Planner
+from runtime.planner_config import PlannerConfig, load_config as load_planner_config
 from semantic.models import SemanticScene
 from skills_learning.config import (
     SkillAuthoringConfig,
@@ -56,8 +58,10 @@ class TaskRunner:
         skill_authoring_config: SkillAuthoringConfig | None = None,
         skill_store: SkillStore | None = None,
         skill_embedding_client: EmbeddingClient | None = None,
+        planner_config: PlannerConfig | None = None,
     ) -> None:
-        self.planner = planner or Planner()
+        self.planner_config = planner_config or load_planner_config()
+        self.planner = planner or self._default_planner()
         self.executor = executor or Executor()
         self.timeline = timeline
         self.metadata_store = metadata_store
@@ -93,9 +97,20 @@ class TaskRunner:
         self._update_task(task, status="running")
 
         for _ in range(self.config.max_steps):
-            scene = self.observer(task.device_id)
-            context.add_scene(scene)
-            steps = self._plan(task.goal, scene, context)
+            try:
+                scene = self.observer(task.device_id)
+                context.add_scene(scene)
+                screenshot = self._planning_screenshot(task.device_id)
+                steps = self._plan(task.goal, scene, context, screenshot=screenshot)
+            except Exception as exc:
+                reason = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
+                self._update_task(
+                    task,
+                    status="failed",
+                    completed=True,
+                    failure_reason=reason,
+                )
+                return task
             if not steps or self.planner.goal_reached(
                 goal=task.goal,
                 scene=scene,
@@ -195,31 +210,41 @@ class TaskRunner:
                 model_name=self.skill_authoring_config.embedding_model,
             )
 
+    def _default_planner(self) -> Planner:
+        if self.planner_config.enabled:
+            return AIPlanner(config=self.planner_config)
+        return Planner()
+
     def _plan(
         self,
         goal: str,
         scene: Scene,
         context: TaskContext,
+        screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
-        if context.world is not None and self._planner_accepts_world():
-            return self.planner.plan(
-                goal=goal,
-                scene=scene,
-                context=context,
-                world=context.world,
-            )
-        return self.planner.plan(goal=goal, scene=scene, context=context)
+        kwargs: dict[str, object] = {}
+        if context.world is not None and self._planner_accepts("world"):
+            kwargs["world"] = context.world
+        if screenshot is not None and self._planner_accepts("screenshot"):
+            kwargs["screenshot"] = screenshot
+        return self.planner.plan(goal=goal, scene=scene, context=context, **kwargs)
 
-    def _planner_accepts_world(self) -> bool:
+    def _planner_accepts(self, name: str) -> bool:
         try:
             parameters = signature(self.planner.plan).parameters
         except (TypeError, ValueError):
             return True
-        return "world" in parameters or any(
+        return name in parameters or any(
             parameter.kind is Parameter.VAR_KEYWORD
             for parameter in parameters.values()
         )
 
+    def _planning_screenshot(self, device_id: str) -> bytes | None:
+        try:
+            return self.screenshot_provider(device_id)
+        except Exception:
+            return None
+
     def _update_world(
         self,
         world_handle: TaskWorldView | None,
@@ -255,10 +280,7 @@ class TaskRunner:
     ) -> None:
         if not self.timeline:
             return
-        try:
-            screenshot = self.screenshot_provider(task.device_id)
-        except Exception:
-            screenshot = None
+        screenshot = self._planning_screenshot(task.device_id)
         self.timeline.append(
             task_id=task.id,
             scene=scene.to_dict(),

runtime/tool_calling_client.py

@@ -0,0 +1,298 @@
+from __future__ import annotations
+
+import base64
+import json
+from dataclasses import dataclass
+from typing import Any, Protocol
+
+from runtime.planner_config import PlannerConfig
+from runtime.tool_specs import ToolSpec
+
+
+class ToolCallUnavailable(Exception):
+    """Internal signal for expected tool-calling transport/response failures."""
+
+
+@dataclass(frozen=True)
+class ToolCallDecision:
+    tool_name: str
+    arguments: dict[str, Any]
+
+
+class ToolCallingClient(Protocol):
+    def decide(
+        self,
+        *,
+        system_prompt: str,
+        user_prompt: str,
+        screenshot: bytes | None,
+        tools: list[ToolSpec],
+        timeout: float,
+    ) -> ToolCallDecision: ...
+
+
+class AnthropicToolCallingClient:
+    def __init__(
+        self,
+        *,
+        model: str,
+        transport: Any | None = None,
+        max_tokens: int = 1024,
+    ) -> None:
+        self.model = model
+        self._transport = transport
+        self.max_tokens = max_tokens
+
+    def decide(
+        self,
+        *,
+        system_prompt: str,
+        user_prompt: str,
+        screenshot: bytes | None,
+        tools: list[ToolSpec],
+        timeout: float,
+    ) -> ToolCallDecision:
+        try:
+            response = self._create_message(
+                system_prompt,
+                user_prompt,
+                screenshot,
+                tools,
+                timeout=timeout,
+            )
+            return _decision_from_anthropic_response(response)
+        except ToolCallUnavailable:
+            raise
+        except Exception as exc:
+            raise ToolCallUnavailable(str(exc)) from exc
+
+    def _create_message(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        screenshot: bytes | None,
+        tools: list[ToolSpec],
+        *,
+        timeout: float,
+    ) -> Any:
+        client = self._client()
+        kwargs = {
+            "model": self.model,
+            "max_tokens": self.max_tokens,
+            "timeout": timeout,
+            "system": [
+                {
+                    "type": "text",
+                    "text": system_prompt,
+                    "cache_control": {"type": "ephemeral"},
+                }
+            ],
+            "messages": [
+                {
+                    "role": "user",
+                    "content": _anthropic_content(user_prompt, screenshot),
+                }
+            ],
+            "tools": [_anthropic_tool(spec) for spec in tools],
+            "tool_choice": {"type": "any", "disable_parallel_tool_use": True},
+        }
+        messages = getattr(client, "messages", None)
+        if messages is not None:
+            return messages.create(**kwargs)
+        return client.create(**kwargs)
+
+    def _client(self) -> Any:
+        if self._transport is not None:
+            return self._transport
+
+        try:
+            import anthropic
+        except Exception as exc:
+            raise ToolCallUnavailable("anthropic SDK is unavailable") from exc
+
+        self._transport = anthropic.Anthropic()
+        return self._transport
+
+
+class OpenAIToolCallingClient:
+    def __init__(
+        self,
+        *,
+        model: str,
+        transport: Any | None = None,
+        max_tokens: int = 1024,
+    ) -> None:
+        self.model = model
+        self._transport = transport
+        self.max_tokens = max_tokens
+
+    def decide(
+        self,
+        *,
+        system_prompt: str,
+        user_prompt: str,
+        screenshot: bytes | None,
+        tools: list[ToolSpec],
+        timeout: float,
+    ) -> ToolCallDecision:
+        try:
+            response = self._create_completion(
+                system_prompt,
+                user_prompt,
+                screenshot,
+                tools,
+                timeout=timeout,
+            )
+            return _decision_from_openai_response(response)
+        except ToolCallUnavailable:
+            raise
+        except Exception as exc:
+            raise ToolCallUnavailable(str(exc)) from exc
+
+    def _create_completion(
+        self,
+        system_prompt: str,
+        user_prompt: str,
+        screenshot: bytes | None,
+        tools: list[ToolSpec],
+        *,
+        timeout: float,
+    ) -> Any:
+        client = self._client()
+        kwargs = {
+            "model": self.model,
+            "max_completion_tokens": self.max_tokens,
+            "timeout": timeout,
+            "messages": [
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": _openai_content(user_prompt, screenshot)},
+            ],
+            "tools": [_openai_tool(spec) for spec in tools],
+            "tool_choice": "required",
+            "parallel_tool_calls": False,
+        }
+        chat = getattr(client, "chat", None)
+        if chat is not None:
+            return chat.completions.create(**kwargs)
+        return client.create(**kwargs)
+
+    def _client(self) -> Any:
+        if self._transport is not None:
+            return self._transport
+
+        try:
+            from openai import OpenAI
+        except Exception as exc:
+            raise ToolCallUnavailable("openai SDK is unavailable") from exc
+
+        self._transport = OpenAI()
+        return self._transport
+
+
+def build_client(config: PlannerConfig) -> ToolCallingClient:
+    model = config.resolved_model()
+    if config.provider == "openai":
+        return OpenAIToolCallingClient(model=model)
+    return AnthropicToolCallingClient(model=model)
+
+
+def _anthropic_content(
+    user_prompt: str,
+    screenshot: bytes | None,
+) -> list[dict[str, Any]]:
+    content: list[dict[str, Any]] = []
+    if screenshot is not None:
+        content.append(
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": "image/png",
+                    "data": base64.b64encode(screenshot).decode("ascii"),
+                },
+            }
+        )
+    content.append({"type": "text", "text": user_prompt})
+    return content
+
+
+def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
+    return {
+        "name": spec.name,
+        "description": spec.description,
+        "input_schema": spec.parameters,
+    }
+
+
+def _decision_from_anthropic_response(response: Any) -> ToolCallDecision:
+    content = _value(response, "content")
+    if not isinstance(content, list):
+        raise ValueError("anthropic tool-call response missing content list")
+    for block in content:
+        if _value(block, "type") != "tool_use":
+            continue
+        name = _value(block, "name")
+        arguments = _value(block, "input")
+        if isinstance(name, str) and isinstance(arguments, dict):
+            return ToolCallDecision(tool_name=name, arguments=arguments)
+    raise ValueError("anthropic response did not include a tool_use block")
+
+
+def _openai_content(
+    user_prompt: str,
+    screenshot: bytes | None,
+) -> str | list[dict[str, Any]]:
+    if screenshot is None:
+        return user_prompt
+    encoded = base64.b64encode(screenshot).decode("ascii")
+    return [
+        {"type": "text", "text": user_prompt},
+        {
+            "type": "image_url",
+            "image_url": {"url": f"data:image/png;base64,{encoded}"},
+        },
+    ]
+
+
+def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
+    return {
+        "type": "function",
+        "function": {
+            "name": spec.name,
+            "description": spec.description,
+            "parameters": spec.parameters,
+        },
+    }
+
+
+def _decision_from_openai_response(response: Any) -> ToolCallDecision:
+    choices = _value(response, "choices")
+    if not isinstance(choices, list) or not choices:
+        raise ValueError("openai tool-call response missing choices")
+    message = _value(choices[0], "message")
+    tool_calls = _value(message, "tool_calls")
+    if not isinstance(tool_calls, list) or not tool_calls:
+        raise ValueError("openai response did not include a tool call")
+    function = _value(tool_calls[0], "function")
+    name = _value(function, "name")
+    if not isinstance(name, str):
+        raise ValueError("openai tool call missing a function name")
+    arguments = _decode_openai_arguments(_value(function, "arguments"))
+    return ToolCallDecision(tool_name=name, arguments=arguments)
+
+
+def _decode_openai_arguments(raw_arguments: Any) -> dict[str, Any]:
+    if isinstance(raw_arguments, dict):
+        return raw_arguments
+    if isinstance(raw_arguments, str):
+        decoded = json.loads(raw_arguments)
+        if isinstance(decoded, dict):
+            return decoded
+    raise ValueError("openai tool call arguments must decode to a JSON object")
+
+
+def _value(source: Any, key: str) -> Any:
+    if isinstance(source, dict):
+        return source.get(key)
+    value = getattr(source, key, None)
+    return None if callable(value) else value

runtime/tool_specs.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ToolSpec:
+    name: str
+    description: str
+    parameters: dict[str, Any]
+
+
+TAP_SPEC = ToolSpec(
+    name="tap",
+    description="Tap a point on the screen, given in Scene pixel coordinates.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["x", "y"],
+        "properties": {
+            "x": {"type": "number", "description": "X coordinate in Scene pixel space."},
+            "y": {"type": "number", "description": "Y coordinate in Scene pixel space."},
+        },
+    },
+)
+
+SWIPE_SPEC = ToolSpec(
+    name="swipe",
+    description=(
+        "Swipe from a start point to an end point on the screen, given in "
+        "Scene pixel coordinates."
+    ),
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["start_x", "start_y", "end_x", "end_y"],
+        "properties": {
+            "start_x": {"type": "number", "description": "Start X coordinate."},
+            "start_y": {"type": "number", "description": "Start Y coordinate."},
+            "end_x": {"type": "number", "description": "End X coordinate."},
+            "end_y": {"type": "number", "description": "End Y coordinate."},
+            "duration_ms": {
+                "type": "integer",
+                "description": "Swipe duration in milliseconds.",
+                "default": 500,
+            },
+        },
+    },
+)
+
+INPUT_TEXT_SPEC = ToolSpec(
+    name="input_text",
+    description="Type text into the currently focused input field.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["text"],
+        "properties": {
+            "text": {"type": "string", "description": "Text to type."},
+        },
+    },
+)
+
+LAUNCH_APP_SPEC = ToolSpec(
+    name="launch_app",
+    description="Launch (foreground) an app by its bundle/package identifier.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["app_id"],
+        "properties": {
+            "app_id": {
+                "type": "string",
+                "description": "App bundle/package identifier.",
+            },
+        },
+    },
+)
+
+TERMINATE_APP_SPEC = ToolSpec(
+    name="terminate_app",
+    description="Terminate a running app by its bundle/package identifier.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["app_id"],
+        "properties": {
+            "app_id": {
+                "type": "string",
+                "description": "App bundle/package identifier.",
+            },
+        },
+    },
+)
+
+FINISH_TASK_SPEC = ToolSpec(
+    name="finish_task",
+    description=(
+        "Signal that the task is finished: either the goal has been reached, "
+        "or it cannot be reached and no further steps should be attempted. "
+        "Call this instead of any other tool once you are done."
+    ),
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["success", "reason"],
+        "properties": {
+            "success": {
+                "type": "boolean",
+                "description": "True if the goal was reached, False otherwise.",
+            },
+            "reason": {
+                "type": "string",
+                "description": "Short explanation of why the task is finished.",
+            },
+        },
+    },
+)
+
+ACTION_TOOL_SPECS: list[ToolSpec] = [
+    TAP_SPEC,
+    SWIPE_SPEC,
+    INPUT_TEXT_SPEC,
+    LAUNCH_APP_SPEC,
+    TERMINATE_APP_SPEC,
+]
+
+ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]