feat(agent-runtime): add LLM-driven AI Planner with dual-provider tool calling

Replaces the stub Planner's fixed describe_screen/[] behavior with a real decision-maker: AIPlanner uses native tool/function calling (Anthropic or OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded action per turn, with an explicit finish_task(success, reason) tool for completion/failure instead of an ambiguous "no tool call" signal. Default disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to the existing stub Planner unchanged when disabled. Amends CONSTITUTION.md's Perception Boundary with one narrow exception: only the AI Planner may receive the current step's raw screenshot bytes alongside Scene, for vision-grounded coordinate grounding. Also fixes a latent gap in TaskRunner.run(): observe/plan exceptions are now caught per iteration and turned into a failed task with a failure_reason, instead of propagating uncaught. openspec change: ai-planner-runtime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-07-12 13:48:50 +08:00
parent b94abde92a
commit 61ff3b425d
19 changed files with 1977 additions and 19 deletions
@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class ToolSpec:
+    name: str
+    description: str
+    parameters: dict[str, Any]
+
+
+TAP_SPEC = ToolSpec(
+    name="tap",
+    description="Tap a point on the screen, given in Scene pixel coordinates.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["x", "y"],
+        "properties": {
+            "x": {"type": "number", "description": "X coordinate in Scene pixel space."},
+            "y": {"type": "number", "description": "Y coordinate in Scene pixel space."},
+        },
+    },
+)
+
+SWIPE_SPEC = ToolSpec(
+    name="swipe",
+    description=(
+        "Swipe from a start point to an end point on the screen, given in "
+        "Scene pixel coordinates."
+    ),
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["start_x", "start_y", "end_x", "end_y"],
+        "properties": {
+            "start_x": {"type": "number", "description": "Start X coordinate."},
+            "start_y": {"type": "number", "description": "Start Y coordinate."},
+            "end_x": {"type": "number", "description": "End X coordinate."},
+            "end_y": {"type": "number", "description": "End Y coordinate."},
+            "duration_ms": {
+                "type": "integer",
+                "description": "Swipe duration in milliseconds.",
+                "default": 500,
+            },
+        },
+    },
+)
+
+INPUT_TEXT_SPEC = ToolSpec(
+    name="input_text",
+    description="Type text into the currently focused input field.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["text"],
+        "properties": {
+            "text": {"type": "string", "description": "Text to type."},
+        },
+    },
+)
+
+LAUNCH_APP_SPEC = ToolSpec(
+    name="launch_app",
+    description="Launch (foreground) an app by its bundle/package identifier.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["app_id"],
+        "properties": {
+            "app_id": {
+                "type": "string",
+                "description": "App bundle/package identifier.",
+            },
+        },
+    },
+)
+
+TERMINATE_APP_SPEC = ToolSpec(
+    name="terminate_app",
+    description="Terminate a running app by its bundle/package identifier.",
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["app_id"],
+        "properties": {
+            "app_id": {
+                "type": "string",
+                "description": "App bundle/package identifier.",
+            },
+        },
+    },
+)
+
+FINISH_TASK_SPEC = ToolSpec(
+    name="finish_task",
+    description=(
+        "Signal that the task is finished: either the goal has been reached, "
+        "or it cannot be reached and no further steps should be attempted. "
+        "Call this instead of any other tool once you are done."
+    ),
+    parameters={
+        "type": "object",
+        "additionalProperties": False,
+        "required": ["success", "reason"],
+        "properties": {
+            "success": {
+                "type": "boolean",
+                "description": "True if the goal was reached, False otherwise.",
+            },
+            "reason": {
+                "type": "string",
+                "description": "Short explanation of why the task is finished.",
+            },
+        },
+    },
+)
+
+ACTION_TOOL_SPECS: list[ToolSpec] = [
+    TAP_SPEC,
+    SWIPE_SPEC,
+    INPUT_TEXT_SPEC,
+    LAUNCH_APP_SPEC,
+    TERMINATE_APP_SPEC,
+]
+
+ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]