q792602257andClaude Sonnet 5
61ff3b425d
Replaces the stub Planner's fixed describe_screen/[] behavior with a real decision-maker: AIPlanner uses native tool/function calling (Anthropic or OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded action per turn, with an explicit finish_task(success, reason) tool for completion/failure instead of an ambiguous "no tool call" signal. Default disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to the existing stub Planner unchanged when disabled. Amends CONSTITUTION.md's Perception Boundary with one narrow exception: only the AI Planner may receive the current step's raw screenshot bytes alongside Scene, for vision-grounded coordinate grounding. Also fixes a latent gap in TaskRunner.run(): observe/plan exceptions are now caught per iteration and turned into a failed task with a failure_reason, instead of propagating uncaught. openspec change: ai-planner-runtime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
130 lines
3.6 KiB
Python
130 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ToolSpec:
|
|
name: str
|
|
description: str
|
|
parameters: dict[str, Any]
|
|
|
|
|
|
TAP_SPEC = ToolSpec(
|
|
name="tap",
|
|
description="Tap a point on the screen, given in Scene pixel coordinates.",
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["x", "y"],
|
|
"properties": {
|
|
"x": {"type": "number", "description": "X coordinate in Scene pixel space."},
|
|
"y": {"type": "number", "description": "Y coordinate in Scene pixel space."},
|
|
},
|
|
},
|
|
)
|
|
|
|
SWIPE_SPEC = ToolSpec(
|
|
name="swipe",
|
|
description=(
|
|
"Swipe from a start point to an end point on the screen, given in "
|
|
"Scene pixel coordinates."
|
|
),
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["start_x", "start_y", "end_x", "end_y"],
|
|
"properties": {
|
|
"start_x": {"type": "number", "description": "Start X coordinate."},
|
|
"start_y": {"type": "number", "description": "Start Y coordinate."},
|
|
"end_x": {"type": "number", "description": "End X coordinate."},
|
|
"end_y": {"type": "number", "description": "End Y coordinate."},
|
|
"duration_ms": {
|
|
"type": "integer",
|
|
"description": "Swipe duration in milliseconds.",
|
|
"default": 500,
|
|
},
|
|
},
|
|
},
|
|
)
|
|
|
|
INPUT_TEXT_SPEC = ToolSpec(
|
|
name="input_text",
|
|
description="Type text into the currently focused input field.",
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["text"],
|
|
"properties": {
|
|
"text": {"type": "string", "description": "Text to type."},
|
|
},
|
|
},
|
|
)
|
|
|
|
LAUNCH_APP_SPEC = ToolSpec(
|
|
name="launch_app",
|
|
description="Launch (foreground) an app by its bundle/package identifier.",
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["app_id"],
|
|
"properties": {
|
|
"app_id": {
|
|
"type": "string",
|
|
"description": "App bundle/package identifier.",
|
|
},
|
|
},
|
|
},
|
|
)
|
|
|
|
TERMINATE_APP_SPEC = ToolSpec(
|
|
name="terminate_app",
|
|
description="Terminate a running app by its bundle/package identifier.",
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["app_id"],
|
|
"properties": {
|
|
"app_id": {
|
|
"type": "string",
|
|
"description": "App bundle/package identifier.",
|
|
},
|
|
},
|
|
},
|
|
)
|
|
|
|
FINISH_TASK_SPEC = ToolSpec(
|
|
name="finish_task",
|
|
description=(
|
|
"Signal that the task is finished: either the goal has been reached, "
|
|
"or it cannot be reached and no further steps should be attempted. "
|
|
"Call this instead of any other tool once you are done."
|
|
),
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["success", "reason"],
|
|
"properties": {
|
|
"success": {
|
|
"type": "boolean",
|
|
"description": "True if the goal was reached, False otherwise.",
|
|
},
|
|
"reason": {
|
|
"type": "string",
|
|
"description": "Short explanation of why the task is finished.",
|
|
},
|
|
},
|
|
},
|
|
)
|
|
|
|
ACTION_TOOL_SPECS: list[ToolSpec] = [
|
|
TAP_SPEC,
|
|
SWIPE_SPEC,
|
|
INPUT_TEXT_SPEC,
|
|
LAUNCH_APP_SPEC,
|
|
TERMINATE_APP_SPEC,
|
|
]
|
|
|
|
ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]
|