from __future__ import annotations from dataclasses import dataclass from typing import Any @dataclass(frozen=True) class ToolSpec: name: str description: str parameters: dict[str, Any] TAP_SPEC = ToolSpec( name="tap", description="Tap a point on the screen, given in Scene pixel coordinates.", parameters={ "type": "object", "additionalProperties": False, "required": ["x", "y"], "properties": { "x": {"type": "number", "description": "X coordinate in Scene pixel space."}, "y": {"type": "number", "description": "Y coordinate in Scene pixel space."}, }, }, ) SWIPE_SPEC = ToolSpec( name="swipe", description=( "Swipe from a start point to an end point on the screen, given in " "Scene pixel coordinates." ), parameters={ "type": "object", "additionalProperties": False, "required": ["start_x", "start_y", "end_x", "end_y"], "properties": { "start_x": {"type": "number", "description": "Start X coordinate."}, "start_y": {"type": "number", "description": "Start Y coordinate."}, "end_x": {"type": "number", "description": "End X coordinate."}, "end_y": {"type": "number", "description": "End Y coordinate."}, "duration_ms": { "type": "integer", "description": "Swipe duration in milliseconds.", "default": 500, }, }, }, ) INPUT_TEXT_SPEC = ToolSpec( name="input_text", description="Type text into the currently focused input field.", parameters={ "type": "object", "additionalProperties": False, "required": ["text"], "properties": { "text": {"type": "string", "description": "Text to type."}, }, }, ) LAUNCH_APP_SPEC = ToolSpec( name="launch_app", description="Launch (foreground) an app by its bundle/package identifier.", parameters={ "type": "object", "additionalProperties": False, "required": ["app_id"], "properties": { "app_id": { "type": "string", "description": "App bundle/package identifier.", }, }, }, ) TERMINATE_APP_SPEC = ToolSpec( name="terminate_app", description="Terminate a running app by its bundle/package identifier.", parameters={ "type": "object", "additionalProperties": False, "required": ["app_id"], "properties": { "app_id": { "type": "string", "description": "App bundle/package identifier.", }, }, }, ) FINISH_TASK_SPEC = ToolSpec( name="finish_task", description=( "Signal that the task is finished: either the goal has been reached, " "or it cannot be reached and no further steps should be attempted. " "Call this instead of any other tool once you are done." ), parameters={ "type": "object", "additionalProperties": False, "required": ["success", "reason"], "properties": { "success": { "type": "boolean", "description": "True if the goal was reached, False otherwise.", }, "reason": { "type": "string", "description": "Short explanation of why the task is finished.", }, }, }, ) ACTION_TOOL_SPECS: list[ToolSpec] = [ TAP_SPEC, SWIPE_SPEC, INPUT_TEXT_SPEC, LAUNCH_APP_SPEC, TERMINATE_APP_SPEC, ] ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]