from __future__ import annotations from dataclasses import dataclass from typing import Any @dataclass(frozen=True) class ToolSpec: name: str description: str parameters: dict[str, Any] ACTION_METADATA_PROPERTIES: dict[str, dict[str, Any]] = { "purpose": { "type": "string", "minLength": 1, "maxLength": 240, "description": "Concise intent for this action, used by later reuse.", }, "expected_outcome": { "type": "string", "minLength": 1, "maxLength": 240, "description": "Observable screen state expected after this action.", }, } def _action_parameters( *, required: list[str], properties: dict[str, dict[str, Any]], ) -> dict[str, Any]: return { "type": "object", "additionalProperties": False, "required": [*required, "purpose", "expected_outcome"], "properties": {**properties, **ACTION_METADATA_PROPERTIES}, } TAP_SPEC = ToolSpec( name="tap", description="Tap a point on the screen, given in Scene pixel coordinates.", parameters=_action_parameters( required=["x", "y"], properties={ "x": { "type": "number", "description": "X coordinate in Scene pixel space.", }, "y": { "type": "number", "description": "Y coordinate in Scene pixel space.", }, }, ), ) SWIPE_SPEC = ToolSpec( name="swipe", description=( "Swipe from a start point to an end point on the screen, given in " "Scene pixel coordinates." ), parameters=_action_parameters( required=["start_x", "start_y", "end_x", "end_y"], properties={ "start_x": {"type": "number", "description": "Start X coordinate."}, "start_y": {"type": "number", "description": "Start Y coordinate."}, "end_x": {"type": "number", "description": "End X coordinate."}, "end_y": {"type": "number", "description": "End Y coordinate."}, "duration_ms": { "type": "integer", "description": "Swipe duration in milliseconds.", "default": 500, }, }, ), ) INPUT_TEXT_SPEC = ToolSpec( name="input_text", description="Type text into the currently focused input field.", parameters=_action_parameters( required=["text"], properties={ "text": {"type": "string", "description": "Text to type."}, }, ), ) LAUNCH_APP_SPEC = ToolSpec( name="launch_app", description="Launch (foreground) an app by its bundle/package identifier.", parameters=_action_parameters( required=["app_id"], properties={ "app_id": { "type": "string", "description": "App bundle/package identifier.", }, }, ), ) TERMINATE_APP_SPEC = ToolSpec( name="terminate_app", description="Terminate a running app by its bundle/package identifier.", parameters=_action_parameters( required=["app_id"], properties={ "app_id": { "type": "string", "description": "App bundle/package identifier.", }, }, ), ) FINISH_TASK_SPEC = ToolSpec( name="finish_task", description=( "Signal that the task is finished: either the goal has been reached, " "or it cannot be reached and no further steps should be attempted. " "Call this instead of any other tool once you are done." ), parameters={ "type": "object", "additionalProperties": False, "required": ["success", "reason"], "properties": { "success": { "type": "boolean", "description": "True if the goal was reached, False otherwise.", }, "reason": { "type": "string", "description": "Short explanation of why the task is finished.", }, }, }, ) ACTION_TOOL_SPECS: list[ToolSpec] = [ TAP_SPEC, SWIPE_SPEC, INPUT_TEXT_SPEC, LAUNCH_APP_SPEC, TERMINATE_APP_SPEC, ] ACTION_TOOL_NAMES = frozenset(spec.name for spec in ACTION_TOOL_SPECS) ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]