157 lines
4.3 KiB
Python
157 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ToolSpec:
|
|
name: str
|
|
description: str
|
|
parameters: dict[str, Any]
|
|
|
|
|
|
ACTION_METADATA_PROPERTIES: dict[str, dict[str, Any]] = {
|
|
"purpose": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"maxLength": 240,
|
|
"description": "Concise intent for this action, used by later reuse.",
|
|
},
|
|
"expected_outcome": {
|
|
"type": "string",
|
|
"minLength": 1,
|
|
"maxLength": 240,
|
|
"description": "Observable screen state expected after this action.",
|
|
},
|
|
}
|
|
|
|
|
|
def _action_parameters(
|
|
*,
|
|
required: list[str],
|
|
properties: dict[str, dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": [*required, "purpose", "expected_outcome"],
|
|
"properties": {**properties, **ACTION_METADATA_PROPERTIES},
|
|
}
|
|
|
|
|
|
TAP_SPEC = ToolSpec(
|
|
name="tap",
|
|
description="Tap a point on the screen, given in Scene pixel coordinates.",
|
|
parameters=_action_parameters(
|
|
required=["x", "y"],
|
|
properties={
|
|
"x": {
|
|
"type": "number",
|
|
"description": "X coordinate in Scene pixel space.",
|
|
},
|
|
"y": {
|
|
"type": "number",
|
|
"description": "Y coordinate in Scene pixel space.",
|
|
},
|
|
},
|
|
),
|
|
)
|
|
|
|
SWIPE_SPEC = ToolSpec(
|
|
name="swipe",
|
|
description=(
|
|
"Swipe from a start point to an end point on the screen, given in "
|
|
"Scene pixel coordinates."
|
|
),
|
|
parameters=_action_parameters(
|
|
required=["start_x", "start_y", "end_x", "end_y"],
|
|
properties={
|
|
"start_x": {"type": "number", "description": "Start X coordinate."},
|
|
"start_y": {"type": "number", "description": "Start Y coordinate."},
|
|
"end_x": {"type": "number", "description": "End X coordinate."},
|
|
"end_y": {"type": "number", "description": "End Y coordinate."},
|
|
"duration_ms": {
|
|
"type": "integer",
|
|
"description": "Swipe duration in milliseconds.",
|
|
"default": 500,
|
|
},
|
|
},
|
|
),
|
|
)
|
|
|
|
INPUT_TEXT_SPEC = ToolSpec(
|
|
name="input_text",
|
|
description="Type text into the currently focused input field.",
|
|
parameters=_action_parameters(
|
|
required=["text"],
|
|
properties={
|
|
"text": {"type": "string", "description": "Text to type."},
|
|
},
|
|
),
|
|
)
|
|
|
|
LAUNCH_APP_SPEC = ToolSpec(
|
|
name="launch_app",
|
|
description="Launch (foreground) an app by its bundle/package identifier.",
|
|
parameters=_action_parameters(
|
|
required=["app_id"],
|
|
properties={
|
|
"app_id": {
|
|
"type": "string",
|
|
"description": "App bundle/package identifier.",
|
|
},
|
|
},
|
|
),
|
|
)
|
|
|
|
TERMINATE_APP_SPEC = ToolSpec(
|
|
name="terminate_app",
|
|
description="Terminate a running app by its bundle/package identifier.",
|
|
parameters=_action_parameters(
|
|
required=["app_id"],
|
|
properties={
|
|
"app_id": {
|
|
"type": "string",
|
|
"description": "App bundle/package identifier.",
|
|
},
|
|
},
|
|
),
|
|
)
|
|
|
|
FINISH_TASK_SPEC = ToolSpec(
|
|
name="finish_task",
|
|
description=(
|
|
"Signal that the task is finished: either the goal has been reached, "
|
|
"or it cannot be reached and no further steps should be attempted. "
|
|
"Call this instead of any other tool once you are done."
|
|
),
|
|
parameters={
|
|
"type": "object",
|
|
"additionalProperties": False,
|
|
"required": ["success", "reason"],
|
|
"properties": {
|
|
"success": {
|
|
"type": "boolean",
|
|
"description": "True if the goal was reached, False otherwise.",
|
|
},
|
|
"reason": {
|
|
"type": "string",
|
|
"description": "Short explanation of why the task is finished.",
|
|
},
|
|
},
|
|
},
|
|
)
|
|
|
|
ACTION_TOOL_SPECS: list[ToolSpec] = [
|
|
TAP_SPEC,
|
|
SWIPE_SPEC,
|
|
INPUT_TEXT_SPEC,
|
|
LAUNCH_APP_SPEC,
|
|
TERMINATE_APP_SPEC,
|
|
]
|
|
|
|
ACTION_TOOL_NAMES = frozenset(spec.name for spec in ACTION_TOOL_SPECS)
|
|
|
|
ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC]
|