diff --git a/runtime/executor.py b/runtime/executor.py index c3666ca..6f468fa 100644 --- a/runtime/executor.py +++ b/runtime/executor.py @@ -113,10 +113,12 @@ def default_tool_registry( ) -> dict[str, ToolCallable]: from runtime.describe_screen_semantic import describe_screen_semantic from tools.describe_screen import describe_screen + from tools.double_tap import double_tap from tools.find_icon import find_icon, find_icon_on_screen from tools.find_text import find_text, find_text_on_screen from tools.input_text import input_text from tools.launch_app import launch_app, terminate_app + from tools.long_press import long_press from tools.screenshot import take_screenshot from tools.swipe import swipe from tools.tap import tap @@ -126,6 +128,8 @@ def default_tool_registry( "take_screenshot": _bind_manager(take_screenshot, manager), "screenshot": _bind_manager(take_screenshot, manager), "tap": _bind_manager(tap, manager), + "long_press": _bind_manager(long_press, manager), + "double_tap": _bind_manager(double_tap, manager), "swipe": _bind_manager(swipe, manager), "input_text": _bind_manager(input_text, manager), "launch_app": _bind_manager(launch_app, manager), diff --git a/runtime/planner_prompts.py b/runtime/planner_prompts.py index fda6f6c..8e1fda1 100644 --- a/runtime/planner_prompts.py +++ b/runtime/planner_prompts.py @@ -26,8 +26,10 @@ Before calling a tool, output a short text block (1-2 sentences): Keep this reflection concise and factual. You must then call exactly one tool: -- One of `tap`, `swipe`, `input_text`, `launch_app`, `terminate_app` to make - progress toward the goal. +- One of `tap`, `long_press`, `double_tap`, `swipe`, `input_text`, + `launch_app`, `terminate_app` to make progress toward the goal. Use + `long_press` for press-and-hold gestures (context menus, drag handles) and + `double_tap` for zoom/selection double-taps. - `finish_task` when the goal has been reached, or when it cannot be reached and no further action would help. diff --git a/runtime/tool_specs.py b/runtime/tool_specs.py index d3217e9..0b3a5f8 100644 --- a/runtime/tool_specs.py +++ b/runtime/tool_specs.py @@ -80,6 +80,40 @@ SWIPE_SPEC = ToolSpec( ), ) +LONG_PRESS_SPEC = ToolSpec( + name="long_press", + description="Press and hold a point on the screen, given in Scene pixel coordinates.", + parameters=_action_parameters( + required=["x", "y"], + properties={ + "x": {"type": "number", "description": "X coordinate in Scene pixel space."}, + "y": {"type": "number", "description": "Y coordinate in Scene pixel space."}, + "duration_ms": { + "type": "integer", + "description": "Hold duration in milliseconds (ignored on some platforms).", + "default": 1200, + }, + }, + ), +) + +DOUBLE_TAP_SPEC = ToolSpec( + name="double_tap", + description="Tap a point twice quickly, given in Scene pixel coordinates.", + parameters=_action_parameters( + required=["x", "y"], + properties={ + "x": {"type": "number", "description": "X coordinate in Scene pixel space."}, + "y": {"type": "number", "description": "Y coordinate in Scene pixel space."}, + "interval_ms": { + "type": "integer", + "description": "Milliseconds between the two taps.", + "default": 80, + }, + }, + ), +) + INPUT_TEXT_SPEC = ToolSpec( name="input_text", description="Type text into the currently focused input field.", @@ -145,6 +179,8 @@ FINISH_TASK_SPEC = ToolSpec( ACTION_TOOL_SPECS: list[ToolSpec] = [ TAP_SPEC, + LONG_PRESS_SPEC, + DOUBLE_TAP_SPEC, SWIPE_SPEC, INPUT_TEXT_SPEC, LAUNCH_APP_SPEC, diff --git a/tests/test_executor.py b/tests/test_executor.py index 72d443c..8e6894e 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -52,3 +52,11 @@ def test_executor_keeps_action_metadata_out_of_device_tool_arguments() -> None: assert ( result.to_dict()["step"]["expected_outcome"] == "The settings page is visible." ) + + +def test_default_registry_dispatches_long_press_and_double_tap(): + from runtime.executor import default_tool_registry + + registry = default_tool_registry() + assert callable(registry["long_press"]) + assert callable(registry["double_tap"]) diff --git a/tests/test_tool_specs.py b/tests/test_tool_specs.py index 05d6cb2..7529e83 100644 --- a/tests/test_tool_specs.py +++ b/tests/test_tool_specs.py @@ -16,8 +16,8 @@ from runtime.tool_specs import ( def test_action_tool_specs_has_five_entries_and_all_tool_specs_adds_finish_task() -> ( None ): - assert len(ACTION_TOOL_SPECS) == 5 - assert len(ALL_TOOL_SPECS) == 6 + assert len(ACTION_TOOL_SPECS) == 7 + assert len(ALL_TOOL_SPECS) == 8 assert ALL_TOOL_SPECS == [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC] assert FINISH_TASK_SPEC not in ACTION_TOOL_SPECS @@ -110,3 +110,28 @@ def test_finish_task_spec_requires_success_and_reason() -> None: def test_no_tool_spec_declares_device_id() -> None: for spec in ALL_TOOL_SPECS: assert "device_id" not in spec.parameters["properties"] + + +def test_long_press_and_double_tap_are_action_tools(): + from runtime.tool_specs import ACTION_TOOL_NAMES + + assert "long_press" in ACTION_TOOL_NAMES + assert "double_tap" in ACTION_TOOL_NAMES + + +def test_long_press_spec_has_duration_with_default(): + from runtime.tool_specs import LONG_PRESS_SPEC + + props = LONG_PRESS_SPEC.parameters["properties"] + assert props["x"]["type"] == "number" + assert props["y"]["type"] == "number" + assert props["duration_ms"]["default"] == 1200 + # purpose/expected_outcome are required metadata + assert "purpose" in LONG_PRESS_SPEC.parameters["required"] + + +def test_double_tap_spec_has_interval_with_default(): + from runtime.tool_specs import DOUBLE_TAP_SPEC + + props = DOUBLE_TAP_SPEC.parameters["properties"] + assert props["interval_ms"]["default"] == 80