Implement Apex Agent MVP scaffold

tools/__init__.py

@@ -0,0 +1,2 @@
+"""Semantic capability wrappers over the active device driver."""
+

tools/_device.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from core.device_manager import DEFAULT_MANAGER, DeviceManager
+from core.driver import Driver
+
+
+def get_driver(
+    device_id: str | None = None,
+    *,
+    manager: DeviceManager | None = None,
+) -> Driver:
+    return (manager or DEFAULT_MANAGER).active_driver(device_id)
+

tools/describe_screen.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from core.models import Scene
+from tools._device import get_driver
+from vision.ocr import PaddleOCREngine, run_ocr
+from vision.scene_builder import build_scene, infer_png_size
+from vision.ui_parser import parse_ui_tree
+
+
+def describe_screen(
+    device_id: str | None = None,
+    *,
+    manager: DeviceManager | None = None,
+    ocr_engine: PaddleOCREngine | None = None,
+) -> Scene:
+    driver = get_driver(device_id, manager=manager)
+    screenshot = driver.screenshot()
+    raw_tree = driver.tree()
+    width, height = infer_png_size(screenshot)
+    return build_scene(
+        screen_width=width,
+        screen_height=height,
+        ui_elements=parse_ui_tree(raw_tree),
+        ocr_elements=run_ocr(screenshot, engine=ocr_engine),
+    )
+

tools/find_icon.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from core.models import Scene
+from tools.describe_screen import describe_screen
+from vision.icon_detector import find_icon as find_icon_in_scene
+
+
+def find_icon(scene: Scene, name: str) -> dict[str, object]:
+    return find_icon_in_scene(scene, name).to_dict()
+
+
+def find_icon_on_screen(
+    name: str,
+    *,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+) -> dict[str, object]:
+    return find_icon(describe_screen(device_id, manager=manager), name)

tools/find_text.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from core.models import Scene, SceneElement
+from tools.describe_screen import describe_screen
+
+
+def find_text(
+    scene: Scene,
+    query: str,
+    *,
+    exact: bool = False,
+    case_sensitive: bool = False,
+) -> dict[str, object]:
+    match = _find_element(scene, query, exact=exact, case_sensitive=case_sensitive)
+    if not match:
+        return {"found": False, "query": query, "reason": "not found"}
+    x, y = match.center
+    return {
+        "found": True,
+        "query": query,
+        "x": x,
+        "y": y,
+        "element": match.to_dict(),
+    }
+
+
+def find_text_on_screen(
+    query: str,
+    *,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+    exact: bool = False,
+    case_sensitive: bool = False,
+) -> dict[str, object]:
+    return find_text(
+        describe_screen(device_id, manager=manager),
+        query,
+        exact=exact,
+        case_sensitive=case_sensitive,
+    )
+
+
+def _find_element(
+    scene: Scene,
+    query: str,
+    *,
+    exact: bool,
+    case_sensitive: bool,
+) -> SceneElement | None:
+    needle = query if case_sensitive else query.casefold()
+    for element in scene.elements:
+        if not element.text:
+            continue
+        haystack = element.text if case_sensitive else element.text.casefold()
+        matched = haystack == needle if exact else needle in haystack
+        if matched:
+            return element
+    return None

tools/input_text.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from tools._device import get_driver
+
+
+def input_text(
+    text: str,
+    *,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+) -> dict[str, object]:
+    get_driver(device_id, manager=manager).input(text)
+    return {"ok": True, "action": "input_text", "text": text}
+

tools/launch_app.py

@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from tools._device import get_driver
+
+
+def launch_app(
+    app_id: str,
+    *,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+) -> dict[str, object]:
+    get_driver(device_id, manager=manager).launch(app_id)
+    return {"ok": True, "action": "launch_app", "app_id": app_id}
+
+
+def terminate_app(
+    app_id: str,
+    *,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+) -> dict[str, object]:
+    get_driver(device_id, manager=manager).terminate(app_id)
+    return {"ok": True, "action": "terminate_app", "app_id": app_id}
+

tools/screenshot.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from tools._device import get_driver
+
+
+def take_screenshot(
+    device_id: str | None = None,
+    *,
+    manager: DeviceManager | None = None,
+) -> bytes:
+    return get_driver(device_id, manager=manager).screenshot()
+

tools/swipe.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from tools._device import get_driver
+
+
+def swipe(
+    start_x: float,
+    start_y: float,
+    end_x: float,
+    end_y: float,
+    *,
+    duration_ms: int = 500,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+) -> dict[str, object]:
+    get_driver(device_id, manager=manager).swipe(
+        start_x,
+        start_y,
+        end_x,
+        end_y,
+        duration_ms,
+    )
+    return {
+        "ok": True,
+        "action": "swipe",
+        "start": {"x": start_x, "y": start_y},
+        "end": {"x": end_x, "y": end_y},
+        "duration_ms": duration_ms,
+    }
+

tools/tap.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from core.device_manager import DeviceManager
+from tools._device import get_driver
+
+
+def tap(
+    x: float,
+    y: float,
+    *,
+    device_id: str | None = None,
+    manager: DeviceManager | None = None,
+) -> dict[str, object]:
+    get_driver(device_id, manager=manager).tap(x, y)
+    return {"ok": True, "action": "tap", "x": x, "y": y}
+

tools/ui_tree.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from typing import Any
+
+from core.device_manager import DeviceManager
+from tools._device import get_driver
+from vision.ui_parser import parse_ui_tree
+
+
+def get_raw_ui_tree(
+    device_id: str | None = None,
+    *,
+    manager: DeviceManager | None = None,
+) -> Any:
+    return get_driver(device_id, manager=manager).tree()
+
+
+def get_ui_tree(
+    device_id: str | None = None,
+    *,
+    manager: DeviceManager | None = None,
+) -> list[dict[str, object]]:
+    return [
+        element.to_dict()
+        for element in parse_ui_tree(get_raw_ui_tree(device_id, manager=manager))
+    ]