feat: checkpoint device agent runtime milestones

tools/_device.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from core.device_manager import DEFAULT_MANAGER, DeviceManager
-from core.driver import Driver
+from device.manager import DEFAULT_MANAGER, DeviceManager
+from driver.base import Driver
 
 
 def get_driver(
@@ -10,4 +10,3 @@ def get_driver(
     manager: DeviceManager | None = None,
 ) -> Driver:
     return (manager or DEFAULT_MANAGER).active_driver(device_id)
-

tools/describe_screen.py

@@ -1,11 +1,10 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
 from core.models import Scene
+from device.manager import DeviceManager
+from perception.ocr import PaddleOCREngine
+from perception.provider import DefaultPerceptionProvider, PerceptionProvider
 from tools._device import get_driver
-from vision.ocr import PaddleOCREngine, run_ocr
-from vision.scene_builder import build_scene, infer_png_size
-from vision.ui_parser import parse_ui_tree
 
 
 def describe_screen(
@@ -13,15 +12,10 @@ def describe_screen(
     *,
     manager: DeviceManager | None = None,
     ocr_engine: PaddleOCREngine | None = None,
+    perception_provider: PerceptionProvider | None = None,
 ) -> Scene:
     driver = get_driver(device_id, manager=manager)
     screenshot = driver.screenshot()
     raw_tree = driver.tree()
-    width, height = infer_png_size(screenshot)
-    return build_scene(
-        screen_width=width,
-        screen_height=height,
-        ui_elements=parse_ui_tree(raw_tree),
-        ocr_elements=run_ocr(screenshot, engine=ocr_engine),
-    )
-
+    provider = perception_provider or DefaultPerceptionProvider(ocr_engine=ocr_engine)
+    return provider.build_scene(screenshot, raw_tree)

tools/describe_screen_semantic.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from core.models import Scene
+from device.manager import DeviceManager
+from perception.ocr import PaddleOCREngine
+from perception.provider import PerceptionProvider
+from semantic.config import SemanticConfig
+from semantic.enricher import SemanticLLMClient, enrich_scene
+from semantic.models import SemanticScene
+from tools.describe_screen import describe_screen
+
+
+def describe_screen_semantic(
+    device_id: str | None = None,
+    *,
+    manager: DeviceManager | None = None,
+    ocr_engine: PaddleOCREngine | None = None,
+    perception_provider: PerceptionProvider | None = None,
+    client: SemanticLLMClient | None = None,
+    semantic_config: SemanticConfig | None = None,
+) -> dict[str, Scene | SemanticScene | None]:
+    scene = describe_screen(
+        device_id,
+        manager=manager,
+        ocr_engine=ocr_engine,
+        perception_provider=perception_provider,
+    )
+    semantic_scene = enrich_scene(scene, client=client, config=semantic_config)
+    return {
+        "scene": scene,
+        "semantic_scene": semantic_scene,
+    }

tools/find_icon.py

@@ -1,9 +1,9 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
 from core.models import Scene
+from device.manager import DeviceManager
+from perception.icon_detector import find_icon as find_icon_in_scene
 from tools.describe_screen import describe_screen
-from vision.icon_detector import find_icon as find_icon_in_scene
 
 
 def find_icon(scene: Scene, name: str) -> dict[str, object]:

tools/find_text.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
 from core.models import Scene, SceneElement
+from device.manager import DeviceManager
 from tools.describe_screen import describe_screen
 
 

tools/input_text.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from tools._device import get_driver
 
 
@@ -12,4 +12,3 @@ def input_text(
 ) -> dict[str, object]:
     get_driver(device_id, manager=manager).input(text)
     return {"ok": True, "action": "input_text", "text": text}
-

tools/launch_app.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from tools._device import get_driver
 
 
@@ -22,4 +22,3 @@ def terminate_app(
 ) -> dict[str, object]:
     get_driver(device_id, manager=manager).terminate(app_id)
     return {"ok": True, "action": "terminate_app", "app_id": app_id}
-

tools/screenshot.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from tools._device import get_driver
 
 
@@ -10,4 +10,3 @@ def take_screenshot(
     manager: DeviceManager | None = None,
 ) -> bytes:
     return get_driver(device_id, manager=manager).screenshot()
-

tools/swipe.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from tools._device import get_driver
 
 
@@ -28,4 +28,3 @@ def swipe(
         "end": {"x": end_x, "y": end_y},
         "duration_ms": duration_ms,
     }
-

tools/tap.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from tools._device import get_driver
 
 
@@ -13,4 +13,3 @@ def tap(
 ) -> dict[str, object]:
     get_driver(device_id, manager=manager).tap(x, y)
     return {"ok": True, "action": "tap", "x": x, "y": y}
-

tools/ui_tree.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 from typing import Any
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
+from perception.ui_parser import parse_ui_tree
 from tools._device import get_driver
-from vision.ui_parser import parse_ui_tree
 
 
 def get_raw_ui_tree(