feat: checkpoint device agent runtime milestones

tests/test_describe_screen_semantic.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from typing import Any
+
+from core.models import Bounds, Scene, SceneElement
+from device.manager import DeviceManager
+from semantic.config import SemanticConfig
+from semantic.llm_client import EnrichmentUnavailable
+from tools.describe_screen import describe_screen
+from tools.describe_screen_semantic import describe_screen_semantic
+from tests.fakes import FakeDriver
+
+
+class FakePerceptionProvider:
+    def __init__(self, scene: Scene) -> None:
+        self.scene = scene
+        self.calls: list[tuple[bytes, Any]] = []
+
+    def build_scene(self, screenshot: bytes, raw_tree: Any) -> Scene:
+        self.calls.append((screenshot, raw_tree))
+        return self.scene
+
+
+class FakeClient:
+    def __init__(
+        self,
+        *,
+        response: dict[str, Any] | None = None,
+        error: Exception | None = None,
+    ) -> None:
+        self.response = response
+        self.error = error
+
+    def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
+        if self.error:
+            raise self.error
+        assert self.response is not None
+        return self.response
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="send",
+                type="button",
+                text="Send",
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def _connected_manager(driver: FakeDriver) -> DeviceManager:
+    manager = DeviceManager()
+    manager.register_device("phone", lambda: driver)
+    manager.connect("phone", max_retries=1)
+    return manager
+
+
+def test_describe_screen_existing_behavior_is_unchanged() -> None:
+    scene = _scene()
+    provider = FakePerceptionProvider(scene)
+    driver = FakeDriver()
+    manager = _connected_manager(driver)
+
+    result = describe_screen(
+        "phone",
+        manager=manager,
+        perception_provider=provider,
+    )
+
+    assert result is scene
+    assert isinstance(result, Scene)
+    assert len(provider.calls) == 1
+    assert ("screenshot", ()) in driver.calls
+    assert ("tree", ()) in driver.calls
+
+
+def test_describe_screen_semantic_returns_scene_and_semantic_scene() -> None:
+    scene = _scene()
+    provider = FakePerceptionProvider(scene)
+    manager = _connected_manager(FakeDriver())
+    client = FakeClient(
+        response={
+            "page": "Chat",
+            "intents": ["send a message"],
+            "widgets": [{"element_id": "send", "purpose": "send message"}],
+        }
+    )
+
+    result = describe_screen_semantic(
+        "phone",
+        manager=manager,
+        perception_provider=provider,
+        client=client,
+        semantic_config=SemanticConfig(enabled=True),
+    )
+
+    assert result["scene"] is scene
+    assert result["semantic_scene"] is not None
+    assert result["semantic_scene"].to_dict()["page"] == "Chat"  # type: ignore[union-attr]
+
+
+def test_describe_screen_semantic_degrades_semantic_scene_to_none() -> None:
+    scene = _scene()
+    provider = FakePerceptionProvider(scene)
+    manager = _connected_manager(FakeDriver())
+
+    result = describe_screen_semantic(
+        "phone",
+        manager=manager,
+        perception_provider=provider,
+        client=FakeClient(error=EnrichmentUnavailable("timeout")),
+        semantic_config=SemanticConfig(enabled=True),
+    )
+
+    assert result["scene"] is scene
+    assert result["semantic_scene"] is None