from __future__ import annotations from typing import Any from core.models import Bounds, Scene, SceneElement from device.manager import DeviceManager from semantic.config import SemanticConfig from semantic.llm_client import EnrichmentUnavailable from tools.describe_screen import describe_screen from tools.describe_screen_semantic import describe_screen_semantic from tests.fakes import FakeDriver class FakePerceptionProvider: def __init__(self, scene: Scene) -> None: self.scene = scene self.calls: list[tuple[bytes, Any]] = [] def build_scene(self, screenshot: bytes, raw_tree: Any) -> Scene: self.calls.append((screenshot, raw_tree)) return self.scene class FakeClient: def __init__( self, *, response: dict[str, Any] | None = None, error: Exception | None = None, ) -> None: self.response = response self.error = error def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]: if self.error: raise self.error assert self.response is not None return self.response def _scene() -> Scene: return Scene( width=10, height=20, elements=[ SceneElement( id="send", type="button", text="Send", bounds=Bounds(1, 2, 3, 4), ) ], ) def _connected_manager(driver: FakeDriver) -> DeviceManager: manager = DeviceManager() manager.register_device("phone", lambda: driver) manager.connect("phone", max_retries=1) return manager def test_describe_screen_existing_behavior_is_unchanged() -> None: scene = _scene() provider = FakePerceptionProvider(scene) driver = FakeDriver() manager = _connected_manager(driver) result = describe_screen( "phone", manager=manager, perception_provider=provider, ) assert result is scene assert isinstance(result, Scene) assert len(provider.calls) == 1 assert ("screenshot", ()) in driver.calls assert ("tree", ()) in driver.calls def test_describe_screen_semantic_returns_scene_and_semantic_scene() -> None: scene = _scene() provider = FakePerceptionProvider(scene) manager = _connected_manager(FakeDriver()) client = FakeClient( response={ "page": "Chat", "intents": ["send a message"], "widgets": [{"element_id": "send", "purpose": "send message"}], } ) result = describe_screen_semantic( "phone", manager=manager, perception_provider=provider, client=client, semantic_config=SemanticConfig(enabled=True), ) assert result["scene"] is scene assert result["semantic_scene"] is not None assert result["semantic_scene"].to_dict()["page"] == "Chat" # type: ignore[union-attr] def test_describe_screen_semantic_degrades_semantic_scene_to_none() -> None: scene = _scene() provider = FakePerceptionProvider(scene) manager = _connected_manager(FakeDriver()) result = describe_screen_semantic( "phone", manager=manager, perception_provider=provider, client=FakeClient(error=EnrichmentUnavailable("timeout")), semantic_config=SemanticConfig(enabled=True), ) assert result["scene"] is scene assert result["semantic_scene"] is None