122 lines
3.4 KiB
Python
122 lines
3.4 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from core.models import Bounds, Scene, SceneElement
|
|
from device.manager import DeviceManager
|
|
from semantic.config import SemanticConfig
|
|
from semantic.llm_client import EnrichmentUnavailable
|
|
from tools.describe_screen import describe_screen
|
|
from tools.describe_screen_semantic import describe_screen_semantic
|
|
from tests.fakes import FakeDriver
|
|
|
|
|
|
class FakePerceptionProvider:
|
|
def __init__(self, scene: Scene) -> None:
|
|
self.scene = scene
|
|
self.calls: list[tuple[bytes, Any]] = []
|
|
|
|
def build_scene(self, screenshot: bytes, raw_tree: Any) -> Scene:
|
|
self.calls.append((screenshot, raw_tree))
|
|
return self.scene
|
|
|
|
|
|
class FakeClient:
|
|
def __init__(
|
|
self,
|
|
*,
|
|
response: dict[str, Any] | None = None,
|
|
error: Exception | None = None,
|
|
) -> None:
|
|
self.response = response
|
|
self.error = error
|
|
|
|
def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
|
|
if self.error:
|
|
raise self.error
|
|
assert self.response is not None
|
|
return self.response
|
|
|
|
|
|
def _scene() -> Scene:
|
|
return Scene(
|
|
width=10,
|
|
height=20,
|
|
elements=[
|
|
SceneElement(
|
|
id="send",
|
|
type="button",
|
|
text="Send",
|
|
bounds=Bounds(1, 2, 3, 4),
|
|
)
|
|
],
|
|
)
|
|
|
|
|
|
def _connected_manager(driver: FakeDriver) -> DeviceManager:
|
|
manager = DeviceManager()
|
|
manager.register_device("phone", lambda: driver)
|
|
manager.connect("phone", max_retries=1)
|
|
return manager
|
|
|
|
|
|
def test_describe_screen_existing_behavior_is_unchanged() -> None:
|
|
scene = _scene()
|
|
provider = FakePerceptionProvider(scene)
|
|
driver = FakeDriver()
|
|
manager = _connected_manager(driver)
|
|
|
|
result = describe_screen(
|
|
"phone",
|
|
manager=manager,
|
|
perception_provider=provider,
|
|
)
|
|
|
|
assert result is scene
|
|
assert isinstance(result, Scene)
|
|
assert len(provider.calls) == 1
|
|
assert ("screenshot", ()) in driver.calls
|
|
assert ("tree", ()) in driver.calls
|
|
|
|
|
|
def test_describe_screen_semantic_returns_scene_and_semantic_scene() -> None:
|
|
scene = _scene()
|
|
provider = FakePerceptionProvider(scene)
|
|
manager = _connected_manager(FakeDriver())
|
|
client = FakeClient(
|
|
response={
|
|
"page": "Chat",
|
|
"intents": ["send a message"],
|
|
"widgets": [{"element_id": "send", "purpose": "send message"}],
|
|
}
|
|
)
|
|
|
|
result = describe_screen_semantic(
|
|
"phone",
|
|
manager=manager,
|
|
perception_provider=provider,
|
|
client=client,
|
|
semantic_config=SemanticConfig(enabled=True),
|
|
)
|
|
|
|
assert result["scene"] is scene
|
|
assert result["semantic_scene"] is not None
|
|
assert result["semantic_scene"].to_dict()["page"] == "Chat" # type: ignore[union-attr]
|
|
|
|
|
|
def test_describe_screen_semantic_degrades_semantic_scene_to_none() -> None:
|
|
scene = _scene()
|
|
provider = FakePerceptionProvider(scene)
|
|
manager = _connected_manager(FakeDriver())
|
|
|
|
result = describe_screen_semantic(
|
|
"phone",
|
|
manager=manager,
|
|
perception_provider=provider,
|
|
client=FakeClient(error=EnrichmentUnavailable("timeout")),
|
|
semantic_config=SemanticConfig(enabled=True),
|
|
)
|
|
|
|
assert result["scene"] is scene
|
|
assert result["semantic_scene"] is None
|