diff --git a/perception/provider.py b/perception/provider.py index 00cb3ac..58c4009 100644 --- a/perception/provider.py +++ b/perception/provider.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging from abc import ABC, abstractmethod from typing import Any @@ -9,6 +10,8 @@ from perception.scene_builder import build_scene as build_fused_scene from perception.scene_builder import infer_png_size from perception.ui_parser import parse_ui_tree +logger = logging.getLogger(__name__) + class PerceptionProvider(ABC): @abstractmethod @@ -22,10 +25,17 @@ class DefaultPerceptionProvider(PerceptionProvider): def build_scene(self, screenshot: bytes, tree: Any) -> Scene: width, height = infer_png_size(screenshot) + try: + ui_elements = parse_ui_tree(tree) + except Exception: + logger.warning( + "UI tree parsing failed; continuing without UI tree", exc_info=True + ) + ui_elements = [] return build_fused_scene( screen_width=width, screen_height=height, - ui_elements=parse_ui_tree(tree), + ui_elements=ui_elements, ocr_elements=run_ocr(screenshot, engine=self._ocr_engine), ) diff --git a/tests/test_describe_screen.py b/tests/test_describe_screen.py new file mode 100644 index 0000000..934932a --- /dev/null +++ b/tests/test_describe_screen.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from pathlib import Path + +from core.errors import DriverError +from core.models import Bounds +from device.manager import DeviceManager +from driver.base import Driver +from perception.ocr import OCRBox +from tools.describe_screen import describe_screen + +PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14" + + +class FakeOCREngine: + def __init__(self, boxes: list[OCRBox]) -> None: + self._boxes = boxes + + def extract(self, image: bytes | str | Path) -> list[OCRBox]: + return self._boxes + + +class TreeFailingDriver(Driver): + def connect(self) -> None: + return None + + def disconnect(self) -> None: + return None + + def screenshot(self) -> bytes: + return PNG_10X20 + + def tap(self, x: float, y: float) -> None: + return None + + def swipe( + self, + start_x: float, + start_y: float, + end_x: float, + end_y: float, + duration_ms: int = 500, + ) -> None: + return None + + def input(self, text: str) -> None: + return None + + def launch(self, app_id: str) -> None: + return None + + def terminate(self, app_id: str) -> None: + return None + + def tree(self): + raise DriverError("ui tree retrieval failed") + + def home(self) -> None: + return None + + def lock(self) -> None: + return None + + def unlock(self) -> None: + return None + + +def test_describe_screen_degrades_to_ocr_only_when_ui_tree_unavailable() -> None: + manager = DeviceManager() + manager.register_device("phone-1", lambda: TreeFailingDriver()) + manager.connect("phone-1") + fake_box = OCRBox(text="Search", bounds=Bounds(1, 2, 4, 4), confidence=0.9) + fake_engine = FakeOCREngine([fake_box]) + + scene = describe_screen( + "phone-1", + manager=manager, + ocr_engine=fake_engine, # type: ignore[arg-type] + ) + + assert [element.text for element in scene.elements] == ["Search"] + assert all(element.source == "ocr" for element in scene.elements) diff --git a/tests/test_perception_provider.py b/tests/test_perception_provider.py index 6f93208..306d35b 100644 --- a/tests/test_perception_provider.py +++ b/tests/test_perception_provider.py @@ -9,13 +9,7 @@ from perception.scene_builder import build_scene as build_fused_scene from perception.scene_builder import infer_png_size from perception.ui_parser import parse_ui_tree -PNG_10X20 = ( - b"\x89PNG\r\n\x1a\n" - b"\x00\x00\x00\r" - b"IHDR" - b"\x00\x00\x00\x0a" - b"\x00\x00\x00\x14" -) +PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14" TREE_XML = """ @@ -57,6 +51,27 @@ def test_null_perception_provider_infers_zero_size_for_non_png() -> None: assert scene == Scene(width=0, height=0, elements=[]) +def test_default_perception_provider_degrades_when_ui_tree_parsing_fails() -> None: + fake_boxes = [ + OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9), + ] + fake_engine = FakeOCREngine(fake_boxes) + provider = DefaultPerceptionProvider(ocr_engine=fake_engine) # type: ignore[arg-type] + + scene = provider.build_scene(PNG_10X20, " None: fake_boxes = [ OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9), diff --git a/tools/describe_screen.py b/tools/describe_screen.py index 15c079e..cbda080 100644 --- a/tools/describe_screen.py +++ b/tools/describe_screen.py @@ -1,11 +1,15 @@ from __future__ import annotations +import logging + from core.models import Scene from device.manager import DeviceManager from perception.ocr import PaddleOCREngine from perception.provider import DefaultPerceptionProvider, PerceptionProvider from tools._device import get_driver +logger = logging.getLogger(__name__) + def describe_screen( device_id: str | None = None, @@ -16,6 +20,12 @@ def describe_screen( ) -> Scene: driver = get_driver(device_id, manager=manager) screenshot = driver.screenshot() - raw_tree = driver.tree() + try: + raw_tree = driver.tree() + except Exception: + logger.warning( + "UI tree retrieval failed; continuing without UI tree", exc_info=True + ) + raw_tree = None provider = perception_provider or DefaultPerceptionProvider(ocr_engine=ocr_engine) return provider.build_scene(screenshot, raw_tree)