driver.tree() failures (WDA/Appium session errors) previously raised uncaught, killing describe_screen() before OCR ever ran. Malformed tree content (invalid XML) had the same problem inside parse_ui_tree(). Both are now caught and logged, falling back to an empty ui_elements list so the scene degrades to OCR-only, mirroring the existing OCR-failure fallback in run_ocr().
This commit is contained in:
@@ -0,0 +1,82 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from core.errors import DriverError
|
||||
from core.models import Bounds
|
||||
from device.manager import DeviceManager
|
||||
from driver.base import Driver
|
||||
from perception.ocr import OCRBox
|
||||
from tools.describe_screen import describe_screen
|
||||
|
||||
PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14"
|
||||
|
||||
|
||||
class FakeOCREngine:
|
||||
def __init__(self, boxes: list[OCRBox]) -> None:
|
||||
self._boxes = boxes
|
||||
|
||||
def extract(self, image: bytes | str | Path) -> list[OCRBox]:
|
||||
return self._boxes
|
||||
|
||||
|
||||
class TreeFailingDriver(Driver):
|
||||
def connect(self) -> None:
|
||||
return None
|
||||
|
||||
def disconnect(self) -> None:
|
||||
return None
|
||||
|
||||
def screenshot(self) -> bytes:
|
||||
return PNG_10X20
|
||||
|
||||
def tap(self, x: float, y: float) -> None:
|
||||
return None
|
||||
|
||||
def swipe(
|
||||
self,
|
||||
start_x: float,
|
||||
start_y: float,
|
||||
end_x: float,
|
||||
end_y: float,
|
||||
duration_ms: int = 500,
|
||||
) -> None:
|
||||
return None
|
||||
|
||||
def input(self, text: str) -> None:
|
||||
return None
|
||||
|
||||
def launch(self, app_id: str) -> None:
|
||||
return None
|
||||
|
||||
def terminate(self, app_id: str) -> None:
|
||||
return None
|
||||
|
||||
def tree(self):
|
||||
raise DriverError("ui tree retrieval failed")
|
||||
|
||||
def home(self) -> None:
|
||||
return None
|
||||
|
||||
def lock(self) -> None:
|
||||
return None
|
||||
|
||||
def unlock(self) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def test_describe_screen_degrades_to_ocr_only_when_ui_tree_unavailable() -> None:
|
||||
manager = DeviceManager()
|
||||
manager.register_device("phone-1", lambda: TreeFailingDriver())
|
||||
manager.connect("phone-1")
|
||||
fake_box = OCRBox(text="Search", bounds=Bounds(1, 2, 4, 4), confidence=0.9)
|
||||
fake_engine = FakeOCREngine([fake_box])
|
||||
|
||||
scene = describe_screen(
|
||||
"phone-1",
|
||||
manager=manager,
|
||||
ocr_engine=fake_engine, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
assert [element.text for element in scene.elements] == ["Search"]
|
||||
assert all(element.source == "ocr" for element in scene.elements)
|
||||
@@ -9,13 +9,7 @@ from perception.scene_builder import build_scene as build_fused_scene
|
||||
from perception.scene_builder import infer_png_size
|
||||
from perception.ui_parser import parse_ui_tree
|
||||
|
||||
PNG_10X20 = (
|
||||
b"\x89PNG\r\n\x1a\n"
|
||||
b"\x00\x00\x00\r"
|
||||
b"IHDR"
|
||||
b"\x00\x00\x00\x0a"
|
||||
b"\x00\x00\x00\x14"
|
||||
)
|
||||
PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14"
|
||||
|
||||
TREE_XML = """
|
||||
<AppiumAUT x="0" y="0" width="10" height="20">
|
||||
@@ -57,6 +51,27 @@ def test_null_perception_provider_infers_zero_size_for_non_png() -> None:
|
||||
assert scene == Scene(width=0, height=0, elements=[])
|
||||
|
||||
|
||||
def test_default_perception_provider_degrades_when_ui_tree_parsing_fails() -> None:
|
||||
fake_boxes = [
|
||||
OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9),
|
||||
]
|
||||
fake_engine = FakeOCREngine(fake_boxes)
|
||||
provider = DefaultPerceptionProvider(ocr_engine=fake_engine) # type: ignore[arg-type]
|
||||
|
||||
scene = provider.build_scene(PNG_10X20, "<unclosed-tag")
|
||||
|
||||
width, height = infer_png_size(PNG_10X20)
|
||||
expected = build_fused_scene(
|
||||
screen_width=width,
|
||||
screen_height=height,
|
||||
ui_elements=[],
|
||||
ocr_elements=[fake_boxes[0].to_scene_element("ocr-000")],
|
||||
)
|
||||
|
||||
assert scene == expected
|
||||
assert scene.elements == [fake_boxes[0].to_scene_element("ocr-000")]
|
||||
|
||||
|
||||
def test_default_perception_provider_wires_through_scene_builder() -> None:
|
||||
fake_boxes = [
|
||||
OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9),
|
||||
|
||||
Reference in New Issue
Block a user