fix(perception): degrade to OCR-only when UI tree is unavailable
Tests / Test passed: 858

driver.tree() failures (WDA/Appium session errors) previously raised
uncaught, killing describe_screen() before OCR ever ran. Malformed
tree content (invalid XML) had the same problem inside
parse_ui_tree(). Both are now caught and logged, falling back to an
empty ui_elements list so the scene degrades to OCR-only, mirroring
the existing OCR-failure fallback in run_ocr().
This commit is contained in:
2026-07-15 10:03:30 +08:00
parent 778af2da53
commit 8d5b02e37f
4 changed files with 126 additions and 9 deletions
+22 -7
View File
@@ -9,13 +9,7 @@ from perception.scene_builder import build_scene as build_fused_scene
from perception.scene_builder import infer_png_size
from perception.ui_parser import parse_ui_tree
PNG_10X20 = (
b"\x89PNG\r\n\x1a\n"
b"\x00\x00\x00\r"
b"IHDR"
b"\x00\x00\x00\x0a"
b"\x00\x00\x00\x14"
)
PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14"
TREE_XML = """
<AppiumAUT x="0" y="0" width="10" height="20">
@@ -57,6 +51,27 @@ def test_null_perception_provider_infers_zero_size_for_non_png() -> None:
assert scene == Scene(width=0, height=0, elements=[])
def test_default_perception_provider_degrades_when_ui_tree_parsing_fails() -> None:
fake_boxes = [
OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9),
]
fake_engine = FakeOCREngine(fake_boxes)
provider = DefaultPerceptionProvider(ocr_engine=fake_engine) # type: ignore[arg-type]
scene = provider.build_scene(PNG_10X20, "<unclosed-tag")
width, height = infer_png_size(PNG_10X20)
expected = build_fused_scene(
screen_width=width,
screen_height=height,
ui_elements=[],
ocr_elements=[fake_boxes[0].to_scene_element("ocr-000")],
)
assert scene == expected
assert scene.elements == [fake_boxes[0].to_scene_element("ocr-000")]
def test_default_perception_provider_wires_through_scene_builder() -> None:
fake_boxes = [
OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9),