q792602257
8d5b02e37f
Tests / Test passed: 858
driver.tree() failures (WDA/Appium session errors) previously raised uncaught, killing describe_screen() before OCR ever ran. Malformed tree content (invalid XML) had the same problem inside parse_ui_tree(). Both are now caught and logged, falling back to an empty ui_elements list so the scene degrades to OCR-only, mirroring the existing OCR-failure fallback in run_ocr().
95 lines
3.1 KiB
Python
95 lines
3.1 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from core.models import Scene
|
|
from perception.ocr import OCRBox
|
|
from perception.provider import DefaultPerceptionProvider, NullPerceptionProvider
|
|
from perception.scene_builder import build_scene as build_fused_scene
|
|
from perception.scene_builder import infer_png_size
|
|
from perception.ui_parser import parse_ui_tree
|
|
|
|
PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14"
|
|
|
|
TREE_XML = """
|
|
<AppiumAUT x="0" y="0" width="10" height="20">
|
|
<XCUIElementTypeButton name="Search" label="Search" x="1" y="2" width="4" height="4" />
|
|
<XCUIElementTypeImage name="Settings" label="Settings" x="6" y="2" width="3" height="3" />
|
|
</AppiumAUT>
|
|
"""
|
|
|
|
|
|
class FakeOCREngine:
|
|
"""Stand-in for PaddleOCREngine that never touches the real dependency."""
|
|
|
|
def __init__(self, boxes: list[OCRBox]) -> None:
|
|
self._boxes = boxes
|
|
self.calls: list[object] = []
|
|
|
|
def extract(self, image: bytes | str | Path) -> list[OCRBox]:
|
|
self.calls.append(image)
|
|
return self._boxes
|
|
|
|
|
|
def test_null_perception_provider_returns_empty_scene_without_ocr() -> None:
|
|
provider = NullPerceptionProvider()
|
|
|
|
scene = provider.build_scene(PNG_10X20, TREE_XML)
|
|
|
|
width, height = infer_png_size(PNG_10X20)
|
|
assert scene == Scene(width=width, height=height, elements=[])
|
|
assert width == 10
|
|
assert height == 20
|
|
assert scene.elements == []
|
|
|
|
|
|
def test_null_perception_provider_infers_zero_size_for_non_png() -> None:
|
|
provider = NullPerceptionProvider()
|
|
|
|
scene = provider.build_scene(b"not-a-png", None)
|
|
|
|
assert scene == Scene(width=0, height=0, elements=[])
|
|
|
|
|
|
def test_default_perception_provider_degrades_when_ui_tree_parsing_fails() -> None:
|
|
fake_boxes = [
|
|
OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9),
|
|
]
|
|
fake_engine = FakeOCREngine(fake_boxes)
|
|
provider = DefaultPerceptionProvider(ocr_engine=fake_engine) # type: ignore[arg-type]
|
|
|
|
scene = provider.build_scene(PNG_10X20, "<unclosed-tag")
|
|
|
|
width, height = infer_png_size(PNG_10X20)
|
|
expected = build_fused_scene(
|
|
screen_width=width,
|
|
screen_height=height,
|
|
ui_elements=[],
|
|
ocr_elements=[fake_boxes[0].to_scene_element("ocr-000")],
|
|
)
|
|
|
|
assert scene == expected
|
|
assert scene.elements == [fake_boxes[0].to_scene_element("ocr-000")]
|
|
|
|
|
|
def test_default_perception_provider_wires_through_scene_builder() -> None:
|
|
fake_boxes = [
|
|
OCRBox(text="Search", bounds=parse_ui_tree(TREE_XML)[0].bounds, confidence=0.9),
|
|
]
|
|
fake_engine = FakeOCREngine(fake_boxes)
|
|
provider = DefaultPerceptionProvider(ocr_engine=fake_engine) # type: ignore[arg-type]
|
|
|
|
scene = provider.build_scene(PNG_10X20, TREE_XML)
|
|
|
|
width, height = infer_png_size(PNG_10X20)
|
|
expected = build_fused_scene(
|
|
screen_width=width,
|
|
screen_height=height,
|
|
ui_elements=parse_ui_tree(TREE_XML),
|
|
ocr_elements=[fake_boxes[0].to_scene_element("ocr-000")],
|
|
)
|
|
|
|
assert scene == expected
|
|
assert len(scene.elements) == len(parse_ui_tree(TREE_XML))
|
|
assert fake_engine.calls == [PNG_10X20]
|