from __future__ import annotations from core.models import Bounds, SceneElement from perception.scene_builder import bbox_iou, build_scene def test_scene_builder_merges_overlapping_ocr_into_ui_element() -> None: ui_button = SceneElement( id="ui-button", type="button", text=None, bounds=Bounds(10, 10, 100, 40), confidence=1.0, source="ui", ) ocr_label = SceneElement( id="ocr-label", type="text", text="Search", bounds=Bounds(12, 12, 96, 36), confidence=0.92, source="ocr", ) ocr_only = SceneElement( id="ocr-only", type="text", text="Footer", bounds=Bounds(0, 100, 40, 20), confidence=0.8, source="ocr", ) scene = build_scene( screen_width=120, screen_height=140, ui_elements=[ui_button], ocr_elements=[ocr_label, ocr_only], ) assert len(scene.elements) == 2 assert scene.elements[0].type == "button" assert scene.elements[0].text == "Search" assert scene.elements[1].text == "Footer" assert [element.text for element in scene.ocr_elements] == ["Search", "Footer"] assert bbox_iou(ui_button.bounds, ocr_label.bounds) > 0.5 def test_build_scene_rescales_ocr_bounds_from_points_to_pixels_mismatch() -> None: # Real numbers observed on an iOS Retina (2x) device: XCUITest reports the # UI tree in points (414x896) while the screenshot (and therefore OCR) is # in pixels (828x1792). root = SceneElement( id="ui-000", type="application", text=None, bounds=Bounds(0, 0, 414, 896), confidence=1.0, source="ui", ) button = SceneElement( id="ui-button", type="button", text=None, bounds=Bounds(20, 40, 100, 40), confidence=1.0, source="ui", ) ocr_label = SceneElement( id="ocr-label", type="text", text="Search", bounds=Bounds(40, 80, 200, 80), # pixel-space, matches `button` once halved confidence=0.9, source="ocr", ) scene = build_scene( screen_width=828, screen_height=1792, ui_elements=[root, button], ocr_elements=[ocr_label], ) assert scene.width == 414 assert scene.height == 896 fused_button = next(e for e in scene.elements if e.id == "ui-button") assert fused_button.text == "Search" ocr_only = SceneElement( id="ocr-only", type="text", text="Footer", bounds=Bounds(200, 1600, 100, 40), confidence=0.8, source="ocr", ) scene_with_extra = build_scene( screen_width=828, screen_height=1792, ui_elements=[root, button], ocr_elements=[ocr_label, ocr_only], ) footer = next( e for e in scene_with_extra.ocr_elements if e.text == "Footer" ) assert footer.bounds == Bounds(100, 800, 50, 20) def test_build_scene_is_noop_when_ui_tree_already_matches_pixel_scale() -> None: # Android's UiAutomator2 bounds already match screenshot pixels 1:1. root = SceneElement( id="ui-000", type="application", text=None, bounds=Bounds(0, 0, 1080, 2280), confidence=1.0, source="ui", ) ocr_label = SceneElement( id="ocr-label", type="text", text="Search", bounds=Bounds(40, 80, 200, 80), confidence=0.9, source="ocr", ) scene = build_scene( screen_width=1080, screen_height=2280, ui_elements=[root], ocr_elements=[ocr_label], ) assert scene.width == 1080 assert scene.height == 2280 assert scene.ocr_elements[0].bounds == Bounds(40, 80, 200, 80)