feat(perception): surface accessibility interaction state on UI-tree elements
SceneElement gains enabled/clickable/selected/checked/focused (bool | None), populated from the literal attributes Appium's XCUITest and UiAutomator2 page_source already emit (iOS: enabled only; Android: all five). None means "not reported by this platform", not false. to_dict() omits unset fields to keep the LLM-facing scene JSON compact; planner_prompts.py documents the new fields so the AI planner knows how to use them (e.g. don't tap disabled elements, use selected/checked to judge whether a toggle already matches the goal).
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from perception.ui_parser import parse_ui_tree
|
||||
|
||||
|
||||
def test_ios_page_source_reports_enabled_only() -> None:
|
||||
xml = """
|
||||
<AppiumAUT x="0" y="0" width="100" height="200">
|
||||
<XCUIElementTypeButton name="Submit" label="Submit" enabled="false"
|
||||
visible="true" x="1" y="2" width="4" height="4" />
|
||||
</AppiumAUT>
|
||||
"""
|
||||
|
||||
elements = parse_ui_tree(xml)
|
||||
|
||||
button = next(element for element in elements if element.type == "button")
|
||||
assert button.enabled is False
|
||||
# iOS page_source never emits these, so they must stay unknown (None),
|
||||
# not be defaulted to False.
|
||||
assert button.clickable is None
|
||||
assert button.selected is None
|
||||
assert button.checked is None
|
||||
assert button.focused is None
|
||||
|
||||
|
||||
def test_android_page_source_reports_full_interaction_state() -> None:
|
||||
xml = """
|
||||
<hierarchy>
|
||||
<node class="android.widget.CheckBox" text="Remember me"
|
||||
enabled="true" clickable="true" selected="false" checked="true"
|
||||
focused="false" bounds="[10,20][60,50]" />
|
||||
</hierarchy>
|
||||
"""
|
||||
|
||||
elements = parse_ui_tree(xml)
|
||||
|
||||
assert len(elements) == 1
|
||||
checkbox = elements[0]
|
||||
assert checkbox.enabled is True
|
||||
assert checkbox.clickable is True
|
||||
assert checkbox.selected is False
|
||||
assert checkbox.checked is True
|
||||
assert checkbox.focused is False
|
||||
|
||||
|
||||
def test_dict_tree_passes_through_native_bool_state() -> None:
|
||||
tree = {
|
||||
"type": "button",
|
||||
"text": "Send",
|
||||
"x": 0,
|
||||
"y": 0,
|
||||
"width": 10,
|
||||
"height": 10,
|
||||
"enabled": True,
|
||||
"clickable": False,
|
||||
}
|
||||
|
||||
elements = parse_ui_tree(tree)
|
||||
|
||||
assert len(elements) == 1
|
||||
assert elements[0].enabled is True
|
||||
assert elements[0].clickable is False
|
||||
assert elements[0].selected is None
|
||||
|
||||
|
||||
def test_to_dict_omits_unknown_state_fields_but_keeps_reported_false() -> None:
|
||||
xml = """
|
||||
<AppiumAUT x="0" y="0" width="10" height="10">
|
||||
<XCUIElementTypeButton name="Submit" enabled="false" x="0" y="0" width="4" height="4" />
|
||||
</AppiumAUT>
|
||||
"""
|
||||
|
||||
elements = parse_ui_tree(xml)
|
||||
element = next(item for item in elements if item.type == "button")
|
||||
data = element.to_dict()
|
||||
|
||||
assert data["enabled"] is False
|
||||
assert "clickable" not in data
|
||||
assert "selected" not in data
|
||||
assert "checked" not in data
|
||||
assert "focused" not in data
|
||||
Reference in New Issue
Block a user