feat(perception): surface accessibility interaction state on UI-tree elements

SceneElement gains enabled/clickable/selected/checked/focused (bool | None),
populated from the literal attributes Appium's XCUITest and UiAutomator2
page_source already emit (iOS: enabled only; Android: all five). None means
"not reported by this platform", not false. to_dict() omits unset fields to
keep the LLM-facing scene JSON compact; planner_prompts.py documents the new
fields so the AI planner knows how to use them (e.g. don't tap disabled
elements, use selected/checked to judge whether a toggle already matches the
goal).
This commit is contained in:
2026-07-15 18:10:53 +08:00
parent 7c6cdc5b67
commit 361dada276
4 changed files with 142 additions and 8 deletions
+27 -8
View File
@@ -2,12 +2,19 @@ from __future__ import annotations
import re
import xml.etree.ElementTree as ET
from dataclasses import replace
from typing import Any
from core.models import Bounds, SceneElement
_ANDROID_BOUNDS = re.compile(r"\[(?P<x1>-?\d+),(?P<y1>-?\d+)\]\[(?P<x2>-?\d+),(?P<y2>-?\d+)\]")
# Accessibility interaction-state attributes, as literally emitted by
# WDA/XCUITest page_source (enabled) and Appium UiAutomator2 page_source
# (enabled, clickable, selected, checked, focused). iOS page_source does not
# emit selected/checked/focused/clickable, so those stay None for iOS trees.
_STATE_ATTR_KEYS = ("enabled", "clickable", "selected", "checked", "focused")
def parse_ui_tree(raw_tree: Any) -> list[SceneElement]:
if raw_tree is None:
@@ -44,6 +51,7 @@ def _parse_xml(raw_tree: str) -> list[SceneElement]:
bounds=bounds,
confidence=1.0,
source="ui",
**_state_from_attrs(node.attrib),
)
)
return _with_stable_ids(elements, "ui")
@@ -65,6 +73,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None
bounds=bounds,
confidence=1.0,
source="ui",
**_state_from_attrs(attrs),
)
)
for child in children:
@@ -74,14 +83,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None
def _with_stable_ids(elements: list[SceneElement], prefix: str) -> list[SceneElement]:
return [
SceneElement(
id=element.id or f"{prefix}-{index:03d}",
type=element.type,
text=element.text,
bounds=element.bounds,
confidence=element.confidence,
source=element.source,
)
element if element.id else replace(element, id=f"{prefix}-{index:03d}")
for index, element in enumerate(elements)
]
@@ -122,6 +124,23 @@ def _text_from_attrs(attrs: dict[str, Any]) -> str | None:
return None
def _state_from_attrs(attrs: dict[str, Any]) -> dict[str, bool | None]:
return {key: _parse_bool(attrs.get(key)) for key in _STATE_ATTR_KEYS}
def _parse_bool(value: Any) -> bool | None:
if isinstance(value, bool):
return value
if value is None:
return None
text = str(value).strip().lower()
if text in ("true", "1", "yes"):
return True
if text in ("false", "0", "no"):
return False
return None
def _normalize_type(raw_type: str, attrs: dict[str, Any]) -> str:
candidate = (
attrs.get("role")