feat(perception): surface accessibility interaction state on UI-tree elements
SceneElement gains enabled/clickable/selected/checked/focused (bool | None), populated from the literal attributes Appium's XCUITest and UiAutomator2 page_source already emit (iOS: enabled only; Android: all five). None means "not reported by this platform", not false. to_dict() omits unset fields to keep the LLM-facing scene JSON compact; planner_prompts.py documents the new fields so the AI planner knows how to use them (e.g. don't tap disabled elements, use selected/checked to judge whether a toggle already matches the goal).
This commit is contained in:
+27
-8
@@ -2,12 +2,19 @@ from __future__ import annotations
|
||||
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import replace
|
||||
from typing import Any
|
||||
|
||||
from core.models import Bounds, SceneElement
|
||||
|
||||
_ANDROID_BOUNDS = re.compile(r"\[(?P<x1>-?\d+),(?P<y1>-?\d+)\]\[(?P<x2>-?\d+),(?P<y2>-?\d+)\]")
|
||||
|
||||
# Accessibility interaction-state attributes, as literally emitted by
|
||||
# WDA/XCUITest page_source (enabled) and Appium UiAutomator2 page_source
|
||||
# (enabled, clickable, selected, checked, focused). iOS page_source does not
|
||||
# emit selected/checked/focused/clickable, so those stay None for iOS trees.
|
||||
_STATE_ATTR_KEYS = ("enabled", "clickable", "selected", "checked", "focused")
|
||||
|
||||
|
||||
def parse_ui_tree(raw_tree: Any) -> list[SceneElement]:
|
||||
if raw_tree is None:
|
||||
@@ -44,6 +51,7 @@ def _parse_xml(raw_tree: str) -> list[SceneElement]:
|
||||
bounds=bounds,
|
||||
confidence=1.0,
|
||||
source="ui",
|
||||
**_state_from_attrs(node.attrib),
|
||||
)
|
||||
)
|
||||
return _with_stable_ids(elements, "ui")
|
||||
@@ -65,6 +73,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None
|
||||
bounds=bounds,
|
||||
confidence=1.0,
|
||||
source="ui",
|
||||
**_state_from_attrs(attrs),
|
||||
)
|
||||
)
|
||||
for child in children:
|
||||
@@ -74,14 +83,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None
|
||||
|
||||
def _with_stable_ids(elements: list[SceneElement], prefix: str) -> list[SceneElement]:
|
||||
return [
|
||||
SceneElement(
|
||||
id=element.id or f"{prefix}-{index:03d}",
|
||||
type=element.type,
|
||||
text=element.text,
|
||||
bounds=element.bounds,
|
||||
confidence=element.confidence,
|
||||
source=element.source,
|
||||
)
|
||||
element if element.id else replace(element, id=f"{prefix}-{index:03d}")
|
||||
for index, element in enumerate(elements)
|
||||
]
|
||||
|
||||
@@ -122,6 +124,23 @@ def _text_from_attrs(attrs: dict[str, Any]) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _state_from_attrs(attrs: dict[str, Any]) -> dict[str, bool | None]:
|
||||
return {key: _parse_bool(attrs.get(key)) for key in _STATE_ATTR_KEYS}
|
||||
|
||||
|
||||
def _parse_bool(value: Any) -> bool | None:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if value is None:
|
||||
return None
|
||||
text = str(value).strip().lower()
|
||||
if text in ("true", "1", "yes"):
|
||||
return True
|
||||
if text in ("false", "0", "no"):
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
def _normalize_type(raw_type: str, attrs: dict[str, Any]) -> str:
|
||||
candidate = (
|
||||
attrs.get("role")
|
||||
|
||||
Reference in New Issue
Block a user