feat(perception): surface accessibility interaction state on UI-tree elements

SceneElement gains enabled/clickable/selected/checked/focused (bool | None),
populated from the literal attributes Appium's XCUITest and UiAutomator2
page_source already emit (iOS: enabled only; Android: all five). None means
"not reported by this platform", not false. to_dict() omits unset fields to
keep the LLM-facing scene JSON compact; planner_prompts.py documents the new
fields so the AI planner knows how to use them (e.g. don't tap disabled
elements, use selected/checked to judge whether a toggle already matches the
goal).
This commit is contained in:
2026-07-15 18:10:53 +08:00
parent 7c6cdc5b67
commit 361dada276
4 changed files with 142 additions and 8 deletions
+19
View File
@@ -71,6 +71,9 @@ class Device:
}
_STATE_FIELDS = ("enabled", "clickable", "selected", "checked", "focused")
@dataclass
class SceneElement:
id: str
@@ -79,6 +82,13 @@ class SceneElement:
text: str | None = None
confidence: float | None = None
source: str | None = None
# Accessibility-tree interaction state, when the platform reports it.
# None means "not reported by this platform/element", not "false".
enabled: bool | None = None
clickable: bool | None = None
selected: bool | None = None
checked: bool | None = None
focused: bool | None = None
@property
def center(self) -> tuple[float, float]:
@@ -94,6 +104,10 @@ class SceneElement:
}
if self.source:
data["source"] = self.source
for field_name in _STATE_FIELDS:
value = getattr(self, field_name)
if value is not None:
data[field_name] = value
return data
@classmethod
@@ -105,6 +119,11 @@ class SceneElement:
bounds=Bounds.from_dict(data["bounds"]),
confidence=data.get("confidence"),
source=data.get("source"),
enabled=data.get("enabled"),
clickable=data.get("clickable"),
selected=data.get("selected"),
checked=data.get("checked"),
focused=data.get("focused"),
)