feat(perception): surface accessibility interaction state on UI-tree elements
SceneElement gains enabled/clickable/selected/checked/focused (bool | None), populated from the literal attributes Appium's XCUITest and UiAutomator2 page_source already emit (iOS: enabled only; Android: all five). None means "not reported by this platform", not false. to_dict() omits unset fields to keep the LLM-facing scene JSON compact; planner_prompts.py documents the new fields so the AI planner knows how to use them (e.g. don't tap disabled elements, use selected/checked to judge whether a toggle already matches the goal).
This commit is contained in:
@@ -71,6 +71,9 @@ class Device:
|
||||
}
|
||||
|
||||
|
||||
_STATE_FIELDS = ("enabled", "clickable", "selected", "checked", "focused")
|
||||
|
||||
|
||||
@dataclass
|
||||
class SceneElement:
|
||||
id: str
|
||||
@@ -79,6 +82,13 @@ class SceneElement:
|
||||
text: str | None = None
|
||||
confidence: float | None = None
|
||||
source: str | None = None
|
||||
# Accessibility-tree interaction state, when the platform reports it.
|
||||
# None means "not reported by this platform/element", not "false".
|
||||
enabled: bool | None = None
|
||||
clickable: bool | None = None
|
||||
selected: bool | None = None
|
||||
checked: bool | None = None
|
||||
focused: bool | None = None
|
||||
|
||||
@property
|
||||
def center(self) -> tuple[float, float]:
|
||||
@@ -94,6 +104,10 @@ class SceneElement:
|
||||
}
|
||||
if self.source:
|
||||
data["source"] = self.source
|
||||
for field_name in _STATE_FIELDS:
|
||||
value = getattr(self, field_name)
|
||||
if value is not None:
|
||||
data[field_name] = value
|
||||
return data
|
||||
|
||||
@classmethod
|
||||
@@ -105,6 +119,11 @@ class SceneElement:
|
||||
bounds=Bounds.from_dict(data["bounds"]),
|
||||
confidence=data.get("confidence"),
|
||||
source=data.get("source"),
|
||||
enabled=data.get("enabled"),
|
||||
clickable=data.get("clickable"),
|
||||
selected=data.get("selected"),
|
||||
checked=data.get("checked"),
|
||||
focused=data.get("focused"),
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user