feat(perception): surface accessibility interaction state on UI-tree elements

SceneElement gains enabled/clickable/selected/checked/focused (bool | None),
populated from the literal attributes Appium's XCUITest and UiAutomator2
page_source already emit (iOS: enabled only; Android: all five). None means
"not reported by this platform", not false. to_dict() omits unset fields to
keep the LLM-facing scene JSON compact; planner_prompts.py documents the new
fields so the AI planner knows how to use them (e.g. don't tap disabled
elements, use selected/checked to judge whether a toggle already matches the
goal).

core/models.py

@@ -71,6 +71,9 @@ class Device:
         }
 
 
+_STATE_FIELDS = ("enabled", "clickable", "selected", "checked", "focused")
+
+
 @dataclass
 class SceneElement:
     id: str
@@ -79,6 +82,13 @@ class SceneElement:
     text: str | None = None
     confidence: float | None = None
     source: str | None = None
+    # Accessibility-tree interaction state, when the platform reports it.
+    # None means "not reported by this platform/element", not "false".
+    enabled: bool | None = None
+    clickable: bool | None = None
+    selected: bool | None = None
+    checked: bool | None = None
+    focused: bool | None = None
 
     @property
     def center(self) -> tuple[float, float]:
@@ -94,6 +104,10 @@ class SceneElement:
         }
         if self.source:
             data["source"] = self.source
+        for field_name in _STATE_FIELDS:
+            value = getattr(self, field_name)
+            if value is not None:
+                data[field_name] = value
         return data
 
     @classmethod
@@ -105,6 +119,11 @@ class SceneElement:
             bounds=Bounds.from_dict(data["bounds"]),
             confidence=data.get("confidence"),
             source=data.get("source"),
+            enabled=data.get("enabled"),
+            clickable=data.get("clickable"),
+            selected=data.get("selected"),
+            checked=data.get("checked"),
+            focused=data.get("focused"),
         )
 
 

perception/ui_parser.py

@@ -2,12 +2,19 @@ from __future__ import annotations
 
 import re
 import xml.etree.ElementTree as ET
+from dataclasses import replace
 from typing import Any
 
 from core.models import Bounds, SceneElement
 
 _ANDROID_BOUNDS = re.compile(r"\[(?P<x1>-?\d+),(?P<y1>-?\d+)\]\[(?P<x2>-?\d+),(?P<y2>-?\d+)\]")
 
+# Accessibility interaction-state attributes, as literally emitted by
+# WDA/XCUITest page_source (enabled) and Appium UiAutomator2 page_source
+# (enabled, clickable, selected, checked, focused). iOS page_source does not
+# emit selected/checked/focused/clickable, so those stay None for iOS trees.
+_STATE_ATTR_KEYS = ("enabled", "clickable", "selected", "checked", "focused")
+
 
 def parse_ui_tree(raw_tree: Any) -> list[SceneElement]:
     if raw_tree is None:
@@ -44,6 +51,7 @@ def _parse_xml(raw_tree: str) -> list[SceneElement]:
                 bounds=bounds,
                 confidence=1.0,
                 source="ui",
+                **_state_from_attrs(node.attrib),
             )
         )
     return _with_stable_ids(elements, "ui")
@@ -65,6 +73,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None
                 bounds=bounds,
                 confidence=1.0,
                 source="ui",
+                **_state_from_attrs(attrs),
             )
         )
     for child in children:
@@ -74,14 +83,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None
 
 def _with_stable_ids(elements: list[SceneElement], prefix: str) -> list[SceneElement]:
     return [
-        SceneElement(
+        element if element.id else replace(element, id=f"{prefix}-{index:03d}")
-            id=element.id or f"{prefix}-{index:03d}",
-            type=element.type,
-            text=element.text,
-            bounds=element.bounds,
-            confidence=element.confidence,
-            source=element.source,
-        )
         for index, element in enumerate(elements)
     ]
 
@@ -122,6 +124,23 @@ def _text_from_attrs(attrs: dict[str, Any]) -> str | None:
     return None
 
 
+def _state_from_attrs(attrs: dict[str, Any]) -> dict[str, bool | None]:
+    return {key: _parse_bool(attrs.get(key)) for key in _STATE_ATTR_KEYS}
+
+
+def _parse_bool(value: Any) -> bool | None:
+    if isinstance(value, bool):
+        return value
+    if value is None:
+        return None
+    text = str(value).strip().lower()
+    if text in ("true", "1", "yes"):
+        return True
+    if text in ("false", "0", "no"):
+        return False
+    return None
+
+
 def _normalize_type(raw_type: str, attrs: dict[str, Any]) -> str:
     candidate = (
         attrs.get("role")

runtime/planner_prompts.py

@@ -10,6 +10,15 @@ list of UI elements with id, type, text, and pixel bounds), and — when
 available — a screenshot of the same screen and a short history of recent
 actions and their outcomes.
 
+Some elements also carry accessibility state fields when the platform
+reports them: `enabled`, `clickable`, `selected`, `checked`, `focused`.
+A field is omitted entirely when the platform does not report it for that
+element — omitted does NOT mean false, treat it as unknown. When present,
+`enabled: false` or `clickable: false` means the element cannot currently be
+interacted with (do not tap it); `selected`/`checked`/`focused` describe its
+current toggle/focus state and are useful for deciding whether an action is
+already done or still needed.
+
 Before calling a tool, output a short text block (1-2 sentences):
 1. If this is the first step, state what you intend to do and why.
 2. Otherwise, first assess whether the previous action achieved its intended
@@ -22,6 +31,12 @@ You must then call exactly one tool:
 - `finish_task` when the goal has been reached, or when it cannot be reached
   and no further action would help.
 
+For every device-action tool call, you must provide both required structured
+fields in addition to the physical-action arguments:
+- `purpose`: one concise sentence describing why this action advances the goal.
+- `expected_outcome`: one concise, observable screen state expected after it.
+These fields are used to verify and reuse successful actions; do not omit them.
+
 Ground every coordinate you choose in the Scene element bounds (and the
 screenshot, if provided) for the current turn only — never reuse coordinates
 from history, since the screen may have changed. Only call `finish_task` with

tests/test_ui_parser.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+from perception.ui_parser import parse_ui_tree
+
+
+def test_ios_page_source_reports_enabled_only() -> None:
+    xml = """
+    <AppiumAUT x="0" y="0" width="100" height="200">
+      <XCUIElementTypeButton name="Submit" label="Submit" enabled="false"
+                             visible="true" x="1" y="2" width="4" height="4" />
+    </AppiumAUT>
+    """
+
+    elements = parse_ui_tree(xml)
+
+    button = next(element for element in elements if element.type == "button")
+    assert button.enabled is False
+    # iOS page_source never emits these, so they must stay unknown (None),
+    # not be defaulted to False.
+    assert button.clickable is None
+    assert button.selected is None
+    assert button.checked is None
+    assert button.focused is None
+
+
+def test_android_page_source_reports_full_interaction_state() -> None:
+    xml = """
+    <hierarchy>
+      <node class="android.widget.CheckBox" text="Remember me"
+            enabled="true" clickable="true" selected="false" checked="true"
+            focused="false" bounds="[10,20][60,50]" />
+    </hierarchy>
+    """
+
+    elements = parse_ui_tree(xml)
+
+    assert len(elements) == 1
+    checkbox = elements[0]
+    assert checkbox.enabled is True
+    assert checkbox.clickable is True
+    assert checkbox.selected is False
+    assert checkbox.checked is True
+    assert checkbox.focused is False
+
+
+def test_dict_tree_passes_through_native_bool_state() -> None:
+    tree = {
+        "type": "button",
+        "text": "Send",
+        "x": 0,
+        "y": 0,
+        "width": 10,
+        "height": 10,
+        "enabled": True,
+        "clickable": False,
+    }
+
+    elements = parse_ui_tree(tree)
+
+    assert len(elements) == 1
+    assert elements[0].enabled is True
+    assert elements[0].clickable is False
+    assert elements[0].selected is None
+
+
+def test_to_dict_omits_unknown_state_fields_but_keeps_reported_false() -> None:
+    xml = """
+    <AppiumAUT x="0" y="0" width="10" height="10">
+      <XCUIElementTypeButton name="Submit" enabled="false" x="0" y="0" width="4" height="4" />
+    </AppiumAUT>
+    """
+
+    elements = parse_ui_tree(xml)
+    element = next(item for item in elements if item.type == "button")
+    data = element.to_dict()
+
+    assert data["enabled"] is False
+    assert "clickable" not in data
+    assert "selected" not in data
+    assert "checked" not in data
+    assert "focused" not in data