From 361dada276b78129308688e8575e9c87c95280d2 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 15 Jul 2026 18:10:53 +0800 Subject: [PATCH] feat(perception): surface accessibility interaction state on UI-tree elements SceneElement gains enabled/clickable/selected/checked/focused (bool | None), populated from the literal attributes Appium's XCUITest and UiAutomator2 page_source already emit (iOS: enabled only; Android: all five). None means "not reported by this platform", not false. to_dict() omits unset fields to keep the LLM-facing scene JSON compact; planner_prompts.py documents the new fields so the AI planner knows how to use them (e.g. don't tap disabled elements, use selected/checked to judge whether a toggle already matches the goal). --- core/models.py | 19 +++++++++ perception/ui_parser.py | 35 ++++++++++++---- runtime/planner_prompts.py | 15 +++++++ tests/test_ui_parser.py | 81 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 142 insertions(+), 8 deletions(-) create mode 100644 tests/test_ui_parser.py diff --git a/core/models.py b/core/models.py index ecae5ad..307cee3 100644 --- a/core/models.py +++ b/core/models.py @@ -71,6 +71,9 @@ class Device: } +_STATE_FIELDS = ("enabled", "clickable", "selected", "checked", "focused") + + @dataclass class SceneElement: id: str @@ -79,6 +82,13 @@ class SceneElement: text: str | None = None confidence: float | None = None source: str | None = None + # Accessibility-tree interaction state, when the platform reports it. + # None means "not reported by this platform/element", not "false". + enabled: bool | None = None + clickable: bool | None = None + selected: bool | None = None + checked: bool | None = None + focused: bool | None = None @property def center(self) -> tuple[float, float]: @@ -94,6 +104,10 @@ class SceneElement: } if self.source: data["source"] = self.source + for field_name in _STATE_FIELDS: + value = getattr(self, field_name) + if value is not None: + data[field_name] = value return data @classmethod @@ -105,6 +119,11 @@ class SceneElement: bounds=Bounds.from_dict(data["bounds"]), confidence=data.get("confidence"), source=data.get("source"), + enabled=data.get("enabled"), + clickable=data.get("clickable"), + selected=data.get("selected"), + checked=data.get("checked"), + focused=data.get("focused"), ) diff --git a/perception/ui_parser.py b/perception/ui_parser.py index 2c92788..3f8f0f7 100644 --- a/perception/ui_parser.py +++ b/perception/ui_parser.py @@ -2,12 +2,19 @@ from __future__ import annotations import re import xml.etree.ElementTree as ET +from dataclasses import replace from typing import Any from core.models import Bounds, SceneElement _ANDROID_BOUNDS = re.compile(r"\[(?P-?\d+),(?P-?\d+)\]\[(?P-?\d+),(?P-?\d+)\]") +# Accessibility interaction-state attributes, as literally emitted by +# WDA/XCUITest page_source (enabled) and Appium UiAutomator2 page_source +# (enabled, clickable, selected, checked, focused). iOS page_source does not +# emit selected/checked/focused/clickable, so those stay None for iOS trees. +_STATE_ATTR_KEYS = ("enabled", "clickable", "selected", "checked", "focused") + def parse_ui_tree(raw_tree: Any) -> list[SceneElement]: if raw_tree is None: @@ -44,6 +51,7 @@ def _parse_xml(raw_tree: str) -> list[SceneElement]: bounds=bounds, confidence=1.0, source="ui", + **_state_from_attrs(node.attrib), ) ) return _with_stable_ids(elements, "ui") @@ -65,6 +73,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None bounds=bounds, confidence=1.0, source="ui", + **_state_from_attrs(attrs), ) ) for child in children: @@ -74,14 +83,7 @@ def _parse_dict_node(node: dict[str, Any], elements: list[SceneElement]) -> None def _with_stable_ids(elements: list[SceneElement], prefix: str) -> list[SceneElement]: return [ - SceneElement( - id=element.id or f"{prefix}-{index:03d}", - type=element.type, - text=element.text, - bounds=element.bounds, - confidence=element.confidence, - source=element.source, - ) + element if element.id else replace(element, id=f"{prefix}-{index:03d}") for index, element in enumerate(elements) ] @@ -122,6 +124,23 @@ def _text_from_attrs(attrs: dict[str, Any]) -> str | None: return None +def _state_from_attrs(attrs: dict[str, Any]) -> dict[str, bool | None]: + return {key: _parse_bool(attrs.get(key)) for key in _STATE_ATTR_KEYS} + + +def _parse_bool(value: Any) -> bool | None: + if isinstance(value, bool): + return value + if value is None: + return None + text = str(value).strip().lower() + if text in ("true", "1", "yes"): + return True + if text in ("false", "0", "no"): + return False + return None + + def _normalize_type(raw_type: str, attrs: dict[str, Any]) -> str: candidate = ( attrs.get("role") diff --git a/runtime/planner_prompts.py b/runtime/planner_prompts.py index c245118..fda6f6c 100644 --- a/runtime/planner_prompts.py +++ b/runtime/planner_prompts.py @@ -10,6 +10,15 @@ list of UI elements with id, type, text, and pixel bounds), and — when available — a screenshot of the same screen and a short history of recent actions and their outcomes. +Some elements also carry accessibility state fields when the platform +reports them: `enabled`, `clickable`, `selected`, `checked`, `focused`. +A field is omitted entirely when the platform does not report it for that +element — omitted does NOT mean false, treat it as unknown. When present, +`enabled: false` or `clickable: false` means the element cannot currently be +interacted with (do not tap it); `selected`/`checked`/`focused` describe its +current toggle/focus state and are useful for deciding whether an action is +already done or still needed. + Before calling a tool, output a short text block (1-2 sentences): 1. If this is the first step, state what you intend to do and why. 2. Otherwise, first assess whether the previous action achieved its intended @@ -22,6 +31,12 @@ You must then call exactly one tool: - `finish_task` when the goal has been reached, or when it cannot be reached and no further action would help. +For every device-action tool call, you must provide both required structured +fields in addition to the physical-action arguments: +- `purpose`: one concise sentence describing why this action advances the goal. +- `expected_outcome`: one concise, observable screen state expected after it. +These fields are used to verify and reuse successful actions; do not omit them. + Ground every coordinate you choose in the Scene element bounds (and the screenshot, if provided) for the current turn only — never reuse coordinates from history, since the screen may have changed. Only call `finish_task` with diff --git a/tests/test_ui_parser.py b/tests/test_ui_parser.py new file mode 100644 index 0000000..0054c8b --- /dev/null +++ b/tests/test_ui_parser.py @@ -0,0 +1,81 @@ +from __future__ import annotations + +from perception.ui_parser import parse_ui_tree + + +def test_ios_page_source_reports_enabled_only() -> None: + xml = """ + + + + """ + + elements = parse_ui_tree(xml) + + button = next(element for element in elements if element.type == "button") + assert button.enabled is False + # iOS page_source never emits these, so they must stay unknown (None), + # not be defaulted to False. + assert button.clickable is None + assert button.selected is None + assert button.checked is None + assert button.focused is None + + +def test_android_page_source_reports_full_interaction_state() -> None: + xml = """ + + + + """ + + elements = parse_ui_tree(xml) + + assert len(elements) == 1 + checkbox = elements[0] + assert checkbox.enabled is True + assert checkbox.clickable is True + assert checkbox.selected is False + assert checkbox.checked is True + assert checkbox.focused is False + + +def test_dict_tree_passes_through_native_bool_state() -> None: + tree = { + "type": "button", + "text": "Send", + "x": 0, + "y": 0, + "width": 10, + "height": 10, + "enabled": True, + "clickable": False, + } + + elements = parse_ui_tree(tree) + + assert len(elements) == 1 + assert elements[0].enabled is True + assert elements[0].clickable is False + assert elements[0].selected is None + + +def test_to_dict_omits_unknown_state_fields_but_keeps_reported_false() -> None: + xml = """ + + + + """ + + elements = parse_ui_tree(xml) + element = next(item for item in elements if item.type == "button") + data = element.to_dict() + + assert data["enabled"] is False + assert "clickable" not in data + assert "selected" not in data + assert "checked" not in data + assert "focused" not in data