feat(perception): expose active app metadata in UI tree
Tests / Test passed: 971

This commit is contained in:
2026-07-16 08:13:04 +08:00
parent 059fb272bb
commit 9a17297f1e
13 changed files with 246 additions and 17 deletions
+43 -1
View File
@@ -75,6 +75,37 @@ _STATE_FIELDS = ("enabled", "clickable", "selected", "checked", "focused")
_COLOR_FIELDS = ("foreground_color", "background_color")
@dataclass(frozen=True)
class ActiveApp:
"""Native identifier for the application currently in the foreground."""
platform: str
bundle_id: str | None = None
package: str | None = None
activity: str | None = None
def to_dict(self) -> dict[str, str]:
data = {"platform": self.platform}
for field_name in ("bundle_id", "package", "activity"):
value = getattr(self, field_name)
if value is not None:
data[field_name] = value
return data
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "ActiveApp":
def text(key: str) -> str | None:
value = data.get(key)
return value if isinstance(value, str) and value else None
return cls(
platform=text("platform") or "unknown",
bundle_id=text("bundle_id"),
package=text("package"),
activity=text("activity"),
)
@dataclass
class SceneElement:
id: str
@@ -143,12 +174,19 @@ class Scene:
# Keep raw OCR observations for local execution evidence without duplicating
# them in the normalized, LLM-facing scene payload.
ocr_elements: list[SceneElement] = field(default_factory=list)
# The foreground app is supplied by the Driver, separately from the
# accessibility tree, and is absent for drivers that cannot query it.
# Kept last to preserve Scene's existing positional constructor arguments.
active_app: ActiveApp | None = None
def to_dict(self) -> dict[str, Any]:
return {
data: dict[str, Any] = {
"screen": {"width": self.width, "height": self.height},
"elements": [element.to_dict() for element in self.elements],
}
if self.active_app is not None:
data["app"] = self.active_app.to_dict()
return data
def ocr_results_to_dict(self) -> list[dict[str, Any]]:
return [element.to_dict() for element in self.ocr_elements]
@@ -156,12 +194,16 @@ class Scene:
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "Scene":
screen = data.get("screen") or {}
raw_app = data.get("app")
return cls(
width=int(screen.get("width") or data.get("width") or 0),
height=int(screen.get("height") or data.get("height") or 0),
elements=[
SceneElement.from_dict(element) for element in data.get("elements", [])
],
active_app=ActiveApp.from_dict(raw_app)
if isinstance(raw_app, dict)
else None,
ocr_elements=[
SceneElement.from_dict(element)
for element in data.get("ocr_elements", [])