From 9a17297f1eb15e204e649581d4068fde63eca1a5 Mon Sep 17 00:00:00 2001
From: Jerry Yan <792602257@qq.com>
Date: Thu, 16 Jul 2026 08:13:04 +0800
Subject: [PATCH] feat(perception): expose active app metadata in UI tree

---
 api/mcp.py                    | 21 ++++++++++----
 core/models.py                | 44 +++++++++++++++++++++++++++-
 driver/android_driver.py      | 14 +++++++++
 driver/base.py                |  7 ++++-
 driver/wda_driver.py          | 14 +++++++++
 runtime/planner_prompts.py    |  5 ++++
 tests/fakes.py                |  7 +++++
 tests/test_active_app.py      | 54 +++++++++++++++++++++++++++++++++++
 tests/test_describe_screen.py | 28 +++++++++++++++++-
 tests/test_mcp.py             | 28 ++++++++++++++++++
 tools/_device.py              | 17 +++++++++++
 tools/describe_screen.py      |  6 ++--
 tools/ui_tree.py              | 18 ++++++++----
 13 files changed, 246 insertions(+), 17 deletions(-)
 create mode 100644 tests/test_active_app.py

diff --git a/api/mcp.py b/api/mcp.py
index c7a8491..f2861e9 100644
--- a/api/mcp.py
+++ b/api/mcp.py
@@ -75,10 +75,13 @@ def tool_handlers(
             device_id=device_id,
             manager=device_manager,
         ),
-        "get_ui_tree": lambda device_id=None: call_with_semantic_errors(
-            get_ui_tree,
-            device_id,
-            manager=device_manager,
+        "get_ui_tree": lambda device_id=None, include_app_info=False: (
+            call_with_semantic_errors(
+                get_ui_tree,
+                device_id,
+                manager=device_manager,
+                include_app_info=include_app_info,
+            )
         ),
         "describe_screen": lambda device_id=None: call_with_semantic_errors(
             lambda: describe_screen(device_id, manager=device_manager).to_dict()
@@ -150,8 +153,14 @@ def create_mcp_server(
         return handlers["find_icon"](name=name, device_id=device_id)
 
     @server.tool(name="get_ui_tree")
-    def _get_ui_tree(device_id: str | None = None) -> Any:
-        return handlers["get_ui_tree"](device_id=device_id)
+    def _get_ui_tree(
+        device_id: str | None = None,
+        include_app_info: bool = False,
+    ) -> Any:
+        return handlers["get_ui_tree"](
+            device_id=device_id,
+            include_app_info=include_app_info,
+        )
 
     @server.tool(name="describe_screen")
     def _describe_screen(device_id: str | None = None) -> dict[str, Any]:
diff --git a/core/models.py b/core/models.py
index df07189..7608942 100644
--- a/core/models.py
+++ b/core/models.py
@@ -75,6 +75,37 @@ _STATE_FIELDS = ("enabled", "clickable", "selected", "checked", "focused")
 _COLOR_FIELDS = ("foreground_color", "background_color")
 
 
+@dataclass(frozen=True)
+class ActiveApp:
+    """Native identifier for the application currently in the foreground."""
+
+    platform: str
+    bundle_id: str | None = None
+    package: str | None = None
+    activity: str | None = None
+
+    def to_dict(self) -> dict[str, str]:
+        data = {"platform": self.platform}
+        for field_name in ("bundle_id", "package", "activity"):
+            value = getattr(self, field_name)
+            if value is not None:
+                data[field_name] = value
+        return data
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "ActiveApp":
+        def text(key: str) -> str | None:
+            value = data.get(key)
+            return value if isinstance(value, str) and value else None
+
+        return cls(
+            platform=text("platform") or "unknown",
+            bundle_id=text("bundle_id"),
+            package=text("package"),
+            activity=text("activity"),
+        )
+
+
 @dataclass
 class SceneElement:
     id: str
@@ -143,12 +174,19 @@ class Scene:
     # Keep raw OCR observations for local execution evidence without duplicating
     # them in the normalized, LLM-facing scene payload.
     ocr_elements: list[SceneElement] = field(default_factory=list)
+    # The foreground app is supplied by the Driver, separately from the
+    # accessibility tree, and is absent for drivers that cannot query it.
+    # Kept last to preserve Scene's existing positional constructor arguments.
+    active_app: ActiveApp | None = None
 
     def to_dict(self) -> dict[str, Any]:
-        return {
+        data: dict[str, Any] = {
             "screen": {"width": self.width, "height": self.height},
             "elements": [element.to_dict() for element in self.elements],
         }
+        if self.active_app is not None:
+            data["app"] = self.active_app.to_dict()
+        return data
 
     def ocr_results_to_dict(self) -> list[dict[str, Any]]:
         return [element.to_dict() for element in self.ocr_elements]
@@ -156,12 +194,16 @@ class Scene:
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "Scene":
         screen = data.get("screen") or {}
+        raw_app = data.get("app")
         return cls(
             width=int(screen.get("width") or data.get("width") or 0),
             height=int(screen.get("height") or data.get("height") or 0),
             elements=[
                 SceneElement.from_dict(element) for element in data.get("elements", [])
             ],
+            active_app=ActiveApp.from_dict(raw_app)
+            if isinstance(raw_app, dict)
+            else None,
             ocr_elements=[
                 SceneElement.from_dict(element)
                 for element in data.get("ocr_elements", [])
diff --git a/driver/android_driver.py b/driver/android_driver.py
index bc226de..160530d 100644
--- a/driver/android_driver.py
+++ b/driver/android_driver.py
@@ -5,6 +5,7 @@ from dataclasses import dataclass, field
 from typing import Any
 
 from core.errors import DeviceOfflineError, DriverError
+from core.models import ActiveApp
 from driver.base import Driver
 
 # Android KeyEvent.KEYCODE_HOME. Kept as a literal rather than importing the
@@ -170,6 +171,19 @@ class AndroidDriver(Driver):
         except Exception as exc:
             raise DriverError("ui tree retrieval failed") from exc
 
+    def active_app(self) -> ActiveApp | None:
+        client = self._require_client()
+        try:
+            package = client.current_package
+            activity = client.current_activity
+        except Exception as exc:
+            raise DriverError("active app retrieval failed") from exc
+        package = package if isinstance(package, str) and package else None
+        activity = activity if isinstance(activity, str) and activity else None
+        if package is None and activity is None:
+            return None
+        return ActiveApp(platform="android", package=package, activity=activity)
+
     def home(self) -> None:
         client = self._require_client()
         try:
diff --git a/driver/base.py b/driver/base.py
index c203960..f4cc0e7 100644
--- a/driver/base.py
+++ b/driver/base.py
@@ -3,6 +3,8 @@ from __future__ import annotations
 from abc import ABC, abstractmethod
 from typing import Any
 
+from core.models import ActiveApp
+
 
 class Driver(ABC):
     """Driver-independent device capability interface.
@@ -71,6 +73,10 @@ class Driver(ABC):
     def tree(self) -> Any:
         """Return the raw UI tree from the device driver."""
 
+    def active_app(self) -> ActiveApp | None:
+        """Return foreground application metadata when the driver supports it."""
+        return None
+
     @abstractmethod
     def home(self) -> None:
         """Press the device home button."""
@@ -82,4 +88,3 @@ class Driver(ABC):
     @abstractmethod
     def unlock(self) -> None:
         """Unlock the device."""
-
diff --git a/driver/wda_driver.py b/driver/wda_driver.py
index 5f8c05b..169daca 100644
--- a/driver/wda_driver.py
+++ b/driver/wda_driver.py
@@ -4,6 +4,7 @@ from dataclasses import dataclass, field
 from typing import Any
 
 from core.errors import DeviceOfflineError, DriverError
+from core.models import ActiveApp
 from driver.base import Driver
 
 
@@ -161,6 +162,19 @@ class WDADriver(Driver):
         except Exception as exc:
             raise DriverError("ui tree retrieval failed") from exc
 
+    def active_app(self) -> ActiveApp | None:
+        client = self._require_client()
+        try:
+            response = client.execute_script("mobile: activeAppInfo")
+        except Exception as exc:
+            raise DriverError("active app retrieval failed") from exc
+        if not isinstance(response, dict):
+            return None
+        bundle_id = response.get("bundleId")
+        if not isinstance(bundle_id, str) or not bundle_id:
+            return None
+        return ActiveApp(platform="ios", bundle_id=bundle_id)
+
     def home(self) -> None:
         client = self._require_client()
         try:
diff --git a/runtime/planner_prompts.py b/runtime/planner_prompts.py
index b465d2d..be8b0b6 100644
--- a/runtime/planner_prompts.py
+++ b/runtime/planner_prompts.py
@@ -10,6 +10,11 @@ list of UI elements with id, type, text, and pixel bounds), and — when
 available — a screenshot of the same screen and a short history of recent
 actions and their outcomes.
 
+The Scene may include an `app` object with the live foreground application's
+native identifier. On iOS, use `bundle_id`; on Android, use `package` and
+`activity`. This metadata may be omitted when the current driver cannot query
+it.
+
 Some elements also carry accessibility state fields when the platform
 reports them: `enabled`, `clickable`, `selected`, `checked`, `focused`.
 A field is omitted entirely when the platform does not report it for that
diff --git a/tests/fakes.py b/tests/fakes.py
index 038ea5b..e72f750 100644
--- a/tests/fakes.py
+++ b/tests/fakes.py
@@ -2,6 +2,7 @@ from __future__ import annotations
 
 from typing import Any
 
+from core.models import ActiveApp
 from driver.base import Driver
 
 PNG_10X20 = (
@@ -27,12 +28,14 @@ class FakeDriver(Driver):
         fail_connect: bool = False,
         tree: Any = TREE_XML,
         screenshot: bytes = PNG_10X20,
+        active_app: ActiveApp | None = None,
     ) -> None:
         self.fail_connect = fail_connect
         self.connected = False
         self.calls: list[tuple[str, tuple[Any, ...]]] = []
         self._tree = tree
         self._screenshot = screenshot
+        self._active_app = active_app
 
     def connect(self) -> None:
         self.calls.append(("connect", ()))
@@ -85,6 +88,10 @@ class FakeDriver(Driver):
         self.calls.append(("tree", ()))
         return self._tree
 
+    def active_app(self) -> ActiveApp | None:
+        self.calls.append(("active_app", ()))
+        return self._active_app
+
     def home(self) -> None:
         self.calls.append(("home", ()))
 
diff --git a/tests/test_active_app.py b/tests/test_active_app.py
new file mode 100644
index 0000000..072c160
--- /dev/null
+++ b/tests/test_active_app.py
@@ -0,0 +1,54 @@
+from __future__ import annotations
+
+from unittest.mock import MagicMock
+
+from core.models import ActiveApp
+from driver.android_driver import AndroidDriver
+from driver.wda_driver import WDADriver
+from tools.ui_tree import get_ui_tree
+from device.manager import DeviceManager
+from tests.fakes import FakeDriver
+
+
+def test_android_driver_reads_current_package_and_activity() -> None:
+    driver = AndroidDriver()
+    driver._client = MagicMock()
+    driver._client.current_package = "com.example.mobile"
+    driver._client.current_activity = ".MainActivity"
+
+    assert driver.active_app() == ActiveApp(
+        platform="android",
+        package="com.example.mobile",
+        activity=".MainActivity",
+    )
+
+
+def test_wda_driver_reads_current_bundle_id() -> None:
+    driver = WDADriver()
+    driver._client = MagicMock()
+    driver._client.execute_script.return_value = {
+        "bundleId": "com.example.mobile",
+        "pid": 123,
+    }
+
+    assert driver.active_app() == ActiveApp(
+        platform="ios",
+        bundle_id="com.example.mobile",
+    )
+    driver._client.execute_script.assert_called_once_with("mobile: activeAppInfo")
+
+
+def test_ui_tree_keeps_list_response_without_active_app_option() -> None:
+    manager = DeviceManager()
+    manager.register_device(
+        "phone-1",
+        lambda: FakeDriver(
+            active_app=ActiveApp(platform="ios", bundle_id="com.example.mobile")
+        ),
+    )
+    manager.connect("phone-1")
+
+    response = get_ui_tree("phone-1", manager=manager)
+
+    assert isinstance(response, list)
+    assert any(node["text"] == "Search" for node in response)
diff --git a/tests/test_describe_screen.py b/tests/test_describe_screen.py
index 0a6ded2..d1a3a1b 100644
--- a/tests/test_describe_screen.py
+++ b/tests/test_describe_screen.py
@@ -3,11 +3,12 @@ from __future__ import annotations
 from pathlib import Path
 
 from core.errors import DriverError
-from core.models import Bounds
+from core.models import ActiveApp, Bounds, Scene
 from device.manager import DeviceManager
 from driver.base import Driver
 from perception.ocr import OCRBox
 from tools.describe_screen import describe_screen
+from tests.fakes import FakeDriver
 
 PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14"
 
@@ -91,3 +92,28 @@ def test_describe_screen_degrades_to_ocr_only_when_ui_tree_unavailable() -> None
 
     assert [element.text for element in scene.elements] == ["Search"]
     assert all(element.source == "ocr" for element in scene.elements)
+
+
+def test_describe_screen_includes_live_active_app_metadata() -> None:
+    active_app = ActiveApp(
+        platform="android",
+        package="com.example.mobile",
+        activity=".MainActivity",
+    )
+    manager = DeviceManager()
+    manager.register_device("phone-1", lambda: FakeDriver(active_app=active_app))
+    manager.connect("phone-1")
+
+    scene = describe_screen(
+        "phone-1",
+        manager=manager,
+        ocr_engine=FakeOCREngine([]),  # type: ignore[arg-type]
+    )
+
+    assert scene.active_app == active_app
+    assert scene.to_dict()["app"] == {
+        "platform": "android",
+        "package": "com.example.mobile",
+        "activity": ".MainActivity",
+    }
+    assert Scene.from_dict(scene.to_dict()).active_app == active_app
diff --git a/tests/test_mcp.py b/tests/test_mcp.py
index 4ee02b3..049f3a3 100644
--- a/tests/test_mcp.py
+++ b/tests/test_mcp.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from api.mcp import tool_handlers
+from core.models import ActiveApp
 from device.manager import DeviceManager
 from tests.fakes import FakeDriver
 
@@ -36,3 +37,30 @@ def test_mcp_handlers_are_semantic_and_hide_driver_terms() -> None:
     assert "Appium" not in serialized
     assert "XCUI" not in serialized
     assert all(response is not None for response in responses)
+
+
+def test_mcp_ui_tree_can_include_active_app_info() -> None:
+    manager = DeviceManager()
+    manager.register_device(
+        "android-1",
+        lambda: FakeDriver(
+            active_app=ActiveApp(
+                platform="android",
+                package="com.example.mobile",
+                activity=".MainActivity",
+            )
+        ),
+    )
+    manager.connect("android-1", max_retries=1)
+
+    response = tool_handlers(manager=manager)["get_ui_tree"](
+        device_id="android-1",
+        include_app_info=True,
+    )
+
+    assert response["app"] == {
+        "platform": "android",
+        "package": "com.example.mobile",
+        "activity": ".MainActivity",
+    }
+    assert isinstance(response["nodes"], list)
diff --git a/tools/_device.py b/tools/_device.py
index 9709253..342c772 100644
--- a/tools/_device.py
+++ b/tools/_device.py
@@ -1,8 +1,13 @@
 from __future__ import annotations
 
+import logging
+
+from core.models import ActiveApp
 from device.manager import DEFAULT_MANAGER, DeviceManager
 from driver.base import Driver
 
+logger = logging.getLogger(__name__)
+
 
 def get_driver(
     device_id: str | None = None,
@@ -10,3 +15,15 @@ def get_driver(
     manager: DeviceManager | None = None,
 ) -> Driver:
     return (manager or DEFAULT_MANAGER).active_driver(device_id)
+
+
+def get_active_app_info(driver: Driver) -> ActiveApp | None:
+    """Best-effort foreground application lookup that cannot block observation."""
+    try:
+        return driver.active_app()
+    except Exception:
+        logger.warning(
+            "active app retrieval failed; continuing without app metadata",
+            exc_info=True,
+        )
+        return None
diff --git a/tools/describe_screen.py b/tools/describe_screen.py
index cbda080..2c44f49 100644
--- a/tools/describe_screen.py
+++ b/tools/describe_screen.py
@@ -6,7 +6,7 @@ from core.models import Scene
 from device.manager import DeviceManager
 from perception.ocr import PaddleOCREngine
 from perception.provider import DefaultPerceptionProvider, PerceptionProvider
-from tools._device import get_driver
+from tools._device import get_active_app_info, get_driver
 
 logger = logging.getLogger(__name__)
 
@@ -28,4 +28,6 @@ def describe_screen(
         )
         raw_tree = None
     provider = perception_provider or DefaultPerceptionProvider(ocr_engine=ocr_engine)
-    return provider.build_scene(screenshot, raw_tree)
+    scene = provider.build_scene(screenshot, raw_tree)
+    scene.active_app = get_active_app_info(driver)
+    return scene
diff --git a/tools/ui_tree.py b/tools/ui_tree.py
index a318ca5..7eb864d 100644
--- a/tools/ui_tree.py
+++ b/tools/ui_tree.py
@@ -4,7 +4,7 @@ from typing import Any
 
 from device.manager import DeviceManager
 from perception.ui_parser import parse_ui_tree
-from tools._device import get_driver
+from tools._device import get_active_app_info, get_driver
 
 
 def get_raw_ui_tree(
@@ -19,8 +19,14 @@ def get_ui_tree(
     device_id: str | None = None,
     *,
     manager: DeviceManager | None = None,
-) -> list[dict[str, object]]:
-    return [
-        element.to_dict()
-        for element in parse_ui_tree(get_raw_ui_tree(device_id, manager=manager))
-    ]
+    include_app_info: bool = False,
+) -> list[dict[str, object]] | dict[str, object]:
+    driver = get_driver(device_id, manager=manager)
+    nodes = [element.to_dict() for element in parse_ui_tree(driver.tree())]
+    if not include_app_info:
+        return nodes
+    active_app = get_active_app_info(driver)
+    return {
+        "app": active_app.to_dict() if active_app is not None else None,
+        "nodes": nodes,
+    }