feat(perception): expose active app metadata in UI tree
Tests / Test passed: 971

This commit is contained in:
2026-07-16 08:13:04 +08:00
parent 059fb272bb
commit 9a17297f1e
13 changed files with 246 additions and 17 deletions
+7
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
from typing import Any
from core.models import ActiveApp
from driver.base import Driver
PNG_10X20 = (
@@ -27,12 +28,14 @@ class FakeDriver(Driver):
fail_connect: bool = False,
tree: Any = TREE_XML,
screenshot: bytes = PNG_10X20,
active_app: ActiveApp | None = None,
) -> None:
self.fail_connect = fail_connect
self.connected = False
self.calls: list[tuple[str, tuple[Any, ...]]] = []
self._tree = tree
self._screenshot = screenshot
self._active_app = active_app
def connect(self) -> None:
self.calls.append(("connect", ()))
@@ -85,6 +88,10 @@ class FakeDriver(Driver):
self.calls.append(("tree", ()))
return self._tree
def active_app(self) -> ActiveApp | None:
self.calls.append(("active_app", ()))
return self._active_app
def home(self) -> None:
self.calls.append(("home", ()))
+54
View File
@@ -0,0 +1,54 @@
from __future__ import annotations
from unittest.mock import MagicMock
from core.models import ActiveApp
from driver.android_driver import AndroidDriver
from driver.wda_driver import WDADriver
from tools.ui_tree import get_ui_tree
from device.manager import DeviceManager
from tests.fakes import FakeDriver
def test_android_driver_reads_current_package_and_activity() -> None:
driver = AndroidDriver()
driver._client = MagicMock()
driver._client.current_package = "com.example.mobile"
driver._client.current_activity = ".MainActivity"
assert driver.active_app() == ActiveApp(
platform="android",
package="com.example.mobile",
activity=".MainActivity",
)
def test_wda_driver_reads_current_bundle_id() -> None:
driver = WDADriver()
driver._client = MagicMock()
driver._client.execute_script.return_value = {
"bundleId": "com.example.mobile",
"pid": 123,
}
assert driver.active_app() == ActiveApp(
platform="ios",
bundle_id="com.example.mobile",
)
driver._client.execute_script.assert_called_once_with("mobile: activeAppInfo")
def test_ui_tree_keeps_list_response_without_active_app_option() -> None:
manager = DeviceManager()
manager.register_device(
"phone-1",
lambda: FakeDriver(
active_app=ActiveApp(platform="ios", bundle_id="com.example.mobile")
),
)
manager.connect("phone-1")
response = get_ui_tree("phone-1", manager=manager)
assert isinstance(response, list)
assert any(node["text"] == "Search" for node in response)
+27 -1
View File
@@ -3,11 +3,12 @@ from __future__ import annotations
from pathlib import Path
from core.errors import DriverError
from core.models import Bounds
from core.models import ActiveApp, Bounds, Scene
from device.manager import DeviceManager
from driver.base import Driver
from perception.ocr import OCRBox
from tools.describe_screen import describe_screen
from tests.fakes import FakeDriver
PNG_10X20 = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x0a\x00\x00\x00\x14"
@@ -91,3 +92,28 @@ def test_describe_screen_degrades_to_ocr_only_when_ui_tree_unavailable() -> None
assert [element.text for element in scene.elements] == ["Search"]
assert all(element.source == "ocr" for element in scene.elements)
def test_describe_screen_includes_live_active_app_metadata() -> None:
active_app = ActiveApp(
platform="android",
package="com.example.mobile",
activity=".MainActivity",
)
manager = DeviceManager()
manager.register_device("phone-1", lambda: FakeDriver(active_app=active_app))
manager.connect("phone-1")
scene = describe_screen(
"phone-1",
manager=manager,
ocr_engine=FakeOCREngine([]), # type: ignore[arg-type]
)
assert scene.active_app == active_app
assert scene.to_dict()["app"] == {
"platform": "android",
"package": "com.example.mobile",
"activity": ".MainActivity",
}
assert Scene.from_dict(scene.to_dict()).active_app == active_app
+28
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from api.mcp import tool_handlers
from core.models import ActiveApp
from device.manager import DeviceManager
from tests.fakes import FakeDriver
@@ -36,3 +37,30 @@ def test_mcp_handlers_are_semantic_and_hide_driver_terms() -> None:
assert "Appium" not in serialized
assert "XCUI" not in serialized
assert all(response is not None for response in responses)
def test_mcp_ui_tree_can_include_active_app_info() -> None:
manager = DeviceManager()
manager.register_device(
"android-1",
lambda: FakeDriver(
active_app=ActiveApp(
platform="android",
package="com.example.mobile",
activity=".MainActivity",
)
),
)
manager.connect("android-1", max_retries=1)
response = tool_handlers(manager=manager)["get_ui_tree"](
device_id="android-1",
include_app_info=True,
)
assert response["app"] == {
"platform": "android",
"package": "com.example.mobile",
"activity": ".MainActivity",
}
assert isinstance(response["nodes"], list)