feat(perception): expose active app metadata in UI tree
Tests / Test passed: 971

This commit is contained in:
2026-07-16 08:13:04 +08:00
parent 059fb272bb
commit 9a17297f1e
13 changed files with 246 additions and 17 deletions
+17
View File
@@ -1,8 +1,13 @@
from __future__ import annotations
import logging
from core.models import ActiveApp
from device.manager import DEFAULT_MANAGER, DeviceManager
from driver.base import Driver
logger = logging.getLogger(__name__)
def get_driver(
device_id: str | None = None,
@@ -10,3 +15,15 @@ def get_driver(
manager: DeviceManager | None = None,
) -> Driver:
return (manager or DEFAULT_MANAGER).active_driver(device_id)
def get_active_app_info(driver: Driver) -> ActiveApp | None:
"""Best-effort foreground application lookup that cannot block observation."""
try:
return driver.active_app()
except Exception:
logger.warning(
"active app retrieval failed; continuing without app metadata",
exc_info=True,
)
return None
+4 -2
View File
@@ -6,7 +6,7 @@ from core.models import Scene
from device.manager import DeviceManager
from perception.ocr import PaddleOCREngine
from perception.provider import DefaultPerceptionProvider, PerceptionProvider
from tools._device import get_driver
from tools._device import get_active_app_info, get_driver
logger = logging.getLogger(__name__)
@@ -28,4 +28,6 @@ def describe_screen(
)
raw_tree = None
provider = perception_provider or DefaultPerceptionProvider(ocr_engine=ocr_engine)
return provider.build_scene(screenshot, raw_tree)
scene = provider.build_scene(screenshot, raw_tree)
scene.active_app = get_active_app_info(driver)
return scene
+12 -6
View File
@@ -4,7 +4,7 @@ from typing import Any
from device.manager import DeviceManager
from perception.ui_parser import parse_ui_tree
from tools._device import get_driver
from tools._device import get_active_app_info, get_driver
def get_raw_ui_tree(
@@ -19,8 +19,14 @@ def get_ui_tree(
device_id: str | None = None,
*,
manager: DeviceManager | None = None,
) -> list[dict[str, object]]:
return [
element.to_dict()
for element in parse_ui_tree(get_raw_ui_tree(device_id, manager=manager))
]
include_app_info: bool = False,
) -> list[dict[str, object]] | dict[str, object]:
driver = get_driver(device_id, manager=manager)
nodes = [element.to_dict() for element in parse_ui_tree(driver.tree())]
if not include_app_info:
return nodes
active_app = get_active_app_info(driver)
return {
"app": active_app.to_dict() if active_app is not None else None,
"nodes": nodes,
}