feat(host-agent): persist UI-tree evidence and add overlay/action visualization
Tests / Test passed: 863
Tests / Test passed: 863
Fixes issue 3: the host-agent console showed OCR results but never real UI-tree data, because _ui_tree_nodes() checked for a get_ui_tree/ui_tree tool action that has never existed anywhere in the codebase. - storage/timeline.py: add a ui_tree_results field to TimelineRecord and Timeline.append(), mirroring the existing ocr_results field. - runtime/task.py: _append_timeline() now extracts scene.elements with source == "ui" into ui_tree_results (scene_builder.build_scene() already preserved these; they were just never persisted). - host_agent/web/app.py: _ui_tree_nodes() reads the new field directly instead of the dead tool-action check. New _overlay_payload() exposes each step's scene dimensions and fused element list for client-side rendering. - task_detail.html: adds a toggle to overlay OCR (orange) and UI-tree (blue) bounding boxes on the before-action screenshot, plus a visual marker for the actually executed action (tap circle, or an animated swipe path) using an SVG viewBox so no manual coordinate-scaling JS is needed. Legacy/incomplete records degrade to no overlay, never an error. Also corrects openspec/specs/runtime-task-evidence and host-agent-console-task-pages, which had encoded the same nonexistent-tool assumption, via the new host-agent-console-visual-evidence change. 600 tests passing; ruff/compileall/openspec validate all clean.
This commit is contained in:
@@ -94,20 +94,30 @@ def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
|
||||
|
||||
def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
tool_call = record.get("tool_call")
|
||||
if not isinstance(tool_call, dict):
|
||||
return []
|
||||
if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}:
|
||||
return []
|
||||
step_result = record.get("result")
|
||||
if not isinstance(step_result, dict):
|
||||
return []
|
||||
raw_nodes = step_result.get("result")
|
||||
raw_nodes = record.get("ui_tree_results")
|
||||
if not isinstance(raw_nodes, list):
|
||||
return []
|
||||
return [node for node in raw_nodes if isinstance(node, dict)]
|
||||
|
||||
|
||||
def _overlay_payload(record: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Combined perception elements + screen size for client-side bounding-box
|
||||
overlay and action-effect rendering on the before-screenshot.
|
||||
"""
|
||||
scene = record.get("scene")
|
||||
screen = scene.get("screen") if isinstance(scene, dict) else None
|
||||
width = screen.get("width") if isinstance(screen, dict) else None
|
||||
height = screen.get("height") if isinstance(screen, dict) else None
|
||||
elements = scene.get("elements") if isinstance(scene, dict) else None
|
||||
return {
|
||||
"width": width if isinstance(width, (int, float)) else 0,
|
||||
"height": height if isinstance(height, (int, float)) else 0,
|
||||
"elements": [element for element in elements if isinstance(element, dict)]
|
||||
if isinstance(elements, list)
|
||||
else [],
|
||||
}
|
||||
|
||||
|
||||
def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]:
|
||||
tool_call = record.get("tool_call")
|
||||
result = record.get("result")
|
||||
@@ -126,6 +136,7 @@ def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]:
|
||||
or _screenshot_data_uri(record),
|
||||
"ocr_results": _ocr_results(record),
|
||||
"ui_tree_nodes": _ui_tree_nodes(record),
|
||||
"overlay": _overlay_payload(record),
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user