feat(host-agent): persist UI-tree evidence and add overlay/action visualization
Tests / Test passed: 863
Tests / Test passed: 863
Fixes issue 3: the host-agent console showed OCR results but never real UI-tree data, because _ui_tree_nodes() checked for a get_ui_tree/ui_tree tool action that has never existed anywhere in the codebase. - storage/timeline.py: add a ui_tree_results field to TimelineRecord and Timeline.append(), mirroring the existing ocr_results field. - runtime/task.py: _append_timeline() now extracts scene.elements with source == "ui" into ui_tree_results (scene_builder.build_scene() already preserved these; they were just never persisted). - host_agent/web/app.py: _ui_tree_nodes() reads the new field directly instead of the dead tool-action check. New _overlay_payload() exposes each step's scene dimensions and fused element list for client-side rendering. - task_detail.html: adds a toggle to overlay OCR (orange) and UI-tree (blue) bounding boxes on the before-action screenshot, plus a visual marker for the actually executed action (tap circle, or an animated swipe path) using an SVG viewBox so no manual coordinate-scaling JS is needed. Legacy/incomplete records degrade to no overlay, never an error. Also corrects openspec/specs/runtime-task-evidence and host-agent-console-task-pages, which had encoded the same nonexistent-tool assumption, via the new host-agent-console-visual-evidence change. 600 tests passing; ruff/compileall/openspec validate all clean.
This commit is contained in:
@@ -223,18 +223,8 @@ def test_authenticated_task_detail_renders_complete_step_evidence(tmp_path) -> N
|
||||
task_id=task.id,
|
||||
scene={"screen": {"width": 10, "height": 20}, "elements": []},
|
||||
prompt="inspect the current UI tree",
|
||||
tool_call={"action": "get_ui_tree", "description": "inspect UI tree"},
|
||||
result={
|
||||
"result": [
|
||||
{
|
||||
"id": "search",
|
||||
"type": "button",
|
||||
"text": "Search",
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
"confidence": 0.98,
|
||||
}
|
||||
]
|
||||
},
|
||||
tool_call={"action": "tap", "description": "tap search"},
|
||||
result={"ok": True},
|
||||
before_screenshot=PNG_10X20,
|
||||
after_screenshot=PNG_10X20 + b"after",
|
||||
ocr_results=[
|
||||
@@ -244,6 +234,15 @@ def test_authenticated_task_detail_renders_complete_step_evidence(tmp_path) -> N
|
||||
"confidence": 0.95,
|
||||
}
|
||||
],
|
||||
ui_tree_results=[
|
||||
{
|
||||
"id": "search",
|
||||
"type": "button",
|
||||
"text": "Search",
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
"confidence": 0.98,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
client, _ = _build_client(
|
||||
@@ -278,3 +277,51 @@ def test_tasks_list_shows_empty_state(tmp_path) -> None:
|
||||
response = client.get("/tasks")
|
||||
assert response.status_code == 200
|
||||
assert "No executions recorded yet" in response.text
|
||||
|
||||
|
||||
def test_ui_tree_nodes_reads_persisted_field_directly() -> None:
|
||||
from host_agent.web.app import _ui_tree_nodes
|
||||
|
||||
record = {
|
||||
"ui_tree_results": [
|
||||
{"id": "search", "type": "button", "text": "Search"},
|
||||
"not-a-dict",
|
||||
]
|
||||
}
|
||||
assert _ui_tree_nodes(record) == [
|
||||
{"id": "search", "type": "button", "text": "Search"}
|
||||
]
|
||||
|
||||
|
||||
def test_ui_tree_nodes_defaults_to_empty_for_legacy_record() -> None:
|
||||
from host_agent.web.app import _ui_tree_nodes
|
||||
|
||||
assert _ui_tree_nodes({}) == []
|
||||
assert _ui_tree_nodes({"ui_tree_results": "not-a-list"}) == []
|
||||
|
||||
|
||||
def test_overlay_payload_combines_scene_dimensions_and_elements() -> None:
|
||||
from host_agent.web.app import _overlay_payload
|
||||
|
||||
record = {
|
||||
"scene": {
|
||||
"screen": {"width": 100, "height": 200},
|
||||
"elements": [{"id": "ui-000", "source": "ui"}, "not-a-dict"],
|
||||
}
|
||||
}
|
||||
assert _overlay_payload(record) == {
|
||||
"width": 100,
|
||||
"height": 200,
|
||||
"elements": [{"id": "ui-000", "source": "ui"}],
|
||||
}
|
||||
|
||||
|
||||
def test_overlay_payload_defaults_gracefully_for_legacy_record() -> None:
|
||||
from host_agent.web.app import _overlay_payload
|
||||
|
||||
assert _overlay_payload({}) == {"width": 0, "height": 0, "elements": []}
|
||||
assert _overlay_payload({"scene": "not-a-dict"}) == {
|
||||
"width": 0,
|
||||
"height": 0,
|
||||
"elements": [],
|
||||
}
|
||||
|
||||
@@ -162,3 +162,69 @@ def test_task_runner_persists_action_evidence_and_raw_ocr(tmp_path) -> None:
|
||||
assert Path(record["after_screenshot_path"]).read_bytes() == b"after-action"
|
||||
assert record["tool_call"]["description"] == "tap search"
|
||||
assert record["ocr_results"][0]["text"] == "Search"
|
||||
|
||||
|
||||
def test_task_runner_persists_ui_tree_elements_from_fused_scene(tmp_path) -> None:
|
||||
scene = Scene(
|
||||
width=10,
|
||||
height=20,
|
||||
elements=[
|
||||
SceneElement(
|
||||
id="ui-000",
|
||||
type="button",
|
||||
text="Search",
|
||||
bounds=Bounds(1, 2, 3, 4),
|
||||
source="ui",
|
||||
),
|
||||
SceneElement(
|
||||
id="ocr-000",
|
||||
type="text",
|
||||
text="Unrelated label",
|
||||
bounds=Bounds(5, 6, 3, 4),
|
||||
source="ocr",
|
||||
),
|
||||
],
|
||||
)
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
runner = TaskRunner(
|
||||
planner=ScriptedPlanner(
|
||||
[PlannedStep(action="tap", description="tap search", args={})]
|
||||
),
|
||||
executor=Executor(
|
||||
tools={"tap": lambda **kwargs: {"ok": True}},
|
||||
config=ExecutorConfig(max_retries=1, backoff_seconds=0),
|
||||
),
|
||||
timeline=timeline,
|
||||
config=TaskRunnerConfig(max_steps=2),
|
||||
observer=lambda device_id: scene,
|
||||
screenshot_provider=lambda device_id: PNG_10X20,
|
||||
)
|
||||
|
||||
result = runner.run(Task(goal="tap search", device_id="iphone-1"))
|
||||
|
||||
record = timeline.read(result.id)[0]
|
||||
assert [element["text"] for element in record["ui_tree_results"]] == ["Search"]
|
||||
assert [element["text"] for element in record["ocr_results"]] == ["Unrelated label"]
|
||||
|
||||
|
||||
def test_task_runner_persists_empty_ui_tree_results_when_scene_has_no_ui_elements(
|
||||
tmp_path,
|
||||
) -> None:
|
||||
scene = Scene(width=10, height=20, elements=[])
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
runner = TaskRunner(
|
||||
planner=ScriptedPlanner(
|
||||
[PlannedStep(action="tap", description="tap search", args={})]
|
||||
),
|
||||
executor=Executor(
|
||||
tools={"tap": lambda **kwargs: {"ok": True}},
|
||||
config=ExecutorConfig(max_retries=1, backoff_seconds=0),
|
||||
),
|
||||
timeline=timeline,
|
||||
config=TaskRunnerConfig(max_steps=2),
|
||||
observer=lambda device_id: scene,
|
||||
screenshot_provider=lambda device_id: PNG_10X20,
|
||||
)
|
||||
result = runner.run(Task(goal="tap search", device_id="iphone-1"))
|
||||
record = timeline.read(result.id)[0]
|
||||
assert record["ui_tree_results"] == []
|
||||
|
||||
@@ -83,3 +83,44 @@ def test_timeline_records_before_and_after_screenshots_with_ocr(tmp_path) -> Non
|
||||
assert Path(record["after_screenshot_path"]).read_bytes() == after
|
||||
assert record["screenshot_path"] == record["after_screenshot_path"]
|
||||
assert record["ocr_results"][0]["text"] == "Search"
|
||||
|
||||
|
||||
def test_timeline_records_ui_tree_results(tmp_path) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
timeline.append(
|
||||
task_id="task-ui-tree",
|
||||
scene={"screen": {"width": 100, "height": 200}, "elements": []},
|
||||
prompt="goal",
|
||||
tool_call={"action": "tap", "description": "tap search"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
ui_tree_results=[
|
||||
{
|
||||
"id": "ui-000",
|
||||
"type": "button",
|
||||
"text": "Search",
|
||||
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
||||
"confidence": None,
|
||||
}
|
||||
],
|
||||
)
|
||||
|
||||
record = timeline.read("task-ui-tree")[0]
|
||||
assert record["ui_tree_results"][0]["text"] == "Search"
|
||||
|
||||
|
||||
def test_timeline_defaults_ui_tree_results_to_empty_list(tmp_path) -> None:
|
||||
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
||||
|
||||
timeline.append(
|
||||
task_id="task-no-ui-tree",
|
||||
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
||||
prompt="goal",
|
||||
tool_call={"action": "input_text"},
|
||||
result={"ok": True},
|
||||
screenshot=PNG_10X20,
|
||||
)
|
||||
|
||||
record = timeline.read("task-no-ui-tree")[0]
|
||||
assert record["ui_tree_results"] == []
|
||||
|
||||
Reference in New Issue
Block a user