q792602257
8162509158
Tests / Test passed: 863
Fixes issue 3: the host-agent console showed OCR results but never real UI-tree data, because _ui_tree_nodes() checked for a get_ui_tree/ui_tree tool action that has never existed anywhere in the codebase. - storage/timeline.py: add a ui_tree_results field to TimelineRecord and Timeline.append(), mirroring the existing ocr_results field. - runtime/task.py: _append_timeline() now extracts scene.elements with source == "ui" into ui_tree_results (scene_builder.build_scene() already preserved these; they were just never persisted). - host_agent/web/app.py: _ui_tree_nodes() reads the new field directly instead of the dead tool-action check. New _overlay_payload() exposes each step's scene dimensions and fused element list for client-side rendering. - task_detail.html: adds a toggle to overlay OCR (orange) and UI-tree (blue) bounding boxes on the before-action screenshot, plus a visual marker for the actually executed action (tap circle, or an animated swipe path) using an SVG viewBox so no manual coordinate-scaling JS is needed. Legacy/incomplete records degrade to no overlay, never an error. Also corrects openspec/specs/runtime-task-evidence and host-agent-console-task-pages, which had encoded the same nonexistent-tool assumption, via the new host-agent-console-visual-evidence change. 600 tests passing; ruff/compileall/openspec validate all clean.
127 lines
4.1 KiB
Python
127 lines
4.1 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
from storage.artifact_store import ArtifactStore
|
|
from storage.timeline import Timeline
|
|
from tests.fakes import PNG_10X20
|
|
|
|
|
|
def test_timeline_records_survive_reopening_store(tmp_path) -> None:
|
|
store = ArtifactStore(tmp_path / "history")
|
|
timeline = Timeline(store)
|
|
timeline.append(
|
|
task_id="task-1",
|
|
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
|
prompt="goal",
|
|
tool_call={"action": "tap"},
|
|
result={"ok": True},
|
|
screenshot=PNG_10X20,
|
|
)
|
|
timeline.append(
|
|
task_id="task-1",
|
|
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
|
prompt="goal",
|
|
tool_call={"action": "input_text"},
|
|
result={"ok": True},
|
|
screenshot=PNG_10X20,
|
|
)
|
|
|
|
reopened = Timeline(ArtifactStore(tmp_path / "history"))
|
|
records = reopened.read("task-1")
|
|
|
|
assert [record["index"] for record in records] == [1, 2]
|
|
assert records[0]["screenshot_path"].endswith("001.png")
|
|
|
|
|
|
def test_timeline_records_per_step_prompt_not_task_goal(tmp_path) -> None:
|
|
"""The prompt field should persist exactly what was passed to append(),
|
|
not a pre-D9 task goal fallback."""
|
|
store = ArtifactStore(tmp_path / "history")
|
|
timeline = Timeline(store)
|
|
|
|
per_step_prompt = "Goal:\nsend a message\n\nCurrent Scene (JSON):\n{...}\n\nCall exactly one tool."
|
|
timeline.append(
|
|
task_id="task-42",
|
|
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
|
prompt=per_step_prompt,
|
|
tool_call={"action": "tap"},
|
|
result={"ok": True},
|
|
screenshot=PNG_10X20,
|
|
)
|
|
|
|
records = timeline.read("task-42")
|
|
assert len(records) == 1
|
|
assert records[0]["prompt"] == per_step_prompt
|
|
assert "Call exactly one tool" in records[0]["prompt"]
|
|
|
|
|
|
def test_timeline_records_before_and_after_screenshots_with_ocr(tmp_path) -> None:
|
|
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
|
before = b"before-image"
|
|
after = b"after-image"
|
|
|
|
timeline.append(
|
|
task_id="task-evidence",
|
|
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
|
prompt="goal",
|
|
tool_call={"action": "tap", "description": "tap search"},
|
|
result={"ok": True},
|
|
before_screenshot=before,
|
|
after_screenshot=after,
|
|
ocr_results=[
|
|
{
|
|
"text": "Search",
|
|
"confidence": 0.98,
|
|
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
|
}
|
|
],
|
|
)
|
|
|
|
record = timeline.read("task-evidence")[0]
|
|
assert Path(record["before_screenshot_path"]).read_bytes() == before
|
|
assert Path(record["after_screenshot_path"]).read_bytes() == after
|
|
assert record["screenshot_path"] == record["after_screenshot_path"]
|
|
assert record["ocr_results"][0]["text"] == "Search"
|
|
|
|
|
|
def test_timeline_records_ui_tree_results(tmp_path) -> None:
|
|
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
|
|
|
timeline.append(
|
|
task_id="task-ui-tree",
|
|
scene={"screen": {"width": 100, "height": 200}, "elements": []},
|
|
prompt="goal",
|
|
tool_call={"action": "tap", "description": "tap search"},
|
|
result={"ok": True},
|
|
screenshot=PNG_10X20,
|
|
ui_tree_results=[
|
|
{
|
|
"id": "ui-000",
|
|
"type": "button",
|
|
"text": "Search",
|
|
"bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
|
|
"confidence": None,
|
|
}
|
|
],
|
|
)
|
|
|
|
record = timeline.read("task-ui-tree")[0]
|
|
assert record["ui_tree_results"][0]["text"] == "Search"
|
|
|
|
|
|
def test_timeline_defaults_ui_tree_results_to_empty_list(tmp_path) -> None:
|
|
timeline = Timeline(ArtifactStore(tmp_path / "history"))
|
|
|
|
timeline.append(
|
|
task_id="task-no-ui-tree",
|
|
scene={"screen": {"width": 1, "height": 1}, "elements": []},
|
|
prompt="goal",
|
|
tool_call={"action": "input_text"},
|
|
result={"ok": True},
|
|
screenshot=PNG_10X20,
|
|
)
|
|
|
|
record = timeline.read("task-no-ui-tree")[0]
|
|
assert record["ui_tree_results"] == []
|