q792602257
8162509158
Tests / Test passed: 863
Fixes issue 3: the host-agent console showed OCR results but never real UI-tree data, because _ui_tree_nodes() checked for a get_ui_tree/ui_tree tool action that has never existed anywhere in the codebase. - storage/timeline.py: add a ui_tree_results field to TimelineRecord and Timeline.append(), mirroring the existing ocr_results field. - runtime/task.py: _append_timeline() now extracts scene.elements with source == "ui" into ui_tree_results (scene_builder.build_scene() already preserved these; they were just never persisted). - host_agent/web/app.py: _ui_tree_nodes() reads the new field directly instead of the dead tool-action check. New _overlay_payload() exposes each step's scene dimensions and fused element list for client-side rendering. - task_detail.html: adds a toggle to overlay OCR (orange) and UI-tree (blue) bounding boxes on the before-action screenshot, plus a visual marker for the actually executed action (tap circle, or an animated swipe path) using an SVG viewBox so no manual coordinate-scaling JS is needed. Legacy/incomplete records degrade to no overlay, never an error. Also corrects openspec/specs/runtime-task-evidence and host-agent-console-task-pages, which had encoded the same nonexistent-tool assumption, via the new host-agent-console-visual-evidence change. 600 tests passing; ruff/compileall/openspec validate all clean.
88 lines
3.0 KiB
Python
88 lines
3.0 KiB
Python
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
from storage.artifact_store import ArtifactStore
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TimelineRecord:
|
|
index: int
|
|
scene: dict[str, Any]
|
|
# The prompt actually sent to the LLM for this step. For non-LLM planners
|
|
# (or older records persisted before D9), this falls back to the task goal.
|
|
prompt: str
|
|
tool_call: dict[str, Any]
|
|
result: dict[str, Any]
|
|
timestamp: str
|
|
ocr_results: list[dict[str, Any]] = field(default_factory=list)
|
|
ui_tree_results: list[dict[str, Any]] = field(default_factory=list)
|
|
before_screenshot_path: str | None = None
|
|
after_screenshot_path: str | None = None
|
|
screenshot_path: str | None = None
|
|
|
|
|
|
class Timeline:
|
|
def __init__(self, artifact_store: ArtifactStore | None = None) -> None:
|
|
self.artifact_store = artifact_store or ArtifactStore()
|
|
|
|
def append(
|
|
self,
|
|
*,
|
|
task_id: str,
|
|
scene: Any,
|
|
prompt: str,
|
|
tool_call: dict[str, Any],
|
|
result: dict[str, Any],
|
|
screenshot: bytes | None = None,
|
|
before_screenshot: bytes | None = None,
|
|
after_screenshot: bytes | None = None,
|
|
ocr_results: list[dict[str, Any]] | None = None,
|
|
ui_tree_results: list[dict[str, Any]] | None = None,
|
|
) -> TimelineRecord:
|
|
index = len(self.read(task_id)) + 1
|
|
resolved_after_screenshot = (
|
|
after_screenshot if after_screenshot is not None else screenshot
|
|
)
|
|
resolved_ocr_results = list(ocr_results or [])
|
|
resolved_ui_tree_results = list(ui_tree_results or [])
|
|
record = {
|
|
"index": index,
|
|
"scene": scene,
|
|
"prompt": prompt,
|
|
"tool_call": tool_call,
|
|
"result": result,
|
|
"ocr_results": resolved_ocr_results,
|
|
"ui_tree_results": resolved_ui_tree_results,
|
|
"timestamp": datetime.now().astimezone().isoformat(),
|
|
}
|
|
paths = self.artifact_store.write_step(
|
|
task_id=task_id,
|
|
index=index,
|
|
before_screenshot=before_screenshot,
|
|
after_screenshot=resolved_after_screenshot,
|
|
record=record,
|
|
)
|
|
return TimelineRecord(
|
|
index=index,
|
|
scene=record["scene"],
|
|
prompt=prompt,
|
|
tool_call=tool_call,
|
|
result=result,
|
|
timestamp=record["timestamp"],
|
|
ocr_results=resolved_ocr_results,
|
|
ui_tree_results=resolved_ui_tree_results,
|
|
before_screenshot_path=paths["before_screenshot_path"],
|
|
after_screenshot_path=paths["after_screenshot_path"],
|
|
screenshot_path=paths["screenshot_path"],
|
|
)
|
|
|
|
def read(self, task_id: str) -> list[dict[str, Any]]:
|
|
return self.artifact_store.read_steps(task_id)
|
|
|
|
def delete_task(self, task_id: str) -> None:
|
|
"""Delete all timeline records and screenshots for a task."""
|
|
self.artifact_store.delete_task(task_id)
|