diff --git a/apps/device-host-agent/host_agent/web/app.py b/apps/device-host-agent/host_agent/web/app.py index c39e492..af47f6b 100644 --- a/apps/device-host-agent/host_agent/web/app.py +++ b/apps/device-host-agent/host_agent/web/app.py @@ -94,20 +94,30 @@ def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]: def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]: - tool_call = record.get("tool_call") - if not isinstance(tool_call, dict): - return [] - if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}: - return [] - step_result = record.get("result") - if not isinstance(step_result, dict): - return [] - raw_nodes = step_result.get("result") + raw_nodes = record.get("ui_tree_results") if not isinstance(raw_nodes, list): return [] return [node for node in raw_nodes if isinstance(node, dict)] +def _overlay_payload(record: dict[str, Any]) -> dict[str, Any]: + """Combined perception elements + screen size for client-side bounding-box + overlay and action-effect rendering on the before-screenshot. + """ + scene = record.get("scene") + screen = scene.get("screen") if isinstance(scene, dict) else None + width = screen.get("width") if isinstance(screen, dict) else None + height = screen.get("height") if isinstance(screen, dict) else None + elements = scene.get("elements") if isinstance(scene, dict) else None + return { + "width": width if isinstance(width, (int, float)) else 0, + "height": height if isinstance(height, (int, float)) else 0, + "elements": [element for element in elements if isinstance(element, dict)] + if isinstance(elements, list) + else [], + } + + def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]: tool_call = record.get("tool_call") result = record.get("result") @@ -126,6 +136,7 @@ def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]: or _screenshot_data_uri(record), "ocr_results": _ocr_results(record), "ui_tree_nodes": _ui_tree_nodes(record), + "overlay": _overlay_payload(record), } diff --git a/apps/device-host-agent/host_agent/web/templates/task_detail.html b/apps/device-host-agent/host_agent/web/templates/task_detail.html index 5abae18..e569b57 100644 --- a/apps/device-host-agent/host_agent/web/templates/task_detail.html +++ b/apps/device-host-agent/host_agent/web/templates/task_detail.html @@ -8,8 +8,18 @@ .evidence-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; margin-bottom: 1rem; } .evidence-pane { margin: 0; min-width: 0; } .evidence-pane h3 { font-size: 1rem; margin: 0 0 0.35rem; } -.screenshot-frame { min-height: 6rem; border: 1px solid #c8d0d6; background: #f8fafb; display: grid; place-items: center; overflow: hidden; color: #5e6b73; } +.screenshot-frame { min-height: 6rem; border: 1px solid #c8d0d6; background: #f8fafb; display: grid; place-items: center; overflow: hidden; color: #5e6b73; position: relative; } .screenshot-frame img { display: block; width: 100%; height: auto; } +.overlay-svg { position: absolute; top: 0; left: 0; width: 100%; height: 100%; pointer-events: none; } +.overlay-svg .overlay-box { fill: none; stroke-width: 2; vector-effect: non-scaling-stroke; } +.overlay-svg .overlay-box.source-ui { stroke: #1e88e5; } +.overlay-svg .overlay-box.source-ocr { stroke: #fb8c00; } +.overlay-svg .overlay-boxes { display: none; } +.overlay-svg.show-boxes .overlay-boxes { display: inline; } +.overlay-svg .action-tap { fill: #e53935; fill-opacity: 0.25; stroke: #e53935; stroke-width: 2; vector-effect: non-scaling-stroke; } +.overlay-svg .action-swipe-line { stroke: #e53935; stroke-width: 3; vector-effect: non-scaling-stroke; fill: none; } +.overlay-svg .action-swipe-dot { fill: #e53935; } +.overlay-toggle { margin-bottom: 1rem; } .step-details { margin-top: 0.75rem; } .step-details summary { cursor: pointer; font-weight: 600; } .step-details pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 0.65rem 0 0; padding: 0.65rem; border: 1px solid #d5dce0; background: #f8fafb; } @@ -36,6 +46,10 @@ {% if not timeline_steps %}

No timeline records.

{% else %} +

+ +

+ {% endif %} {% for step in timeline_steps %}
@@ -48,6 +62,12 @@
{% if step.before_screenshot_src %} Screenshot before action + {% if step.overlay.width and step.overlay.height %} + + + {% endif %} {% else %} No screenshot {% endif %} @@ -113,5 +133,104 @@ {% endif %}
{% endfor %} - {% endif %} + {% endblock %} diff --git a/openspec/changes/host-agent-console-visual-evidence/design.md b/openspec/changes/host-agent-console-visual-evidence/design.md new file mode 100644 index 0000000..292f839 --- /dev/null +++ b/openspec/changes/host-agent-console-visual-evidence/design.md @@ -0,0 +1,177 @@ +## Context + +Every planning step already computes a fused `Scene` via +`perception/scene_builder.py::build_scene()`. Confirmed by reading the +fusion logic: every UI-tree element passed in as `ui_elements` survives +into `Scene.elements` with `source == "ui"` unchanged (whether or not it +matched an OCR box — matching only overwrites `text`/`confidence`, never +`source` or drops the element). Only OCR elements that never matched a +UI-tree element are additionally kept, separately, in `Scene.ocr_elements`. + +`runtime/task.py::_append_timeline()` today only ever extracts +`source == "ocr"` elements (via `scene.ocr_results_to_dict()`, with a +fallback scan of `scene.elements`) into the Timeline. No equivalent +extraction of `source == "ui"` elements exists, and `TimelineRecord` has no +field to hold them even if extracted. + +`host_agent/web/app.py::_ui_tree_nodes()` was written against a different, +incorrect assumption: that UI-tree data only exists when a step's tool call +is `get_ui_tree`/`ui_tree`. That action has never existed in +`runtime/tool_specs.py::ALL_TOOL_SPECS` (`tap`, `swipe`, `input_text`, +`launch_app`, `terminate_app`, `finish_task` only) — so `_ui_tree_nodes()` +returns `[]` for every real task. `task_detail.html` already has working +Jinja2 markup for `step.ui_tree_nodes` (a `
` block, same shape as +the OCR one), so once the data is wired through it needs no template +rewrite. + +`tap`/`swipe` tool arguments (`runtime/tool_specs.py`) are specified in +"Scene pixel coordinates" — the same coordinate space as +`SceneElement.bounds` and `Scene.width`/`Scene.height`. This is what makes +drawing both the perception boxes and the action-effect marker from the +same coordinate space consistent. + +## Goals / Non-Goals + +**Goals:** +- Make UI-tree elements flow from the fused `Scene` into the Timeline and + render as a list on the task-detail page, replacing dead code. +- Let an operator toggle an overlay of OCR/UI-tree bounding boxes directly + on the screenshot that scene data was actually captured from. +- Visualize the actual spatial effect of `tap`/`swipe` actions on that same + screenshot. +- Do this without any new backend endpoint or additional persisted fields + beyond one new Timeline field (`ui_tree_results`), since bounds/text/tool + args are already computed and already serialized to the page today. + +**Non-Goals:** +- Not changing `perception/scene_builder.py`'s fusion algorithm. +- Not adding overlay/animation to the after-screenshot — the persisted + `Scene` and its bounds describe the state the action was planned + against (the before-screenshot), not the resulting state. Overlaying + boxes on the after-image would misleadingly imply they describe + post-action element positions. +- Not persisting rendered overlay images; overlay/animation are computed + client-side from data already in the page. +- Not adding overlay/animation for non-spatial actions (`input_text`, + `launch_app`, `terminate_app`, `finish_task`). + +## Decisions + +### D1: UI-tree evidence sourced from `Scene.elements` where `source == "ui"`, not a new Scene field + +**Decision**: `_append_timeline()` computes +`ui_tree_results = [element.to_dict() for element in scene.elements if element.source == "ui"]` +and passes it to `Timeline.append(..., ui_tree_results=ui_tree_results)`. +`TimelineRecord` gains `ui_tree_results: list[dict[str, Any]] = field(default_factory=list)`, +mirroring the existing `ocr_results` field exactly. + +**Why not add a raw `ui_elements` field to `Scene`**: `scene_builder.build_scene()` +already preserves every UI-tree element in `Scene.elements` untouched aside +from an OCR-provided `text`/`confidence` merge; filtering by `source` at the +Timeline layer needs no change to the perception layer, and matches the +existing pattern for how `ocr_results` gets a fallback scan of +`scene.elements` by `source`. + +**Why not reuse the old `get_ui_tree`/`ui_tree` tool-name check**: There is +no such tool; it was never callable. Removing the dead branch entirely +rather than keeping it alongside the new path avoids two divergent, only +one of which is reachable, code paths for the same concept. + +### D2: `host_agent/web/app.py::_ui_tree_nodes()` reads the new field directly + +**Decision**: `_ui_tree_nodes(record)` becomes a direct, unconditional read +of `record.get("ui_tree_results", [])` filtered to dicts — the same shape +as the existing `_ocr_results(record)`. The tool-name filter is removed. + +**Why**: `task_detail.html` already renders `step.ui_tree_nodes` as a +collapsible list identical in structure to `step.ocr_results`; no template +change is needed once the data source is correct. + +### D3: Overlay is rendered entirely client-side from data already in the page + +**Decision**: The task-detail page already embeds each step's OCR/UI-tree +results and screenshot as inline data (base64 image `src`, and the +per-step context dict rendered into the template). Add a `