feat(host-agent): persist UI-tree evidence and add overlay/action visualization

Fixes issue 3: the host-agent console showed OCR results but never real
UI-tree data, because _ui_tree_nodes() checked for a get_ui_tree/ui_tree
tool action that has never existed anywhere in the codebase.

- storage/timeline.py: add a ui_tree_results field to TimelineRecord and
  Timeline.append(), mirroring the existing ocr_results field.
- runtime/task.py: _append_timeline() now extracts scene.elements with
  source == "ui" into ui_tree_results (scene_builder.build_scene() already
  preserved these; they were just never persisted).
- host_agent/web/app.py: _ui_tree_nodes() reads the new field directly
  instead of the dead tool-action check. New _overlay_payload() exposes
  each step's scene dimensions and fused element list for client-side
  rendering.
- task_detail.html: adds a toggle to overlay OCR (orange) and UI-tree
  (blue) bounding boxes on the before-action screenshot, plus a visual
  marker for the actually executed action (tap circle, or an animated
  swipe path) using an SVG viewBox so no manual coordinate-scaling JS is
  needed. Legacy/incomplete records degrade to no overlay, never an error.

Also corrects openspec/specs/runtime-task-evidence and
host-agent-console-task-pages, which had encoded the same nonexistent-tool
assumption, via the new host-agent-console-visual-evidence change.

600 tests passing; ruff/compileall/openspec validate all clean.

apps/device-host-agent/host_agent/web/app.py

@@ -94,20 +94,30 @@ def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]:
 
 
 def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]:
-    tool_call = record.get("tool_call")
+    raw_nodes = record.get("ui_tree_results")
-    if not isinstance(tool_call, dict):
-        return []
-    if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}:
-        return []
-    step_result = record.get("result")
-    if not isinstance(step_result, dict):
-        return []
-    raw_nodes = step_result.get("result")
     if not isinstance(raw_nodes, list):
         return []
     return [node for node in raw_nodes if isinstance(node, dict)]
 
 
+def _overlay_payload(record: dict[str, Any]) -> dict[str, Any]:
+    """Combined perception elements + screen size for client-side bounding-box
+    overlay and action-effect rendering on the before-screenshot.
+    """
+    scene = record.get("scene")
+    screen = scene.get("screen") if isinstance(scene, dict) else None
+    width = screen.get("width") if isinstance(screen, dict) else None
+    height = screen.get("height") if isinstance(screen, dict) else None
+    elements = scene.get("elements") if isinstance(scene, dict) else None
+    return {
+        "width": width if isinstance(width, (int, float)) else 0,
+        "height": height if isinstance(height, (int, float)) else 0,
+        "elements": [element for element in elements if isinstance(element, dict)]
+        if isinstance(elements, list)
+        else [],
+    }
+
+
 def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]:
     tool_call = record.get("tool_call")
     result = record.get("result")
@@ -126,6 +136,7 @@ def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]:
         or _screenshot_data_uri(record),
         "ocr_results": _ocr_results(record),
         "ui_tree_nodes": _ui_tree_nodes(record),
+        "overlay": _overlay_payload(record),
     }
 
 

apps/device-host-agent/host_agent/web/templates/task_detail.html

@@ -8,8 +8,18 @@
 .evidence-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; margin-bottom: 1rem; }
 .evidence-pane { margin: 0; min-width: 0; }
 .evidence-pane h3 { font-size: 1rem; margin: 0 0 0.35rem; }
-.screenshot-frame { min-height: 6rem; border: 1px solid #c8d0d6; background: #f8fafb; display: grid; place-items: center; overflow: hidden; color: #5e6b73; }
+.screenshot-frame { min-height: 6rem; border: 1px solid #c8d0d6; background: #f8fafb; display: grid; place-items: center; overflow: hidden; color: #5e6b73; position: relative; }
 .screenshot-frame img { display: block; width: 100%; height: auto; }
+.overlay-svg { position: absolute; top: 0; left: 0; width: 100%; height: 100%; pointer-events: none; }
+.overlay-svg .overlay-box { fill: none; stroke-width: 2; vector-effect: non-scaling-stroke; }
+.overlay-svg .overlay-box.source-ui { stroke: #1e88e5; }
+.overlay-svg .overlay-box.source-ocr { stroke: #fb8c00; }
+.overlay-svg .overlay-boxes { display: none; }
+.overlay-svg.show-boxes .overlay-boxes { display: inline; }
+.overlay-svg .action-tap { fill: #e53935; fill-opacity: 0.25; stroke: #e53935; stroke-width: 2; vector-effect: non-scaling-stroke; }
+.overlay-svg .action-swipe-line { stroke: #e53935; stroke-width: 3; vector-effect: non-scaling-stroke; fill: none; }
+.overlay-svg .action-swipe-dot { fill: #e53935; }
+.overlay-toggle { margin-bottom: 1rem; }
 .step-details { margin-top: 0.75rem; }
 .step-details summary { cursor: pointer; font-weight: 600; }
 .step-details pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 0.65rem 0 0; padding: 0.65rem; border: 1px solid #d5dce0; background: #f8fafb; }
@@ -36,6 +46,10 @@
     {% if not timeline_steps %}
     <p>No timeline records.</p>
     {% else %}
+    <p class="overlay-toggle">
+      <label><input type="checkbox" id="overlay-boxes-toggle"> Show OCR/UI-tree bounding boxes on before-action screenshots</label>
+    </p>
+    {% endif %}
     {% for step in timeline_steps %}
     <section class="timeline-step">
       <div class="step-heading">
@@ -48,6 +62,12 @@
           <div class="screenshot-frame">
             {% if step.before_screenshot_src %}
             <img src="{{ step.before_screenshot_src }}" alt="Screenshot before action">
+            {% if step.overlay.width and step.overlay.height %}
+            <svg class="overlay-svg" data-step-overlay
+                 viewBox="0 0 {{ step.overlay.width }} {{ step.overlay.height }}"
+                 preserveAspectRatio="none"></svg>
+            <script type="application/json" class="step-overlay-data">{{ {"overlay": step.overlay, "tool_call": step.tool_call} | tojson }}</script>
+            {% endif %}
             {% else %}
             <span>No screenshot</span>
             {% endif %}
@@ -113,5 +133,104 @@
       {% endif %}
     </section>
     {% endfor %}
-    {% endif %}
+    <script>
+    (function () {
+      const SVG_NS = "http://www.w3.org/2000/svg";
+
+      function rect(bounds, className) {
+        const el = document.createElementNS(SVG_NS, "rect");
+        el.setAttribute("x", bounds.x);
+        el.setAttribute("y", bounds.y);
+        el.setAttribute("width", bounds.width);
+        el.setAttribute("height", bounds.height);
+        el.setAttribute("class", className);
+        return el;
+      }
+
+      function buildBoxesGroup(elements) {
+        const group = document.createElementNS(SVG_NS, "g");
+        group.setAttribute("class", "overlay-boxes");
+        (elements || []).forEach(function (element) {
+          const bounds = element.bounds;
+          if (!bounds) return;
+          const source = element.source === "ui" ? "source-ui" : "source-ocr";
+          const box = rect(bounds, "overlay-box " + source);
+          const label = element.text || element.id || "";
+          if (label) {
+            const title = document.createElementNS(SVG_NS, "title");
+            title.textContent = label;
+            box.appendChild(title);
+          }
+          group.appendChild(box);
+        });
+        return group;
+      }
+
+      function buildActionGroup(toolCall) {
+        const group = document.createElementNS(SVG_NS, "g");
+        group.setAttribute("class", "overlay-action");
+        if (!toolCall) return group;
+        const action = toolCall.action;
+        const args = toolCall.args || {};
+        if (action === "tap" && isFinite(args.x) && isFinite(args.y)) {
+          const circle = document.createElementNS(SVG_NS, "circle");
+          circle.setAttribute("cx", args.x);
+          circle.setAttribute("cy", args.y);
+          circle.setAttribute("r", 14);
+          circle.setAttribute("class", "action-tap");
+          group.appendChild(circle);
+        } else if (
+          action === "swipe" &&
+          isFinite(args.start_x) &&
+          isFinite(args.start_y) &&
+          isFinite(args.end_x) &&
+          isFinite(args.end_y)
+        ) {
+          const line = document.createElementNS(SVG_NS, "line");
+          line.setAttribute("x1", args.start_x);
+          line.setAttribute("y1", args.start_y);
+          line.setAttribute("x2", args.end_x);
+          line.setAttribute("y2", args.end_y);
+          line.setAttribute("class", "action-swipe-line");
+          group.appendChild(line);
+
+          const dot = document.createElementNS(SVG_NS, "circle");
+          dot.setAttribute("r", 8);
+          dot.setAttribute("class", "action-swipe-dot");
+          const motion = document.createElementNS(SVG_NS, "animateMotion");
+          motion.setAttribute("dur", "1.2s");
+          motion.setAttribute("repeatCount", "indefinite");
+          motion.setAttribute(
+            "path",
+            "M" + args.start_x + "," + args.start_y + " L" + args.end_x + "," + args.end_y
+          );
+          dot.appendChild(motion);
+          group.appendChild(dot);
+        }
+        return group;
+      }
+
+      document.querySelectorAll("svg.overlay-svg[data-step-overlay]").forEach(function (svg) {
+        const dataScript = svg.nextElementSibling;
+        if (!dataScript || !dataScript.classList.contains("step-overlay-data")) return;
+        let payload;
+        try {
+          payload = JSON.parse(dataScript.textContent);
+        } catch (err) {
+          return;
+        }
+        svg.appendChild(buildBoxesGroup(payload.overlay && payload.overlay.elements));
+        svg.appendChild(buildActionGroup(payload.tool_call));
+      });
+
+      const toggle = document.getElementById("overlay-boxes-toggle");
+      if (toggle) {
+        toggle.addEventListener("change", function () {
+          document.querySelectorAll("svg.overlay-svg").forEach(function (svg) {
+            svg.classList.toggle("show-boxes", toggle.checked);
+          });
+        });
+      }
+    })();
+    </script>
 {% endblock %}

openspec/changes/host-agent-console-visual-evidence/design.md

@@ -0,0 +1,177 @@
+## Context
+
+Every planning step already computes a fused `Scene` via
+`perception/scene_builder.py::build_scene()`. Confirmed by reading the
+fusion logic: every UI-tree element passed in as `ui_elements` survives
+into `Scene.elements` with `source == "ui"` unchanged (whether or not it
+matched an OCR box — matching only overwrites `text`/`confidence`, never
+`source` or drops the element). Only OCR elements that never matched a
+UI-tree element are additionally kept, separately, in `Scene.ocr_elements`.
+
+`runtime/task.py::_append_timeline()` today only ever extracts
+`source == "ocr"` elements (via `scene.ocr_results_to_dict()`, with a
+fallback scan of `scene.elements`) into the Timeline. No equivalent
+extraction of `source == "ui"` elements exists, and `TimelineRecord` has no
+field to hold them even if extracted.
+
+`host_agent/web/app.py::_ui_tree_nodes()` was written against a different,
+incorrect assumption: that UI-tree data only exists when a step's tool call
+is `get_ui_tree`/`ui_tree`. That action has never existed in
+`runtime/tool_specs.py::ALL_TOOL_SPECS` (`tap`, `swipe`, `input_text`,
+`launch_app`, `terminate_app`, `finish_task` only) — so `_ui_tree_nodes()`
+returns `[]` for every real task. `task_detail.html` already has working
+Jinja2 markup for `step.ui_tree_nodes` (a `<details>` block, same shape as
+the OCR one), so once the data is wired through it needs no template
+rewrite.
+
+`tap`/`swipe` tool arguments (`runtime/tool_specs.py`) are specified in
+"Scene pixel coordinates" — the same coordinate space as
+`SceneElement.bounds` and `Scene.width`/`Scene.height`. This is what makes
+drawing both the perception boxes and the action-effect marker from the
+same coordinate space consistent.
+
+## Goals / Non-Goals
+
+**Goals:**
+- Make UI-tree elements flow from the fused `Scene` into the Timeline and
+  render as a list on the task-detail page, replacing dead code.
+- Let an operator toggle an overlay of OCR/UI-tree bounding boxes directly
+  on the screenshot that scene data was actually captured from.
+- Visualize the actual spatial effect of `tap`/`swipe` actions on that same
+  screenshot.
+- Do this without any new backend endpoint or additional persisted fields
+  beyond one new Timeline field (`ui_tree_results`), since bounds/text/tool
+  args are already computed and already serialized to the page today.
+
+**Non-Goals:**
+- Not changing `perception/scene_builder.py`'s fusion algorithm.
+- Not adding overlay/animation to the after-screenshot — the persisted
+  `Scene` and its bounds describe the state the action was planned
+  against (the before-screenshot), not the resulting state. Overlaying
+  boxes on the after-image would misleadingly imply they describe
+  post-action element positions.
+- Not persisting rendered overlay images; overlay/animation are computed
+  client-side from data already in the page.
+- Not adding overlay/animation for non-spatial actions (`input_text`,
+  `launch_app`, `terminate_app`, `finish_task`).
+
+## Decisions
+
+### D1: UI-tree evidence sourced from `Scene.elements` where `source == "ui"`, not a new Scene field
+
+**Decision**: `_append_timeline()` computes
+`ui_tree_results = [element.to_dict() for element in scene.elements if element.source == "ui"]`
+and passes it to `Timeline.append(..., ui_tree_results=ui_tree_results)`.
+`TimelineRecord` gains `ui_tree_results: list[dict[str, Any]] = field(default_factory=list)`,
+mirroring the existing `ocr_results` field exactly.
+
+**Why not add a raw `ui_elements` field to `Scene`**: `scene_builder.build_scene()`
+already preserves every UI-tree element in `Scene.elements` untouched aside
+from an OCR-provided `text`/`confidence` merge; filtering by `source` at the
+Timeline layer needs no change to the perception layer, and matches the
+existing pattern for how `ocr_results` gets a fallback scan of
+`scene.elements` by `source`.
+
+**Why not reuse the old `get_ui_tree`/`ui_tree` tool-name check**: There is
+no such tool; it was never callable. Removing the dead branch entirely
+rather than keeping it alongside the new path avoids two divergent, only
+one of which is reachable, code paths for the same concept.
+
+### D2: `host_agent/web/app.py::_ui_tree_nodes()` reads the new field directly
+
+**Decision**: `_ui_tree_nodes(record)` becomes a direct, unconditional read
+of `record.get("ui_tree_results", [])` filtered to dicts — the same shape
+as the existing `_ocr_results(record)`. The tool-name filter is removed.
+
+**Why**: `task_detail.html` already renders `step.ui_tree_nodes` as a
+collapsible list identical in structure to `step.ocr_results`; no template
+change is needed once the data source is correct.
+
+### D3: Overlay is rendered entirely client-side from data already in the page
+
+**Decision**: The task-detail page already embeds each step's OCR/UI-tree
+results and screenshot as inline data (base64 image `src`, and the
+per-step context dict rendered into the template). Add a `<template>` or
+inline JSON blob per step with `{elements: [...], scene: {width, height}}`
+already available from `_timeline_step_context()`, and a toggle checkbox
+(one global control, applying to all steps) that adds/removes a CSS class
+showing/hiding an absolutely-positioned overlay `<div>` layer per
+bounding box. Box position/size are computed in JS as
+`left = box.x / scene.width * renderedImageWidth`, and analogously for
+`top`/`width`/`height`, recomputed on image load and on window resize.
+
+**Why client-side rather than server-rendered overlay images**: The boxes
+and coordinates are already present in the page; server-side image
+compositing would require a new image-processing dependency and a new
+per-step render step for a purely presentational feature. Client-side
+positioning also lets the operator toggle without a page reload.
+
+**Why one global toggle rather than per-step**: Operators reviewing a task
+want a consistent view across all steps while scanning; a per-step toggle
+adds UI clutter for no clear benefit. (A follow-on can add per-source
+toggles — OCR only vs. UI-tree only — if requested; out of scope here.)
+
+### D4: Action-effect marker/path derived from `tool_call.args`, drawn on the before-screenshot
+
+**Decision**: For a step whose `tool_call.action` is `"tap"`, render a
+marker (e.g. a small pulsing ring) centered at `(args.x, args.y)` in the
+same Scene-pixel-to-rendered-pixel scaling as D3. For `"swipe"`, render a
+line/arrow from `(args.start_x, args.start_y)` to `(args.end_x, args.end_y)`
+with a small dot animated along the path on load, indicating direction.
+For any other action, no marker is rendered.
+
+**Why the before-screenshot and not the after-screenshot**: `tap`/`swipe`
+coordinates are the input to the action, computed against the `Scene` that
+was captured before the action executed — i.e., the before-screenshot.
+Drawing them there is literally accurate ("this is where the AI aimed");
+drawing them on the after-screenshot would incorrectly suggest the
+coordinates describe the resulting frame.
+
+**Why derive from `tool_call.args` rather than a new persisted field**:
+`_append_timeline()` already persists `tool_call={"action":..., "args":...}`
+for every step; `args` already contains exactly the coordinates needed. No
+new Timeline field is needed for this part.
+
+### D5: Legacy/incomplete records degrade to no overlay/animation, not an error
+
+**Decision**: If a step's record has no `scene` (legacy timeline record —
+see `runtime-task-evidence`'s existing legacy-screenshot handling) or the
+`tool_call.args` lacks the expected coordinate keys, the overlay toggle and
+action marker simply render nothing for that step; the rest of the page
+(screenshots, OCR/UI-tree lists, result) renders exactly as it does today.
+
+**Why**: Consistent with the repo's existing degrade-gracefully pattern for
+legacy Timeline records (single-screenshot records, missing OCR) — a
+presentational enhancement must never make an old record fail to render.
+
+## Risks / Trade-offs
+
+- **Coordinate scaling drift**: if the actual screenshot's pixel dimensions
+  ever differ from `scene.screen.width`/`height` (should not normally
+  happen — both originate from the same captured screenshot), overlay
+  boxes and the action marker would be positioned slightly off. Accepted:
+  this mirrors an existing assumption already relied upon by the AI planner
+  itself (tool args are specified "in Scene pixel coordinates").
+  Rendered-size fallback re-measures on image `load`, so a slow-loading
+  image doesn't produce zero-sized boxes.
+- **Overlay only on before-screenshot may surprise operators expecting it
+  on both**: accepted per D3/D4 — showing it on the after-screenshot would
+  be actively misleading, not just unhelpful, since element positions may
+  have moved once the action executed.
+- **No per-source (OCR vs. UI-tree) toggle in this iteration**: operators
+  who only want one or the other must visually distinguish by box style
+  (e.g. color/label per `source`). Follow-on work can split the toggle if
+  requested.
+
+## Migration Plan
+
+1. Ship `TimelineRecord.ui_tree_results` (additive, defaults to `[]` —
+   existing serialized records without the field remain readable).
+2. Deploy Host Agent with the updated `_append_timeline()`,
+   `_ui_tree_nodes()`, and `task_detail.html`/static JS.
+3. Rollback: revert Host Agent to prior version; no data migration to
+   reverse since the field is additive and unused by older code.
+
+## Open Questions
+
+- None blocking implementation.

openspec/changes/host-agent-console-visual-evidence/proposal.md

@@ -0,0 +1,72 @@
+## Why
+
+The Host Agent console's task-detail page only ever shows OCR observations,
+never UI-tree elements, even though every planning step fuses both into the
+step's `Scene`. The page's UI-tree rendering has been dead code since it was
+written: `_ui_tree_nodes()` only returns data for a step whose tool call is
+named `get_ui_tree`/`ui_tree` — an action that has never existed in
+`ALL_TOOL_SPECS` (the planner only ever calls `tap`/`swipe`/`input_text`/
+`launch_app`/`terminate_app`/`finish_task`). The `runtime-task-evidence` spec
+itself was written against this same wrong premise.
+
+Separately, operators reviewing a task's evidence today can only look at
+raw before/after screenshots side by side. They asked for two additional
+ways to see the AI's actual perception and effect at a glance: (1) an
+on/off toggle that overlays the OCR/UI-tree bounding boxes directly on the
+screenshot, and (2) a visualization of what the executed action actually
+did — where a `tap` landed, or where a `swipe` started and ended — so a
+wrong tap/swipe target is visible without cross-referencing raw JSON
+coordinates against the image by hand.
+
+## What Changes
+
+- Fix UI-tree evidence capture: `TaskRunner._append_timeline()` extracts the
+  fused Scene's `source == "ui"` elements (already computed by
+  `scene_builder.build_scene()` for every step) into a new
+  `ui_tree_results` field on `TimelineRecord`/`Timeline.append()`, mirroring
+  the existing `ocr_results` field.
+- Replace `host_agent/web/app.py::_ui_tree_nodes()`'s dead tool-name check
+  with a direct read of the new `ui_tree_results` field, so the
+  already-existing (previously unreachable) list markup in
+  `task_detail.html` renders real data.
+- Add an operator-facing toggle on the task-detail page that overlays OCR
+  and UI-tree bounding boxes (with label/text and source) directly on the
+  **before**-screenshot — the frame the persisted `Scene` and its bounds
+  actually describe. Purely client-side (HTML/CSS/JS): boxes are
+  positioned from the bounds/coordinates already serialized into the
+  rendered page; no new backend endpoint or persisted data.
+- Add an action-effect visualization on the same before-screenshot: a
+  marker at the `tap` coordinates, or an animated start→end path for
+  `swipe`, derived from the step's already-persisted `tool_call.args`.
+  Non-spatial actions (`input_text`, `launch_app`, `terminate_app`,
+  `finish_task`) render no marker.
+- Correct the `runtime-task-evidence` spec requirement that assumed a
+  `get_ui_tree`/`ui_tree` tool call triggers UI-tree capture; it is
+  rewritten to describe capture from every step's fused Scene.
+
+## Capabilities
+
+### Modified Capabilities
+
+- `runtime-task-evidence`: UI-tree inspection requirement rewritten to
+  source elements from the per-step fused Scene (`source == "ui"`
+  elements) instead of a nonexistent `get_ui_tree`/`ui_tree` tool call.
+- `host-agent-console-task-pages`: task-detail page renders real UI-tree
+  elements as a list (previously always empty); adds the OCR/UI-tree
+  bounding-box overlay toggle and the tap/swipe action-effect
+  visualization, both on the before-screenshot.
+
+## Impact
+
+- `runtime/task.py` — `TaskRunner._append_timeline()`
+- `storage/timeline.py` — `TimelineRecord`, `Timeline.append()`
+- `apps/device-host-agent/host_agent/web/app.py` — `_ui_tree_nodes()`,
+  `_timeline_step_context()`
+- `apps/device-host-agent/host_agent/web/templates/task_detail.html` —
+  overlay toggle control, overlay layer markup, action-effect markup
+- `apps/device-host-agent/host_agent/web/static/` — new client-side JS for
+  overlay positioning/scaling and the action-effect animation (no new
+  backend endpoint; all data is already inline in the rendered page)
+- No changes to `perception/scene_builder.py`'s fusion logic — it already
+  preserves every UI-tree element with `source == "ui"` in `Scene.elements`
+- No new external dependencies

openspec/changes/host-agent-console-visual-evidence/specs/host-agent-console-task-pages/spec.md

@@ -0,0 +1,124 @@
+## MODIFIED Requirements
+
+### Requirement: Host Agent local console exposes read-only task history with per-step detail and screenshots
+
+The Host Agent's local console SHALL provide authenticated, read-only pages
+listing recently executed local tasks and, for a selected task, its full
+per-step history from the Host-local metadata store and Timeline. The task
+detail SHALL show available before and after screenshots, operation details and
+arguments, execution result, OCR observations, and normalized UI-tree results.
+It SHALL render legacy Timeline records that only have a single screenshot as
+a post-action image.
+
+#### Scenario: Operator lists recent Host executions
+
+- **WHEN** an authenticated operator opens the Host Agent local console's task
+  list page
+- **THEN** it shows local executions most recent first, including terminal
+  tasks and any available Cloud task ID and attempt correlation
+
+#### Scenario: Operator inspects a completed task's step history
+
+- **WHEN** an authenticated operator opens the detail page for a completed
+  Host execution
+- **THEN** the page shows each recorded step in order with its tool call,
+  result, and available before/after screenshots
+
+#### Scenario: OCR was captured for a step
+
+- **WHEN** the selected Timeline record contains OCR observations
+- **THEN** the detail page shows each observation's text, confidence, and
+  bounds
+
+#### Scenario: A step's fused scene contains UI-tree elements
+
+- **WHEN** the selected Timeline record's `ui_tree_results` is non-empty
+- **THEN** the detail page exposes a structured, collapsible node view
+  (type, text/identifier, bounds, confidence) while retaining the
+  persisted result JSON
+
+#### Scenario: A legacy Timeline record is displayed
+
+- **WHEN** a Timeline record has only `screenshot_path`
+- **THEN** the detail page renders it as the post-action image without failing
+
+#### Scenario: Unauthenticated request
+
+- **WHEN** a request to the task list or task detail pages is made without a
+  valid Host Agent console session
+- **THEN** the Host Agent rejects the request the same way it rejects
+  unauthenticated requests to its other console pages
+
+## ADDED Requirements
+
+### Requirement: Task-detail page can overlay OCR/UI-tree bounding boxes on the before-screenshot
+
+The Host Agent local console's task-detail page SHALL provide an operator
+toggle that overlays each step's OCR and UI-tree element bounding boxes
+directly on that step's before-screenshot, computed client-side from the
+bounds and screen dimensions already present in the rendered page, without
+a new backend endpoint or additional persisted data.
+
+#### Scenario: Operator enables the overlay toggle
+
+- **WHEN** an authenticated operator turns on the bounding-box overlay
+  toggle on a task-detail page
+- **THEN** every step's before-screenshot shows a box for each of that
+  step's OCR and UI-tree elements, positioned and sized proportionally to
+  the element's bounds and the scene's screen dimensions
+
+#### Scenario: Operator disables the overlay toggle
+
+- **WHEN** an authenticated operator turns off the bounding-box overlay
+  toggle
+- **THEN** the before-screenshots render without any bounding-box overlay
+
+#### Scenario: A step has no OCR or UI-tree elements
+
+- **WHEN** the overlay toggle is on and a step's `ocr_results` and
+  `ui_tree_results` are both empty
+- **THEN** that step's before-screenshot renders with no overlay boxes and
+  without error
+
+#### Scenario: A legacy step has no scene dimensions
+
+- **WHEN** the overlay toggle is on and a step's Timeline record predates
+  scene/screen-dimension persistence
+- **THEN** that step's before-screenshot renders unmodified, with no
+  overlay boxes, without error
+
+### Requirement: Task-detail page visualizes the executed action's spatial effect
+
+The Host Agent local console's task-detail page SHALL render a visual
+indicator of a `tap` or `swipe` step's target coordinates on that step's
+before-screenshot, derived from the step's persisted `tool_call.args`. A
+`tap` step SHALL show a marker at the tapped point. A `swipe` step SHALL
+show a path from the start point to the end point. Steps for other actions
+SHALL render no such indicator.
+
+#### Scenario: A tap step is displayed
+
+- **WHEN** a task step's tool call is `tap` with `x`/`y` arguments
+- **THEN** the step's before-screenshot shows a marker at the point
+  corresponding to those coordinates
+
+#### Scenario: A swipe step is displayed
+
+- **WHEN** a task step's tool call is `swipe` with `start_x`/`start_y`/
+  `end_x`/`end_y` arguments
+- **THEN** the step's before-screenshot shows a path from the start point
+  to the end point corresponding to those coordinates
+
+#### Scenario: A non-spatial step is displayed
+
+- **WHEN** a task step's tool call is `input_text`, `launch_app`,
+  `terminate_app`, or `finish_task`
+- **THEN** the step's before-screenshot renders with no action-effect
+  marker or path
+
+#### Scenario: A tap/swipe step is missing expected coordinate arguments
+
+- **WHEN** a task step's tool call is `tap` or `swipe` but its persisted
+  `args` lacks the expected coordinate keys
+- **THEN** the step's before-screenshot renders unmodified, with no
+  action-effect marker or path, without error

openspec/changes/host-agent-console-visual-evidence/specs/runtime-task-evidence/spec.md

@@ -0,0 +1,34 @@
+## MODIFIED Requirements
+
+### Requirement: Runtime task evidence retains UI-tree inspection results
+
+The Runtime SHALL retain, for each step, the UI-tree elements present in
+that step's fused `Scene` (the elements produced by
+`perception/scene_builder.py::build_scene()` with `source == "ui"`) as a
+`ui_tree_results` list on the Timeline record. The Host Agent task-detail
+UI SHALL render those elements in a structured, collapsible view. The
+Runtime SHALL NOT require a dedicated UI-tree tool call to capture this
+data, and SHALL NOT change the tool response contract or duplicate the
+result in a separate persistence field.
+
+#### Scenario: A step's scene contains UI-tree elements
+
+- **WHEN** a task step's planning `Scene` contains one or more elements
+  with `source == "ui"`
+- **THEN** the Timeline record's `ui_tree_results` includes each such
+  element's type, visible text or identifier, bounds, and available
+  confidence, and the Host Agent task-detail page displays them
+
+#### Scenario: A step's scene has no UI-tree elements
+
+- **WHEN** a task step's planning `Scene` contains no elements with
+  `source == "ui"` (e.g. UI-tree parsing failed and degraded to an empty
+  list)
+- **THEN** the Timeline record's `ui_tree_results` is an empty list and the
+  Host Agent task-detail page does not render an empty UI-tree section
+
+#### Scenario: A legacy Timeline record has no `ui_tree_results` field
+
+- **WHEN** a Timeline record was persisted before this field existed
+- **THEN** the Runtime exposes it as an empty `ui_tree_results` list
+  without failing to render the record

openspec/changes/host-agent-console-visual-evidence/tasks.md

@@ -0,0 +1,44 @@
+## 1. Timeline — persist UI-tree elements per step
+
+- [x] 1.1 Add `ui_tree_results: list[dict[str, Any]] = field(default_factory=list)` to `TimelineRecord` in `storage/timeline.py`, mirroring `ocr_results`
+- [x] 1.2 Add `ui_tree_results: list[dict[str, Any]] | None = None` parameter to `Timeline.append()`, stored the same way as `ocr_results`
+- [x] 1.3 Add unit tests: `Timeline.append()` with `ui_tree_results` → record round-trips it; without it → record defaults to `[]`; a record serialized before this field existed → reads as `[]` without error
+
+## 2. Runtime — extract UI-tree elements into the Timeline
+
+- [x] 2.1 In `runtime/task.py::_append_timeline()`, compute `ui_tree_results = [element.to_dict() for element in scene.elements if element.source == "ui"]` and pass it to `self.timeline.append(..., ui_tree_results=ui_tree_results)`
+- [x] 2.2 Add unit tests: a scene with `source == "ui"` elements → `ui_tree_results` populated in the appended record; a scene with no UI elements → `ui_tree_results` is `[]`; a scene with mixed `ui`/`ocr`/merged elements → only `source == "ui"` elements appear in `ui_tree_results` and only `source == "ocr"` (or fallback) elements appear in `ocr_results`
+
+## 3. Host Agent console — render real UI-tree data
+
+- [x] 3.1 Rewrite `_ui_tree_nodes()` in `apps/device-host-agent/host_agent/web/app.py` to read `record.get("ui_tree_results", [])` directly (list of dicts), removing the dead `tool_call.get("action") in {"get_ui_tree", "ui_tree"}` check
+- [x] 3.2 Confirm `_timeline_step_context()` passes the corrected `ui_tree_nodes` through unchanged; no `task_detail.html` markup changes needed for the list view itself (existing `step.ui_tree_nodes` block already renders it)
+- [x] 3.3 Add/update unit tests for `_ui_tree_nodes()`: a record with `ui_tree_results` → returns those dicts; a record without the field (legacy) → returns `[]`; a record with non-dict entries → those entries are filtered out
+
+## 4. Overlay toggle — OCR/UI-tree bounding boxes on the before-screenshot
+
+- [x] 4.1 Add `_overlay_payload(record)` to `app.py` exposing each step's `scene` screen dimensions (`width`/`height`) and its fused element list (`bounds`/`text`/`source`, straight from `scene.elements` — already the combined OCR+UI-tree view) as `step.overlay`; embedded per-step as an inline `<script type="application/json">` blob in `task_detail.html`
+- [x] 4.2 Add a single page-level toggle control (checkbox `#overlay-boxes-toggle`) in `task_detail.html` that adds/removes a `show-boxes` class on every step's overlay `<svg>`
+- [x] 4.3 Render each step's overlay as an inline `<svg viewBox="0 0 {scene.width} {scene.height}">` absolutely positioned over the before-screenshot `<img>`; boxes are drawn as `<rect>` elements directly in scene-pixel coordinates, so the SVG viewBox scaling handles image-size scaling natively — no manual JS scale computation or resize listener needed (simpler than the originally planned separate static JS file with manual scaling)
+- [x] 4.4 OCR boxes (`source="ocr"`) and UI-tree boxes (`source="ui"`) get distinct stroke colors (orange vs. blue) via CSS classes
+- [x] 4.5 Verified: `_overlay_payload()` degrades to `{"width": 0, "height": 0, "elements": []}` for a record with no/malformed `scene`; the template only emits the `<svg>` when `overlay.width`/`overlay.height` are truthy, so legacy records render the screenshot with no overlay markup at all
+
+## 5. Action-effect visualization — tap marker / swipe path
+
+- [x] 5.1 Inline JS in `task_detail.html` reads each step's already-embedded `tool_call.action`/`args`; draws an SVG `<circle>` marker for `tap`, or an SVG `<line>` plus an `<animateMotion>`-animated dot for `swipe`; any other action draws nothing
+- [x] 5.2 Action markers are drawn in the same `<svg viewBox>` as the overlay boxes (task 4.3), so no separate scaling logic was needed
+- [x] 5.3 Verified: `buildActionGroup()` checks `isFinite()` on every required coordinate before drawing; missing/non-numeric args (or no `<svg>` at all, for a legacy record) render nothing, no error
+- [x] 5.4 Rendered a real task-detail page (via the FastAPI test client with fabricated `tap`/`swipe` Timeline records) and inspected the generated HTML/JSON: `viewBox`, box coordinates, tap marker coordinates, and swipe line/path coordinates all match the source data; opened the page in a browser for visual confirmation
+
+## 6. Spec corrections
+
+- [x] 6.1 `openspec/specs/runtime-task-evidence/spec.md` (via this change's spec delta): UI-tree requirement no longer references a `get_ui_tree`/`ui_tree` tool call
+- [x] 6.2 `openspec/specs/host-agent-console-task-pages/spec.md` (via this change's spec delta): UI-tree scenario corrected; overlay toggle and action-effect requirements added
+
+## 7. End-to-end verification
+
+- [x] 7.1 Run full non-integration test suite (`uv run --all-packages pytest -m "not integration"`) and confirm no regressions
+- [x] 7.2 Run `ruff check` and `ruff format --check` on all modified Python files
+- [x] 7.3 Run `python -m compileall` on modified packages
+- [x] 7.4 Run `openspec validate --strict host-agent-console-visual-evidence` and confirm all artifacts pass validation
+- [ ] 7.5 Manual smoke test (requires Host Agent + Appium + a device): run a live task, open its task-detail page, confirm UI-tree elements render, the overlay toggle shows/hides boxes correctly on the before-screenshot, and the tap/swipe action-effect marker/path renders correctly

runtime/task.py

@@ -364,6 +364,9 @@ class TaskRunner:
                 for element in scene.elements
                 if element.source == "ocr"
             ]
+        ui_tree_results = [
+            element.to_dict() for element in scene.elements if element.source == "ui"
+        ]
         self.timeline.append(
             task_id=task.id,
             scene=scene.to_dict(),
@@ -379,6 +382,7 @@ class TaskRunner:
             before_screenshot=before_screenshot,
             after_screenshot=after_screenshot,
             ocr_results=ocr_results,
+            ui_tree_results=ui_tree_results,
         )
 
     def _step_for_device(self, step: PlannedStep, device_id: str) -> PlannedStep:

storage/timeline.py

@@ -18,6 +18,7 @@ class TimelineRecord:
     result: dict[str, Any]
     timestamp: str
     ocr_results: list[dict[str, Any]] = field(default_factory=list)
+    ui_tree_results: list[dict[str, Any]] = field(default_factory=list)
     before_screenshot_path: str | None = None
     after_screenshot_path: str | None = None
     screenshot_path: str | None = None
@@ -39,12 +40,14 @@ class Timeline:
         before_screenshot: bytes | None = None,
         after_screenshot: bytes | None = None,
         ocr_results: list[dict[str, Any]] | None = None,
+        ui_tree_results: list[dict[str, Any]] | None = None,
     ) -> TimelineRecord:
         index = len(self.read(task_id)) + 1
         resolved_after_screenshot = (
             after_screenshot if after_screenshot is not None else screenshot
         )
         resolved_ocr_results = list(ocr_results or [])
+        resolved_ui_tree_results = list(ui_tree_results or [])
         record = {
             "index": index,
             "scene": scene,
@@ -52,6 +55,7 @@ class Timeline:
             "tool_call": tool_call,
             "result": result,
             "ocr_results": resolved_ocr_results,
+            "ui_tree_results": resolved_ui_tree_results,
             "timestamp": datetime.now().astimezone().isoformat(),
         }
         paths = self.artifact_store.write_step(
@@ -69,6 +73,7 @@ class Timeline:
             result=result,
             timestamp=record["timestamp"],
             ocr_results=resolved_ocr_results,
+            ui_tree_results=resolved_ui_tree_results,
             before_screenshot_path=paths["before_screenshot_path"],
             after_screenshot_path=paths["after_screenshot_path"],
             screenshot_path=paths["screenshot_path"],

tests/test_host_agent_console_tasks.py

@@ -223,18 +223,8 @@ def test_authenticated_task_detail_renders_complete_step_evidence(tmp_path) -> N
         task_id=task.id,
         scene={"screen": {"width": 10, "height": 20}, "elements": []},
         prompt="inspect the current UI tree",
-        tool_call={"action": "get_ui_tree", "description": "inspect UI tree"},
+        tool_call={"action": "tap", "description": "tap search"},
-        result={
+        result={"ok": True},
-            "result": [
-                {
-                    "id": "search",
-                    "type": "button",
-                    "text": "Search",
-                    "bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
-                    "confidence": 0.98,
-                }
-            ]
-        },
         before_screenshot=PNG_10X20,
         after_screenshot=PNG_10X20 + b"after",
         ocr_results=[
@@ -244,6 +234,15 @@ def test_authenticated_task_detail_renders_complete_step_evidence(tmp_path) -> N
                 "confidence": 0.95,
             }
         ],
+        ui_tree_results=[
+            {
+                "id": "search",
+                "type": "button",
+                "text": "Search",
+                "bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
+                "confidence": 0.98,
+            }
+        ],
     )
 
     client, _ = _build_client(
@@ -278,3 +277,51 @@ def test_tasks_list_shows_empty_state(tmp_path) -> None:
     response = client.get("/tasks")
     assert response.status_code == 200
     assert "No executions recorded yet" in response.text
+
+
+def test_ui_tree_nodes_reads_persisted_field_directly() -> None:
+    from host_agent.web.app import _ui_tree_nodes
+
+    record = {
+        "ui_tree_results": [
+            {"id": "search", "type": "button", "text": "Search"},
+            "not-a-dict",
+        ]
+    }
+    assert _ui_tree_nodes(record) == [
+        {"id": "search", "type": "button", "text": "Search"}
+    ]
+
+
+def test_ui_tree_nodes_defaults_to_empty_for_legacy_record() -> None:
+    from host_agent.web.app import _ui_tree_nodes
+
+    assert _ui_tree_nodes({}) == []
+    assert _ui_tree_nodes({"ui_tree_results": "not-a-list"}) == []
+
+
+def test_overlay_payload_combines_scene_dimensions_and_elements() -> None:
+    from host_agent.web.app import _overlay_payload
+
+    record = {
+        "scene": {
+            "screen": {"width": 100, "height": 200},
+            "elements": [{"id": "ui-000", "source": "ui"}, "not-a-dict"],
+        }
+    }
+    assert _overlay_payload(record) == {
+        "width": 100,
+        "height": 200,
+        "elements": [{"id": "ui-000", "source": "ui"}],
+    }
+
+
+def test_overlay_payload_defaults_gracefully_for_legacy_record() -> None:
+    from host_agent.web.app import _overlay_payload
+
+    assert _overlay_payload({}) == {"width": 0, "height": 0, "elements": []}
+    assert _overlay_payload({"scene": "not-a-dict"}) == {
+        "width": 0,
+        "height": 0,
+        "elements": [],
+    }

tests/test_task_loop.py

@@ -162,3 +162,69 @@ def test_task_runner_persists_action_evidence_and_raw_ocr(tmp_path) -> None:
     assert Path(record["after_screenshot_path"]).read_bytes() == b"after-action"
     assert record["tool_call"]["description"] == "tap search"
     assert record["ocr_results"][0]["text"] == "Search"
+
+
+def test_task_runner_persists_ui_tree_elements_from_fused_scene(tmp_path) -> None:
+    scene = Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="ui-000",
+                type="button",
+                text="Search",
+                bounds=Bounds(1, 2, 3, 4),
+                source="ui",
+            ),
+            SceneElement(
+                id="ocr-000",
+                type="text",
+                text="Unrelated label",
+                bounds=Bounds(5, 6, 3, 4),
+                source="ocr",
+            ),
+        ],
+    )
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    runner = TaskRunner(
+        planner=ScriptedPlanner(
+            [PlannedStep(action="tap", description="tap search", args={})]
+        ),
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        timeline=timeline,
+        config=TaskRunnerConfig(max_steps=2),
+        observer=lambda device_id: scene,
+        screenshot_provider=lambda device_id: PNG_10X20,
+    )
+
+    result = runner.run(Task(goal="tap search", device_id="iphone-1"))
+
+    record = timeline.read(result.id)[0]
+    assert [element["text"] for element in record["ui_tree_results"]] == ["Search"]
+    assert [element["text"] for element in record["ocr_results"]] == ["Unrelated label"]
+
+
+def test_task_runner_persists_empty_ui_tree_results_when_scene_has_no_ui_elements(
+    tmp_path,
+) -> None:
+    scene = Scene(width=10, height=20, elements=[])
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    runner = TaskRunner(
+        planner=ScriptedPlanner(
+            [PlannedStep(action="tap", description="tap search", args={})]
+        ),
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        timeline=timeline,
+        config=TaskRunnerConfig(max_steps=2),
+        observer=lambda device_id: scene,
+        screenshot_provider=lambda device_id: PNG_10X20,
+    )
+    result = runner.run(Task(goal="tap search", device_id="iphone-1"))
+    record = timeline.read(result.id)[0]
+    assert record["ui_tree_results"] == []

tests/test_timeline.py

@@ -83,3 +83,44 @@ def test_timeline_records_before_and_after_screenshots_with_ocr(tmp_path) -> Non
     assert Path(record["after_screenshot_path"]).read_bytes() == after
     assert record["screenshot_path"] == record["after_screenshot_path"]
     assert record["ocr_results"][0]["text"] == "Search"
+
+
+def test_timeline_records_ui_tree_results(tmp_path) -> None:
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+
+    timeline.append(
+        task_id="task-ui-tree",
+        scene={"screen": {"width": 100, "height": 200}, "elements": []},
+        prompt="goal",
+        tool_call={"action": "tap", "description": "tap search"},
+        result={"ok": True},
+        screenshot=PNG_10X20,
+        ui_tree_results=[
+            {
+                "id": "ui-000",
+                "type": "button",
+                "text": "Search",
+                "bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
+                "confidence": None,
+            }
+        ],
+    )
+
+    record = timeline.read("task-ui-tree")[0]
+    assert record["ui_tree_results"][0]["text"] == "Search"
+
+
+def test_timeline_defaults_ui_tree_results_to_empty_list(tmp_path) -> None:
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+
+    timeline.append(
+        task_id="task-no-ui-tree",
+        scene={"screen": {"width": 1, "height": 1}, "elements": []},
+        prompt="goal",
+        tool_call={"action": "input_text"},
+        result={"ok": True},
+        screenshot=PNG_10X20,
+    )
+
+    record = timeline.read("task-no-ui-tree")[0]
+    assert record["ui_tree_results"] == []