feat(host-agent): make execution history authoritative

apps/device-host-agent/host_agent/web/app.py

@@ -70,9 +70,13 @@ def _device_display_status(device: Any, *, busy_device_id: str | None) -> str:
     return device.status
 
 
-def _screenshot_data_uri(record: dict[str, Any]) -> str | None:
-    """Return a ``data:`` URI for the step's screenshot, or ``None``."""
-    screenshot_path = record.get("screenshot_path")
+def _screenshot_data_uri(
+    record: dict[str, Any],
+    *,
+    path_key: str = "screenshot_path",
+) -> str | None:
+    """Return a ``data:`` URI for one step screenshot, or ``None``."""
+    screenshot_path = record.get(path_key)
     if not screenshot_path:
         return None
     path = Path(str(screenshot_path))
@@ -82,6 +86,49 @@ def _screenshot_data_uri(record: dict[str, Any]) -> str | None:
     return f"data:image/png;base64,{encoded}"
 
 
+def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]:
+    raw_results = record.get("ocr_results")
+    if not isinstance(raw_results, list):
+        return []
+    return [result for result in raw_results if isinstance(result, dict)]
+
+
+def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]:
+    tool_call = record.get("tool_call")
+    if not isinstance(tool_call, dict):
+        return []
+    if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}:
+        return []
+    step_result = record.get("result")
+    if not isinstance(step_result, dict):
+        return []
+    raw_nodes = step_result.get("result")
+    if not isinstance(raw_nodes, list):
+        return []
+    return [node for node in raw_nodes if isinstance(node, dict)]
+
+
+def _timeline_step_context(record: dict[str, Any]) -> dict[str, Any]:
+    tool_call = record.get("tool_call")
+    result = record.get("result")
+    return {
+        "index": record.get("index", ""),
+        "timestamp": record.get("timestamp", ""),
+        "prompt": record.get("prompt") or "",
+        "tool_call": tool_call if isinstance(tool_call, dict) else {},
+        "result": result if isinstance(result, dict) else {},
+        "before_screenshot_src": _screenshot_data_uri(
+            record, path_key="before_screenshot_path"
+        ),
+        "after_screenshot_src": _screenshot_data_uri(
+            record, path_key="after_screenshot_path"
+        )
+        or _screenshot_data_uri(record),
+        "ocr_results": _ocr_results(record),
+        "ui_tree_nodes": _ui_tree_nodes(record),
+    }
+
+
 def _safe_submission_error(detail: str) -> str:
     """Return a safe, single-line error message for the operator.
 
@@ -525,7 +572,7 @@ def create_console_app(
             if task_id:
                 notice = (
                     f"Task submitted. Cloud task ID: {task_id}. "
-                    "Track it from the Cloud console for execution progress."
+                    "It will appear below when this Host begins executing it."
                 )
         elif request.query_params.get("outcome") == "unknown":
             unknown = True
@@ -664,36 +711,22 @@ def create_console_app(
         if timeline is not None:
             timeline_records = await asyncio.to_thread(timeline.read, task_id)
         task_rows = [
-            (key, task[key])
-            for key in (
-                "id",
-                "goal",
-                "device_id",
-                "status",
-                "created_at",
-                "updated_at",
+            (label, task[key])
+            for key, label in (
+                ("source_task_id", "Cloud task ID"),
+                ("source_attempt", "Cloud attempt"),
+                ("id", "Execution ID"),
+                ("goal", "Goal"),
+                ("device_id", "Device"),
+                ("status", "Status"),
+                ("created_at", "Created"),
+                ("updated_at", "Updated"),
+                ("completed_at", "Completed"),
+                ("failure_reason", "Failure reason"),
             )
             if task.get(key) is not None
         ]
-        timeline_steps = [
-            {
-                "index": record.get("index", ""),
-                "timestamp": record.get("timestamp", ""),
-                "prompt": record.get("prompt") or "",
-                "tool_call_text": (
-                    json.dumps(record.get("tool_call"), ensure_ascii=False)
-                    if record.get("tool_call")
-                    else ""
-                ),
-                "result_text": (
-                    json.dumps(record.get("result"), ensure_ascii=False)
-                    if record.get("result")
-                    else ""
-                ),
-                "screenshot_src": _screenshot_data_uri(record),
-            }
-            for record in timeline_records
-        ]
+        timeline_steps = [_timeline_step_context(record) for record in timeline_records]
         return _render(
             "task_detail.html",
             title=f"Task {task_id}",

apps/device-host-agent/host_agent/web/templates/task_detail.html

@@ -1,6 +1,33 @@
 {% extends "base.html" %}
+{% block styles %}
+{{ super() }}
+.task-back { margin-top: 0; }
+.timeline-step { border: 1px solid #c8d0d6; background: #fff; padding: 1rem; margin-bottom: 1rem; }
+.step-heading { display: flex; flex-wrap: wrap; gap: 0.5rem 1rem; align-items: baseline; margin-bottom: 0.75rem; }
+.step-heading p { margin: 0; color: #4d5a63; }
+.evidence-grid { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); gap: 1rem; margin-bottom: 1rem; }
+.evidence-pane { margin: 0; min-width: 0; }
+.evidence-pane h3 { font-size: 1rem; margin: 0 0 0.35rem; }
+.screenshot-frame { min-height: 6rem; border: 1px solid #c8d0d6; background: #f8fafb; display: grid; place-items: center; overflow: hidden; color: #5e6b73; }
+.screenshot-frame img { display: block; width: 100%; height: auto; }
+.step-details { margin-top: 0.75rem; }
+.step-details summary { cursor: pointer; font-weight: 600; }
+.step-details pre { white-space: pre-wrap; overflow-wrap: anywhere; margin: 0.65rem 0 0; padding: 0.65rem; border: 1px solid #d5dce0; background: #f8fafb; }
+.operation-grid { display: grid; grid-template-columns: minmax(7rem, 0.4fr) minmax(0, 1fr); gap: 0.35rem 0.75rem; margin: 0.65rem 0 0; }
+.operation-grid dt { font-weight: 600; }
+.operation-grid dd { margin: 0; overflow-wrap: anywhere; }
+.observation-list, .ui-tree-nodes { margin: 0.65rem 0 0; padding-left: 1.25rem; }
+.observation-list li, .ui-tree-nodes li { margin-bottom: 0.45rem; overflow-wrap: anywhere; }
+.observation-list span, .ui-tree-nodes span { color: #4d5a63; margin-left: 0.4rem; }
+.ui-tree-nodes code { overflow-wrap: anywhere; }
+@media (max-width: 640px) {
+  .evidence-grid { grid-template-columns: minmax(0, 1fr); }
+  .timeline-step { padding: 0.75rem; }
+}
+{% endblock %}
 {% block body %}
-    <h1>Task {{ task.get("id") or "" }}</h1>
+    <p class="task-back"><a href="/tasks">&larr; Back to executions</a></p>
+    <h1>Execution</h1>
     <table>
       <thead><tr><th>Field</th><th>Value</th></tr></thead>
       <tbody>{% for row in task_rows %}<tr><td>{{ row[0] }}</td><td>{{ row[1] }}</td></tr>{% endfor %}</tbody>
@@ -10,13 +37,81 @@
     <p>No timeline records.</p>
     {% else %}
     {% for step in timeline_steps %}
-    <div style="border:1px solid #ccc;background:#fff;padding:0.75rem;margin-bottom:0.75rem;">
-      <p><strong>Step {{ step.index }}</strong> &mdash; {{ step.timestamp }}</p>
-      <p>Prompt: {{ step.prompt }}</p>
-      <p>Tool call: <code>{{ step.tool_call_text }}</code></p>
-      <p>Result: <code>{{ step.result_text }}</code></p>
-      {% if step.screenshot_src %}<img src="{{ step.screenshot_src }}" alt="screenshot" style="max-width:100%;border:1px solid #ccc;margin-top:0.5rem;">{% endif %}
-    </div>
+    <section class="timeline-step">
+      <div class="step-heading">
+        <strong>Step {{ step.index }}</strong>
+        <p>{{ step.timestamp }}</p>
+      </div>
+      <div class="evidence-grid">
+        <figure class="evidence-pane">
+          <h3>Before action</h3>
+          <div class="screenshot-frame">
+            {% if step.before_screenshot_src %}
+            <img src="{{ step.before_screenshot_src }}" alt="Screenshot before action">
+            {% else %}
+            <span>No screenshot</span>
+            {% endif %}
+          </div>
+        </figure>
+        <figure class="evidence-pane">
+          <h3>After action</h3>
+          <div class="screenshot-frame">
+            {% if step.after_screenshot_src %}
+            <img src="{{ step.after_screenshot_src }}" alt="Screenshot after action">
+            {% else %}
+            <span>No screenshot</span>
+            {% endif %}
+          </div>
+        </figure>
+      </div>
+      <details class="step-details" open>
+        <summary>Operation</summary>
+        <dl class="operation-grid">
+          <dt>Action</dt><dd>{{ step.tool_call.get("action") or "-" }}</dd>
+          <dt>Description</dt><dd>{{ step.tool_call.get("description") or "-" }}</dd>
+        </dl>
+        <pre>{{ step.tool_call | tojson(indent=2) }}</pre>
+      </details>
+      <details class="step-details">
+        <summary>Result</summary>
+        <pre>{{ step.result | tojson(indent=2) }}</pre>
+      </details>
+      {% if step.prompt %}
+      <details class="step-details">
+        <summary>Planner prompt</summary>
+        <pre>{{ step.prompt }}</pre>
+      </details>
+      {% endif %}
+      {% if step.ocr_results %}
+      <details class="step-details" open>
+        <summary>OCR results ({{ step.ocr_results|length }})</summary>
+        <ul class="observation-list">
+          {% for ocr in step.ocr_results %}
+          <li>
+            <strong>{{ ocr.get("text") or "-" }}</strong>
+            <span>{{ ocr.get("bounds") | tojson }}</span>
+            {% if ocr.get("confidence") is not none %}<span>confidence {{ "%.3f" | format(ocr.get("confidence")) }}</span>{% endif %}
+          </li>
+          {% endfor %}
+        </ul>
+      </details>
+      {% endif %}
+      {% if step.ui_tree_nodes %}
+      <details class="step-details">
+        <summary>UI tree ({{ step.ui_tree_nodes|length }} normalized nodes)</summary>
+        <ul class="ui-tree-nodes">
+          {% for node in step.ui_tree_nodes %}
+          <li>
+            <strong>{{ node.get("type") or "unknown" }}</strong>
+            <span>{{ node.get("text") or node.get("id") or "-" }}</span>
+            <code>{{ node.get("bounds") | tojson }}</code>
+            {% if node.get("confidence") is not none %}<span>confidence {{ "%.3f" | format(node.get("confidence")) }}</span>{% endif %}
+          </li>
+          {% endfor %}
+        </ul>
+      </details>
+      {% endif %}
+    </section>
     {% endfor %}
     {% endif %}
 {% endblock %}

apps/device-host-agent/host_agent/web/templates/tasks_list.html

@@ -29,17 +29,19 @@
     </section>
 
     <section id="local-tasks">
-      <h2>Local Runtime tasks</h2>
+      <h2>Executed tasks on this Host</h2>
       {% if metadata_store_missing %}
       <p class="error">Task metadata store is not configured.</p>
       {% elif not tasks %}
-      <p>No tasks recorded.</p>
+      <p>No executions recorded yet.</p>
       {% else %}
       <table>
-        <thead><tr><th>Task ID</th><th>Status</th><th>Device</th><th>Created</th><th>Updated</th></tr></thead>
+        <thead><tr><th>Execution ID</th><th>Cloud task</th><th>Attempt</th><th>Status</th><th>Device</th><th>Created</th><th>Updated</th></tr></thead>
         <tbody>{% for task in tasks %}
           <tr>
             <td><a href="/tasks/{{ task["id"] }}">{{ task["id"] }}</a></td>
+            <td>{{ task.get("source_task_id") or "" }}</td>
+            <td>{{ task.get("source_attempt") if task.get("source_attempt") is not none else "" }}</td>
             <td>{{ task.get("status") or "" }}</td>
             <td>{{ task.get("device_id") or "" }}</td>
             <td>{{ task.get("created_at") or "" }}</td>
@@ -49,4 +51,4 @@
       </table>
       {% endif %}
     </section>
-{% endblock %}
+{% endblock %}