feat(runtime): capture step evidence in console

api/console.py

@@ -105,7 +105,7 @@ class ConsoleService:
         if self._metadata_store.get_task(task_id) is None:
             raise KeyError("task not found")
         return [
-            self._inline_screenshot(record) for record in self._timeline.read(task_id)
+            self._inline_screenshots(record) for record in self._timeline.read(task_id)
         ]
 
     def get_runtime_config(self) -> dict[str, int]:
@@ -124,15 +124,23 @@ class ConsoleService:
         )
 
     @staticmethod
-    def _inline_screenshot(record: dict[str, Any]) -> dict[str, Any]:
+    def _inline_screenshots(record: dict[str, Any]) -> dict[str, Any]:
         payload = dict(record)
-        screenshot_path = payload.get("screenshot_path")
-        if screenshot_path:
+        for path_key, image_key in (
+            ("before_screenshot_path", "before_image_base64"),
+            ("after_screenshot_path", "after_image_base64"),
+            ("screenshot_path", "image_base64"),
+        ):
+            screenshot_path = payload.get(path_key)
+            if not screenshot_path:
+                continue
             path = Path(str(screenshot_path))
             if path.exists():
-                payload["image_base64"] = base64.b64encode(path.read_bytes()).decode(
-                    "ascii"
-                )
+                payload[image_key] = base64.b64encode(path.read_bytes()).decode("ascii")
+        if "after_image_base64" not in payload and "image_base64" in payload:
+            payload["after_image_base64"] = payload["image_base64"]
+        elif "after_image_base64" in payload:
+            payload["image_base64"] = payload["after_image_base64"]
         return payload
 
     def _runner_max_steps(self) -> int:

api/console_web.py

@@ -49,6 +49,28 @@ def _default_driver_type() -> str:
     return _DEFAULT_FORM_VALUES["driver_type"]
 
 
+def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]:
+    raw_results = record.get("ocr_results")
+    if not isinstance(raw_results, list):
+        return []
+    return [result for result in raw_results if isinstance(result, dict)]
+
+
+def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]:
+    tool_call = record.get("tool_call")
+    if not isinstance(tool_call, dict):
+        return []
+    if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}:
+        return []
+    step_result = record.get("result")
+    if not isinstance(step_result, dict):
+        return []
+    raw_nodes = step_result.get("result")
+    if not isinstance(raw_nodes, list):
+        return []
+    return [node for node in raw_nodes if isinstance(node, dict)]
+
+
 def _config_context(
     request: Request,
     service: ConsoleService,
@@ -183,6 +205,12 @@ def create_console_web_router(service: ConsoleService) -> APIRouter:
                 timeline=timeline,
                 current_step=current_step if current_step is not None else {},
                 current_step_index=index,
+                ocr_results=_ocr_results(current_step)
+                if current_step is not None
+                else [],
+                ui_tree_nodes=_ui_tree_nodes(current_step)
+                if current_step is not None
+                else [],
             )
         )
 

api/static/runtime_console/console.css

@@ -409,14 +409,28 @@ select {
 
 .timeline-stage {
   display: grid;
-  grid-template-columns: minmax(220px, 360px) minmax(0, 1fr);
   gap: 14px;
 }
 
+.evidence-grid {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 14px;
+}
+
+.evidence-pane {
+  min-width: 0;
+}
+
+.evidence-pane h3 {
+  margin: 0 0 6px;
+  font-size: 14px;
+}
+
 .screenshot-frame {
   display: grid;
   place-items: center;
-  min-height: 360px;
+  min-height: 280px;
   border: 1px solid #d9dde5;
   border-radius: 8px;
   background: #111827;
@@ -442,6 +456,92 @@ select {
   font-size: 14px;
 }
 
+.operation-grid {
+  display: grid;
+  grid-template-columns: repeat(2, minmax(0, 1fr));
+  gap: 8px;
+  margin: 0 0 8px;
+}
+
+.operation-grid div {
+  min-width: 0;
+}
+
+.operation-grid dt {
+  color: #667085;
+  font-size: 12px;
+}
+
+.operation-grid dd {
+  margin: 2px 0 0;
+  overflow-wrap: anywhere;
+}
+
+.ocr-results {
+  display: grid;
+  gap: 8px;
+  margin: 0;
+  padding: 0;
+  list-style: none;
+}
+
+.ocr-results li {
+  display: grid;
+  gap: 3px;
+  border-left: 3px solid #2f7c67;
+  background: #f5faf7;
+  padding: 8px 10px;
+  overflow-wrap: anywhere;
+}
+
+.ocr-results span,
+.muted {
+  color: #667085;
+  font-size: 12px;
+}
+
+.ui-tree {
+  border: 1px solid #e2e6ec;
+  border-radius: 8px;
+  background: #f9fafb;
+  padding: 10px;
+}
+
+.ui-tree summary {
+  cursor: pointer;
+  font-size: 13px;
+  font-weight: 650;
+}
+
+.ui-tree-nodes {
+  display: grid;
+  gap: 8px;
+  margin: 10px 0 0;
+  padding: 0;
+  list-style: none;
+}
+
+.ui-tree-nodes li {
+  display: grid;
+  grid-template-columns: auto minmax(0, 1fr);
+  gap: 3px 8px;
+  border-left: 3px solid #5f7fb0;
+  background: #f1f5fb;
+  padding: 8px 10px;
+  overflow-wrap: anywhere;
+}
+
+.ui-tree-nodes code,
+.ui-tree-nodes li > span {
+  grid-column: 1 / -1;
+  color: #667085;
+  font-size: 12px;
+}
+
+.muted {
+  margin: 0;
+}
+
 pre {
   max-height: 248px;
   overflow: auto;
@@ -511,6 +611,11 @@ pre {
     grid-template-columns: 1fr;
   }
 
+  .evidence-grid,
+  .operation-grid {
+    grid-template-columns: 1fr;
+  }
+
   .detail-grid,
   .form-grid {
     grid-template-columns: 1fr;

api/templates/runtime_console/task_detail.html

@@ -64,25 +64,91 @@
   </form>
 
   <div class="timeline-stage">
-    <div class="screenshot-frame">
-      {% if current_step.image_base64 %}
-      <img
-        src="data:image/png;base64,{{ current_step.image_base64 }}"
-        alt="Task step screenshot"
-      />
-      {% else %}
-      <span>No screenshot</span>
-      {% endif %}
+    <div class="evidence-grid">
+      <section class="evidence-pane">
+        <h3>Before action</h3>
+        <div class="screenshot-frame">
+          {% if current_step.before_image_base64 %}
+          <img
+            src="data:image/png;base64,{{ current_step.before_image_base64 }}"
+            alt="Screenshot captured before the action"
+          />
+          {% else %}
+          <span>No screenshot</span>
+          {% endif %}
+        </div>
+      </section>
+      <section class="evidence-pane">
+        <h3>After action</h3>
+        <div class="screenshot-frame">
+          {% if current_step.after_image_base64 %}
+          <img
+            src="data:image/png;base64,{{ current_step.after_image_base64 }}"
+            alt="Screenshot captured after the action"
+          />
+          {% else %}
+          <span>No screenshot</span>
+          {% endif %}
+        </div>
+      </section>
     </div>
     <div class="step-data">
       <div>
-        <h3>Tool Call</h3>
+        <h3>Operation</h3>
+        <dl class="operation-grid">
+          <div>
+            <dt>Action</dt>
+            <dd>{{ current_step.tool_call.get('action', '-') }}</dd>
+          </div>
+          <div>
+            <dt>Description</dt>
+            <dd>{{ current_step.tool_call.get('description', '-') }}</dd>
+          </div>
+        </dl>
         <pre>{{ current_step.tool_call | tojson(indent=2) }}</pre>
       </div>
       <div>
-        <h3>Result</h3>
+        <h3>Execution result</h3>
         <pre>{{ current_step.result | tojson(indent=2) }}</pre>
       </div>
+      <div>
+        <h3>OCR results</h3>
+        {% if ocr_results %}
+        <ul class="ocr-results">
+          {% for ocr in ocr_results %}
+          <li>
+            <strong>{{ ocr.get('text') or '-' }}</strong>
+            <span>{{ ocr.get('bounds') | tojson }}</span>
+            {% if ocr.get('confidence') is not none %}
+            <span>confidence {{ '%.3f' | format(ocr.get('confidence')) }}</span>
+            {% endif %}
+          </li>
+          {% endfor %}
+        </ul>
+        {% else %}
+        <p class="muted">No OCR output captured for this step.</p>
+        {% endif %}
+      </div>
+      {% if ui_tree_nodes %}
+      <div>
+        <h3>UI tree</h3>
+        <details class="ui-tree" open>
+          <summary>{{ ui_tree_nodes|length }} normalized nodes</summary>
+          <ul class="ui-tree-nodes">
+            {% for node in ui_tree_nodes %}
+            <li>
+              <strong>{{ node.get('type') or 'unknown' }}</strong>
+              <span>{{ node.get('text') or node.get('id') or '-' }}</span>
+              <code>{{ node.get('bounds') | tojson }}</code>
+              {% if node.get('confidence') is not none %}
+              <span>confidence {{ '%.3f' | format(node.get('confidence')) }}</span>
+              {% endif %}
+            </li>
+            {% endfor %}
+          </ul>
+        </details>
+      </div>
+      {% endif %}
     </div>
   </div>
   {% endif %}