feat(runtime): capture step evidence in console

tests/test_agents_collab_runner.py

@@ -5,7 +5,12 @@ from unittest.mock import MagicMock, patch
 
 from agents.collab_runner import CollaborativeTaskRunner, CollaborativeTaskRunnerConfig
 from agents.config import CollaborationConfig
-from agents.models import Observation, ReflectionAction, ReflectionOutcome, VerificationVerdict
+from agents.models import (
+    Observation,
+    ReflectionAction,
+    ReflectionOutcome,
+    VerificationVerdict,
+)
 from core.models import Bounds, Scene, SceneElement, Task
 from runtime.executor import StepResult
 from runtime.planner import PlannedStep
@@ -13,6 +18,7 @@ from runtime.task import TaskRunner, TaskRunnerConfig
 from storage.artifact_store import ArtifactStore
 from storage.task_metadata import TaskMetadataStore
 from storage.timeline import Timeline
+from tests.fakes import PNG_10X20
 from world.config import WorldConfig
 
 
@@ -20,7 +26,11 @@ def _scene() -> Scene:
     return Scene(
         width=1080,
         height=1920,
-        elements=[SceneElement(id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="OK")],
+        elements=[
+            SceneElement(
+                id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="OK"
+            )
+        ],
     )
 
 
@@ -55,7 +65,9 @@ def _replan_outcome() -> ReflectionOutcome:
 def _recovery_outcome() -> ReflectionOutcome:
     return ReflectionOutcome(
         replan=False,
-        action=ReflectionAction(action="swipe", description="Scroll", args={"direction": "up"}),
+        action=ReflectionAction(
+            action="swipe", description="Scroll", args={"direction": "up"}
+        ),
         reasoning="Try scrolling.",
     )
 
@@ -204,6 +216,7 @@ def test_collaborative_run_shares_bookkeeping_with_task_runner(tmp_path) -> None
         on_task_succeeded=on_task_succeeded,
         world_config=WorldConfig(enabled=True),
         config=TaskRunnerConfig(max_steps=5),
+        screenshot_provider=lambda device_id: PNG_10X20,
     )
 
     runner = CollaborativeTaskRunner(
@@ -225,6 +238,9 @@ def test_collaborative_run_shares_bookkeeping_with_task_runner(tmp_path) -> None
 
     assert result.status == "completed"
     assert len(timeline.read(task.id)) == 1
+    record = timeline.read(task.id)[0]
+    assert record["before_screenshot_path"]
+    assert record["after_screenshot_path"]
     assert metadata.get_task(task.id)["status"] == "completed"
     on_task_succeeded.assert_called_once_with(task.id, task.goal, timeline)
 

tests/test_console_api.py

@@ -114,6 +114,28 @@ def test_console_timeline_inlines_screenshot_and_handles_empty_history(
     assert client.get("/console/tasks/missing/timeline").status_code == 404
 
 
+def test_console_timeline_inlines_before_and_after_screenshots(tmp_path) -> None:
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    client, metadata_store = _client(tmp_path, timeline=timeline)
+    metadata_store.create_task(Task(id="task-evidence", goal="tap", device_id="phone"))
+    before = b"before"
+    after = b"after"
+    timeline.append(
+        task_id="task-evidence",
+        scene={"screen": {"width": 10, "height": 20}, "elements": []},
+        prompt="tap",
+        tool_call={"action": "tap", "description": "tap search"},
+        result={"ok": True},
+        before_screenshot=before,
+        after_screenshot=after,
+    )
+
+    record = client.get("/console/tasks/task-evidence/timeline").json()[0]
+    assert record["before_image_base64"] == base64.b64encode(before).decode("ascii")
+    assert record["after_image_base64"] == base64.b64encode(after).decode("ascii")
+    assert record["image_base64"] == base64.b64encode(after).decode("ascii")
+
+
 def test_console_device_registration_and_unregistration(tmp_path) -> None:
     config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
     manager = DeviceManager()

tests/test_runtime_console_web.py

@@ -238,16 +238,37 @@ def test_task_detail_renders_timeline_with_screenshot(tmp_path) -> None:
         task_id="task-with-timeline",
         scene={"screen": {"width": 10, "height": 20}, "elements": []},
         prompt="tap search",
-        tool_call={"action": "tap", "args": {"x": 1, "y": 2}},
+        tool_call={
+            "action": "tap",
+            "description": "tap search",
+            "args": {"x": 1, "y": 2},
+        },
         result={"ok": True},
-        screenshot=PNG_10X20,
+        before_screenshot=PNG_10X20 + b"before",
+        after_screenshot=PNG_10X20 + b"after",
+        ocr_results=[
+            {
+                "text": "Search",
+                "confidence": 0.98,
+                "bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
+            }
+        ],
     )
     body = client.get("/ui/tasks/task-with-timeline").text
-    expected_data_uri = "data:image/png;base64," + base64.b64encode(PNG_10X20).decode(
-        "ascii"
-    )
-    assert expected_data_uri in body
-    assert "tap" in body
+    before_data_uri = "data:image/png;base64," + base64.b64encode(
+        PNG_10X20 + b"before"
+    ).decode("ascii")
+    after_data_uri = "data:image/png;base64," + base64.b64encode(
+        PNG_10X20 + b"after"
+    ).decode("ascii")
+    assert before_data_uri in body
+    assert after_data_uri in body
+    assert "Before action" in body
+    assert "After action" in body
+    assert "Operation" in body
+    assert "OCR results" in body
+    assert "Search" in body
+    assert "UI tree" not in body
 
 
 def test_task_detail_404_for_unknown_task(tmp_path) -> None:
@@ -256,6 +277,39 @@ def test_task_detail_404_for_unknown_task(tmp_path) -> None:
     assert response.status_code == 404
 
 
+def test_task_detail_renders_normalized_ui_tree_result(tmp_path) -> None:
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    client, metadata_store = _client(tmp_path, timeline=timeline)
+    metadata_store.create_task(
+        Task(id="task-ui-tree", goal="inspect the screen", device_id="iphone-1")
+    )
+    timeline.append(
+        task_id="task-ui-tree",
+        scene={"screen": {"width": 10, "height": 20}, "elements": []},
+        prompt="inspect the screen",
+        tool_call={"action": "get_ui_tree", "description": "inspect UI tree"},
+        result={
+            "success": True,
+            "result": [
+                {
+                    "id": "ui-000",
+                    "type": "button",
+                    "text": "Search",
+                    "bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
+                    "confidence": 1.0,
+                }
+            ],
+        },
+    )
+
+    body = client.get("/ui/tasks/task-ui-tree").text
+
+    assert "UI tree" in body
+    assert "1 normalized nodes" in body
+    assert "button" in body
+    assert "Search" in body
+
+
 def test_config_page_lists_supported_drivers_and_current_max_steps(tmp_path) -> None:
     config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
     config_store.set_setting("max_steps", 5)

tests/test_scene_builder.py

@@ -41,4 +41,5 @@ def test_scene_builder_merges_overlapping_ocr_into_ui_element() -> None:
     assert scene.elements[0].type == "button"
     assert scene.elements[0].text == "Search"
     assert scene.elements[1].text == "Footer"
+    assert [element.text for element in scene.ocr_elements] == ["Search", "Footer"]
     assert bbox_iou(ui_button.bounds, ocr_label.bounds) > 0.5

tests/test_task_loop.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from core.models import Bounds, Scene, SceneElement, Task
 from runtime.executor import Executor, ExecutorConfig
 from runtime.planner import PlannedStep, Planner
@@ -112,3 +114,52 @@ def test_task_runner_stops_before_the_next_planned_action() -> None:
     assert result.status == "failed"
     assert result.failure_reason == "execution interrupted"
     assert actions == ["tap"]
+
+
+def test_task_runner_persists_action_evidence_and_raw_ocr(tmp_path) -> None:
+    scene = Scene(
+        width=10,
+        height=20,
+        elements=[],
+        ocr_elements=[
+            SceneElement(
+                id="ocr-001",
+                type="text",
+                text="Search",
+                bounds=Bounds(1, 2, 3, 4),
+                confidence=0.98,
+                source="ocr",
+            )
+        ],
+    )
+    screenshots = iter(
+        [
+            b"planning",
+            b"before-action",
+            b"after-action",
+            b"completion-check",
+        ]
+    )
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    runner = TaskRunner(
+        planner=ScriptedPlanner(
+            [PlannedStep(action="tap", description="tap search", args={})]
+        ),
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        timeline=timeline,
+        config=TaskRunnerConfig(max_steps=2),
+        observer=lambda device_id: scene,
+        screenshot_provider=lambda device_id: next(screenshots),
+    )
+
+    result = runner.run(Task(goal="tap search", device_id="iphone-1"))
+
+    assert result.status == "completed"
+    record = timeline.read(result.id)[0]
+    assert Path(record["before_screenshot_path"]).read_bytes() == b"before-action"
+    assert Path(record["after_screenshot_path"]).read_bytes() == b"after-action"
+    assert record["tool_call"]["description"] == "tap search"
+    assert record["ocr_results"][0]["text"] == "Search"

tests/test_timeline.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from pathlib import Path
+
 from storage.artifact_store import ArtifactStore
 from storage.timeline import Timeline
 from tests.fakes import PNG_10X20
@@ -52,3 +54,32 @@ def test_timeline_records_per_step_prompt_not_task_goal(tmp_path) -> None:
     assert len(records) == 1
     assert records[0]["prompt"] == per_step_prompt
     assert "Call exactly one tool" in records[0]["prompt"]
+
+
+def test_timeline_records_before_and_after_screenshots_with_ocr(tmp_path) -> None:
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    before = b"before-image"
+    after = b"after-image"
+
+    timeline.append(
+        task_id="task-evidence",
+        scene={"screen": {"width": 1, "height": 1}, "elements": []},
+        prompt="goal",
+        tool_call={"action": "tap", "description": "tap search"},
+        result={"ok": True},
+        before_screenshot=before,
+        after_screenshot=after,
+        ocr_results=[
+            {
+                "text": "Search",
+                "confidence": 0.98,
+                "bounds": {"x": 1, "y": 2, "width": 3, "height": 4},
+            }
+        ],
+    )
+
+    record = timeline.read("task-evidence")[0]
+    assert Path(record["before_screenshot_path"]).read_bytes() == before
+    assert Path(record["after_screenshot_path"]).read_bytes() == after
+    assert record["screenshot_path"] == record["after_screenshot_path"]
+    assert record["ocr_results"][0]["text"] == "Search"