From ccde30e37862ad61b6dd7f436cc038b1ba4f20d1 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 15 Jul 2026 10:12:09 +0800 Subject: [PATCH] feat(runtime): capture step evidence in console --- agents/collab_runner.py | 29 ++++- api/console.py | 22 ++-- api/console_web.py | 28 +++++ api/static/runtime_console/console.css | 109 +++++++++++++++++- .../runtime_console/task_detail.html | 88 ++++++++++++-- core/models.py | 12 +- .../design.md | 10 +- .../proposal.md | 6 +- .../specs/runtime-task-evidence/spec.md | 38 ++++++ .../tasks.md | 8 ++ perception/scene_builder.py | 8 +- runtime/task.py | 38 +++++- storage/artifact_store.py | 37 ++++-- storage/timeline.py | 19 ++- tests/test_agents_collab_runner.py | 22 +++- tests/test_console_api.py | 22 ++++ tests/test_runtime_console_web.py | 68 +++++++++-- tests/test_scene_builder.py | 1 + tests/test_task_loop.py | 51 ++++++++ tests/test_timeline.py | 31 +++++ 20 files changed, 594 insertions(+), 53 deletions(-) create mode 100644 openspec/changes/task-execution-progress-visibility/specs/runtime-task-evidence/spec.md diff --git a/agents/collab_runner.py b/agents/collab_runner.py index b3458a5..4cf39e9 100644 --- a/agents/collab_runner.py +++ b/agents/collab_runner.py @@ -4,7 +4,6 @@ import logging from dataclasses import dataclass, replace from agents.config import CollaborationConfig, load_config -from agents.models import Observation, VerificationVerdict from agents.observer import Observer from agents.reflector import Reflector from agents.verifier import Verifier @@ -94,10 +93,21 @@ class CollaborativeTaskRunner: for step in steps: executable_step = self._step_for_device(step, task.device_id) + before_screenshot = self.task_runner._planning_screenshot( + task.device_id + ) result = self.executor.execute(executable_step, context=context) + after_screenshot = self.task_runner._planning_screenshot(task.device_id) context.add_step_result(result) self.task_runner._record_step_result( - world_handle, context, task, scene, step, result + world_handle, + context, + task, + scene, + step, + result, + before_screenshot=before_screenshot, + after_screenshot=after_screenshot, ) post_scene = self._observe_scene(task.device_id) @@ -146,13 +156,26 @@ class CollaborativeTaskRunner: description=outcome.action.description, args=outcome.action.args, ) + before_screenshot = self.task_runner._planning_screenshot( + task.device_id + ) recovery_result = self.executor.execute( self._step_for_device(recovery_step, task.device_id), context=context, ) + after_screenshot = self.task_runner._planning_screenshot( + task.device_id + ) context.add_step_result(recovery_result) self.task_runner._record_step_result( - world_handle, context, task, post_scene, recovery_step, recovery_result + world_handle, + context, + task, + post_scene, + recovery_step, + recovery_result, + before_screenshot=before_screenshot, + after_screenshot=after_screenshot, ) if not recovery_result.success: return self._fail( diff --git a/api/console.py b/api/console.py index d47a518..3d923f1 100644 --- a/api/console.py +++ b/api/console.py @@ -105,7 +105,7 @@ class ConsoleService: if self._metadata_store.get_task(task_id) is None: raise KeyError("task not found") return [ - self._inline_screenshot(record) for record in self._timeline.read(task_id) + self._inline_screenshots(record) for record in self._timeline.read(task_id) ] def get_runtime_config(self) -> dict[str, int]: @@ -124,15 +124,23 @@ class ConsoleService: ) @staticmethod - def _inline_screenshot(record: dict[str, Any]) -> dict[str, Any]: + def _inline_screenshots(record: dict[str, Any]) -> dict[str, Any]: payload = dict(record) - screenshot_path = payload.get("screenshot_path") - if screenshot_path: + for path_key, image_key in ( + ("before_screenshot_path", "before_image_base64"), + ("after_screenshot_path", "after_image_base64"), + ("screenshot_path", "image_base64"), + ): + screenshot_path = payload.get(path_key) + if not screenshot_path: + continue path = Path(str(screenshot_path)) if path.exists(): - payload["image_base64"] = base64.b64encode(path.read_bytes()).decode( - "ascii" - ) + payload[image_key] = base64.b64encode(path.read_bytes()).decode("ascii") + if "after_image_base64" not in payload and "image_base64" in payload: + payload["after_image_base64"] = payload["image_base64"] + elif "after_image_base64" in payload: + payload["image_base64"] = payload["after_image_base64"] return payload def _runner_max_steps(self) -> int: diff --git a/api/console_web.py b/api/console_web.py index 945e483..44d3a21 100644 --- a/api/console_web.py +++ b/api/console_web.py @@ -49,6 +49,28 @@ def _default_driver_type() -> str: return _DEFAULT_FORM_VALUES["driver_type"] +def _ocr_results(record: dict[str, Any]) -> list[dict[str, Any]]: + raw_results = record.get("ocr_results") + if not isinstance(raw_results, list): + return [] + return [result for result in raw_results if isinstance(result, dict)] + + +def _ui_tree_nodes(record: dict[str, Any]) -> list[dict[str, Any]]: + tool_call = record.get("tool_call") + if not isinstance(tool_call, dict): + return [] + if tool_call.get("action") not in {"get_ui_tree", "ui_tree"}: + return [] + step_result = record.get("result") + if not isinstance(step_result, dict): + return [] + raw_nodes = step_result.get("result") + if not isinstance(raw_nodes, list): + return [] + return [node for node in raw_nodes if isinstance(node, dict)] + + def _config_context( request: Request, service: ConsoleService, @@ -183,6 +205,12 @@ def create_console_web_router(service: ConsoleService) -> APIRouter: timeline=timeline, current_step=current_step if current_step is not None else {}, current_step_index=index, + ocr_results=_ocr_results(current_step) + if current_step is not None + else [], + ui_tree_nodes=_ui_tree_nodes(current_step) + if current_step is not None + else [], ) ) diff --git a/api/static/runtime_console/console.css b/api/static/runtime_console/console.css index 134377b..12ae741 100644 --- a/api/static/runtime_console/console.css +++ b/api/static/runtime_console/console.css @@ -409,14 +409,28 @@ select { .timeline-stage { display: grid; - grid-template-columns: minmax(220px, 360px) minmax(0, 1fr); gap: 14px; } +.evidence-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 14px; +} + +.evidence-pane { + min-width: 0; +} + +.evidence-pane h3 { + margin: 0 0 6px; + font-size: 14px; +} + .screenshot-frame { display: grid; place-items: center; - min-height: 360px; + min-height: 280px; border: 1px solid #d9dde5; border-radius: 8px; background: #111827; @@ -442,6 +456,92 @@ select { font-size: 14px; } +.operation-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 8px; + margin: 0 0 8px; +} + +.operation-grid div { + min-width: 0; +} + +.operation-grid dt { + color: #667085; + font-size: 12px; +} + +.operation-grid dd { + margin: 2px 0 0; + overflow-wrap: anywhere; +} + +.ocr-results { + display: grid; + gap: 8px; + margin: 0; + padding: 0; + list-style: none; +} + +.ocr-results li { + display: grid; + gap: 3px; + border-left: 3px solid #2f7c67; + background: #f5faf7; + padding: 8px 10px; + overflow-wrap: anywhere; +} + +.ocr-results span, +.muted { + color: #667085; + font-size: 12px; +} + +.ui-tree { + border: 1px solid #e2e6ec; + border-radius: 8px; + background: #f9fafb; + padding: 10px; +} + +.ui-tree summary { + cursor: pointer; + font-size: 13px; + font-weight: 650; +} + +.ui-tree-nodes { + display: grid; + gap: 8px; + margin: 10px 0 0; + padding: 0; + list-style: none; +} + +.ui-tree-nodes li { + display: grid; + grid-template-columns: auto minmax(0, 1fr); + gap: 3px 8px; + border-left: 3px solid #5f7fb0; + background: #f1f5fb; + padding: 8px 10px; + overflow-wrap: anywhere; +} + +.ui-tree-nodes code, +.ui-tree-nodes li > span { + grid-column: 1 / -1; + color: #667085; + font-size: 12px; +} + +.muted { + margin: 0; +} + pre { max-height: 248px; overflow: auto; @@ -511,6 +611,11 @@ pre { grid-template-columns: 1fr; } + .evidence-grid, + .operation-grid { + grid-template-columns: 1fr; + } + .detail-grid, .form-grid { grid-template-columns: 1fr; diff --git a/api/templates/runtime_console/task_detail.html b/api/templates/runtime_console/task_detail.html index 1f6b0cd..542a170 100644 --- a/api/templates/runtime_console/task_detail.html +++ b/api/templates/runtime_console/task_detail.html @@ -64,25 +64,91 @@
-
- {% if current_step.image_base64 %} - Task step screenshot - {% else %} - No screenshot - {% endif %} +
+
+

Before action

+
+ {% if current_step.before_image_base64 %} + Screenshot captured before the action + {% else %} + No screenshot + {% endif %} +
+
+
+

After action

+
+ {% if current_step.after_image_base64 %} + Screenshot captured after the action + {% else %} + No screenshot + {% endif %} +
+
-

Tool Call

+

Operation

+
+
+
Action
+
{{ current_step.tool_call.get('action', '-') }}
+
+
+
Description
+
{{ current_step.tool_call.get('description', '-') }}
+
+ 
{{ current_step.tool_call | tojson(indent=2) }}
-

Result

+

Execution result

{{ current_step.result | tojson(indent=2) }}
+
+

OCR results

+ {% if ocr_results %} +
    + {% for ocr in ocr_results %} +
  • + {{ ocr.get('text') or '-' }} + {{ ocr.get('bounds') | tojson }} + {% if ocr.get('confidence') is not none %} + confidence {{ '%.3f' | format(ocr.get('confidence')) }} + {% endif %} +
    • + {% endfor %} +
+ {% else %} +

No OCR output captured for this step.

+ {% endif %} +
+ {% if ui_tree_nodes %} +
+

UI tree

+
+ {{ ui_tree_nodes|length }} normalized nodes +
    + {% for node in ui_tree_nodes %} +
  • + {{ node.get('type') or 'unknown' }} + {{ node.get('text') or node.get('id') or '-' }} + {{ node.get('bounds') | tojson }} + {% if node.get('confidence') is not none %} + confidence {{ '%.3f' | format(node.get('confidence')) }} + {% endif %} +
    • + {% endfor %} +
+
+
+ {% endif %}
{% endif %} diff --git a/core/models.py b/core/models.py index b37bf13..ecae5ad 100644 --- a/core/models.py +++ b/core/models.py @@ -113,6 +113,9 @@ class Scene: width: int height: int elements: list[SceneElement] = field(default_factory=list) + # Keep raw OCR observations for local execution evidence without duplicating + # them in the normalized, LLM-facing scene payload. + ocr_elements: list[SceneElement] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return { @@ -120,6 +123,9 @@ class Scene: "elements": [element.to_dict() for element in self.elements], } + def ocr_results_to_dict(self) -> list[dict[str, Any]]: + return [element.to_dict() for element in self.ocr_elements] + @classmethod def from_dict(cls, data: dict[str, Any]) -> "Scene": screen = data.get("screen") or {} @@ -127,8 +133,11 @@ class Scene: width=int(screen.get("width") or data.get("width") or 0), height=int(screen.get("height") or data.get("height") or 0), elements=[ + SceneElement.from_dict(element) for element in data.get("elements", []) + ], + ocr_elements=[ SceneElement.from_dict(element) - for element in data.get("elements", []) + for element in data.get("ocr_elements", []) ], ) @@ -179,4 +188,3 @@ class Step: "result": self.result, "error": self.error, } - diff --git a/openspec/changes/task-execution-progress-visibility/design.md b/openspec/changes/task-execution-progress-visibility/design.md index bef650f..4f9b125 100644 --- a/openspec/changes/task-execution-progress-visibility/design.md +++ b/openspec/changes/task-execution-progress-visibility/design.md @@ -19,7 +19,7 @@ Three surfaces currently cannot show a task's in-progress status: - No SSE/WebSocket push. All three surfaces keep polling (Host Agent console 5s, Cloud Console/`cloud-console` and `console/` at their existing intervals). Nothing here requires push infra, and adding it would be a bigger, separate change. - The coarse live step index/status/summary reported via lease renewal (D4/D5) stays *latest-snapshot-only* on the Cloud side (overwrite semantics) — it is a "what's happening right now" indicator, not a history mechanism, and is unaffected by the LLM-content decisions below. - Full step-by-step **LLM interaction** history *is* now synced to and durably persisted by Cloud (see D7/D8) — this is a deliberate reversal of this change's earlier draft, which had scoped Cloud to latest-only. The motivation: Host Agent deployments already route every AI Planner decision through Cloud API's existing `cloud-planner-proxy` endpoint when configured for the `cloud` transport, so Cloud is already on the natural path for this data and centralizing it there (rather than only on whichever edge host happens to still be running) is the more useful place for an operator doing centralized troubleshooting. This durable log is scoped strictly to LLM prompt/response text — it is not a general-purpose duplicate of the Host Agent's full `Timeline` (no screenshots, no scene JSON dumps beyond what's embedded in the prompt text itself). -- No change to the Runtime `console/` SPA or `runtime/`-owned packages. The multi-backend-SPA alternative was considered and rejected (see Decisions). +- No multi-backend Runtime console or cross-origin Host Agent integration. Runtime changes remain local to its existing same-origin API/UI and shared timeline evidence model. - No change to `driver/`/`device/`/`core/` device-control internals. - No screenshot or full-scene data leaves the Host Agent process as part of progress reporting, and no screenshot bytes are ever persisted by the Cloud Control Plane. (Full LLM prompt/response *text* is, by contrast, an explicit Goal below — see D7/D8 — which partially supersedes the original `cloud-planner-proxy` proposal's "does not durably persist ... full prompt text" statement. Screenshots remain excluded; prompts no longer are.) @@ -96,6 +96,14 @@ This is a pre-existing gap in a component shared by Runtime and Host Agent alike **Decision**: extend `ToolCallDecision` with the actual `user_prompt`/`system_prompt` it was given (or have `AIPlanner.plan()` return a small side-channel result instead of changing the `Planner` interface's return type) so `TaskRunner._append_timeline()` can pass the real per-step prompt into `Timeline.append()`. Rename `Timeline`/`TimelineRecord`'s `prompt` field's meaning (or add a new field) to unambiguously mean "the prompt actually sent to the LLM for this step." This fix lands in the shared `runtime`/`storage` packages, so both Host Agent's local console (D3) and the existing Runtime `console/` automatically benefit — it is not Host-Agent-specific plumbing. +### D10: Persist and render before/after action evidence in the Runtime timeline + +The existing Timeline captures only one screenshot after an action, while the planning screenshot used before the action is transient. Extend each Timeline record with distinct before/after screenshot paths, preserving the existing `screenshot_path` as a backward-compatible alias for the after screenshot. `TaskRunner` captures each image immediately before and after invoking the executor, including failed action attempts. The Runtime console API inlines both images and the existing task-detail page renders them beside the recorded action and result. + +OCR is already available during perception but fused into the normalized Scene, where an OCR value that overlaps a UI-tree node can lose its raw provenance. Preserve raw OCR observations on `Scene` for local timeline capture only; keep them out of `Scene.to_dict()` so the LLM-facing planner payload does not grow with duplicated text. The Runtime task-detail page renders the persisted OCR list when it is available and handles legacy records without it. + +The existing `get_ui_tree`/`ui_tree` tool returns a normalized flat list of UI nodes, not the driver-specific raw XML hierarchy. Its StepResult is already persisted in the Timeline. Detect those action names in the Runtime UI and render the returned nodes in a collapsible structured view while retaining the full JSON result below it. This makes the inspection result readable without changing the tool response contract or storing a second tree copy. + ## Risks / Trade-offs - [Risk] Per-step SQLite writes in a long-lived Host Agent process add I/O overhead → Mitigation: this is the same write pattern Runtime's own console has always used per step; no new proof of acceptability needed. If profiling later shows it matters for very high step-rate tasks, batching/debouncing is a follow-up, not a blocker here. diff --git a/openspec/changes/task-execution-progress-visibility/proposal.md b/openspec/changes/task-execution-progress-visibility/proposal.md index 46081f0..189aa1a 100644 --- a/openspec/changes/task-execution-progress-visibility/proposal.md +++ b/openspec/changes/task-execution-progress-visibility/proposal.md @@ -12,6 +12,7 @@ Nobody can see a task while it is running. Host Agent's local console only shows - All three surfaces continue to use polling (matching current behavior); this change does not introduce SSE/WebSocket infrastructure unless design.md finds a compelling reason to. - Fix the shared `Timeline`/`TaskRunner`/`AIPlanner` recording path so a persisted step's "prompt" is the *actual* prompt sent to the LLM for that step (not the task's overall goal) and the model's resulting decision is captured too — this pre-existing gap affects Runtime and Host Agent alike and undermines the step-level detail this change otherwise adds. - Persist a durable, per-step log of full LLM prompt/response content on the Cloud Control Plane, for centralized troubleshooting — reusing the already-existing `cloud-planner-proxy` decide endpoint as the capture point (no new protocol/endpoint) rather than the coarse lease-renewal piggyback used for live index/status. This durable log is populated only for hosts using the `cloud` planner transport; hosts on the `direct` transport still get only the coarse index/status via lease renewal. Cloud Console gains a view to browse a task's full LLM interaction history. +- Extend the local Runtime timeline so every executed action retains a before screenshot, operation detail, after screenshot, and any available raw OCR observations; render that evidence in the Runtime task-detail UI and expose it from the existing Runtime console API. When a step uses the existing UI-tree tool, render its normalized node result as a structured, collapsible view as well. ## Capabilities @@ -19,6 +20,7 @@ Nobody can see a task while it is running. Host Agent's local console only shows - `host-agent-task-progress`: Host Agent captures step-level execution progress for its in-flight assignment (via a wired `metadata_store`/`timeline`) and exposes it through its local console/API. - `cloud-task-progress-visibility`: Cloud Control Plane receives, persists, and exposes near-real-time step-level progress for assignments it has dispatched to a Host Agent, and Cloud Console renders it. - `host-agent-console-task-pages`: Host Agent's local server-rendered console gains read-only task list/detail/timeline pages (mirroring `api/console.py`'s task query shape) so an operator can inspect a Host-Agent-executed task's progress and history without needing the separate Runtime `console/` SPA or violating the `runtime/`-package host/cloud isolation boundary. +- `runtime-task-evidence`: Runtime task history retains and renders pre/post action evidence, raw OCR observations, and existing UI-tree inspection results. ### Modified Capabilities - `host-agent-protocol`: add a requirement that the Host Agent reports in-progress step-level status updates to the control plane (in addition to the existing heartbeat/claim/renewal/result operations), and that the control plane accepts and stores them per active assignment. @@ -27,8 +29,8 @@ Nobody can see a task while it is running. Host Agent's local console only shows ## Impact - `apps/device-host-agent/host_agent/app.py`, `execution.py`, `assignment.py`, `status.py`, `web/app.py` — wire a real metadata/timeline store into the in-process `TaskRunner`, extend status tracking and the local console UI/API. -- `runtime/task.py`, `runtime/ai_planner.py`, `runtime/tool_calling_client.py`, `storage/timeline.py` — fix the shared step-recording path so the real per-step prompt and the model's response are captured, not just the task goal and parsed tool call (a correctness fix in already-existing, shared code, not new behavior scope). +- `runtime/task.py`, `runtime/ai_planner.py`, `runtime/tool_calling_client.py`, `storage/timeline.py`, `storage/artifact_store.py`, `core/models.py`, `perception/scene_builder.py` — fix the shared step-recording path so the real per-step prompt and the model's response are captured, retain pre/post action screenshots and raw OCR observations, not just the task goal and parsed tool call. - `packages/cloud-platform/cloud/internal_api/models.py`, `packages/cloud-platform/cloud/internal_api/api.py` (`decide_planner_call`), `repository.py`/`sql_repository.py`, a new Alembic migration — new progress-reporting request/response models and a persistence + query path for latest per-assignment coarse progress, *and* a new durable per-step `planner_decision_log` table (with its own retention job) populated from the existing planner-decision endpoint. - `cloud-console/` (Vue3 SPA) — new UI to render live per-assignment progress, and a new view to browse a task's full LLM interaction history. -- `apps/device-host-agent/host_agent/web/app.py` — new read-only task list/detail/timeline pages backed by the Host-Agent-local `TaskMetadataStore`/`Timeline`; no changes anticipated to `console/` (Vue3 SPA). +- `api/console.py`, `api/console_web.py`, `api/templates/runtime_console/*` — expose and render Runtime-local step evidence; Host Agent task pages receive the data through the shared timeline shape. - No changes anticipated to `driver/`, `device/`, `core/` device-control internals. diff --git a/openspec/changes/task-execution-progress-visibility/specs/runtime-task-evidence/spec.md b/openspec/changes/task-execution-progress-visibility/specs/runtime-task-evidence/spec.md new file mode 100644 index 0000000..25d6d6c --- /dev/null +++ b/openspec/changes/task-execution-progress-visibility/specs/runtime-task-evidence/spec.md @@ -0,0 +1,38 @@ +## ADDED Requirements + +### Requirement: Runtime persists complete evidence for each executed action +The Runtime SHALL persist, for each action it attempts, a screenshot captured immediately before the executor call, the action description and arguments, the execution result, and a screenshot captured immediately after the executor call. Existing timeline records that contain only the legacy single screenshot SHALL remain readable, with that screenshot treated as the post-action image. + +#### Scenario: An action succeeds +- **WHEN** the Runtime executes an action for a task +- **THEN** its timeline record includes distinct before and after screenshots, the action detail, and the execution result + +#### Scenario: An action fails +- **WHEN** the Runtime executor exhausts its retries for an action +- **THEN** the action's timeline record still includes any screenshots that were captured and the failure result before the task is marked failed + +#### Scenario: A legacy timeline record is read +- **WHEN** a timeline record has only the prior `screenshot_path` field +- **THEN** the Runtime exposes it as the post-action screenshot without failing to render the record + +### Requirement: Runtime task evidence exposes available OCR observations +The Runtime SHALL persist raw OCR observations associated with the scene used to plan an action when available, without adding duplicate OCR data to the LLM-facing normalized Scene payload. The Runtime task-detail UI SHALL render available OCR text, confidence, and bounds, and SHALL render normally when no OCR result exists. + +#### Scenario: OCR found text while planning an action +- **WHEN** perception produced one or more OCR observations for the action's planning scene +- **THEN** the corresponding timeline record includes those observations and the Runtime task-detail UI displays them + +#### Scenario: OCR was unavailable or found no text +- **WHEN** perception yields no OCR observations +- **THEN** the Runtime records the action evidence and renders the task detail without an OCR result list + +### Requirement: Runtime task evidence renders UI-tree inspection results +When a Runtime step invokes the existing `get_ui_tree` or `ui_tree` tool and the persisted result contains normalized UI nodes, the Runtime task-detail UI SHALL render those nodes in a structured, collapsible view while retaining the recorded JSON result. The Runtime SHALL NOT change the tool's response contract or duplicate the result in a separate persistence field. + +#### Scenario: UI-tree inspection succeeds +- **WHEN** a task step uses `get_ui_tree` or `ui_tree` and returns one or more normalized nodes +- **THEN** the task-detail UI displays each node's type, visible text or identifier, bounds, and available confidence + +#### Scenario: A non-UI-tree step is displayed +- **WHEN** a task step did not invoke a UI-tree tool +- **THEN** the task-detail UI does not render an empty UI-tree section diff --git a/openspec/changes/task-execution-progress-visibility/tasks.md b/openspec/changes/task-execution-progress-visibility/tasks.md index d9c84e5..95ffc79 100644 --- a/openspec/changes/task-execution-progress-visibility/tasks.md +++ b/openspec/changes/task-execution-progress-visibility/tasks.md @@ -66,3 +66,11 @@ - [x] 9.2 Add a `cloud-console/` view rendering a task's full LLM interaction history (prompt + resulting decision per step). - [x] 9.3 When a task's host used the `direct` transport (no persisted decisions and the host's configured transport is known to be `direct`), show an explicit "not reported by this host's transport" state rather than an empty list. - [x] 9.4 Add Vitest coverage for populated history, empty-but-cloud-transport (task hasn't produced any decisions yet), and direct-transport-hidden cases. + +## 10. Runtime per-step evidence (D10) + +- [x] 10.1 Extend the shared Scene/Timeline/ArtifactStore model to retain raw OCR observations and separate before/after screenshots while preserving compatibility with existing single-screenshot records. +- [x] 10.2 Capture before and after screenshots around every TaskRunner executor call, then persist the action, result, and available OCR observations in the same timeline record. +- [x] 10.3 Extend the Runtime console JSON API and task-detail UI to render before/after screenshots, operation details, and available OCR results. +- [x] 10.4 Render persisted normalized UI-tree tool results in the Runtime task-detail UI without changing the tool contract or duplicating stored data. +- [x] 10.5 Add focused Timeline, TaskRunner, perception, JSON API, Runtime UI, and UI-tree regression coverage; run the relevant format, lint, test, and strict OpenSpec validation commands. diff --git a/perception/scene_builder.py b/perception/scene_builder.py index dcb34f8..a9a5542 100644 --- a/perception/scene_builder.py +++ b/perception/scene_builder.py @@ -47,7 +47,12 @@ def build_scene( continue merged.append(_with_id(ocr_element, f"ocr-{ocr_index:03d}")) - return Scene(width=screen_width, height=screen_height, elements=merged) + return Scene( + width=screen_width, + height=screen_height, + elements=merged, + ocr_elements=list(ocr_elements), + ) def bbox_iou(first: Bounds, second: Bounds) -> float: @@ -88,4 +93,3 @@ def _best_confidence(first: float | None, second: float | None) -> float | None: if not values: return None return max(values) - diff --git a/runtime/task.py b/runtime/task.py index c012f4a..f09ef08 100644 --- a/runtime/task.py +++ b/runtime/task.py @@ -136,13 +136,22 @@ class TaskRunner: if should_stop is not None and should_stop(): return self._interrupt_task(task) executable_step = self._step_for_device(step, task.device_id) + before_screenshot = self._planning_screenshot(task.device_id) result = self.executor.execute( executable_step, context=context, ) + after_screenshot = self._planning_screenshot(task.device_id) context.add_step_result(result) self._record_step_result( - world_handle, context, task, scene, step, result + world_handle, + context, + task, + scene, + step, + result, + before_screenshot=before_screenshot, + after_screenshot=after_screenshot, ) self._emit_step_progress( len(context.step_results), @@ -204,6 +213,9 @@ class TaskRunner: scene: Scene, step: PlannedStep, result: object, + *, + before_screenshot: bytes | None = None, + after_screenshot: bytes | None = None, ) -> None: """Record one executed step's bookkeeping: world-model observe + timeline append. @@ -212,7 +224,14 @@ class TaskRunner: cannot drift apart on what gets recorded for a step. """ self._update_world(world_handle, context, scene, step, result) - self._append_timeline(task, scene, step, result) + self._append_timeline( + task, + scene, + step, + result, + before_screenshot=before_screenshot, + after_screenshot=after_screenshot, + ) def _emit_step_progress( self, step_index: int, step_status: str, summary: str @@ -330,10 +349,19 @@ class TaskRunner: scene: Scene, step: PlannedStep, result: object, + *, + before_screenshot: bytes | None = None, + after_screenshot: bytes | None = None, ) -> None: if not self.timeline: return - screenshot = self._planning_screenshot(task.device_id) + ocr_results = scene.ocr_results_to_dict() + if not ocr_results: + ocr_results = [ + element.to_dict() + for element in scene.elements + if element.source == "ocr" + ] self.timeline.append( task_id=task.id, scene=scene.to_dict(), @@ -346,7 +374,9 @@ class TaskRunner: result=result.to_dict() if hasattr(result, "to_dict") else {"result": result}, - screenshot=screenshot, + before_screenshot=before_screenshot, + after_screenshot=after_screenshot, + ocr_results=ocr_results, ) def _step_for_device(self, step: PlannedStep, device_id: str) -> PlannedStep: diff --git a/storage/artifact_store.py b/storage/artifact_store.py index bc7b973..c37f94f 100644 --- a/storage/artifact_store.py +++ b/storage/artifact_store.py @@ -20,21 +20,36 @@ class ArtifactStore: *, task_id: str, index: int, - screenshot: bytes | None, + before_screenshot: bytes | None, + after_screenshot: bytes | None, record: dict[str, Any], ) -> dict[str, str | None]: task_dir = self.task_dir(task_id) task_dir.mkdir(parents=True, exist_ok=True) stem = f"{index:03d}" - screenshot_path: Path | None = None - if screenshot is not None: - screenshot_path = task_dir / f"{stem}.png" - screenshot_path.write_bytes(screenshot) + before_screenshot_path: Path | None = None + after_screenshot_path: Path | None = None + if before_screenshot is not None: + before_screenshot_path = task_dir / f"{stem}-before.png" + before_screenshot_path.write_bytes(before_screenshot) + if after_screenshot is not None: + # Preserve the original filename as the compatibility alias for the + # screenshot captured after a step. + after_screenshot_path = task_dir / f"{stem}.png" + after_screenshot_path.write_bytes(after_screenshot) json_path = task_dir / f"{stem}.json" payload = { **record, - "screenshot_path": str(screenshot_path) if screenshot_path else None, + "before_screenshot_path": ( + str(before_screenshot_path) if before_screenshot_path else None + ), + "after_screenshot_path": ( + str(after_screenshot_path) if after_screenshot_path else None + ), + "screenshot_path": ( + str(after_screenshot_path) if after_screenshot_path else None + ), } json_path.write_text( json.dumps(_jsonable(payload), ensure_ascii=False, indent=2), @@ -42,7 +57,15 @@ class ArtifactStore: ) return { "json_path": str(json_path), - "screenshot_path": str(screenshot_path) if screenshot_path else None, + "before_screenshot_path": ( + str(before_screenshot_path) if before_screenshot_path else None + ), + "after_screenshot_path": ( + str(after_screenshot_path) if after_screenshot_path else None + ), + "screenshot_path": ( + str(after_screenshot_path) if after_screenshot_path else None + ), } def read_steps(self, task_id: str) -> list[dict[str, Any]]: diff --git a/storage/timeline.py b/storage/timeline.py index 1e0adfb..a590ead 100644 --- a/storage/timeline.py +++ b/storage/timeline.py @@ -1,6 +1,6 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime from typing import Any @@ -17,6 +17,9 @@ class TimelineRecord: tool_call: dict[str, Any] result: dict[str, Any] timestamp: str + ocr_results: list[dict[str, Any]] = field(default_factory=list) + before_screenshot_path: str | None = None + after_screenshot_path: str | None = None screenshot_path: str | None = None @@ -33,20 +36,29 @@ class Timeline: tool_call: dict[str, Any], result: dict[str, Any], screenshot: bytes | None = None, + before_screenshot: bytes | None = None, + after_screenshot: bytes | None = None, + ocr_results: list[dict[str, Any]] | None = None, ) -> TimelineRecord: index = len(self.read(task_id)) + 1 + resolved_after_screenshot = ( + after_screenshot if after_screenshot is not None else screenshot + ) + resolved_ocr_results = list(ocr_results or []) record = { "index": index, "scene": scene, "prompt": prompt, "tool_call": tool_call, "result": result, + "ocr_results": resolved_ocr_results, "timestamp": datetime.now().astimezone().isoformat(), } paths = self.artifact_store.write_step( task_id=task_id, index=index, - screenshot=screenshot, + before_screenshot=before_screenshot, + after_screenshot=resolved_after_screenshot, record=record, ) return TimelineRecord( @@ -56,6 +68,9 @@ class Timeline: tool_call=tool_call, result=result, timestamp=record["timestamp"], + ocr_results=resolved_ocr_results, + before_screenshot_path=paths["before_screenshot_path"], + after_screenshot_path=paths["after_screenshot_path"], screenshot_path=paths["screenshot_path"], ) diff --git a/tests/test_agents_collab_runner.py b/tests/test_agents_collab_runner.py index d129ad5..c656285 100644 --- a/tests/test_agents_collab_runner.py +++ b/tests/test_agents_collab_runner.py @@ -5,7 +5,12 @@ from unittest.mock import MagicMock, patch from agents.collab_runner import CollaborativeTaskRunner, CollaborativeTaskRunnerConfig from agents.config import CollaborationConfig -from agents.models import Observation, ReflectionAction, ReflectionOutcome, VerificationVerdict +from agents.models import ( + Observation, + ReflectionAction, + ReflectionOutcome, + VerificationVerdict, +) from core.models import Bounds, Scene, SceneElement, Task from runtime.executor import StepResult from runtime.planner import PlannedStep @@ -13,6 +18,7 @@ from runtime.task import TaskRunner, TaskRunnerConfig from storage.artifact_store import ArtifactStore from storage.task_metadata import TaskMetadataStore from storage.timeline import Timeline +from tests.fakes import PNG_10X20 from world.config import WorldConfig @@ -20,7 +26,11 @@ def _scene() -> Scene: return Scene( width=1080, height=1920, - elements=[SceneElement(id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="OK")], + elements=[ + SceneElement( + id="btn1", type="button", bounds=Bounds(10, 20, 100, 50), text="OK" + ) + ], ) @@ -55,7 +65,9 @@ def _replan_outcome() -> ReflectionOutcome: def _recovery_outcome() -> ReflectionOutcome: return ReflectionOutcome( replan=False, - action=ReflectionAction(action="swipe", description="Scroll", args={"direction": "up"}), + action=ReflectionAction( + action="swipe", description="Scroll", args={"direction": "up"} + ), reasoning="Try scrolling.", ) @@ -204,6 +216,7 @@ def test_collaborative_run_shares_bookkeeping_with_task_runner(tmp_path) -> None on_task_succeeded=on_task_succeeded, world_config=WorldConfig(enabled=True), config=TaskRunnerConfig(max_steps=5), + screenshot_provider=lambda device_id: PNG_10X20, ) runner = CollaborativeTaskRunner( @@ -225,6 +238,9 @@ def test_collaborative_run_shares_bookkeeping_with_task_runner(tmp_path) -> None assert result.status == "completed" assert len(timeline.read(task.id)) == 1 + record = timeline.read(task.id)[0] + assert record["before_screenshot_path"] + assert record["after_screenshot_path"] assert metadata.get_task(task.id)["status"] == "completed" on_task_succeeded.assert_called_once_with(task.id, task.goal, timeline) diff --git a/tests/test_console_api.py b/tests/test_console_api.py index 1ba7bff..af33d45 100644 --- a/tests/test_console_api.py +++ b/tests/test_console_api.py @@ -114,6 +114,28 @@ def test_console_timeline_inlines_screenshot_and_handles_empty_history( assert client.get("/console/tasks/missing/timeline").status_code == 404 +def test_console_timeline_inlines_before_and_after_screenshots(tmp_path) -> None: + timeline = Timeline(ArtifactStore(tmp_path / "history")) + client, metadata_store = _client(tmp_path, timeline=timeline) + metadata_store.create_task(Task(id="task-evidence", goal="tap", device_id="phone")) + before = b"before" + after = b"after" + timeline.append( + task_id="task-evidence", + scene={"screen": {"width": 10, "height": 20}, "elements": []}, + prompt="tap", + tool_call={"action": "tap", "description": "tap search"}, + result={"ok": True}, + before_screenshot=before, + after_screenshot=after, + ) + + record = client.get("/console/tasks/task-evidence/timeline").json()[0] + assert record["before_image_base64"] == base64.b64encode(before).decode("ascii") + assert record["after_image_base64"] == base64.b64encode(after).decode("ascii") + assert record["image_base64"] == base64.b64encode(after).decode("ascii") + + def test_console_device_registration_and_unregistration(tmp_path) -> None: config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3") manager = DeviceManager() diff --git a/tests/test_runtime_console_web.py b/tests/test_runtime_console_web.py index 1a18869..9cdaa53 100644 --- a/tests/test_runtime_console_web.py +++ b/tests/test_runtime_console_web.py @@ -238,16 +238,37 @@ def test_task_detail_renders_timeline_with_screenshot(tmp_path) -> None: task_id="task-with-timeline", scene={"screen": {"width": 10, "height": 20}, "elements": []}, prompt="tap search", - tool_call={"action": "tap", "args": {"x": 1, "y": 2}}, + tool_call={ + "action": "tap", + "description": "tap search", + "args": {"x": 1, "y": 2}, + }, result={"ok": True}, - screenshot=PNG_10X20, + before_screenshot=PNG_10X20 + b"before", + after_screenshot=PNG_10X20 + b"after", + ocr_results=[ + { + "text": "Search", + "confidence": 0.98, + "bounds": {"x": 1, "y": 2, "width": 3, "height": 4}, + } + ], ) body = client.get("/ui/tasks/task-with-timeline").text - expected_data_uri = "data:image/png;base64," + base64.b64encode(PNG_10X20).decode( - "ascii" - ) - assert expected_data_uri in body - assert "tap" in body + before_data_uri = "data:image/png;base64," + base64.b64encode( + PNG_10X20 + b"before" + ).decode("ascii") + after_data_uri = "data:image/png;base64," + base64.b64encode( + PNG_10X20 + b"after" + ).decode("ascii") + assert before_data_uri in body + assert after_data_uri in body + assert "Before action" in body + assert "After action" in body + assert "Operation" in body + assert "OCR results" in body + assert "Search" in body + assert "UI tree" not in body def test_task_detail_404_for_unknown_task(tmp_path) -> None: @@ -256,6 +277,39 @@ def test_task_detail_404_for_unknown_task(tmp_path) -> None: assert response.status_code == 404 +def test_task_detail_renders_normalized_ui_tree_result(tmp_path) -> None: + timeline = Timeline(ArtifactStore(tmp_path / "history")) + client, metadata_store = _client(tmp_path, timeline=timeline) + metadata_store.create_task( + Task(id="task-ui-tree", goal="inspect the screen", device_id="iphone-1") + ) + timeline.append( + task_id="task-ui-tree", + scene={"screen": {"width": 10, "height": 20}, "elements": []}, + prompt="inspect the screen", + tool_call={"action": "get_ui_tree", "description": "inspect UI tree"}, + result={ + "success": True, + "result": [ + { + "id": "ui-000", + "type": "button", + "text": "Search", + "bounds": {"x": 1, "y": 2, "width": 3, "height": 4}, + "confidence": 1.0, + } + ], + }, + ) + + body = client.get("/ui/tasks/task-ui-tree").text + + assert "UI tree" in body + assert "1 normalized nodes" in body + assert "button" in body + assert "Search" in body + + def test_config_page_lists_supported_drivers_and_current_max_steps(tmp_path) -> None: config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3") config_store.set_setting("max_steps", 5) diff --git a/tests/test_scene_builder.py b/tests/test_scene_builder.py index cdaee04..e7d4566 100644 --- a/tests/test_scene_builder.py +++ b/tests/test_scene_builder.py @@ -41,4 +41,5 @@ def test_scene_builder_merges_overlapping_ocr_into_ui_element() -> None: assert scene.elements[0].type == "button" assert scene.elements[0].text == "Search" assert scene.elements[1].text == "Footer" + assert [element.text for element in scene.ocr_elements] == ["Search", "Footer"] assert bbox_iou(ui_button.bounds, ocr_label.bounds) > 0.5 diff --git a/tests/test_task_loop.py b/tests/test_task_loop.py index f04f31a..a0da184 100644 --- a/tests/test_task_loop.py +++ b/tests/test_task_loop.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pathlib import Path + from core.models import Bounds, Scene, SceneElement, Task from runtime.executor import Executor, ExecutorConfig from runtime.planner import PlannedStep, Planner @@ -112,3 +114,52 @@ def test_task_runner_stops_before_the_next_planned_action() -> None: assert result.status == "failed" assert result.failure_reason == "execution interrupted" assert actions == ["tap"] + + +def test_task_runner_persists_action_evidence_and_raw_ocr(tmp_path) -> None: + scene = Scene( + width=10, + height=20, + elements=[], + ocr_elements=[ + SceneElement( + id="ocr-001", + type="text", + text="Search", + bounds=Bounds(1, 2, 3, 4), + confidence=0.98, + source="ocr", + ) + ], + ) + screenshots = iter( + [ + b"planning", + b"before-action", + b"after-action", + b"completion-check", + ] + ) + timeline = Timeline(ArtifactStore(tmp_path / "history")) + runner = TaskRunner( + planner=ScriptedPlanner( + [PlannedStep(action="tap", description="tap search", args={})] + ), + executor=Executor( + tools={"tap": lambda **kwargs: {"ok": True}}, + config=ExecutorConfig(max_retries=1, backoff_seconds=0), + ), + timeline=timeline, + config=TaskRunnerConfig(max_steps=2), + observer=lambda device_id: scene, + screenshot_provider=lambda device_id: next(screenshots), + ) + + result = runner.run(Task(goal="tap search", device_id="iphone-1")) + + assert result.status == "completed" + record = timeline.read(result.id)[0] + assert Path(record["before_screenshot_path"]).read_bytes() == b"before-action" + assert Path(record["after_screenshot_path"]).read_bytes() == b"after-action" + assert record["tool_call"]["description"] == "tap search" + assert record["ocr_results"][0]["text"] == "Search" diff --git a/tests/test_timeline.py b/tests/test_timeline.py index f2ac53a..5486e70 100644 --- a/tests/test_timeline.py +++ b/tests/test_timeline.py @@ -1,5 +1,7 @@ from __future__ import annotations +from pathlib import Path + from storage.artifact_store import ArtifactStore from storage.timeline import Timeline from tests.fakes import PNG_10X20 @@ -52,3 +54,32 @@ def test_timeline_records_per_step_prompt_not_task_goal(tmp_path) -> None: assert len(records) == 1 assert records[0]["prompt"] == per_step_prompt assert "Call exactly one tool" in records[0]["prompt"] + + +def test_timeline_records_before_and_after_screenshots_with_ocr(tmp_path) -> None: + timeline = Timeline(ArtifactStore(tmp_path / "history")) + before = b"before-image" + after = b"after-image" + + timeline.append( + task_id="task-evidence", + scene={"screen": {"width": 1, "height": 1}, "elements": []}, + prompt="goal", + tool_call={"action": "tap", "description": "tap search"}, + result={"ok": True}, + before_screenshot=before, + after_screenshot=after, + ocr_results=[ + { + "text": "Search", + "confidence": 0.98, + "bounds": {"x": 1, "y": 2, "width": 3, "height": 4}, + } + ], + ) + + record = timeline.read("task-evidence")[0] + assert Path(record["before_screenshot_path"]).read_bytes() == before + assert Path(record["after_screenshot_path"]).read_bytes() == after + assert record["screenshot_path"] == record["after_screenshot_path"] + assert record["ocr_results"][0]["text"] == "Search"