From d69be48f96fd5aa690126bf0c80ad91ca6677169 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 15 Jul 2026 18:14:28 +0800 Subject: [PATCH] feat(planner): persist reusable action semantics --- .../host_agent/cloud_planner_client.py | 4 + .../tests/test_cloud_planner_client.py | 18 +++- cloud-console/src/types.ts | 4 + cloud-console/src/views/TasksView.vue | 16 ++++ .../planner-reflection-history/design.md | 33 ++++++- .../planner-reflection-history/proposal.md | 11 ++- .../specs/agent-runtime/spec.md | 10 +- .../cloud-task-progress-visibility/spec.md | 8 +- .../specs/planner-reflection-history/spec.md | 19 +++- .../specs/world-model/spec.md | 10 +- .../planner-reflection-history/tasks.md | 8 ++ packages/cloud-platform/cloud/db_models.py | 2 + .../cloud-platform/cloud/internal_api/api.py | 6 ++ .../cloud/internal_api/models.py | 4 + .../0012_planner_decision_action_metadata.py | 27 ++++++ packages/cloud-platform/cloud/repository.py | 4 + packages/cloud-platform/cloud/schema.py | 2 +- packages/cloud-platform/cloud/sdk/api.py | 4 + packages/cloud-platform/cloud/sdk/models.py | 4 + .../cloud-platform/cloud/sql_repository.py | 30 +++--- runtime/ai_planner.py | 12 ++- runtime/executor.py | 4 +- runtime/planner.py | 4 + runtime/task.py | 2 + runtime/tool_calling_client.py | 44 ++++++++- runtime/tool_specs.py | 91 ++++++++++++------- skills_learning/models.py | 40 ++++++-- skills_learning/synthesis.py | 25 ++++- tests/test_ai_planner.py | 40 +++++++- tests/test_ai_planner_task_runner.py | 13 ++- tests/test_cloud_planner_decision_endpoint.py | 31 ++++++- tests/test_cloud_repository_contract.py | 19 +++- tests/test_cloud_sdk_api.py | 47 +++++++++- tests/test_executor.py | 28 ++++++ tests/test_skill_synthesis.py | 49 +++++++++- tests/test_tool_calling_client.py | 66 ++++++++++++++ tests/test_tool_specs.py | 56 ++++++++++-- tests/test_world_model.py | 23 ++++- tests/test_world_models.py | 9 ++ world/model.py | 12 ++- world/models.py | 10 +- 41 files changed, 733 insertions(+), 116 deletions(-) create mode 100644 packages/cloud-platform/cloud/migrations/versions/0012_planner_decision_action_metadata.py diff --git a/apps/device-host-agent/host_agent/cloud_planner_client.py b/apps/device-host-agent/host_agent/cloud_planner_client.py index aef2304..99da56e 100644 --- a/apps/device-host-agent/host_agent/cloud_planner_client.py +++ b/apps/device-host-agent/host_agent/cloud_planner_client.py @@ -124,6 +124,10 @@ class CloudProxyToolCallingClient: ) else None ), + text_output=decoded.rationale, + thinking=decoded.thinking, + purpose=decoded.purpose, + expected_outcome=decoded.expected_outcome, ) raise ToolCallUnavailable(_error_detail(response)) diff --git a/apps/device-host-agent/tests/test_cloud_planner_client.py b/apps/device-host-agent/tests/test_cloud_planner_client.py index 7157455..5846c8e 100644 --- a/apps/device-host-agent/tests/test_cloud_planner_client.py +++ b/apps/device-host-agent/tests/test_cloud_planner_client.py @@ -36,7 +36,14 @@ def test_decide_returns_tool_call_decision_on_success() -> None: seen_requests.append(request) return httpx.Response( 200, - json={"tool_name": "tap", "arguments": {"x": 1, "y": 2}}, + json={ + "tool_name": "tap", + "arguments": {"x": 1, "y": 2}, + "rationale": "The button is visible. Opening it.", + "thinking": "A tap should navigate to the next page.", + "purpose": "Open the next page.", + "expected_outcome": "The next page is visible.", + }, ) client = _client(handler) @@ -49,7 +56,14 @@ def test_decide_returns_tool_call_decision_on_success() -> None: timeout=30.0, ) - assert decision == ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}) + assert decision == ToolCallDecision( + tool_name="tap", + arguments={"x": 1, "y": 2}, + text_output="The button is visible. Opening it.", + thinking="A tap should navigate to the next page.", + purpose="Open the next page.", + expected_outcome="The next page is visible.", + ) assert len(seen_requests) == 1 request = seen_requests[0] assert request.url.path == "/internal/v1/hosts/host-a/planner/decide" diff --git a/cloud-console/src/types.ts b/cloud-console/src/types.ts index 7ee4c50..4910e2c 100644 --- a/cloud-console/src/types.ts +++ b/cloud-console/src/types.ts @@ -190,6 +190,10 @@ export interface PlannerDecisionItem { user_prompt: string; tool_name: string; arguments: Record; + rationale?: string | null; + thinking?: string | null; + purpose?: string | null; + expected_outcome?: string | null; created_at: string; } diff --git a/cloud-console/src/views/TasksView.vue b/cloud-console/src/views/TasksView.vue index e7732b2..77d6aa1 100644 --- a/cloud-console/src/views/TasksView.vue +++ b/cloud-console/src/views/TasksView.vue @@ -504,6 +504,22 @@ function formatArguments(args: Record): string { Arguments 
{{ formatArguments(decision.arguments) }}
+
+ Action purpose + 
{{ decision.purpose }}
+
+
+ Expected outcome + 
{{ decision.expected_outcome }}
+
+
+ Rationale + 
{{ decision.rationale }}
+
+
+ Thinking + 
{{ decision.thinking }}
+
diff --git a/openspec/changes/planner-reflection-history/design.md b/openspec/changes/planner-reflection-history/design.md index 19bfdd2..1b13386 100644 --- a/openspec/changes/planner-reflection-history/design.md +++ b/openspec/changes/planner-reflection-history/design.md @@ -36,9 +36,7 @@ Constraints: **Non-Goals:** - Not adding a dedicated reflection LLM call (explicit method B) — method C (pre-tool text block) achieves the intent within the existing call budget. -- Not changing `CloudProxyToolCallingClient` — it transparently proxies and is not aware of thinking/text blocks. - Not changing the `scene_summary` field in `WorldEvent` for stored/displayed task history in the Console — only the prompt-construction path (`_history_summary()`) is changed. -- Not surfacing thinking/rationale in the Cloud Console UI (that's a follow-on concern). - Not making `thinking_budget_tokens` configurable per-task at runtime (only via environment variable). ## Decisions @@ -112,13 +110,42 @@ Constraints: **Why separate columns rather than a JSON blob**: The existing table uses discrete columns for `system_prompt`, `user_prompt`, `tool_name`, `tool_arguments` — consistency favours discrete columns. Both fields are optional (cloud-proxy path only; `direct` transport never produces cloud decision records). +### D9: Device-action tool calls carry required purpose and expected outcome + +**Decision**: Every device-action schema (`tap`, `swipe`, `input_text`, +`launch_app`, and `terminate_app`) adds required non-empty `purpose` and +`expected_outcome` string fields. The tool-call response parser removes these +metadata fields from executable arguments and assigns them to +`ToolCallDecision`; `AIPlanner` carries them on `PlannedStep`. The completion +signal remains unchanged because it already has a required terminal `reason`. + +The Cloud planner response returns rationale, thinking, purpose, and expected +outcome to the Host. The Cloud decision log persists the four values as +nullable, additive columns so historic rows and non-AI callers remain +readable. `WorldEvent` retains the executable action arguments together with +purpose and expected outcome; its compact history summary includes that action +record. Timeline records and learned `FlowStep` instances retain the same +arguments and metadata; flow embedding text includes available semantics so +retrieval can use them. + +**Why structured tool arguments rather than rationale**: the pre-tool text +block is optional by API design and can be suppressed by a forced retry. Tool +schemas are the provider-enforced structured-output boundary, so requiring +purpose and expected outcome there makes them available for every submitted +device action without relying on free-form rationale. + +**Why strip metadata before execution**: device tool functions only accept +their physical-action arguments. Keeping metadata off `step.args` preserves +their API contracts while still making it available to execution history, +verification, Cloud audit, and skill synthesis. + ## Risks / Trade-offs - **AI may not always output a text block**: even with `tool_choice: "auto"` (D8), the model is not guaranteed to prefix a text block before the tool call. When absent, `text_output` is `None` and `rationale` is `None`. History degrades gracefully to `{page, rationale: null, action, success}`. - **`tool_choice: "auto"` occasionally yields no tool call at all**: unlike forced `tool_choice`, `"auto"` permits the model to respond with text only and no tool call. D8's forced retry (no thinking, no rationale on that path) guards this case so a step never stalls; this trades away rationale/thinking for that single step, not overall reliability. - **Extended thinking increases latency**: `budget_tokens` directly adds to minimum response time. This is opt-in and accepted by the operator who enables it. - **`WorldEvent` schema divergence from stored data**: Existing `WorldEvent` instances in memory or serialised timelines lack `rationale`/`thinking`. The `to_dict()` method will emit `null` for these fields; downstream consumers should treat `null` as absent, not as a failure. -- **Cloud-proxy transport never produces thinking/rationale at the client layer**: The proxy returns only `tool_name`/`arguments`. `ToolCallDecision.thinking` and `.text_output` will always be `None` for cloud-transport tasks. The `planner_decision_log` on the cloud side will be populated from the cloud-proxied call itself (D7), which does see the full LLM response. +- **Cloud/Host deployment order**: a new Host requires a Cloud API that returns the additive metadata fields to preserve them locally. The Cloud response models remain nullable so an old peer remains readable during a rolling deployment, but it cannot provide the new semantic records. ## Migration Plan diff --git a/openspec/changes/planner-reflection-history/proposal.md b/openspec/changes/planner-reflection-history/proposal.md index 173e282..a43c9ff 100644 --- a/openspec/changes/planner-reflection-history/proposal.md +++ b/openspec/changes/planner-reflection-history/proposal.md @@ -8,7 +8,8 @@ The AI planner's execution history currently stores raw `scene_summary` (full UI - **Rationale capture**: `ToolCallDecision` captures the AI's pre-tool text output (`text_output`) and, when extended thinking is enabled, the thinking block (`thinking`). Both flow through `PlannedStep` into `WorldEvent`. - **Extended thinking support**: `AnthropicToolCallingClient` gains optional `thinking_budget_tokens` config. When set, the Anthropic API is called with `thinking` enabled (interleaved thinking beta); the thinking block is extracted and stored. - **OpenAI reasoning capture**: `OpenAIToolCallingClient` extracts `reasoning_content` from responses when present (o-series models). -- **WorldEvent schema change**: `scene_summary` is replaced by `rationale` (from `text_output`) and `thinking` (from thinking block), plus `current_page` from `WorldState` for minimal page-level verification context. This reduces per-step history token cost by an order of magnitude. +- **Required action metadata**: every device-action tool call must return a concise `purpose` and observable `expected_outcome`; the Runtime stores them separately from executable tool arguments so successful executions can be reused as semantically meaningful flows. +- **WorldEvent schema change**: `scene_summary` is replaced by rationale/thinking plus the action name, executable arguments, purpose, expected outcome, and `current_page` from `WorldState`, giving later planning and skill reuse a compact but complete action record. - **History format**: `_history_summary()` in `ai_planner.py` switches from full `WorldEvent.to_dict()` to a compact `{page, rationale, action, success}` format. - **planner_decision_log extension**: The Cloud-side decision log table adds `thinking` and `rationale` columns to persist these fields alongside existing prompt/tool records. @@ -28,6 +29,7 @@ The AI planner's execution history currently stores raw `scene_summary` (full UI - `runtime/tool_calling_client.py` — `ToolCallDecision`, `AnthropicToolCallingClient`, `OpenAIToolCallingClient`, response parsers - `runtime/planner.py` — `PlannedStep` +- `runtime/tool_specs.py` — required purpose/expected-outcome fields for device actions - `runtime/ai_planner.py` — `AIPlanner.plan()`, `_history_summary()` - `runtime/planner_prompts.py` — `PLANNER_SYSTEM_PROMPT`, `planner_user_prompt` - `runtime/planner_config.py` — new `thinking_budget_tokens` field @@ -35,7 +37,8 @@ The AI planner's execution history currently stores raw `scene_summary` (full UI - `world/model.py` — `_append_history()` - `packages/cloud-platform/cloud/db_models.py` — `planner_decision_log` table - `packages/cloud-platform/cloud/schema.py` — Alembic migration -- `packages/cloud-platform/cloud/internal_api/api.py` — `record_planner_decision()` -- `packages/cloud-platform/cloud/internal_api/models.py` — `PlannerDecisionRecord` -- No changes to `CloudProxyToolCallingClient` — thinking/text are surfaced at the local client layer only; the cloud proxy is transparent to them. +- `packages/cloud-platform/cloud/internal_api/api.py` — `record_planner_decision()` and planner decision response +- `packages/cloud-platform/cloud/internal_api/models.py` — planner decision transport models +- `apps/device-host-agent/host_agent/cloud_planner_client.py` — return reflection and action metadata from the Cloud proxy +- `skills_learning/` — retain action purpose/expected outcome in synthesized flow steps - No new external dependencies; Anthropic extended thinking uses existing SDK via beta header. diff --git a/openspec/changes/planner-reflection-history/specs/agent-runtime/spec.md b/openspec/changes/planner-reflection-history/specs/agent-runtime/spec.md index 39edfbc..499abf1 100644 --- a/openspec/changes/planner-reflection-history/specs/agent-runtime/spec.md +++ b/openspec/changes/planner-reflection-history/specs/agent-runtime/spec.md @@ -23,6 +23,10 @@ The system SHALL provide a Planner implementation that, given a goal, the curren - **WHEN** the LLM response contains only a tool call with no preceding text block - **THEN** `PlannedStep.rationale` is `None` and the step is returned normally +#### Scenario: Device action includes reusable purpose and expected outcome +- **WHEN** the AI Planner selects a device action (`tap`, `swipe`, `input_text`, `launch_app`, or `terminate_app`) +- **THEN** its tool call requires non-empty `purpose` and `expected_outcome` values, and the returned `PlannedStep` carries both separately from the executable action arguments + ### Requirement: Pluggable dual-provider tool-calling abstraction The system SHALL support at least two interchangeable LLM providers (Anthropic native tool use and OpenAI function calling) for the AI Planner's decision calls, selectable via configuration, with both providers constrained to return exactly one tool call per request. The Anthropic client SHALL additionally support optional extended thinking via a configurable `thinking_budget_tokens` value. The OpenAI client SHALL capture `reasoning_content` from responses when present. Independently of provider selection, the system SHALL support at least two transports for making that decision call — direct-to-provider and cloud-proxy — selectable via configuration without requiring any change to `AIPlanner`'s own decision logic. @@ -50,6 +54,6 @@ The system SHALL support at least two interchangeable LLM providers (Anthropic n - **WHEN** the Host Agent is configured with the cloud-proxy transport - **THEN** its tool-calling client sends the decision request to the Cloud Control Plane's planner-decision endpoint instead of constructing a local Anthropic or OpenAI SDK client -#### Scenario: Cloud-proxy transport returns None for thinking and text_output -- **WHEN** the Host Agent uses cloud-proxy transport -- **THEN** `ToolCallDecision.thinking` and `ToolCallDecision.text_output` are `None` because the proxy surface does not expose them +#### Scenario: Cloud-proxy transport returns planner metadata +- **WHEN** the Host Agent uses cloud-proxy transport and the Cloud Planner returns a decision +- **THEN** the proxy returns its rationale, thinking, purpose, and expected outcome alongside the tool name and executable arguments diff --git a/openspec/changes/planner-reflection-history/specs/cloud-task-progress-visibility/spec.md b/openspec/changes/planner-reflection-history/specs/cloud-task-progress-visibility/spec.md index c8b170e..fb987d2 100644 --- a/openspec/changes/planner-reflection-history/specs/cloud-task-progress-visibility/spec.md +++ b/openspec/changes/planner-reflection-history/specs/cloud-task-progress-visibility/spec.md @@ -1,7 +1,7 @@ ## MODIFIED Requirements ### Requirement: Cloud Console displays a task's full LLM interaction history -Cloud Console SHALL provide a view, for a given task, listing each persisted planner decision in step order, including its full prompt, resulting decision, and — when available — the AI's rationale (pre-tool text reflection) and thinking (extended thinking block), sourced from the Cloud Control Plane's persisted planner-decision log. +Cloud Console SHALL provide a view, for a given task, listing each persisted planner decision in step order, including its full prompt, resulting decision, and — when available — the AI's rationale (pre-tool text reflection), thinking (extended thinking block), action purpose, and expected outcome, sourced from the Cloud Control Plane's persisted planner-decision log. #### Scenario: Task has persisted planner decisions with rationale - **WHEN** an operator opens the LLM interaction history view for a task that has one or more persisted planner decisions with non-null rationale @@ -20,7 +20,7 @@ Cloud Console SHALL provide a view, for a given task, listing each persisted pla - **THEN** Cloud Console indicates that no LLM interaction history is available because the Host does not report it, rather than showing an empty history with no explanation ### Requirement: Cloud Control Plane persists rationale and thinking in the planner decision log -The Cloud Control Plane's planner-decision log SHALL store the AI's rationale and thinking fields alongside the existing prompt and tool-call fields for each persisted decision. Both fields SHALL be nullable; absence of either field SHALL NOT prevent a decision record from being stored or queried. +The Cloud Control Plane's planner-decision log SHALL store the AI's rationale, thinking, action purpose, and expected outcome fields alongside the existing prompt and tool-call fields for each persisted decision. These fields SHALL be nullable for backward compatibility; absence of a legacy or non-AI value SHALL NOT prevent a decision record from being stored or queried. #### Scenario: Decision record includes rationale - **WHEN** the Host Agent reports a planner decision with a non-null rationale @@ -34,6 +34,10 @@ The Cloud Control Plane's planner-decision log SHALL store the AI's rationale an - **WHEN** the Host Agent reports a planner decision with null rationale and null thinking (e.g., cloud-proxy transport where these are not surfaced) - **THEN** the persisted row stores NULL for both columns without error +#### Scenario: Decision record includes reusable action metadata +- **WHEN** the Host reports a device-action planner decision with a purpose and expected outcome +- **THEN** the persisted row stores both values separately from the executable tool arguments and the task API returns them to authorized readers + #### Scenario: Existing decision records without rationale or thinking remain readable - **WHEN** the system queries a `planner_decision_log` row created before this migration - **THEN** both `rationale` and `thinking` read as NULL, and the row is returned normally diff --git a/openspec/changes/planner-reflection-history/specs/planner-reflection-history/spec.md b/openspec/changes/planner-reflection-history/specs/planner-reflection-history/spec.md index 33caf95..42b6814 100644 --- a/openspec/changes/planner-reflection-history/specs/planner-reflection-history/spec.md +++ b/openspec/changes/planner-reflection-history/specs/planner-reflection-history/spec.md @@ -34,6 +34,17 @@ The system SHALL extract and preserve the AI model's thinking block (when extend - **WHEN** the LLM response contains only a tool call block (no thinking, no text) - **THEN** `ToolCallDecision.thinking` and `ToolCallDecision.text_output` are both `None`, and the decision is returned normally +### Requirement: Device actions return required reusable metadata +The system SHALL require each device-action tool call to include concise, non-empty `purpose` and `expected_outcome` strings in its structured arguments. The Runtime SHALL preserve these values as planner metadata while excluding them from the arguments supplied to the physical device tool. + +#### Scenario: Tool schema requires purpose and expected outcome +- **WHEN** the Planner sends an action tool schema to an LLM provider +- **THEN** each device-action schema requires `purpose` and `expected_outcome` in addition to its physical-action arguments + +#### Scenario: Action metadata is not passed to the device tool +- **WHEN** a planned action is executed +- **THEN** the device tool receives only its physical-action arguments while the purpose and expected outcome remain available on the planned step and execution record + ### Requirement: Extended thinking is opt-in via configuration The system SHALL support enabling Anthropic extended thinking for the AI planner via a `thinking_budget_tokens` configuration value. When not configured, the planner SHALL operate identically to its pre-existing behavior. @@ -50,12 +61,16 @@ The system SHALL support enabling Anthropic extended thinking for the AI planner - **THEN** the OpenAI client does not apply the Anthropic thinking parameter; reasoning content is captured only if the model returns it naturally ### Requirement: Execution history uses compact rationale-based representation -The system SHALL construct the AI planner's history prompt from a compact per-step record containing the page context, rationale, action, and success flag — not the full scene JSON. This compact history SHALL be the sole format used when constructing the `history_summary` passed to `planner_user_prompt`. +The system SHALL construct the AI planner's history prompt from a compact per-step record containing the page context, rationale, action, executable action arguments, purpose, expected outcome, and success flag — not the full scene JSON. This compact history SHALL be the sole format used when constructing the `history_summary` passed to `planner_user_prompt`. #### Scenario: History prompt uses compact format - **WHEN** `_history_summary()` is called with a `WorldState` that has one or more history entries -- **THEN** each entry in the returned list contains `page`, `rationale`, `action`, and `success` fields only, without any scene element data +- **THEN** each entry in the returned list contains `page`, `rationale`, `action`, `arguments`, `purpose`, `expected_outcome`, and `success` fields only, without any scene element data #### Scenario: History prompt handles None rationale - **WHEN** a `WorldEvent` in history has `rationale=None` - **THEN** the compact history entry for that step includes `"rationale": null` without omitting the field or raising an error + +#### Scenario: History prompt preserves executed action arguments +- **WHEN** a prior action tapped a coordinate or otherwise supplied tool arguments +- **THEN** the compact history entry includes those exact executable arguments alongside the action name diff --git a/openspec/changes/planner-reflection-history/specs/world-model/spec.md b/openspec/changes/planner-reflection-history/specs/world-model/spec.md index 08e140b..702386a 100644 --- a/openspec/changes/planner-reflection-history/specs/world-model/spec.md +++ b/openspec/changes/planner-reflection-history/specs/world-model/spec.md @@ -1,7 +1,7 @@ ## MODIFIED Requirements ### Requirement: Bounded history of recent scene/action pairs -The system SHALL maintain `WorldState.history` as a fixed-size, bounded collection of the most recent per-step records, automatically evicting the oldest entry when a new entry is added past the configured bound. Each history record SHALL store the action name, success flag, page context (from `WorldState.current_page` at the time of recording), and optional rationale and thinking fields sourced from the executed `PlannedStep`. The `scene_summary` field SHALL be retained as an optional field for backward compatibility but SHALL NOT be required for new entries. +The system SHALL maintain `WorldState.history` as a fixed-size, bounded collection of the most recent per-step records, automatically evicting the oldest entry when a new entry is added past the configured bound. Each history record SHALL store the action name, executable action arguments, success flag, page context (from `WorldState.current_page` at the time of recording), and optional rationale, thinking, purpose, and expected-outcome fields sourced from the executed `PlannedStep`. The `scene_summary` field SHALL be retained as an optional field for backward compatibility but SHALL NOT be required for new entries. #### Scenario: WorldState survives across steps within a task - **WHEN** a task executes multiple steps in sequence @@ -23,6 +23,14 @@ The system SHALL maintain `WorldState.history` as a fixed-size, bounded collecti - **WHEN** the executed `PlannedStep` carries a non-None `thinking` - **THEN** the resulting `WorldEvent` stores that thinking string +#### Scenario: History record includes reusable action metadata +- **WHEN** an executed `PlannedStep` carries a purpose and expected outcome +- **THEN** the resulting `WorldEvent` stores both values for the next planning turn and later reuse + +#### Scenario: History record includes executed action arguments +- **WHEN** a `PlannedStep` executes with action arguments such as a tap's `x` and `y` coordinates +- **THEN** the resulting `WorldEvent` stores those executable arguments with the action name + #### Scenario: History record captures current page at time of recording - **WHEN** a step is appended to history and `WorldState.current_page` is non-None at that moment - **THEN** `WorldEvent.page` is set to that page value diff --git a/openspec/changes/planner-reflection-history/tasks.md b/openspec/changes/planner-reflection-history/tasks.md index 8f1cb85..f972beb 100644 --- a/openspec/changes/planner-reflection-history/tasks.md +++ b/openspec/changes/planner-reflection-history/tasks.md @@ -60,3 +60,11 @@ - [x] 9.6 Add/rename unit tests in `tests/test_tool_calling_client.py` covering: default request uses `tool_choice: "auto"`; retry sequence when first response has no tool call (Anthropic and OpenAI); thinking/`betas` dropped on the forced retry; OpenAI `text_output` capture - [x] 9.7 Update `design.md` (D1 correction + new D8) and this file to document the bug and fix - [x] 9.8 Re-run `uv run --all-packages pytest -m "not integration"`, `ruff check`/`ruff format --check`, `python -m compileall`, and `openspec validate --strict --change "planner-reflection-history"` after the fix + +## 10. Required reusable action metadata (post-completion amendment) + +- [x] 10.1 Require `purpose` and `expected_outcome` in every device-action tool schema; extract them from executable arguments into `ToolCallDecision` and `PlannedStep`. +- [x] 10.2 Return rationale, thinking, purpose, and expected outcome through the Cloud planner response; persist the new action metadata in `planner_decision_log` with an additive migration and expose it through the task decision API. +- [x] 10.3 Preserve executable arguments, purpose, and expected outcome in `WorldEvent`, Timeline records, and synthesized `FlowStep` values; include available metadata in skill embedding text for semantic retrieval. +- [x] 10.4 Render available rationale, thinking, purpose, and expected outcome in Cloud Console planner-decision history. +- [x] 10.5 Add focused regression tests and run the relevant Python, frontend, migration, lint/format, and OpenSpec validation checks. diff --git a/packages/cloud-platform/cloud/db_models.py b/packages/cloud-platform/cloud/db_models.py index d23783a..7fa9974 100644 --- a/packages/cloud-platform/cloud/db_models.py +++ b/packages/cloud-platform/cloud/db_models.py @@ -157,6 +157,8 @@ class PlannerDecisionLogRow(Base): created_at: Mapped[str] = mapped_column(String, nullable=False) rationale: Mapped[str | None] = mapped_column(Text, nullable=True) thinking: Mapped[str | None] = mapped_column(Text, nullable=True) + purpose: Mapped[str | None] = mapped_column(Text, nullable=True) + expected_outcome: Mapped[str | None] = mapped_column(Text, nullable=True) class PluginRow(Base): diff --git a/packages/cloud-platform/cloud/internal_api/api.py b/packages/cloud-platform/cloud/internal_api/api.py index 190d4e3..a2a7a06 100644 --- a/packages/cloud-platform/cloud/internal_api/api.py +++ b/packages/cloud-platform/cloud/internal_api/api.py @@ -506,6 +506,8 @@ def create_internal_router( now=utc_now(), rationale=getattr(decision, "text_output", None), thinking=getattr(decision, "thinking", None), + purpose=getattr(decision, "purpose", None), + expected_outcome=getattr(decision, "expected_outcome", None), ) logger.info( "planner-decision request resolved", @@ -518,6 +520,10 @@ def create_internal_router( return PlannerDecisionResponse( tool_name=decision.tool_name, arguments=dict(decision.arguments), + rationale=getattr(decision, "text_output", None), + thinking=getattr(decision, "thinking", None), + purpose=getattr(decision, "purpose", None), + expected_outcome=getattr(decision, "expected_outcome", None), input_tokens=(decision.usage.input_tokens if decision.usage else None), output_tokens=(decision.usage.output_tokens if decision.usage else None), total_tokens=(decision.usage.total_tokens if decision.usage else None), diff --git a/packages/cloud-platform/cloud/internal_api/models.py b/packages/cloud-platform/cloud/internal_api/models.py index ee08683..49ce9cc 100644 --- a/packages/cloud-platform/cloud/internal_api/models.py +++ b/packages/cloud-platform/cloud/internal_api/models.py @@ -147,6 +147,10 @@ class PlannerDecisionRequest(BaseModel): class PlannerDecisionResponse(BaseModel): tool_name: str arguments: dict[str, Any] = Field(default_factory=dict) + rationale: str | None = None + thinking: str | None = None + purpose: str | None = None + expected_outcome: str | None = None input_tokens: int | None = Field(default=None, ge=0) output_tokens: int | None = Field(default=None, ge=0) total_tokens: int | None = Field(default=None, ge=0) diff --git a/packages/cloud-platform/cloud/migrations/versions/0012_planner_decision_action_metadata.py b/packages/cloud-platform/cloud/migrations/versions/0012_planner_decision_action_metadata.py new file mode 100644 index 0000000..0953604 --- /dev/null +++ b/packages/cloud-platform/cloud/migrations/versions/0012_planner_decision_action_metadata.py @@ -0,0 +1,27 @@ +"""Add reusable action metadata to planner_decision_log.""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "0012_planner_decision_action_metadata" +down_revision = "0011_planner_decision_log_reflection" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.add_column( + "planner_decision_log", + sa.Column("purpose", sa.Text(), nullable=True), + ) + op.add_column( + "planner_decision_log", + sa.Column("expected_outcome", sa.Text(), nullable=True), + ) + + +def downgrade() -> None: + op.drop_column("planner_decision_log", "expected_outcome") + op.drop_column("planner_decision_log", "purpose") diff --git a/packages/cloud-platform/cloud/repository.py b/packages/cloud-platform/cloud/repository.py index 2929cce..9547a19 100644 --- a/packages/cloud-platform/cloud/repository.py +++ b/packages/cloud-platform/cloud/repository.py @@ -136,6 +136,8 @@ class PlannerDecisionRecord: created_at: datetime rationale: str | None = None thinking: str | None = None + purpose: str | None = None + expected_outcome: str | None = None class CloudRepository(Protocol): @@ -522,6 +524,8 @@ class CloudRepository(Protocol): now: datetime, rationale: str | None = None, thinking: str | None = None, + purpose: str | None = None, + expected_outcome: str | None = None, ) -> int: """Insert one planner-decision log row, returning the assigned step_index. diff --git a/packages/cloud-platform/cloud/schema.py b/packages/cloud-platform/cloud/schema.py index 1764eae..aaef3b4 100644 --- a/packages/cloud-platform/cloud/schema.py +++ b/packages/cloud-platform/cloud/schema.py @@ -9,7 +9,7 @@ from alembic.runtime.migration import MigrationContext from cloud.database import create_database_engine, normalize_database_url -HEAD_REVISION = "0011_planner_decision_log_reflection" +HEAD_REVISION = "0012_planner_decision_action_metadata" class SchemaVersionError(RuntimeError): diff --git a/packages/cloud-platform/cloud/sdk/api.py b/packages/cloud-platform/cloud/sdk/api.py index 9033635..7e914e7 100644 --- a/packages/cloud-platform/cloud/sdk/api.py +++ b/packages/cloud-platform/cloud/sdk/api.py @@ -263,6 +263,10 @@ def create_cloud_router( user_prompt=rec.user_prompt, tool_name=rec.tool_name, arguments=parsed_args, + rationale=rec.rationale, + thinking=rec.thinking, + purpose=rec.purpose, + expected_outcome=rec.expected_outcome, created_at=rec.created_at, ) ) diff --git a/packages/cloud-platform/cloud/sdk/models.py b/packages/cloud-platform/cloud/sdk/models.py index 4ae71b2..a215ab2 100644 --- a/packages/cloud-platform/cloud/sdk/models.py +++ b/packages/cloud-platform/cloud/sdk/models.py @@ -280,6 +280,10 @@ class TaskPlannerDecisionItem(BaseModel): user_prompt: str tool_name: str arguments: dict[str, Any] = Field(default_factory=dict) + rationale: str | None = None + thinking: str | None = None + purpose: str | None = None + expected_outcome: str | None = None created_at: datetime diff --git a/packages/cloud-platform/cloud/sql_repository.py b/packages/cloud-platform/cloud/sql_repository.py index 238099d..0493460 100644 --- a/packages/cloud-platform/cloud/sql_repository.py +++ b/packages/cloud-platform/cloud/sql_repository.py @@ -1696,6 +1696,8 @@ class SQLAlchemyCloudRepository: now: datetime, rationale: str | None = None, thinking: str | None = None, + purpose: str | None = None, + expected_outcome: str | None = None, ) -> int: with self._sessions.begin() as session: current_max = session.scalars( @@ -1718,6 +1720,8 @@ class SQLAlchemyCloudRepository: created_at=_iso(now), rationale=rationale, thinking=thinking, + purpose=purpose, + expected_outcome=expected_outcome, ) ) session.flush() @@ -1772,6 +1776,8 @@ class SQLAlchemyCloudRepository: created_at=_parse_dt(row.created_at), # type: ignore[arg-type] rationale=row.rationale, thinking=row.thinking, + purpose=row.purpose, + expected_outcome=row.expected_outcome, ) for row in rows ] @@ -1894,9 +1900,7 @@ class SQLAlchemyCloudRepository: CloudSkillEntitlementRow.skill_id == skill_id ) ) - session.execute( - delete(CloudSkillRow).where(CloudSkillRow.id == skill_id) - ) + session.execute(delete(CloudSkillRow).where(CloudSkillRow.id == skill_id)) def list_entitlements_for_skill(self, skill_id: str) -> list[str]: with self._sessions() as session: @@ -1922,13 +1926,9 @@ class SQLAlchemyCloudRepository: skills.sort(key=lambda s: s.name_normalized) return skills - def grant_entitlement( - self, skill_id: str, host_id: str, *, now: datetime - ) -> None: + def grant_entitlement(self, skill_id: str, host_id: str, *, now: datetime) -> None: with self._sessions.begin() as session: - existing = session.get( - CloudSkillEntitlementRow, (skill_id, host_id) - ) + existing = session.get(CloudSkillEntitlementRow, (skill_id, host_id)) if existing is not None: return # idempotent session.add( @@ -1940,21 +1940,15 @@ class SQLAlchemyCloudRepository: ) self._bump_host(session, host_id, skill_id, "upsert", now) - def revoke_entitlement( - self, skill_id: str, host_id: str, *, now: datetime - ) -> None: + def revoke_entitlement(self, skill_id: str, host_id: str, *, now: datetime) -> None: with self._sessions.begin() as session: - existing = session.get( - CloudSkillEntitlementRow, (skill_id, host_id) - ) + existing = session.get(CloudSkillEntitlementRow, (skill_id, host_id)) if existing is None: return session.delete(existing) self._bump_host(session, host_id, skill_id, "remove", now) - def fetch_host_delta( - self, host_id: str, since_version: int | None - ) -> Any: + def fetch_host_delta(self, host_id: str, since_version: int | None) -> Any: from cloud.skills import HostSkillDelta with self._sessions.begin() as session: diff --git a/runtime/ai_planner.py b/runtime/ai_planner.py index 305a41e..8efc224 100644 --- a/runtime/ai_planner.py +++ b/runtime/ai_planner.py @@ -57,8 +57,15 @@ class AIPlanner(Planner): return [ PlannedStep( action=decision.tool_name, - description=f"AI planner: {decision.tool_name}({decision.arguments})", + description=( + f"AI planner: {decision.purpose}" + if decision.purpose + else f"AI planner: {decision.tool_name}({decision.arguments})" + ), args=dict(decision.arguments), + expected_text=decision.expected_outcome, + purpose=decision.purpose, + expected_outcome=decision.expected_outcome, prompt=decision.user_prompt or user_prompt, rationale=decision.text_output, thinking=decision.thinking, @@ -78,7 +85,10 @@ def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]: { "page": event.page, "action": event.action, + "arguments": dict(event.arguments), "rationale": event.rationale, + "purpose": event.purpose, + "expected_outcome": event.expected_outcome, "success": event.success, } for event in world.history diff --git a/runtime/executor.py b/runtime/executor.py index 94d31a2..c3666ca 100644 --- a/runtime/executor.py +++ b/runtime/executor.py @@ -1,7 +1,7 @@ from __future__ import annotations from collections.abc import Callable -from dataclasses import dataclass, field +from dataclasses import dataclass from time import sleep from typing import Any @@ -28,6 +28,8 @@ class StepResult: "description": self.step.description, "args": dict(self.step.args), "expected_text": self.step.expected_text, + "purpose": self.step.purpose, + "expected_outcome": self.step.expected_outcome, }, "success": self.success, "attempts": self.attempts, diff --git a/runtime/planner.py b/runtime/planner.py index 5b4f1d1..e4c3d29 100644 --- a/runtime/planner.py +++ b/runtime/planner.py @@ -16,6 +16,10 @@ class PlannedStep: description: str args: dict[str, Any] = field(default_factory=dict) expected_text: str | None = None + # Structured action semantics, populated by AI planner tool calls and + # retained independently from executable arguments for later reuse. + purpose: str | None = None + expected_outcome: str | None = None # The actual prompt sent to the LLM for this step (AI planners only). # ``None`` for non-LLM planners; TaskRunner falls back to the task goal. prompt: str | None = None diff --git a/runtime/task.py b/runtime/task.py index 5a3a867..55656c4 100644 --- a/runtime/task.py +++ b/runtime/task.py @@ -382,6 +382,8 @@ class TaskRunner: "action": step.action, "description": step.description, "args": step.args, + "purpose": step.purpose, + "expected_outcome": step.expected_outcome, }, result=result.to_dict() if hasattr(result, "to_dict") diff --git a/runtime/tool_calling_client.py b/runtime/tool_calling_client.py index 223423d..bdfbe46 100644 --- a/runtime/tool_calling_client.py +++ b/runtime/tool_calling_client.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import Any, Protocol from runtime.planner_config import PlannerConfig -from runtime.tool_specs import ToolSpec +from runtime.tool_specs import ACTION_TOOL_NAMES, ToolSpec class ToolCallUnavailable(Exception): @@ -36,6 +36,10 @@ class ToolCallDecision: # Extended thinking block (Anthropic) or reasoning_content (OpenAI o-series). # None when not enabled or not present in the response. thinking: str | None = None + # Required structured metadata for device actions. These fields are removed + # from ``arguments`` before the Runtime invokes the physical device tool. + purpose: str | None = None + expected_outcome: str | None = None class ToolCallingClient(Protocol): @@ -352,14 +356,19 @@ def _decision_from_anthropic_response( name = _value(block, "name") arguments = _value(block, "input") if isinstance(name, str) and isinstance(arguments, dict): + executable_arguments, purpose, expected_outcome = ( + _split_action_metadata(name, arguments) + ) return ToolCallDecision( tool_name=name, - arguments=arguments, + arguments=executable_arguments, usage=_anthropic_usage(response), system_prompt=system_prompt, user_prompt=user_prompt, text_output="\n".join(text_parts) if text_parts else None, thinking=thinking, + purpose=purpose, + expected_outcome=expected_outcome, ) raise ValueError("anthropic response did not include a tool_use block") @@ -418,6 +427,9 @@ def _decision_from_openai_response( if not isinstance(name, str): raise ValueError("openai tool call missing a function name") arguments = _decode_openai_arguments(_value(function, "arguments")) + executable_arguments, purpose, expected_outcome = _split_action_metadata( + name, arguments + ) # Extract reasoning_content from o-series models when present. raw_reasoning = _value(message, "reasoning_content") thinking: str | None = raw_reasoning if isinstance(raw_reasoning, str) else None @@ -427,12 +439,14 @@ def _decision_from_openai_response( text_output: str | None = raw_content if isinstance(raw_content, str) else None return ToolCallDecision( tool_name=name, - arguments=arguments, + arguments=executable_arguments, usage=_openai_usage(response), system_prompt=system_prompt, user_prompt=user_prompt, thinking=thinking, text_output=text_output, + purpose=purpose, + expected_outcome=expected_outcome, ) @@ -475,6 +489,30 @@ def _decode_openai_arguments(raw_arguments: Any) -> dict[str, Any]: raise ValueError("openai tool call arguments must decode to a JSON object") +def _split_action_metadata( + tool_name: str, + arguments: dict[str, Any], +) -> tuple[dict[str, Any], str | None, str | None]: + executable_arguments = dict(arguments) + if tool_name not in ACTION_TOOL_NAMES: + return executable_arguments, None, None + return ( + { + name: value + for name, value in executable_arguments.items() + if name not in {"purpose", "expected_outcome"} + }, + _metadata_text(executable_arguments.get("purpose")), + _metadata_text(executable_arguments.get("expected_outcome")), + ) + + +def _metadata_text(value: Any) -> str | None: + if not isinstance(value, str): + return None + return value.strip() or None + + def _value(source: Any, key: str) -> Any: if isinstance(source, dict): return source.get(key) diff --git a/runtime/tool_specs.py b/runtime/tool_specs.py index 5565ec7..d3217e9 100644 --- a/runtime/tool_specs.py +++ b/runtime/tool_specs.py @@ -11,18 +11,51 @@ class ToolSpec: parameters: dict[str, Any] +ACTION_METADATA_PROPERTIES: dict[str, dict[str, Any]] = { + "purpose": { + "type": "string", + "minLength": 1, + "maxLength": 240, + "description": "Concise intent for this action, used by later reuse.", + }, + "expected_outcome": { + "type": "string", + "minLength": 1, + "maxLength": 240, + "description": "Observable screen state expected after this action.", + }, +} + + +def _action_parameters( + *, + required: list[str], + properties: dict[str, dict[str, Any]], +) -> dict[str, Any]: + return { + "type": "object", + "additionalProperties": False, + "required": [*required, "purpose", "expected_outcome"], + "properties": {**properties, **ACTION_METADATA_PROPERTIES}, + } + + TAP_SPEC = ToolSpec( name="tap", description="Tap a point on the screen, given in Scene pixel coordinates.", - parameters={ - "type": "object", - "additionalProperties": False, - "required": ["x", "y"], - "properties": { - "x": {"type": "number", "description": "X coordinate in Scene pixel space."}, - "y": {"type": "number", "description": "Y coordinate in Scene pixel space."}, + parameters=_action_parameters( + required=["x", "y"], + properties={ + "x": { + "type": "number", + "description": "X coordinate in Scene pixel space.", + }, + "y": { + "type": "number", + "description": "Y coordinate in Scene pixel space.", + }, }, - }, + ), ) SWIPE_SPEC = ToolSpec( @@ -31,11 +64,9 @@ SWIPE_SPEC = ToolSpec( "Swipe from a start point to an end point on the screen, given in " "Scene pixel coordinates." ), - parameters={ - "type": "object", - "additionalProperties": False, - "required": ["start_x", "start_y", "end_x", "end_y"], - "properties": { + parameters=_action_parameters( + required=["start_x", "start_y", "end_x", "end_y"], + properties={ "start_x": {"type": "number", "description": "Start X coordinate."}, "start_y": {"type": "number", "description": "Start Y coordinate."}, "end_x": {"type": "number", "description": "End X coordinate."}, @@ -46,52 +77,46 @@ SWIPE_SPEC = ToolSpec( "default": 500, }, }, - }, + ), ) INPUT_TEXT_SPEC = ToolSpec( name="input_text", description="Type text into the currently focused input field.", - parameters={ - "type": "object", - "additionalProperties": False, - "required": ["text"], - "properties": { + parameters=_action_parameters( + required=["text"], + properties={ "text": {"type": "string", "description": "Text to type."}, }, - }, + ), ) LAUNCH_APP_SPEC = ToolSpec( name="launch_app", description="Launch (foreground) an app by its bundle/package identifier.", - parameters={ - "type": "object", - "additionalProperties": False, - "required": ["app_id"], - "properties": { + parameters=_action_parameters( + required=["app_id"], + properties={ "app_id": { "type": "string", "description": "App bundle/package identifier.", }, }, - }, + ), ) TERMINATE_APP_SPEC = ToolSpec( name="terminate_app", description="Terminate a running app by its bundle/package identifier.", - parameters={ - "type": "object", - "additionalProperties": False, - "required": ["app_id"], - "properties": { + parameters=_action_parameters( + required=["app_id"], + properties={ "app_id": { "type": "string", "description": "App bundle/package identifier.", }, }, - }, + ), ) FINISH_TASK_SPEC = ToolSpec( @@ -126,4 +151,6 @@ ACTION_TOOL_SPECS: list[ToolSpec] = [ TERMINATE_APP_SPEC, ] +ACTION_TOOL_NAMES = frozenset(spec.name for spec in ACTION_TOOL_SPECS) + ALL_TOOL_SPECS: list[ToolSpec] = [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC] diff --git a/skills_learning/models.py b/skills_learning/models.py index 903c79d..0c10776 100644 --- a/skills_learning/models.py +++ b/skills_learning/models.py @@ -61,18 +61,36 @@ class SkillMetadata: class FlowStep: tool_name: str args: dict[str, Any] = field(default_factory=dict) + purpose: str | None = None + expected_outcome: str | None = None def to_dict(self) -> dict[str, Any]: - return { + payload: dict[str, Any] = { "tool_name": self.tool_name, "args": dict(self.args), } + if self.purpose is not None: + payload["purpose"] = self.purpose + if self.expected_outcome is not None: + payload["expected_outcome"] = self.expected_outcome + return payload @classmethod def from_dict(cls, data: dict[str, Any]) -> "FlowStep": return cls( tool_name=str(data.get("tool_name") or data.get("action") or ""), args=dict(data.get("args") or {}), + purpose=( + data["purpose"] + if isinstance(data.get("purpose"), str) and data["purpose"].strip() + else None + ), + expected_outcome=( + data["expected_outcome"] + if isinstance(data.get("expected_outcome"), str) + and data["expected_outcome"].strip() + else None + ), ) @@ -119,8 +137,7 @@ class FlowTemplateSkill(Skill): **self.metadata.to_dict(), "steps": [step.to_dict() for step in self.steps], "parameters": { - name: dict(schema) - for name, schema in self.parameters.items() + name: dict(schema) for name, schema in self.parameters.items() }, } @@ -128,10 +145,7 @@ class FlowTemplateSkill(Skill): def from_dict(cls, data: dict[str, Any]) -> "FlowTemplateSkill": return cls( metadata=SkillMetadata.from_dict(data), - steps=[ - FlowStep.from_dict(step) - for step in data.get("steps", []) - ], + steps=[FlowStep.from_dict(step) for step in data.get("steps", [])], parameters={ str(name): dict(schema) for name, schema in (data.get("parameters") or {}).items() @@ -189,7 +203,17 @@ class KnowledgeSkill(Skill): def skill_embedding_text(skill: FlowTemplateSkill) -> str: goal = skill.originating_goal or "" - return f"{skill.name}: {skill.description}\nOriginal goal: {goal}" + step_context = "\n".join( + ( + f"{step.tool_name}: purpose={step.purpose}; " + f"expected_outcome={step.expected_outcome}" + ) + for step in skill.steps + if step.purpose is not None or step.expected_outcome is not None + ) + return f"{skill.name}: {skill.description}\nOriginal goal: {goal}" + ( + f"\nAction semantics:\n{step_context}" if step_context else "" + ) def clone_skill(skill: FlowTemplateSkill) -> FlowTemplateSkill: diff --git a/skills_learning/synthesis.py b/skills_learning/synthesis.py index 6520cb1..2b2a49f 100644 --- a/skills_learning/synthesis.py +++ b/skills_learning/synthesis.py @@ -40,7 +40,14 @@ def extract_tool_calls( tool_name = str(tool_call.get("action") or tool_call.get("tool_name") or "") if not tool_name or tool_name in READ_ONLY_TOOL_NAMES: continue - steps.append(FlowStep(tool_name=tool_name, args=dict(tool_call.get("args") or {}))) + steps.append( + FlowStep( + tool_name=tool_name, + args=dict(tool_call.get("args") or {}), + purpose=_optional_text(tool_call.get("purpose")), + expected_outcome=_optional_text(tool_call.get("expected_outcome")), + ) + ) return steps @@ -105,11 +112,15 @@ def promote_parameters( existing_parameters: dict[str, dict[str, Any]] | None = None, ) -> tuple[list[FlowStep], dict[str, dict[str, Any]]]: parameters = { - name: dict(schema) - for name, schema in (existing_parameters or {}).items() + name: dict(schema) for name, schema in (existing_parameters or {}).items() } parameterized_steps = [ - FlowStep(step.tool_name, dict(step.args)) + FlowStep( + step.tool_name, + dict(step.args), + purpose=step.purpose, + expected_outcome=step.expected_outcome, + ) for step in executed_steps ] used_names = set(parameters) @@ -162,6 +173,12 @@ def _record_value(record: Any, key: str) -> Any: return getattr(record, key, None) +def _optional_text(value: Any) -> str | None: + if not isinstance(value, str): + return None + return value.strip() or None + + def _tool_sequence(steps: list[FlowStep]) -> list[str]: return [step.tool_name for step in steps] diff --git a/tests/test_ai_planner.py b/tests/test_ai_planner.py index 2e6a85e..0850a60 100644 --- a/tests/test_ai_planner.py +++ b/tests/test_ai_planner.py @@ -57,7 +57,12 @@ def _context() -> TaskContext: def test_ai_planner_returns_single_planned_step_for_action_decision() -> None: client = FakeToolCallingClient( - ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}) + ToolCallDecision( + tool_name="tap", + arguments={"x": 1, "y": 2}, + purpose="Open the send control.", + expected_outcome="The message composer is focused.", + ) ) planner = AIPlanner(client=client) @@ -66,8 +71,11 @@ def test_ai_planner_returns_single_planned_step_for_action_decision() -> None: assert len(steps) == 1 step = steps[0] assert step.action == "tap" - assert step.description == "AI planner: tap({'x': 1, 'y': 2})" + assert step.description == "AI planner: Open the send control." assert step.args == {"x": 1, "y": 2} + assert step.purpose == "Open the send control." + assert step.expected_outcome == "The message composer is focused." + assert step.expected_text == "The message composer is focused." def test_ai_planner_finish_task_success_returns_empty_plan() -> None: @@ -204,6 +212,8 @@ def test_ai_planner_propagates_rationale_and_thinking_to_planned_step() -> None: arguments={"x": 1, "y": 2}, text_output="Previous step opened settings. Now tapping account.", thinking="I need to navigate deeper.", + purpose="Open account settings.", + expected_outcome="The account settings page is visible.", ) ) planner = AIPlanner(client=client) @@ -212,6 +222,8 @@ def test_ai_planner_propagates_rationale_and_thinking_to_planned_step() -> None: assert steps[0].rationale == "Previous step opened settings. Now tapping account." assert steps[0].thinking == "I need to navigate deeper." + assert steps[0].purpose == "Open account settings." + assert steps[0].expected_outcome == "The account settings page is visible." def test_history_summary_returns_compact_format() -> None: @@ -226,12 +238,16 @@ def test_history_summary_returns_compact_format() -> None: action="tap", success=True, rationale="Opened settings.", + arguments={"x": 1, "y": 2}, + purpose="Open settings.", + expected_outcome="Settings is visible.", page="Home", ), WorldEvent( action="swipe", success=False, rationale=None, + arguments={"start_y": 700, "end_y": 200}, page="Settings", ), ] @@ -241,8 +257,24 @@ def test_history_summary_returns_compact_format() -> None: summary = _history_summary(state) assert summary == [ - {"page": "Home", "action": "tap", "rationale": "Opened settings.", "success": True}, - {"page": "Settings", "action": "swipe", "rationale": None, "success": False}, + { + "page": "Home", + "action": "tap", + "arguments": {"x": 1, "y": 2}, + "rationale": "Opened settings.", + "purpose": "Open settings.", + "expected_outcome": "Settings is visible.", + "success": True, + }, + { + "page": "Settings", + "action": "swipe", + "arguments": {"start_y": 700, "end_y": 200}, + "rationale": None, + "purpose": None, + "expected_outcome": None, + "success": False, + }, ] # Must not contain scene element data for entry in summary: diff --git a/tests/test_ai_planner_task_runner.py b/tests/test_ai_planner_task_runner.py index 62861dd..8323acd 100644 --- a/tests/test_ai_planner_task_runner.py +++ b/tests/test_ai_planner_task_runner.py @@ -193,7 +193,12 @@ def test_multi_step_timeline_records_actual_per_step_prompts(tmp_path) -> None: """When AIPlanner is used, each timeline step's prompt is the real per-step user prompt (containing scene JSON), not the bare task goal.""" decisions = [ - ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}), + ToolCallDecision( + tool_name="tap", + arguments={"x": 1, "y": 2}, + purpose="Open the send control.", + expected_outcome="The message composer is focused.", + ), ToolCallDecision(tool_name="finish_task", arguments={"success": True}), ] client = ScriptedToolCallingClient(decisions) @@ -228,6 +233,12 @@ def test_multi_step_timeline_records_actual_per_step_prompts(tmp_path) -> None: assert "Current Scene (JSON)" in prompt assert "Call exactly one tool" in prompt assert "tap the button" in prompt + assert records[0]["tool_call"]["args"] == {"x": 1, "y": 2} + assert records[0]["tool_call"]["purpose"] == "Open the send control." + assert ( + records[0]["tool_call"]["expected_outcome"] + == "The message composer is focused." + ) def test_non_ai_planner_falls_back_to_task_goal_for_prompt(tmp_path) -> None: diff --git a/tests/test_cloud_planner_decision_endpoint.py b/tests/test_cloud_planner_decision_endpoint.py index 220391e..125d20b 100644 --- a/tests/test_cloud_planner_decision_endpoint.py +++ b/tests/test_cloud_planner_decision_endpoint.py @@ -96,7 +96,14 @@ def _decision_payload(**overrides: object) -> dict[str, object]: def test_authenticated_host_resolves_planner_decision(tmp_path) -> None: fake_client = _FakeToolCallingClient( - decision=ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}) + decision=ToolCallDecision( + tool_name="tap", + arguments={"x": 1, "y": 2}, + text_output="The login button is visible. Opening it.", + thinking="The next screen should be the account page.", + purpose="Open the login screen.", + expected_outcome="The login form is visible.", + ) ) client, fake_client, _pool = _build_client(tmp_path, fake_client=fake_client) @@ -107,7 +114,14 @@ def test_authenticated_host_resolves_planner_decision(tmp_path) -> None: ) assert response.status_code == 200 - assert response.json() == {"tool_name": "tap", "arguments": {"x": 1, "y": 2}} + assert response.json() == { + "tool_name": "tap", + "arguments": {"x": 1, "y": 2}, + "rationale": "The login button is visible. Opening it.", + "thinking": "The next screen should be the account page.", + "purpose": "Open the login screen.", + "expected_outcome": "The login form is visible.", + } assert len(fake_client.calls) == 1 assert fake_client.calls[0]["system_prompt"] == "you are a planner" assert fake_client.calls[0]["timeout"] == 30.0 @@ -232,7 +246,14 @@ def _seed_attempt(pool: DevicePool, task_id: str, host_id: str = "host-a") -> No def test_successful_decision_is_persisted_with_correct_fields(tmp_path) -> None: fake_client = _FakeToolCallingClient( - decision=ToolCallDecision(tool_name="tap", arguments={"x": 10, "y": 20}) + decision=ToolCallDecision( + tool_name="tap", + arguments={"x": 10, "y": 20}, + text_output="The settings tab is visible. Opening it.", + thinking="A tap should navigate to settings.", + purpose="Open settings.", + expected_outcome="The settings page is visible.", + ) ) client, fake_client, pool = _build_client(tmp_path, fake_client=fake_client) _seed_attempt(pool, "task-log-1") @@ -259,6 +280,10 @@ def test_successful_decision_is_persisted_with_correct_fields(tmp_path) -> None: assert row.tool_name == "tap" assert '"x": 10' in row.arguments_json assert '"y": 20' in row.arguments_json + assert row.rationale == "The settings tab is visible. Opening it." + assert row.thinking == "A tap should navigate to settings." + assert row.purpose == "Open settings." + assert row.expected_outcome == "The settings page is visible." def test_failed_decision_persists_nothing(tmp_path) -> None: diff --git a/tests/test_cloud_repository_contract.py b/tests/test_cloud_repository_contract.py index b595e56..b237b19 100644 --- a/tests/test_cloud_repository_contract.py +++ b/tests/test_cloud_repository_contract.py @@ -1773,12 +1773,21 @@ def test_record_planner_decision_stores_rationale_and_thinking( now=now, rationale="Previous step opened settings. Now tapping account.", thinking="I need to navigate to account settings.", + purpose="Open account settings.", + expected_outcome="The account settings page is visible.", ) - decisions = database.repository.list_planner_decisions(task_id=task_id, attempt=1) + decisions = database.repository.list_planner_decisions( + task_id=task_id, attempt=1 + ) assert len(decisions) == 1 - assert decisions[0].rationale == "Previous step opened settings. Now tapping account." + assert ( + decisions[0].rationale + == "Previous step opened settings. Now tapping account." + ) assert decisions[0].thinking == "I need to navigate to account settings." + assert decisions[0].purpose == "Open account settings." + assert decisions[0].expected_outcome == "The account settings page is visible." finally: database.close() @@ -1804,9 +1813,13 @@ def test_record_planner_decision_stores_null_rationale_and_thinking( # rationale and thinking omitted (default None) ) - decisions = database.repository.list_planner_decisions(task_id=task_id, attempt=1) + decisions = database.repository.list_planner_decisions( + task_id=task_id, attempt=1 + ) assert len(decisions) == 1 assert decisions[0].rationale is None assert decisions[0].thinking is None + assert decisions[0].purpose is None + assert decisions[0].expected_outcome is None finally: database.close() diff --git a/tests/test_cloud_sdk_api.py b/tests/test_cloud_sdk_api.py index 7ad0a2c..99566b9 100644 --- a/tests/test_cloud_sdk_api.py +++ b/tests/test_cloud_sdk_api.py @@ -94,6 +94,44 @@ def test_unknown_task_id_returns_404(tmp_path) -> None: assert resp.status_code == 404, resp.text +def test_planner_decision_history_returns_reusable_action_metadata(tmp_path) -> None: + app, pool, scheduler, _ = _build_app(tmp_path) + task_id = scheduler.submit(goal="open settings") + pool.store.record_planner_decision( + host_id="host-a", + task_id=task_id, + attempt=0, + system_prompt="system", + user_prompt="open settings", + tool_name="tap", + arguments_json='{"x": 12, "y": 34}', + now=datetime.now(UTC), + rationale="The settings tab is visible. Opening it.", + thinking="A tap should navigate to settings.", + purpose="Open settings.", + expected_outcome="The settings page is visible.", + ) + + response = _client_for(app).get(f"/v1/tasks/{task_id}/planner-decisions?attempt=0") + + assert response.status_code == 200, response.text + assert response.json()["items"] == [ + { + "step_index": 1, + "attempt": 0, + "system_prompt": "system", + "user_prompt": "open settings", + "tool_name": "tap", + "arguments": {"x": 12, "y": 34}, + "rationale": "The settings tab is visible. Opening it.", + "thinking": "A tap should navigate to settings.", + "purpose": "Open settings.", + "expected_outcome": "The settings page is visible.", + "created_at": response.json()["items"][0]["created_at"], + } + ] + + def test_task_status_exposes_distributed_metadata_without_lease_secret( tmp_path, ) -> None: @@ -299,7 +337,10 @@ def test_submit_with_explicit_target_is_listed_and_not_rerouted(tmp_path) -> Non assert task["assigned_host_id"] == "host-b" assert task["assigned_device_id"] == "device-b" listed = client.get("/v1/tasks").json()["items"] - assert next(item for item in listed if item["id"] == task_id)["target_host_id"] == "host-b" + assert ( + next(item for item in listed if item["id"] == task_id)["target_host_id"] + == "host-b" + ) def test_submit_rejects_incomplete_or_foreign_target(tmp_path) -> None: @@ -429,9 +470,7 @@ def test_list_tasks_returns_summary_with_pagination_and_status_filter( assert [item["id"] for item in queued_only["items"]] == [second_id] assert all(item["status"] == "queued" for item in queued_only["items"]) - assigned_only = client.get( - "/v1/tasks", params={"status": "assigned"} - ).json() + assigned_only = client.get("/v1/tasks", params={"status": "assigned"}).json() assert assigned_only["total"] == 1 assert [item["id"] for item in assigned_only["items"]] == [assigned_id] diff --git a/tests/test_executor.py b/tests/test_executor.py index a7d325a..72d443c 100644 --- a/tests/test_executor.py +++ b/tests/test_executor.py @@ -24,3 +24,31 @@ def test_executor_retries_until_transient_tool_succeeds() -> None: assert result.attempts == 3 assert result.result == {"ok": True} + +def test_executor_keeps_action_metadata_out_of_device_tool_arguments() -> None: + calls: list[tuple[int, int]] = [] + + def tap(*, x: int, y: int) -> dict[str, bool]: + calls.append((x, y)) + return {"ok": True} + + executor = Executor( + tools={"tap": tap}, + config=ExecutorConfig(max_retries=1, backoff_seconds=0), + ) + step = PlannedStep( + action="tap", + description="Open settings.", + args={"x": 12, "y": 34}, + purpose="Open settings.", + expected_outcome="The settings page is visible.", + ) + + result = executor.execute(step) + + assert result.success is True + assert calls == [(12, 34)] + assert result.to_dict()["step"]["purpose"] == "Open settings." + assert ( + result.to_dict()["step"]["expected_outcome"] == "The settings page is visible." + ) diff --git a/tests/test_skill_synthesis.py b/tests/test_skill_synthesis.py index 9143742..1fc2b85 100644 --- a/tests/test_skill_synthesis.py +++ b/tests/test_skill_synthesis.py @@ -1,6 +1,11 @@ from __future__ import annotations -from skills_learning.models import FlowStep, FlowTemplateSkill, SkillMetadata +from skills_learning.models import ( + FlowStep, + FlowTemplateSkill, + SkillMetadata, + skill_embedding_text, +) from skills_learning.store import SkillStore from skills_learning.synthesis import extract_tool_calls, synthesize_flow_skill @@ -10,12 +15,19 @@ def _record( args: dict[str, object] | None = None, *, result: dict[str, object] | None = None, + purpose: str | None = None, + expected_outcome: str | None = None, ) -> dict[str, object]: + tool_call: dict[str, object] = { + "action": action, + "args": args or {}, + } + if purpose is not None: + tool_call["purpose"] = purpose + if expected_outcome is not None: + tool_call["expected_outcome"] = expected_outcome return { - "tool_call": { - "action": action, - "args": args or {}, - }, + "tool_call": tool_call, "result": result or {}, } @@ -50,6 +62,33 @@ def test_first_time_synthesis_has_literal_steps_and_no_parameters() -> None: assert skill.parameters == {} +def test_synthesis_preserves_action_metadata_for_reuse_and_embedding() -> None: + skill = synthesize_flow_skill( + "open settings", + [ + _record( + "tap", + {"x": 12, "y": 34}, + purpose="Open the settings tab.", + expected_outcome="The settings page is visible.", + ) + ], + ) + + assert skill.steps == [ + FlowStep( + "tap", + {"x": 12, "y": 34}, + purpose="Open the settings tab.", + expected_outcome="The settings page is visible.", + ) + ] + assert FlowStep.from_dict(skill.steps[0].to_dict()) == skill.steps[0] + embedding_text = skill_embedding_text(skill) + assert "Open the settings tab." in embedding_text + assert "The settings page is visible." in embedding_text + + def test_second_execution_promotes_differing_argument_to_parameter() -> None: store = SkillStore() store.create_version( diff --git a/tests/test_tool_calling_client.py b/tests/test_tool_calling_client.py index 99cff71..d5a73c4 100644 --- a/tests/test_tool_calling_client.py +++ b/tests/test_tool_calling_client.py @@ -632,6 +632,36 @@ def test_anthropic_client_tool_only_response_has_none_thinking_and_text_output() assert decision.text_output is None +def test_anthropic_client_separates_required_action_metadata_from_arguments() -> None: + messages = FakeMessages( + response={ + "content": [ + { + "type": "tool_use", + "name": "tap", + "input": { + "x": 10, + "y": 20, + "purpose": "Open the account screen.", + "expected_outcome": "The account screen is visible.", + }, + } + ] + } + ) + client = AnthropicToolCallingClient( + model="test-model", transport=FakeTransport(messages) + ) + + decision = client.decide( + system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1 + ) + + assert decision.arguments == {"x": 10, "y": 20} + assert decision.purpose == "Open the account screen." + assert decision.expected_outcome == "The account screen is visible." + + def test_openai_client_captures_reasoning_content() -> None: completions = FakeCompletions( response={ @@ -691,6 +721,42 @@ def test_openai_client_captures_message_content_as_text_output() -> None: assert decision.text_output == "Previous step succeeded. Now tapping login." +def test_openai_client_separates_required_action_metadata_from_arguments() -> None: + completions = FakeCompletions( + response={ + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": "tap", + "arguments": ( + '{"x": 5, "y": 6, ' + '"purpose": "Open settings.", ' + '"expected_outcome": "Settings is visible."}' + ), + } + } + ] + } + } + ] + } + ) + client = OpenAIToolCallingClient( + model="test-model", transport=FakeOpenAITransport(completions) + ) + + decision = client.decide( + system_prompt="s", user_prompt="u", screenshot=None, tools=[TAP_SPEC], timeout=1 + ) + + assert decision.arguments == {"x": 5, "y": 6} + assert decision.purpose == "Open settings." + assert decision.expected_outcome == "Settings is visible." + + def test_openai_client_no_reasoning_content_gives_none_thinking() -> None: completions = FakeCompletions( response={ diff --git a/tests/test_tool_specs.py b/tests/test_tool_specs.py index 1d6b1cc..05d6cb2 100644 --- a/tests/test_tool_specs.py +++ b/tests/test_tool_specs.py @@ -13,7 +13,9 @@ from runtime.tool_specs import ( ) -def test_action_tool_specs_has_five_entries_and_all_tool_specs_adds_finish_task() -> None: +def test_action_tool_specs_has_five_entries_and_all_tool_specs_adds_finish_task() -> ( + None +): assert len(ACTION_TOOL_SPECS) == 5 assert len(ALL_TOOL_SPECS) == 6 assert ALL_TOOL_SPECS == [*ACTION_TOOL_SPECS, FINISH_TASK_SPEC] @@ -33,32 +35,70 @@ def test_every_tool_spec_schema_forbids_additional_properties() -> None: def test_tap_spec_requires_x_and_y() -> None: - assert TAP_SPEC.parameters["required"] == ["x", "y"] - assert set(TAP_SPEC.parameters["properties"]) == {"x", "y"} + assert TAP_SPEC.parameters["required"] == ["x", "y", "purpose", "expected_outcome"] + assert set(TAP_SPEC.parameters["properties"]) == { + "x", + "y", + "purpose", + "expected_outcome", + } def test_swipe_spec_requires_coordinates_and_makes_duration_optional() -> None: - assert SWIPE_SPEC.parameters["required"] == ["start_x", "start_y", "end_x", "end_y"] + assert SWIPE_SPEC.parameters["required"] == [ + "start_x", + "start_y", + "end_x", + "end_y", + "purpose", + "expected_outcome", + ] assert set(SWIPE_SPEC.parameters["properties"]) == { "start_x", "start_y", "end_x", "end_y", "duration_ms", + "purpose", + "expected_outcome", } assert "duration_ms" not in SWIPE_SPEC.parameters["required"] assert SWIPE_SPEC.parameters["properties"]["duration_ms"]["default"] == 500 def test_input_text_spec_requires_text() -> None: - assert INPUT_TEXT_SPEC.parameters["required"] == ["text"] - assert set(INPUT_TEXT_SPEC.parameters["properties"]) == {"text"} + assert INPUT_TEXT_SPEC.parameters["required"] == [ + "text", + "purpose", + "expected_outcome", + ] + assert set(INPUT_TEXT_SPEC.parameters["properties"]) == { + "text", + "purpose", + "expected_outcome", + } def test_launch_and_terminate_app_specs_require_app_id() -> None: for spec in (LAUNCH_APP_SPEC, TERMINATE_APP_SPEC): - assert spec.parameters["required"] == ["app_id"] - assert set(spec.parameters["properties"]) == {"app_id"} + assert spec.parameters["required"] == [ + "app_id", + "purpose", + "expected_outcome", + ] + assert set(spec.parameters["properties"]) == { + "app_id", + "purpose", + "expected_outcome", + } + + +def test_action_metadata_is_required_nonempty_text() -> None: + for spec in ACTION_TOOL_SPECS: + for field_name in ("purpose", "expected_outcome"): + field = spec.parameters["properties"][field_name] + assert field["type"] == "string" + assert field["minLength"] == 1 def test_finish_task_spec_requires_success_and_reason() -> None: diff --git a/tests/test_world_model.py b/tests/test_world_model.py index 2855964..caf270e 100644 --- a/tests/test_world_model.py +++ b/tests/test_world_model.py @@ -61,7 +61,9 @@ def test_observe_leaves_current_page_unchanged_without_semantic_scene() -> None: assert model.state.current_page == "Chat" -def test_observe_updates_current_app_for_successful_launch_and_clear_for_terminate() -> None: +def test_observe_updates_current_app_for_successful_launch_and_clear_for_terminate() -> ( + None +): model = WorldModel(config=WorldConfig(history_size=2)) launch = PlannedStep( action="launch_app", @@ -188,3 +190,22 @@ def test_observe_appends_semantic_or_raw_scene_history_with_eviction() -> None: assert [event.action for event in model.state.history] == ["second", "third"] assert model.state.history[0].scene_summary.to_dict() == _scene().to_dict() assert model.state.history.maxlen == 2 + + +def test_observe_records_executed_action_arguments_and_metadata() -> None: + model = WorldModel(config=WorldConfig(history_size=2)) + step = PlannedStep( + action="tap", + description="Open settings.", + args={"x": 12, "y": 34}, + purpose="Open settings.", + expected_outcome="The settings page is visible.", + ) + + model.observe(_scene(), _semantic_scene("Settings"), step, _result(step)) + + event = model.state.history[0] + assert event.action == "tap" + assert event.arguments == {"x": 12, "y": 34} + assert event.purpose == "Open settings." + assert event.expected_outcome == "The settings page is visible." diff --git a/tests/test_world_models.py b/tests/test_world_models.py index 0c6e586..b11d148 100644 --- a/tests/test_world_models.py +++ b/tests/test_world_models.py @@ -66,12 +66,18 @@ def test_world_event_with_rationale_thinking_page() -> None: success=True, rationale="Previous step opened settings. Now tapping account.", thinking="I should navigate to account settings.", + arguments={"x": 12, "y": 34}, + purpose="Open account settings.", + expected_outcome="The account settings page is visible.", page="Settings", ) data = event.to_dict() assert data["rationale"] == "Previous step opened settings. Now tapping account." assert data["thinking"] == "I should navigate to account settings." + assert data["arguments"] == {"x": 12, "y": 34} + assert data["purpose"] == "Open account settings." + assert data["expected_outcome"] == "The account settings page is visible." assert data["page"] == "Settings" assert data["scene_summary"] is None @@ -82,6 +88,9 @@ def test_world_event_all_optional_fields_none() -> None: assert data["rationale"] is None assert data["thinking"] is None + assert data["arguments"] == {} + assert data["purpose"] is None + assert data["expected_outcome"] is None assert data["page"] is None assert data["scene_summary"] is None assert data["action"] == "swipe" diff --git a/world/model.py b/world/model.py index 8b60eed..1462be5 100644 --- a/world/model.py +++ b/world/model.py @@ -86,7 +86,9 @@ class WorldModel: def _new_state(self) -> WorldState: return WorldState.with_history_bound(self.config.history_size) - def _update_page(self, state: WorldState, semantic_scene: SemanticScene | None) -> None: + def _update_page( + self, state: WorldState, semantic_scene: SemanticScene | None + ) -> None: if semantic_scene is None: return page = semantic_scene.page.strip() @@ -137,14 +139,22 @@ class WorldModel: WorldEvent( action=str(getattr(step, "action", "unknown")), success=bool(getattr(result, "success", False)), + arguments=_step_arguments(step), scene_summary=semantic_scene or scene, rationale=getattr(step, "rationale", None), thinking=getattr(step, "thinking", None), + purpose=getattr(step, "purpose", None), + expected_outcome=getattr(step, "expected_outcome", None), page=state.current_page, ) ) +def _step_arguments(step: "PlannedStep") -> dict[str, Any]: + raw_arguments = getattr(step, "args", {}) + return dict(raw_arguments) if isinstance(raw_arguments, dict) else {} + + class TaskWorldView: """Per-task handle into a `WorldModel`, scoped explicitly by `task_id`. diff --git a/world/models.py b/world/models.py index ddabbe9..873246e 100644 --- a/world/models.py +++ b/world/models.py @@ -14,21 +14,29 @@ from world.config import DEFAULT_HISTORY_SIZE class WorldEvent: action: str success: bool + arguments: dict[str, Any] = field(default_factory=dict) # Optional for backward compatibility; new entries created by AIPlanner # paths leave this as None and use rationale/thinking instead. scene_summary: SemanticScene | Scene | None = None rationale: str | None = None thinking: str | None = None + purpose: str | None = None + expected_outcome: str | None = None page: str | None = None timestamp: datetime = field(default_factory=utc_now) def to_dict(self) -> dict[str, Any]: return { - "scene_summary": self.scene_summary.to_dict() if self.scene_summary is not None else None, + "scene_summary": self.scene_summary.to_dict() + if self.scene_summary is not None + else None, "action": self.action, + "arguments": dict(self.arguments), "success": self.success, "rationale": self.rationale, "thinking": self.thinking, + "purpose": self.purpose, + "expected_outcome": self.expected_outcome, "page": self.page, "timestamp": self.timestamp.isoformat(), }