feat: preserve planner context across task steps

apps/device-host-agent/host_agent/cloud_planner_client.py

@@ -62,11 +62,13 @@ class CloudProxyToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         payload: dict[str, Any] = {
             "host_id": self.config.host_id,
             "system_prompt": system_prompt,
             "user_prompt": user_prompt,
+            "history": history or [],
             "screenshot_base64": (
                 base64.b64encode(screenshot).decode("ascii")
                 if screenshot is not None

apps/device-host-agent/tests/test_cloud_planner_client.py

@@ -96,6 +96,36 @@ def test_decide_base64_encodes_screenshot() -> None:
     assert body["screenshot_base64"] == "aGVsbG8="
 
 
+def test_decide_forwards_planner_history() -> None:
+    seen_requests: list[httpx.Request] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen_requests.append(request)
+        return httpx.Response(200, json={"tool_name": "tap", "arguments": {}})
+
+    client = _client(handler)
+    history = [
+        {
+            "user_prompt": "first screen",
+            "tool_name": "tap",
+            "arguments": {"x": 1, "y": 2},
+            "rationale": "Open it.",
+            "tool_result": {"success": True},
+        }
+    ]
+
+    client.decide(
+        system_prompt="sp",
+        user_prompt="next screen",
+        screenshot=None,
+        tools=_TOOLS,
+        timeout=10.0,
+        history=history,
+    )
+
+    assert json.loads(seen_requests[0].content)["history"] == history
+
+
 def test_decide_clamps_legacy_timeout_and_waits_for_cloud_profile_timeout() -> None:
     seen_requests: list[httpx.Request] = []
 

packages/cloud-platform/cloud/internal_api/api.py

@@ -6,6 +6,7 @@ import json
 import logging
 from collections.abc import Awaitable, Callable
 from datetime import timedelta
+from inspect import Parameter, signature
 from time import monotonic
 from typing import TYPE_CHECKING
 from uuid import uuid4
@@ -505,13 +506,21 @@ def create_internal_router(
 
         started_at = monotonic()
         try:
-            decision = client.decide(
+            decision_kwargs = {
-                system_prompt=payload.system_prompt,
+                "system_prompt": payload.system_prompt,
-                user_prompt=payload.user_prompt,
+                "user_prompt": payload.user_prompt,
-                screenshot=screenshot,
+                "screenshot": screenshot,
-                tools=tools,
+                "tools": tools,
-                timeout=planner_timeout,
+                "timeout": planner_timeout,
-            )
+            }
+            parameters = signature(client.decide).parameters.values()
+            if any(
+                parameter.name == "history"
+                or parameter.kind == Parameter.VAR_KEYWORD
+                for parameter in parameters
+            ):
+                decision_kwargs["history"] = payload.history
+            decision = client.decide(**decision_kwargs)
         except ToolCallUnavailable as exc:
             logger.info(
                 "planner-decision request failed",

packages/cloud-platform/cloud/internal_api/models.py

@@ -143,6 +143,7 @@ class PlannerDecisionRequest(BaseModel):
     host_id: str = Field(min_length=1)
     system_prompt: str
     user_prompt: str
+    history: list[dict[str, Any]] = Field(default_factory=list)
     screenshot_base64: str | None = None
     tools: list[PlannerToolSpecModel] = Field(default_factory=list)
     timeout_seconds: float = Field(default=30.0, gt=0, le=120)

runtime/ai_planner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from inspect import Parameter, signature
 from typing import TYPE_CHECKING, Any
 
 from core.errors import TaskFailedError
@@ -39,11 +40,11 @@ class AIPlanner(Planner):
         world: "WorldState | None" = None,
         screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
+        _sync_tool_results(context)
         scene_json = _without_ocr(scene.to_dict()) if self.config.multimodal else scene.to_dict()
         user_prompt = planner_user_prompt(
             goal=goal,
             scene_json=scene_json,
-            history_summary=_history_summary(world),
             device_platform=context.device_platform,
         )
         if context.step_results and not context.step_results[-1].success:
@@ -56,13 +57,18 @@ class AIPlanner(Planner):
                    "system_prompt": PLANNER_SYSTEM_PROMPT, "user_prompt": user_prompt,
                    "has_screenshot": screenshot is not None})
         try:
-            decision = self.client.decide(
+            kwargs = {
-                system_prompt=PLANNER_SYSTEM_PROMPT,
+                "system_prompt": PLANNER_SYSTEM_PROMPT,
-                user_prompt=user_prompt,
+                "user_prompt": user_prompt,
-                screenshot=screenshot,
+                "screenshot": screenshot,
-                tools=ALL_TOOL_SPECS,
+                "tools": ALL_TOOL_SPECS,
-                timeout=self.config.timeout,
+                "timeout": self.config.timeout,
-            )
+            }
+            if _accepts_history(self.client.decide):
+                kwargs["history"] = context.planner_history[
+                    -self.config.history_max_turns :
+                ]
+            decision = self.client.decide(**kwargs)
         except Exception as exc:
             self._log({"type": "agent_error", "task_id": context.task_id, "error": str(exc)})
             raise
@@ -76,6 +82,16 @@ class AIPlanner(Planner):
                 return []
             raise TaskFailedError(decision.arguments.get("reason") or "task failed")
 
+        context.planner_history.append(
+            {
+                "user_prompt": user_prompt,
+                "tool_name": decision.tool_name,
+                "arguments": _conversation_arguments(decision),
+                "rationale": decision.text_output,
+                "tool_result": None,
+            }
+        )
+
         return [
             PlannedStep(
                 action=decision.tool_name,
@@ -107,23 +123,6 @@ class AIPlanner(Planner):
                 pass
 
 
-def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]:
-    if world is None:
-        return []
-    return [
-        {
-            "page": event.page,
-            "action": event.action,
-            "arguments": dict(event.arguments),
-            "rationale": event.rationale,
-            "purpose": event.purpose,
-            "expected_outcome": event.expected_outcome,
-            "success": event.success,
-        }
-        for event in world.history
-    ]
-
-
 def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
     cleaned = dict(scene_json)
     elements = cleaned.get("elements")
@@ -135,3 +134,26 @@ def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
         ]
     cleaned.pop("ocr_elements", None)
     return cleaned
+
+
+def _sync_tool_results(context: TaskContext) -> None:
+    for turn, result in zip(context.planner_history, context.step_results, strict=False):
+        if turn.get("tool_result") is None:
+            turn["tool_result"] = result.to_dict()
+
+
+def _conversation_arguments(decision: Any) -> dict[str, Any]:
+    arguments = dict(decision.arguments)
+    if decision.purpose is not None:
+        arguments["purpose"] = decision.purpose
+    if decision.expected_outcome is not None:
+        arguments["expected_outcome"] = decision.expected_outcome
+    return arguments
+
+
+def _accepts_history(method: Any) -> bool:
+    parameters = signature(method).parameters.values()
+    return any(
+        parameter.name == "history" or parameter.kind == Parameter.VAR_KEYWORD
+        for parameter in parameters
+    )

runtime/context.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from core.models import Scene
 
@@ -18,6 +18,7 @@ class TaskContext:
     scenes: list[Scene] = field(default_factory=list)
     step_results: list["StepResult"] = field(default_factory=list)
     world: "WorldState | None" = None
+    planner_history: list[dict[str, Any]] = field(default_factory=list)
 
     def add_scene(self, scene: Scene) -> None:
         self.scenes.append(scene)

runtime/planner_config.py

@@ -11,6 +11,7 @@ DEFAULT_MODEL_BY_PROVIDER = {
     "openai_compatible": "local-model",
 }
 DEFAULT_TIMEOUT_SECONDS = 30.0
+DEFAULT_HISTORY_MAX_TURNS = 20
 
 ENABLED_ENV = "AI_PLANNER_ENABLED"
 PROVIDER_ENV = "AI_PLANNER_PROVIDER"
@@ -20,6 +21,7 @@ THINKING_BUDGET_ENV = "AI_PLANNER_THINKING_BUDGET_TOKENS"
 API_KEY_ENV = "AI_PLANNER_API_KEY"
 BASE_URL_ENV = "AI_PLANNER_BASE_URL"
 MULTIMODAL_ENV = "AI_PLANNER_MULTIMODAL"
+HISTORY_MAX_TURNS_ENV = "AI_PLANNER_HISTORY_MAX_TURNS"
 
 SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODEL_BY_PROVIDER)
 
@@ -34,6 +36,7 @@ class PlannerConfig:
     api_key: str | None = None
     base_url: str | None = None
     multimodal: bool = False
+    history_max_turns: int = DEFAULT_HISTORY_MAX_TURNS
 
     def resolved_model(self) -> str:
         return self.model or DEFAULT_MODEL_BY_PROVIDER[self.provider]
@@ -50,6 +53,10 @@ def load_config(env: Mapping[str, str] | None = None) -> PlannerConfig:
         api_key=values.get(API_KEY_ENV) or _provider_key(values),
         base_url=values.get(BASE_URL_ENV) or None,
         multimodal=_parse_bool(values.get(MULTIMODAL_ENV), default=False),
+        history_max_turns=_parse_positive_int(
+            values.get(HISTORY_MAX_TURNS_ENV),
+            default=DEFAULT_HISTORY_MAX_TURNS,
+        ),
     )
 
 
@@ -88,6 +95,16 @@ def _parse_thinking_budget(value: str | None) -> int | None:
     return budget if budget > 0 else None
 
 
+def _parse_positive_int(value: str | None, *, default: int) -> int:
+    if value is None:
+        return default
+    try:
+        parsed = int(value)
+    except ValueError:
+        return default
+    return parsed if parsed > 0 else default
+
+
 def _provider_key(values: Mapping[str, str]) -> str | None:
     provider = (values.get(PROVIDER_ENV) or DEFAULT_PROVIDER).strip().lower()
     if provider == "openai":

runtime/planner_prompts.py

@@ -66,7 +66,6 @@ def planner_user_prompt(
     *,
     goal: str,
     scene_json: dict[str, Any],
-    history_summary: list[dict[str, Any]],
     device_platform: str | None = None,
     now: datetime | None = None,
 ) -> str:
@@ -83,8 +82,6 @@ def planner_user_prompt(
         f"{goal}\n\n"
         "Current Scene (JSON):\n"
         f"{json.dumps(scene_json, ensure_ascii=False, sort_keys=True)}\n\n"
-        "Recent history, oldest first (JSON):\n"
-        f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n"
         "Call exactly one tool for this turn."
     )
 

runtime/tool_calling_client.py

@@ -51,6 +51,7 @@ class ToolCallingClient(Protocol):
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision: ...
 
 
@@ -80,6 +81,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_message(
@@ -87,6 +89,7 @@ class AnthropicToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -103,6 +106,7 @@ class AnthropicToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -123,6 +127,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -146,10 +151,8 @@ class AnthropicToolCallingClient:
                 }
             ],
             "messages": [
-                {
+                *_anthropic_history(history or []),
-                    "role": "user",
+                {"role": "user", "content": _anthropic_content(user_prompt, screenshot)},
-                    "content": _anthropic_content(user_prompt, screenshot),
-                }
             ],
             "tools": [_anthropic_tool(spec) for spec in tools],
             "tool_choice": {
@@ -207,6 +210,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_completion(
@@ -214,6 +218,7 @@ class OpenAIToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -227,6 +232,7 @@ class OpenAIToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -247,6 +253,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -257,6 +264,7 @@ class OpenAIToolCallingClient:
             "timeout": timeout,
             "messages": [
                 {"role": "system", "content": system_prompt},
+                *_openai_history(history or []),
                 {"role": "user", "content": _openai_content(user_prompt, screenshot)},
             ],
             "tools": [_openai_tool(spec) for spec in tools],
@@ -320,6 +328,44 @@ def _anthropic_content(
     return content
 
 
+def _anthropic_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_use_id = f"planner-turn-{index}"
+        assistant_content: list[dict[str, Any]] = []
+        rationale = turn.get("rationale")
+        if isinstance(rationale, str) and rationale:
+            assistant_content.append({"type": "text", "text": rationale})
+        assistant_content.append(
+            {
+                "type": "tool_use",
+                "id": tool_use_id,
+                "name": turn["tool_name"],
+                "input": dict(turn.get("arguments") or {}),
+            }
+        )
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {"role": "assistant", "content": assistant_content},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "tool_result",
+                            "tool_use_id": tool_use_id,
+                            "content": json.dumps(result, ensure_ascii=False, default=str),
+                        }
+                    ],
+                },
+            ]
+        )
+    return messages
+
+
 def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "name": spec.name,
@@ -393,6 +439,43 @@ def _openai_content(
     ]
 
 
+def _openai_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_call_id = f"planner-turn-{index}"
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {
+                    "role": "assistant",
+                    "content": turn.get("rationale"),
+                    "tool_calls": [
+                        {
+                            "id": tool_call_id,
+                            "type": "function",
+                            "function": {
+                                "name": turn["tool_name"],
+                                "arguments": json.dumps(
+                                    turn.get("arguments") or {},
+                                    ensure_ascii=False,
+                                ),
+                            },
+                        }
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": tool_call_id,
+                    "content": json.dumps(result, ensure_ascii=False, default=str),
+                },
+            ]
+        )
+    return messages
+
+
 def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "type": "function",

tests/test_ai_planner.py

@@ -8,6 +8,7 @@ from core.errors import TaskFailedError
 from core.models import Bounds, Scene, SceneElement
 from runtime.ai_planner import AIPlanner
 from runtime.context import TaskContext
+from runtime.executor import StepResult
 from runtime.planner_config import PlannerConfig
 from runtime.tool_calling_client import ToolCallDecision
 from runtime.tool_specs import ALL_TOOL_SPECS
@@ -26,6 +27,7 @@ class FakeToolCallingClient:
         screenshot: bytes | None,
         tools: list[Any],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         self.calls.append(
             {
@@ -34,6 +36,7 @@ class FakeToolCallingClient:
                 "screenshot": screenshot,
                 "tools": tools,
                 "timeout": timeout,
+                "history": list(history) if history is not None else None,
             }
         )
         return self.decision
@@ -242,63 +245,64 @@ def test_ai_planner_propagates_rationale_and_thinking_to_planned_step() -> None:
     assert steps[0].expected_outcome == "The account settings page is visible."
 
 
-def test_history_summary_returns_compact_format() -> None:
+def test_ai_planner_carries_completed_turn_into_the_next_llm_call() -> None:
-    from collections import deque
+    client = FakeToolCallingClient(
-    from runtime.ai_planner import _history_summary
+        ToolCallDecision(
-    from world.models import WorldEvent, WorldState
+            tool_name="tap",
-
+            arguments={"x": 1, "y": 2},
-    state = WorldState(
+            text_output="Opening the send control.",
-        history=deque(
+            purpose="Open the send control.",
-            [
+            expected_outcome="The composer is focused.",
-                WorldEvent(
-                    action="tap",
-                    success=True,
-                    rationale="Opened settings.",
-                    arguments={"x": 1, "y": 2},
-                    purpose="Open settings.",
-                    expected_outcome="Settings is visible.",
-                    page="Home",
-                ),
-                WorldEvent(
-                    action="swipe",
-                    success=False,
-                    rationale=None,
-                    arguments={"start_y": 700, "end_y": 200},
-                    page="Settings",
-                ),
-            ]
         )
     )
+    planner = AIPlanner(client=client)
+    context = _context()
 
-    summary = _history_summary(state)
+    first_step = planner.plan(goal=context.goal, scene=_scene(), context=context)[0]
+    context.add_step_result(
+        StepResult(
+            step=first_step,
+            success=True,
+            attempts=1,
+            result={"ok": True},
+        )
+    )
+    planner.plan(goal=context.goal, scene=_scene(), context=context)
 
-    assert summary == [
+    assert client.calls[0]["history"] == []
+    history = client.calls[1]["history"]
+    assert history is not None
+    assert history[0]["tool_name"] == "tap"
+    assert history[0]["arguments"] == {
+        "x": 1,
+        "y": 2,
+        "purpose": "Open the send control.",
+        "expected_outcome": "The composer is focused.",
+    }
+    assert history[0]["tool_result"]["success"] is True
+    assert history[0]["tool_result"]["result"] == {"ok": True}
+
+
+def test_ai_planner_limits_history_sent_to_the_llm() -> None:
+    client = FakeToolCallingClient(
+        ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
+    )
+    planner = AIPlanner(client=client, config=PlannerConfig(history_max_turns=2))
+    context = _context()
+    context.planner_history.extend(
         {
-            "page": "Home",
+            "user_prompt": f"turn-{index}",
-            "action": "tap",
+            "tool_name": "tap",
-            "arguments": {"x": 1, "y": 2},
+            "arguments": {},
-            "rationale": "Opened settings.",
-            "purpose": "Open settings.",
-            "expected_outcome": "Settings is visible.",
-            "success": True,
-        },
-        {
-            "page": "Settings",
-            "action": "swipe",
-            "arguments": {"start_y": 700, "end_y": 200},
             "rationale": None,
-            "purpose": None,
+            "tool_result": {"success": True},
-            "expected_outcome": None,
+        }
-            "success": False,
+        for index in range(3)
-        },
+    )
+
+    planner.plan(goal=context.goal, scene=_scene(), context=context)
+
+    assert [turn["user_prompt"] for turn in client.calls[0]["history"]] == [
+        "turn-1",
+        "turn-2",
     ]
-    # Must not contain scene element data
-    for entry in summary:
-        assert "scene_summary" not in entry
-        assert "elements" not in entry
-
-
-def test_history_summary_returns_empty_for_none_world() -> None:
-    from runtime.ai_planner import _history_summary
-
-    assert _history_summary(None) == []

tests/test_planner_prompts.py

@@ -9,7 +9,6 @@ def test_planner_user_prompt_includes_time_zone_and_configured_device_type() ->
     prompt = planner_user_prompt(
         goal="open settings",
         scene_json={"screen": {"width": 1, "height": 1}, "elements": []},
-        history_summary=[],
         device_platform="ios",
         now=datetime(
             2026,
@@ -35,8 +34,16 @@ def test_planner_user_prompt_uses_scene_platform_when_context_is_unavailable() -
             "elements": [],
             "app": {"platform": "android"},
         },
-        history_summary=[],
         now=datetime(2026, 7, 16, tzinfo=timezone.utc),
     )
 
     assert "Device type: android" in prompt
+
+
+def test_planner_user_prompt_does_not_duplicate_conversation_history() -> None:
+    prompt = planner_user_prompt(
+        goal="open settings",
+        scene_json={"screen": {"width": 1, "height": 1}, "elements": []},
+    )
+
+    assert "Recent history" not in prompt