feat: preserve planner context across task steps

runtime/ai_planner.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from inspect import Parameter, signature
 from typing import TYPE_CHECKING, Any
 
 from core.errors import TaskFailedError
@@ -39,11 +40,11 @@ class AIPlanner(Planner):
         world: "WorldState | None" = None,
         screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
+        _sync_tool_results(context)
         scene_json = _without_ocr(scene.to_dict()) if self.config.multimodal else scene.to_dict()
         user_prompt = planner_user_prompt(
             goal=goal,
             scene_json=scene_json,
-            history_summary=_history_summary(world),
             device_platform=context.device_platform,
         )
         if context.step_results and not context.step_results[-1].success:
@@ -56,13 +57,18 @@ class AIPlanner(Planner):
                    "system_prompt": PLANNER_SYSTEM_PROMPT, "user_prompt": user_prompt,
                    "has_screenshot": screenshot is not None})
         try:
-            decision = self.client.decide(
-                system_prompt=PLANNER_SYSTEM_PROMPT,
-                user_prompt=user_prompt,
-                screenshot=screenshot,
-                tools=ALL_TOOL_SPECS,
-                timeout=self.config.timeout,
-            )
+            kwargs = {
+                "system_prompt": PLANNER_SYSTEM_PROMPT,
+                "user_prompt": user_prompt,
+                "screenshot": screenshot,
+                "tools": ALL_TOOL_SPECS,
+                "timeout": self.config.timeout,
+            }
+            if _accepts_history(self.client.decide):
+                kwargs["history"] = context.planner_history[
+                    -self.config.history_max_turns :
+                ]
+            decision = self.client.decide(**kwargs)
         except Exception as exc:
             self._log({"type": "agent_error", "task_id": context.task_id, "error": str(exc)})
             raise
@@ -76,6 +82,16 @@ class AIPlanner(Planner):
                 return []
             raise TaskFailedError(decision.arguments.get("reason") or "task failed")
 
+        context.planner_history.append(
+            {
+                "user_prompt": user_prompt,
+                "tool_name": decision.tool_name,
+                "arguments": _conversation_arguments(decision),
+                "rationale": decision.text_output,
+                "tool_result": None,
+            }
+        )
+
         return [
             PlannedStep(
                 action=decision.tool_name,
@@ -107,23 +123,6 @@ class AIPlanner(Planner):
                 pass
 
 
-def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]:
-    if world is None:
-        return []
-    return [
-        {
-            "page": event.page,
-            "action": event.action,
-            "arguments": dict(event.arguments),
-            "rationale": event.rationale,
-            "purpose": event.purpose,
-            "expected_outcome": event.expected_outcome,
-            "success": event.success,
-        }
-        for event in world.history
-    ]
-
-
 def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
     cleaned = dict(scene_json)
     elements = cleaned.get("elements")
@@ -135,3 +134,26 @@ def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
         ]
     cleaned.pop("ocr_elements", None)
     return cleaned
+
+
+def _sync_tool_results(context: TaskContext) -> None:
+    for turn, result in zip(context.planner_history, context.step_results, strict=False):
+        if turn.get("tool_result") is None:
+            turn["tool_result"] = result.to_dict()
+
+
+def _conversation_arguments(decision: Any) -> dict[str, Any]:
+    arguments = dict(decision.arguments)
+    if decision.purpose is not None:
+        arguments["purpose"] = decision.purpose
+    if decision.expected_outcome is not None:
+        arguments["expected_outcome"] = decision.expected_outcome
+    return arguments
+
+
+def _accepts_history(method: Any) -> bool:
+    parameters = signature(method).parameters.values()
+    return any(
+        parameter.name == "history" or parameter.kind == Parameter.VAR_KEYWORD
+        for parameter in parameters
+    )

runtime/context.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from core.models import Scene
 
@@ -18,6 +18,7 @@ class TaskContext:
     scenes: list[Scene] = field(default_factory=list)
     step_results: list["StepResult"] = field(default_factory=list)
     world: "WorldState | None" = None
+    planner_history: list[dict[str, Any]] = field(default_factory=list)
 
     def add_scene(self, scene: Scene) -> None:
         self.scenes.append(scene)

runtime/planner_config.py

@@ -11,6 +11,7 @@ DEFAULT_MODEL_BY_PROVIDER = {
     "openai_compatible": "local-model",
 }
 DEFAULT_TIMEOUT_SECONDS = 30.0
+DEFAULT_HISTORY_MAX_TURNS = 20
 
 ENABLED_ENV = "AI_PLANNER_ENABLED"
 PROVIDER_ENV = "AI_PLANNER_PROVIDER"
@@ -20,6 +21,7 @@ THINKING_BUDGET_ENV = "AI_PLANNER_THINKING_BUDGET_TOKENS"
 API_KEY_ENV = "AI_PLANNER_API_KEY"
 BASE_URL_ENV = "AI_PLANNER_BASE_URL"
 MULTIMODAL_ENV = "AI_PLANNER_MULTIMODAL"
+HISTORY_MAX_TURNS_ENV = "AI_PLANNER_HISTORY_MAX_TURNS"
 
 SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODEL_BY_PROVIDER)
 
@@ -34,6 +36,7 @@ class PlannerConfig:
     api_key: str | None = None
     base_url: str | None = None
     multimodal: bool = False
+    history_max_turns: int = DEFAULT_HISTORY_MAX_TURNS
 
     def resolved_model(self) -> str:
         return self.model or DEFAULT_MODEL_BY_PROVIDER[self.provider]
@@ -50,6 +53,10 @@ def load_config(env: Mapping[str, str] | None = None) -> PlannerConfig:
         api_key=values.get(API_KEY_ENV) or _provider_key(values),
         base_url=values.get(BASE_URL_ENV) or None,
         multimodal=_parse_bool(values.get(MULTIMODAL_ENV), default=False),
+        history_max_turns=_parse_positive_int(
+            values.get(HISTORY_MAX_TURNS_ENV),
+            default=DEFAULT_HISTORY_MAX_TURNS,
+        ),
     )
 
 
@@ -88,6 +95,16 @@ def _parse_thinking_budget(value: str | None) -> int | None:
     return budget if budget > 0 else None
 
 
+def _parse_positive_int(value: str | None, *, default: int) -> int:
+    if value is None:
+        return default
+    try:
+        parsed = int(value)
+    except ValueError:
+        return default
+    return parsed if parsed > 0 else default
+
+
 def _provider_key(values: Mapping[str, str]) -> str | None:
     provider = (values.get(PROVIDER_ENV) or DEFAULT_PROVIDER).strip().lower()
     if provider == "openai":

runtime/planner_prompts.py

@@ -66,7 +66,6 @@ def planner_user_prompt(
     *,
     goal: str,
     scene_json: dict[str, Any],
-    history_summary: list[dict[str, Any]],
     device_platform: str | None = None,
     now: datetime | None = None,
 ) -> str:
@@ -83,8 +82,6 @@ def planner_user_prompt(
         f"{goal}\n\n"
         "Current Scene (JSON):\n"
         f"{json.dumps(scene_json, ensure_ascii=False, sort_keys=True)}\n\n"
-        "Recent history, oldest first (JSON):\n"
-        f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n"
         "Call exactly one tool for this turn."
     )
 

runtime/tool_calling_client.py

@@ -51,6 +51,7 @@ class ToolCallingClient(Protocol):
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision: ...
 
 
@@ -80,6 +81,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_message(
@@ -87,6 +89,7 @@ class AnthropicToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -103,6 +106,7 @@ class AnthropicToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -123,6 +127,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -146,10 +151,8 @@ class AnthropicToolCallingClient:
                 }
             ],
             "messages": [
-                {
-                    "role": "user",
-                    "content": _anthropic_content(user_prompt, screenshot),
-                }
+                *_anthropic_history(history or []),
+                {"role": "user", "content": _anthropic_content(user_prompt, screenshot)},
             ],
             "tools": [_anthropic_tool(spec) for spec in tools],
             "tool_choice": {
@@ -207,6 +210,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_completion(
@@ -214,6 +218,7 @@ class OpenAIToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -227,6 +232,7 @@ class OpenAIToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -247,6 +253,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -257,6 +264,7 @@ class OpenAIToolCallingClient:
             "timeout": timeout,
             "messages": [
                 {"role": "system", "content": system_prompt},
+                *_openai_history(history or []),
                 {"role": "user", "content": _openai_content(user_prompt, screenshot)},
             ],
             "tools": [_openai_tool(spec) for spec in tools],
@@ -320,6 +328,44 @@ def _anthropic_content(
     return content
 
 
+def _anthropic_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_use_id = f"planner-turn-{index}"
+        assistant_content: list[dict[str, Any]] = []
+        rationale = turn.get("rationale")
+        if isinstance(rationale, str) and rationale:
+            assistant_content.append({"type": "text", "text": rationale})
+        assistant_content.append(
+            {
+                "type": "tool_use",
+                "id": tool_use_id,
+                "name": turn["tool_name"],
+                "input": dict(turn.get("arguments") or {}),
+            }
+        )
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {"role": "assistant", "content": assistant_content},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "tool_result",
+                            "tool_use_id": tool_use_id,
+                            "content": json.dumps(result, ensure_ascii=False, default=str),
+                        }
+                    ],
+                },
+            ]
+        )
+    return messages
+
+
 def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "name": spec.name,
@@ -393,6 +439,43 @@ def _openai_content(
     ]
 
 
+def _openai_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_call_id = f"planner-turn-{index}"
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {
+                    "role": "assistant",
+                    "content": turn.get("rationale"),
+                    "tool_calls": [
+                        {
+                            "id": tool_call_id,
+                            "type": "function",
+                            "function": {
+                                "name": turn["tool_name"],
+                                "arguments": json.dumps(
+                                    turn.get("arguments") or {},
+                                    ensure_ascii=False,
+                                ),
+                            },
+                        }
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": tool_call_id,
+                    "content": json.dumps(result, ensure_ascii=False, default=str),
+                },
+            ]
+        )
+    return messages
+
+
 def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "type": "function",