feat: preserve planner context across task steps

runtime/tool_calling_client.py

@@ -51,6 +51,7 @@ class ToolCallingClient(Protocol):
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision: ...
 
 
@@ -80,6 +81,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_message(
@@ -87,6 +89,7 @@ class AnthropicToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -103,6 +106,7 @@ class AnthropicToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -123,6 +127,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -146,10 +151,8 @@ class AnthropicToolCallingClient:
                 }
             ],
             "messages": [
-                {
-                    "role": "user",
-                    "content": _anthropic_content(user_prompt, screenshot),
-                }
+                *_anthropic_history(history or []),
+                {"role": "user", "content": _anthropic_content(user_prompt, screenshot)},
             ],
             "tools": [_anthropic_tool(spec) for spec in tools],
             "tool_choice": {
@@ -207,6 +210,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_completion(
@@ -214,6 +218,7 @@ class OpenAIToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -227,6 +232,7 @@ class OpenAIToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -247,6 +253,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -257,6 +264,7 @@ class OpenAIToolCallingClient:
             "timeout": timeout,
             "messages": [
                 {"role": "system", "content": system_prompt},
+                *_openai_history(history or []),
                 {"role": "user", "content": _openai_content(user_prompt, screenshot)},
             ],
             "tools": [_openai_tool(spec) for spec in tools],
@@ -320,6 +328,44 @@ def _anthropic_content(
     return content
 
 
+def _anthropic_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_use_id = f"planner-turn-{index}"
+        assistant_content: list[dict[str, Any]] = []
+        rationale = turn.get("rationale")
+        if isinstance(rationale, str) and rationale:
+            assistant_content.append({"type": "text", "text": rationale})
+        assistant_content.append(
+            {
+                "type": "tool_use",
+                "id": tool_use_id,
+                "name": turn["tool_name"],
+                "input": dict(turn.get("arguments") or {}),
+            }
+        )
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {"role": "assistant", "content": assistant_content},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "tool_result",
+                            "tool_use_id": tool_use_id,
+                            "content": json.dumps(result, ensure_ascii=False, default=str),
+                        }
+                    ],
+                },
+            ]
+        )
+    return messages
+
+
 def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "name": spec.name,
@@ -393,6 +439,43 @@ def _openai_content(
     ]
 
 
+def _openai_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_call_id = f"planner-turn-{index}"
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {
+                    "role": "assistant",
+                    "content": turn.get("rationale"),
+                    "tool_calls": [
+                        {
+                            "id": tool_call_id,
+                            "type": "function",
+                            "function": {
+                                "name": turn["tool_name"],
+                                "arguments": json.dumps(
+                                    turn.get("arguments") or {},
+                                    ensure_ascii=False,
+                                ),
+                            },
+                        }
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": tool_call_id,
+                    "content": json.dumps(result, ensure_ascii=False, default=str),
+                },
+            ]
+        )
+    return messages
+
+
 def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "type": "function",