From 60ee157e9764dbb00b52cba4c9068965b5cc1124 Mon Sep 17 00:00:00 2001 From: showtan001 <240788545@qq.com> Date: Sun, 30 Aug 2026 22:59:19 +0800 Subject: [PATCH] feat: preserve planner context across task steps --- .../host_agent/cloud_planner_client.py | 2 + .../tests/test_cloud_planner_client.py | 30 +++++ .../cloud-platform/cloud/internal_api/api.py | 23 ++-- .../cloud/internal_api/models.py | 1 + runtime/ai_planner.py | 72 ++++++++---- runtime/context.py | 3 +- runtime/planner_config.py | 17 +++ runtime/planner_prompts.py | 3 - runtime/tool_calling_client.py | 91 ++++++++++++++- tests/test_ai_planner.py | 110 +++++++++--------- tests/test_planner_prompts.py | 11 +- 11 files changed, 268 insertions(+), 95 deletions(-) diff --git a/apps/device-host-agent/host_agent/cloud_planner_client.py b/apps/device-host-agent/host_agent/cloud_planner_client.py index 99da56e..56f8195 100644 --- a/apps/device-host-agent/host_agent/cloud_planner_client.py +++ b/apps/device-host-agent/host_agent/cloud_planner_client.py @@ -62,11 +62,13 @@ class CloudProxyToolCallingClient: screenshot: bytes | None, tools: list[ToolSpec], timeout: float, + history: list[dict[str, Any]] | None = None, ) -> ToolCallDecision: payload: dict[str, Any] = { "host_id": self.config.host_id, "system_prompt": system_prompt, "user_prompt": user_prompt, + "history": history or [], "screenshot_base64": ( base64.b64encode(screenshot).decode("ascii") if screenshot is not None diff --git a/apps/device-host-agent/tests/test_cloud_planner_client.py b/apps/device-host-agent/tests/test_cloud_planner_client.py index 5846c8e..c210e0e 100644 --- a/apps/device-host-agent/tests/test_cloud_planner_client.py +++ b/apps/device-host-agent/tests/test_cloud_planner_client.py @@ -96,6 +96,36 @@ def test_decide_base64_encodes_screenshot() -> None: assert body["screenshot_base64"] == "aGVsbG8=" +def test_decide_forwards_planner_history() -> None: + seen_requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen_requests.append(request) + return httpx.Response(200, json={"tool_name": "tap", "arguments": {}}) + + client = _client(handler) + history = [ + { + "user_prompt": "first screen", + "tool_name": "tap", + "arguments": {"x": 1, "y": 2}, + "rationale": "Open it.", + "tool_result": {"success": True}, + } + ] + + client.decide( + system_prompt="sp", + user_prompt="next screen", + screenshot=None, + tools=_TOOLS, + timeout=10.0, + history=history, + ) + + assert json.loads(seen_requests[0].content)["history"] == history + + def test_decide_clamps_legacy_timeout_and_waits_for_cloud_profile_timeout() -> None: seen_requests: list[httpx.Request] = [] diff --git a/packages/cloud-platform/cloud/internal_api/api.py b/packages/cloud-platform/cloud/internal_api/api.py index fb19fdf..5da47ba 100644 --- a/packages/cloud-platform/cloud/internal_api/api.py +++ b/packages/cloud-platform/cloud/internal_api/api.py @@ -6,6 +6,7 @@ import json import logging from collections.abc import Awaitable, Callable from datetime import timedelta +from inspect import Parameter, signature from time import monotonic from typing import TYPE_CHECKING from uuid import uuid4 @@ -505,13 +506,21 @@ def create_internal_router( started_at = monotonic() try: - decision = client.decide( - system_prompt=payload.system_prompt, - user_prompt=payload.user_prompt, - screenshot=screenshot, - tools=tools, - timeout=planner_timeout, - ) + decision_kwargs = { + "system_prompt": payload.system_prompt, + "user_prompt": payload.user_prompt, + "screenshot": screenshot, + "tools": tools, + "timeout": planner_timeout, + } + parameters = signature(client.decide).parameters.values() + if any( + parameter.name == "history" + or parameter.kind == Parameter.VAR_KEYWORD + for parameter in parameters + ): + decision_kwargs["history"] = payload.history + decision = client.decide(**decision_kwargs) except ToolCallUnavailable as exc: logger.info( "planner-decision request failed", diff --git a/packages/cloud-platform/cloud/internal_api/models.py b/packages/cloud-platform/cloud/internal_api/models.py index 87bf7b3..45e11bd 100644 --- a/packages/cloud-platform/cloud/internal_api/models.py +++ b/packages/cloud-platform/cloud/internal_api/models.py @@ -143,6 +143,7 @@ class PlannerDecisionRequest(BaseModel): host_id: str = Field(min_length=1) system_prompt: str user_prompt: str + history: list[dict[str, Any]] = Field(default_factory=list) screenshot_base64: str | None = None tools: list[PlannerToolSpecModel] = Field(default_factory=list) timeout_seconds: float = Field(default=30.0, gt=0, le=120) diff --git a/runtime/ai_planner.py b/runtime/ai_planner.py index 136cca0..f9625da 100644 --- a/runtime/ai_planner.py +++ b/runtime/ai_planner.py @@ -1,6 +1,7 @@ from __future__ import annotations from collections.abc import Callable +from inspect import Parameter, signature from typing import TYPE_CHECKING, Any from core.errors import TaskFailedError @@ -39,11 +40,11 @@ class AIPlanner(Planner): world: "WorldState | None" = None, screenshot: bytes | None = None, ) -> list[PlannedStep]: + _sync_tool_results(context) scene_json = _without_ocr(scene.to_dict()) if self.config.multimodal else scene.to_dict() user_prompt = planner_user_prompt( goal=goal, scene_json=scene_json, - history_summary=_history_summary(world), device_platform=context.device_platform, ) if context.step_results and not context.step_results[-1].success: @@ -56,13 +57,18 @@ class AIPlanner(Planner): "system_prompt": PLANNER_SYSTEM_PROMPT, "user_prompt": user_prompt, "has_screenshot": screenshot is not None}) try: - decision = self.client.decide( - system_prompt=PLANNER_SYSTEM_PROMPT, - user_prompt=user_prompt, - screenshot=screenshot, - tools=ALL_TOOL_SPECS, - timeout=self.config.timeout, - ) + kwargs = { + "system_prompt": PLANNER_SYSTEM_PROMPT, + "user_prompt": user_prompt, + "screenshot": screenshot, + "tools": ALL_TOOL_SPECS, + "timeout": self.config.timeout, + } + if _accepts_history(self.client.decide): + kwargs["history"] = context.planner_history[ + -self.config.history_max_turns : + ] + decision = self.client.decide(**kwargs) except Exception as exc: self._log({"type": "agent_error", "task_id": context.task_id, "error": str(exc)}) raise @@ -76,6 +82,16 @@ class AIPlanner(Planner): return [] raise TaskFailedError(decision.arguments.get("reason") or "task failed") + context.planner_history.append( + { + "user_prompt": user_prompt, + "tool_name": decision.tool_name, + "arguments": _conversation_arguments(decision), + "rationale": decision.text_output, + "tool_result": None, + } + ) + return [ PlannedStep( action=decision.tool_name, @@ -107,23 +123,6 @@ class AIPlanner(Planner): pass -def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]: - if world is None: - return [] - return [ - { - "page": event.page, - "action": event.action, - "arguments": dict(event.arguments), - "rationale": event.rationale, - "purpose": event.purpose, - "expected_outcome": event.expected_outcome, - "success": event.success, - } - for event in world.history - ] - - def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]: cleaned = dict(scene_json) elements = cleaned.get("elements") @@ -135,3 +134,26 @@ def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]: ] cleaned.pop("ocr_elements", None) return cleaned + + +def _sync_tool_results(context: TaskContext) -> None: + for turn, result in zip(context.planner_history, context.step_results, strict=False): + if turn.get("tool_result") is None: + turn["tool_result"] = result.to_dict() + + +def _conversation_arguments(decision: Any) -> dict[str, Any]: + arguments = dict(decision.arguments) + if decision.purpose is not None: + arguments["purpose"] = decision.purpose + if decision.expected_outcome is not None: + arguments["expected_outcome"] = decision.expected_outcome + return arguments + + +def _accepts_history(method: Any) -> bool: + parameters = signature(method).parameters.values() + return any( + parameter.name == "history" or parameter.kind == Parameter.VAR_KEYWORD + for parameter in parameters + ) diff --git a/runtime/context.py b/runtime/context.py index 31f3651..41f49d4 100644 --- a/runtime/context.py +++ b/runtime/context.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from core.models import Scene @@ -18,6 +18,7 @@ class TaskContext: scenes: list[Scene] = field(default_factory=list) step_results: list["StepResult"] = field(default_factory=list) world: "WorldState | None" = None + planner_history: list[dict[str, Any]] = field(default_factory=list) def add_scene(self, scene: Scene) -> None: self.scenes.append(scene) diff --git a/runtime/planner_config.py b/runtime/planner_config.py index 872263a..abefea1 100644 --- a/runtime/planner_config.py +++ b/runtime/planner_config.py @@ -11,6 +11,7 @@ DEFAULT_MODEL_BY_PROVIDER = { "openai_compatible": "local-model", } DEFAULT_TIMEOUT_SECONDS = 30.0 +DEFAULT_HISTORY_MAX_TURNS = 20 ENABLED_ENV = "AI_PLANNER_ENABLED" PROVIDER_ENV = "AI_PLANNER_PROVIDER" @@ -20,6 +21,7 @@ THINKING_BUDGET_ENV = "AI_PLANNER_THINKING_BUDGET_TOKENS" API_KEY_ENV = "AI_PLANNER_API_KEY" BASE_URL_ENV = "AI_PLANNER_BASE_URL" MULTIMODAL_ENV = "AI_PLANNER_MULTIMODAL" +HISTORY_MAX_TURNS_ENV = "AI_PLANNER_HISTORY_MAX_TURNS" SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODEL_BY_PROVIDER) @@ -34,6 +36,7 @@ class PlannerConfig: api_key: str | None = None base_url: str | None = None multimodal: bool = False + history_max_turns: int = DEFAULT_HISTORY_MAX_TURNS def resolved_model(self) -> str: return self.model or DEFAULT_MODEL_BY_PROVIDER[self.provider] @@ -50,6 +53,10 @@ def load_config(env: Mapping[str, str] | None = None) -> PlannerConfig: api_key=values.get(API_KEY_ENV) or _provider_key(values), base_url=values.get(BASE_URL_ENV) or None, multimodal=_parse_bool(values.get(MULTIMODAL_ENV), default=False), + history_max_turns=_parse_positive_int( + values.get(HISTORY_MAX_TURNS_ENV), + default=DEFAULT_HISTORY_MAX_TURNS, + ), ) @@ -88,6 +95,16 @@ def _parse_thinking_budget(value: str | None) -> int | None: return budget if budget > 0 else None +def _parse_positive_int(value: str | None, *, default: int) -> int: + if value is None: + return default + try: + parsed = int(value) + except ValueError: + return default + return parsed if parsed > 0 else default + + def _provider_key(values: Mapping[str, str]) -> str | None: provider = (values.get(PROVIDER_ENV) or DEFAULT_PROVIDER).strip().lower() if provider == "openai": diff --git a/runtime/planner_prompts.py b/runtime/planner_prompts.py index 549c435..88198ff 100644 --- a/runtime/planner_prompts.py +++ b/runtime/planner_prompts.py @@ -66,7 +66,6 @@ def planner_user_prompt( *, goal: str, scene_json: dict[str, Any], - history_summary: list[dict[str, Any]], device_platform: str | None = None, now: datetime | None = None, ) -> str: @@ -83,8 +82,6 @@ def planner_user_prompt( f"{goal}\n\n" "Current Scene (JSON):\n" f"{json.dumps(scene_json, ensure_ascii=False, sort_keys=True)}\n\n" - "Recent history, oldest first (JSON):\n" - f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n" "Call exactly one tool for this turn." ) diff --git a/runtime/tool_calling_client.py b/runtime/tool_calling_client.py index d41e808..f7caf6d 100644 --- a/runtime/tool_calling_client.py +++ b/runtime/tool_calling_client.py @@ -51,6 +51,7 @@ class ToolCallingClient(Protocol): screenshot: bytes | None, tools: list[ToolSpec], timeout: float, + history: list[dict[str, Any]] | None = None, ) -> ToolCallDecision: ... @@ -80,6 +81,7 @@ class AnthropicToolCallingClient: screenshot: bytes | None, tools: list[ToolSpec], timeout: float, + history: list[dict[str, Any]] | None = None, ) -> ToolCallDecision: try: response = self._create_message( @@ -87,6 +89,7 @@ class AnthropicToolCallingClient: user_prompt, screenshot, tools, + history=history, timeout=timeout, forced=False, ) @@ -103,6 +106,7 @@ class AnthropicToolCallingClient: user_prompt, screenshot, tools, + history=history, timeout=timeout, forced=True, ) @@ -123,6 +127,7 @@ class AnthropicToolCallingClient: screenshot: bytes | None, tools: list[ToolSpec], *, + history: list[dict[str, Any]] | None, timeout: float, forced: bool, ) -> Any: @@ -146,10 +151,8 @@ class AnthropicToolCallingClient: } ], "messages": [ - { - "role": "user", - "content": _anthropic_content(user_prompt, screenshot), - } + *_anthropic_history(history or []), + {"role": "user", "content": _anthropic_content(user_prompt, screenshot)}, ], "tools": [_anthropic_tool(spec) for spec in tools], "tool_choice": { @@ -207,6 +210,7 @@ class OpenAIToolCallingClient: screenshot: bytes | None, tools: list[ToolSpec], timeout: float, + history: list[dict[str, Any]] | None = None, ) -> ToolCallDecision: try: response = self._create_completion( @@ -214,6 +218,7 @@ class OpenAIToolCallingClient: user_prompt, screenshot, tools, + history=history, timeout=timeout, forced=False, ) @@ -227,6 +232,7 @@ class OpenAIToolCallingClient: user_prompt, screenshot, tools, + history=history, timeout=timeout, forced=True, ) @@ -247,6 +253,7 @@ class OpenAIToolCallingClient: screenshot: bytes | None, tools: list[ToolSpec], *, + history: list[dict[str, Any]] | None, timeout: float, forced: bool, ) -> Any: @@ -257,6 +264,7 @@ class OpenAIToolCallingClient: "timeout": timeout, "messages": [ {"role": "system", "content": system_prompt}, + *_openai_history(history or []), {"role": "user", "content": _openai_content(user_prompt, screenshot)}, ], "tools": [_openai_tool(spec) for spec in tools], @@ -320,6 +328,44 @@ def _anthropic_content( return content +def _anthropic_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + for index, turn in enumerate(history): + result = turn.get("tool_result") + if result is None: + continue + tool_use_id = f"planner-turn-{index}" + assistant_content: list[dict[str, Any]] = [] + rationale = turn.get("rationale") + if isinstance(rationale, str) and rationale: + assistant_content.append({"type": "text", "text": rationale}) + assistant_content.append( + { + "type": "tool_use", + "id": tool_use_id, + "name": turn["tool_name"], + "input": dict(turn.get("arguments") or {}), + } + ) + messages.extend( + [ + {"role": "user", "content": turn["user_prompt"]}, + {"role": "assistant", "content": assistant_content}, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": json.dumps(result, ensure_ascii=False, default=str), + } + ], + }, + ] + ) + return messages + + def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]: return { "name": spec.name, @@ -393,6 +439,43 @@ def _openai_content( ] +def _openai_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]: + messages: list[dict[str, Any]] = [] + for index, turn in enumerate(history): + result = turn.get("tool_result") + if result is None: + continue + tool_call_id = f"planner-turn-{index}" + messages.extend( + [ + {"role": "user", "content": turn["user_prompt"]}, + { + "role": "assistant", + "content": turn.get("rationale"), + "tool_calls": [ + { + "id": tool_call_id, + "type": "function", + "function": { + "name": turn["tool_name"], + "arguments": json.dumps( + turn.get("arguments") or {}, + ensure_ascii=False, + ), + }, + } + ], + }, + { + "role": "tool", + "tool_call_id": tool_call_id, + "content": json.dumps(result, ensure_ascii=False, default=str), + }, + ] + ) + return messages + + def _openai_tool(spec: ToolSpec) -> dict[str, Any]: return { "type": "function", diff --git a/tests/test_ai_planner.py b/tests/test_ai_planner.py index ea52290..cb0b962 100644 --- a/tests/test_ai_planner.py +++ b/tests/test_ai_planner.py @@ -8,6 +8,7 @@ from core.errors import TaskFailedError from core.models import Bounds, Scene, SceneElement from runtime.ai_planner import AIPlanner from runtime.context import TaskContext +from runtime.executor import StepResult from runtime.planner_config import PlannerConfig from runtime.tool_calling_client import ToolCallDecision from runtime.tool_specs import ALL_TOOL_SPECS @@ -26,6 +27,7 @@ class FakeToolCallingClient: screenshot: bytes | None, tools: list[Any], timeout: float, + history: list[dict[str, Any]] | None = None, ) -> ToolCallDecision: self.calls.append( { @@ -34,6 +36,7 @@ class FakeToolCallingClient: "screenshot": screenshot, "tools": tools, "timeout": timeout, + "history": list(history) if history is not None else None, } ) return self.decision @@ -242,63 +245,64 @@ def test_ai_planner_propagates_rationale_and_thinking_to_planned_step() -> None: assert steps[0].expected_outcome == "The account settings page is visible." -def test_history_summary_returns_compact_format() -> None: - from collections import deque - from runtime.ai_planner import _history_summary - from world.models import WorldEvent, WorldState - - state = WorldState( - history=deque( - [ - WorldEvent( - action="tap", - success=True, - rationale="Opened settings.", - arguments={"x": 1, "y": 2}, - purpose="Open settings.", - expected_outcome="Settings is visible.", - page="Home", - ), - WorldEvent( - action="swipe", - success=False, - rationale=None, - arguments={"start_y": 700, "end_y": 200}, - page="Settings", - ), - ] +def test_ai_planner_carries_completed_turn_into_the_next_llm_call() -> None: + client = FakeToolCallingClient( + ToolCallDecision( + tool_name="tap", + arguments={"x": 1, "y": 2}, + text_output="Opening the send control.", + purpose="Open the send control.", + expected_outcome="The composer is focused.", ) ) + planner = AIPlanner(client=client) + context = _context() - summary = _history_summary(state) + first_step = planner.plan(goal=context.goal, scene=_scene(), context=context)[0] + context.add_step_result( + StepResult( + step=first_step, + success=True, + attempts=1, + result={"ok": True}, + ) + ) + planner.plan(goal=context.goal, scene=_scene(), context=context) - assert summary == [ + assert client.calls[0]["history"] == [] + history = client.calls[1]["history"] + assert history is not None + assert history[0]["tool_name"] == "tap" + assert history[0]["arguments"] == { + "x": 1, + "y": 2, + "purpose": "Open the send control.", + "expected_outcome": "The composer is focused.", + } + assert history[0]["tool_result"]["success"] is True + assert history[0]["tool_result"]["result"] == {"ok": True} + + +def test_ai_planner_limits_history_sent_to_the_llm() -> None: + client = FakeToolCallingClient( + ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2}) + ) + planner = AIPlanner(client=client, config=PlannerConfig(history_max_turns=2)) + context = _context() + context.planner_history.extend( { - "page": "Home", - "action": "tap", - "arguments": {"x": 1, "y": 2}, - "rationale": "Opened settings.", - "purpose": "Open settings.", - "expected_outcome": "Settings is visible.", - "success": True, - }, - { - "page": "Settings", - "action": "swipe", - "arguments": {"start_y": 700, "end_y": 200}, + "user_prompt": f"turn-{index}", + "tool_name": "tap", + "arguments": {}, "rationale": None, - "purpose": None, - "expected_outcome": None, - "success": False, - }, + "tool_result": {"success": True}, + } + for index in range(3) + ) + + planner.plan(goal=context.goal, scene=_scene(), context=context) + + assert [turn["user_prompt"] for turn in client.calls[0]["history"]] == [ + "turn-1", + "turn-2", ] - # Must not contain scene element data - for entry in summary: - assert "scene_summary" not in entry - assert "elements" not in entry - - -def test_history_summary_returns_empty_for_none_world() -> None: - from runtime.ai_planner import _history_summary - - assert _history_summary(None) == [] diff --git a/tests/test_planner_prompts.py b/tests/test_planner_prompts.py index 5311064..97436fd 100644 --- a/tests/test_planner_prompts.py +++ b/tests/test_planner_prompts.py @@ -9,7 +9,6 @@ def test_planner_user_prompt_includes_time_zone_and_configured_device_type() -> prompt = planner_user_prompt( goal="open settings", scene_json={"screen": {"width": 1, "height": 1}, "elements": []}, - history_summary=[], device_platform="ios", now=datetime( 2026, @@ -35,8 +34,16 @@ def test_planner_user_prompt_uses_scene_platform_when_context_is_unavailable() - "elements": [], "app": {"platform": "android"}, }, - history_summary=[], now=datetime(2026, 7, 16, tzinfo=timezone.utc), ) assert "Device type: android" in prompt + + +def test_planner_user_prompt_does_not_duplicate_conversation_history() -> None: + prompt = planner_user_prompt( + goal="open settings", + scene_json={"screen": {"width": 1, "height": 1}, "elements": []}, + ) + + assert "Recent history" not in prompt