feat(planner): add execution context to prompts
Tests / Test passed: 977

This commit is contained in:
2026-07-16 10:25:20 +08:00
parent 240b7be7b8
commit 358f4623ba
9 changed files with 159 additions and 3 deletions
@@ -50,6 +50,9 @@ def create_execution_factories(
timeline=timeline, timeline=timeline,
planner=_host_agent_planner(resolved_host_agent_config), planner=_host_agent_planner(resolved_host_agent_config),
planner_config=_host_agent_planner_config(), planner_config=_host_agent_planner_config(),
device_platform_provider=lambda device_id: _device_platform(
manager, device_id
),
) )
def create_workflow_runner() -> WorkflowRunner: def create_workflow_runner() -> WorkflowRunner:
@@ -103,3 +106,15 @@ def _host_agent_planner(
client=CloudProxyToolCallingClient(resolved_config), client=CloudProxyToolCallingClient(resolved_config),
config=planner_config, config=planner_config,
) )
def _device_platform(manager: DeviceManager, device_id: str) -> str | None:
for device in manager.list_devices():
if device.id != device_id:
continue
if device.driver_type == "wda":
return "ios"
if device.driver_type == "uiautomator2":
return "android"
return None
return None
@@ -141,6 +141,24 @@ def test_created_task_runner_observer_uses_configured_manager(
assert seen == {"device_id": "phone-1", "manager": manager} assert seen == {"device_id": "phone-1", "manager": manager}
@pytest.mark.parametrize(
("driver_type", "expected_platform"),
[("wda", "ios"), ("uiautomator2", "android")],
)
def test_created_task_runner_resolves_platform_from_configured_driver(
tmp_path, driver_type: str, expected_platform: str
) -> None:
manager = DeviceManager()
manager.register_device("phone-1", lambda: FakeDriver(), driver_type=driver_type)
factories = create_execution_factories(
manager, workflow_store=WorkflowStore(tmp_path / "workflows.sqlite3")
)
task_runner = factories.task_runner_factory()
assert task_runner.device_platform_provider is not None
assert task_runner.device_platform_provider("phone-1") == expected_platform
def test_created_task_runner_defaults_to_ai_planner( def test_created_task_runner_defaults_to_ai_planner(
tmp_path, monkeypatch: pytest.MonkeyPatch tmp_path, monkeypatch: pytest.MonkeyPatch
) -> None: ) -> None:
+1
View File
@@ -40,6 +40,7 @@ class AIPlanner(Planner):
goal=goal, goal=goal,
scene_json=scene.to_dict(), scene_json=scene.to_dict(),
history_summary=_history_summary(world), history_summary=_history_summary(world),
device_platform=context.device_platform,
) )
decision = self.client.decide( decision = self.client.decide(
system_prompt=PLANNER_SYSTEM_PROMPT, system_prompt=PLANNER_SYSTEM_PROMPT,
+1
View File
@@ -14,6 +14,7 @@ if TYPE_CHECKING:
class TaskContext: class TaskContext:
task_id: str task_id: str
goal: str goal: str
device_platform: str | None = None
scenes: list[Scene] = field(default_factory=list) scenes: list[Scene] = field(default_factory=list)
step_results: list["StepResult"] = field(default_factory=list) step_results: list["StepResult"] = field(default_factory=list)
world: "WorldState | None" = None world: "WorldState | None" = None
+24
View File
@@ -1,6 +1,7 @@
from __future__ import annotations from __future__ import annotations
import json import json
from datetime import datetime
from typing import Any from typing import Any
PLANNER_SYSTEM_PROMPT = """You are the planning brain of a mobile device automation agent. PLANNER_SYSTEM_PROMPT = """You are the planning brain of a mobile device automation agent.
@@ -66,8 +67,18 @@ def planner_user_prompt(
goal: str, goal: str,
scene_json: dict[str, Any], scene_json: dict[str, Any],
history_summary: list[dict[str, Any]], history_summary: list[dict[str, Any]],
device_platform: str | None = None,
now: datetime | None = None,
) -> str: ) -> str:
current_time = now or datetime.now().astimezone()
if current_time.tzinfo is None:
current_time = current_time.astimezone()
timezone_name = current_time.tzname() or str(current_time.tzinfo) or "unknown"
return ( return (
"Execution context:\n"
f"Current date and time: {current_time.isoformat(timespec='seconds')}\n"
f"Time zone: {timezone_name}\n"
f"Device type: {_device_type(scene_json, device_platform)}\n\n"
"Goal:\n" "Goal:\n"
f"{goal}\n\n" f"{goal}\n\n"
"Current Scene (JSON):\n" "Current Scene (JSON):\n"
@@ -76,3 +87,16 @@ def planner_user_prompt(
f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n" f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n"
"Call exactly one tool for this turn." "Call exactly one tool for this turn."
) )
def _device_type(scene_json: dict[str, Any], device_platform: str | None) -> str:
platform = device_platform
if platform is None:
app = scene_json.get("app")
if isinstance(app, dict):
raw_platform = app.get("platform")
platform = raw_platform if isinstance(raw_platform, str) else None
if not isinstance(platform, str):
return "unknown"
normalized = platform.strip().lower()
return normalized if normalized in {"ios", "android"} else "unknown"
+23 -2
View File
@@ -42,6 +42,7 @@ TaskSucceededHook = Callable[[str, str, Timeline], None]
StopRequested = Callable[[], bool] StopRequested = Callable[[], bool]
StopReason = Callable[[], "str | None"] StopReason = Callable[[], "str | None"]
StepProgressCallback = Callable[[int, str, str], None] StepProgressCallback = Callable[[int, str, str], None]
DevicePlatformProvider = Callable[[str], str | None]
def is_cancellation_reason(reason: str | None) -> bool: def is_cancellation_reason(reason: str | None) -> bool:
@@ -68,6 +69,7 @@ class TaskRunner:
skill_embedding_client: EmbeddingClient | None = None, skill_embedding_client: EmbeddingClient | None = None,
planner_config: PlannerConfig | None = None, planner_config: PlannerConfig | None = None,
on_step_progress: StepProgressCallback | None = None, on_step_progress: StepProgressCallback | None = None,
device_platform_provider: DevicePlatformProvider | None = None,
) -> None: ) -> None:
self.planner_config = planner_config or load_planner_config() self.planner_config = planner_config or load_planner_config()
self.planner = planner or self._default_planner() self.planner = planner or self._default_planner()
@@ -98,6 +100,7 @@ class TaskRunner:
else: else:
self.on_task_succeeded = None self.on_task_succeeded = None
self.on_step_progress = on_step_progress self.on_step_progress = on_step_progress
self.device_platform_provider = device_platform_provider
def run( def run(
self, self,
@@ -108,7 +111,11 @@ class TaskRunner:
) -> Task: ) -> Task:
if self.metadata_store: if self.metadata_store:
self.metadata_store.create_task(task) self.metadata_store.create_task(task)
context = TaskContext(task_id=task.id, goal=task.goal) context = TaskContext(
task_id=task.id,
goal=task.goal,
device_platform=self._device_platform(task.device_id),
)
world_handle = self._start_world_view(task.id) world_handle = self._start_world_view(task.id)
if world_handle is not None: if world_handle is not None:
context.world = world_handle.state context.world = world_handle.state
@@ -201,7 +208,9 @@ class TaskRunner:
) )
return task return task
def _interrupt_task(self, task: Task, stop_reason: StopReason | None = None) -> Task: def _interrupt_task(
self, task: Task, stop_reason: StopReason | None = None
) -> Task:
reason = stop_reason() if stop_reason is not None else None reason = stop_reason() if stop_reason is not None else None
message = reason or "execution interrupted" message = reason or "execution interrupted"
status = "cancelled" if is_cancellation_reason(reason) else "failed" status = "cancelled" if is_cancellation_reason(reason) else "failed"
@@ -307,6 +316,18 @@ class TaskRunner:
return AIPlanner(config=self.planner_config) return AIPlanner(config=self.planner_config)
return Planner() return Planner()
def _device_platform(self, device_id: str) -> str | None:
if self.device_platform_provider is None:
return None
try:
return self.device_platform_provider(device_id)
except Exception:
logger.warning(
"device platform lookup failed; continuing without configured platform",
exc_info=True,
)
return None
def _plan( def _plan(
self, self,
goal: str, goal: str,
+16
View File
@@ -146,6 +146,22 @@ def test_ai_planner_forwards_tools_screenshot_and_timeout_to_client() -> None:
assert "send a message" in call["user_prompt"] assert "send a message" in call["user_prompt"]
def test_ai_planner_includes_device_platform_from_task_context() -> None:
client = FakeToolCallingClient(
ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
)
planner = AIPlanner(client=client)
context = TaskContext(
task_id="task-1",
goal="send a message",
device_platform="android",
)
planner.plan(goal="send a message", scene=_scene(), context=context)
assert "Device type: android" in client.calls[0]["user_prompt"]
def test_ai_planner_populates_step_prompt_from_user_prompt() -> None: def test_ai_planner_populates_step_prompt_from_user_prompt() -> None:
"""PlannedStep.prompt should carry the actual user prompt sent to the LLM, """PlannedStep.prompt should carry the actual user prompt sent to the LLM,
not the bare task goal.""" not the bare task goal."""
+19 -1
View File
@@ -46,9 +46,11 @@ class NarrowSignaturePlanner(Planner):
class ScreenshotRecordingPlanner(Planner): class ScreenshotRecordingPlanner(Planner):
def __init__(self) -> None: def __init__(self) -> None:
self.screenshots: list[bytes | None] = [] self.screenshots: list[bytes | None] = []
self.device_platforms: list[str | None] = []
def plan(self, *, goal, scene, context, screenshot=None): def plan(self, *, goal, scene, context, screenshot=None):
self.screenshots.append(screenshot) self.screenshots.append(screenshot)
self.device_platforms.append(context.device_platform)
if context.step_results: if context.step_results:
return [] return []
return [PlannedStep(action="tap", description="tap")] return [PlannedStep(action="tap", description="tap")]
@@ -69,7 +71,9 @@ def _scene() -> Scene:
) )
def _runner(*, planner=None, planner_config=None, observer=None) -> TaskRunner: def _runner(
*, planner=None, planner_config=None, observer=None, device_platform_provider=None
) -> TaskRunner:
return TaskRunner( return TaskRunner(
planner=planner, planner=planner,
planner_config=planner_config, planner_config=planner_config,
@@ -80,6 +84,7 @@ def _runner(*, planner=None, planner_config=None, observer=None) -> TaskRunner:
config=TaskRunnerConfig(max_steps=5), config=TaskRunnerConfig(max_steps=5),
observer=observer or (lambda device_id: _scene()), observer=observer or (lambda device_id: _scene()),
screenshot_provider=lambda device_id: PNG_10X20, screenshot_provider=lambda device_id: PNG_10X20,
device_platform_provider=device_platform_provider,
) )
@@ -126,6 +131,19 @@ def test_task_runner_passes_screenshot_to_planner_that_declares_it() -> None:
assert planner.screenshots == [PNG_10X20, PNG_10X20] assert planner.screenshots == [PNG_10X20, PNG_10X20]
def test_task_runner_passes_configured_device_platform_to_planner_context() -> None:
planner = ScreenshotRecordingPlanner()
runner = _runner(
planner=planner,
device_platform_provider=lambda device_id: "ios",
)
result = runner.run(Task(goal="inspect", device_id="phone"))
assert result.status == "completed"
assert planner.device_platforms == ["ios", "ios"]
def test_task_runner_default_planner_is_stub_when_ai_planner_disabled() -> None: def test_task_runner_default_planner_is_stub_when_ai_planner_disabled() -> None:
runner = _runner(planner=None, planner_config=PlannerConfig(enabled=False)) runner = _runner(planner=None, planner_config=PlannerConfig(enabled=False))
+42
View File
@@ -0,0 +1,42 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from runtime.planner_prompts import planner_user_prompt
def test_planner_user_prompt_includes_time_zone_and_configured_device_type() -> None:
prompt = planner_user_prompt(
goal="open settings",
scene_json={"screen": {"width": 1, "height": 1}, "elements": []},
history_summary=[],
device_platform="ios",
now=datetime(
2026,
7,
16,
9,
8,
7,
tzinfo=timezone(timedelta(hours=8), "Asia/Shanghai"),
),
)
assert "Current date and time: 2026-07-16T09:08:07+08:00" in prompt
assert "Time zone: Asia/Shanghai" in prompt
assert "Device type: ios" in prompt
def test_planner_user_prompt_uses_scene_platform_when_context_is_unavailable() -> None:
prompt = planner_user_prompt(
goal="open settings",
scene_json={
"screen": {"width": 1, "height": 1},
"elements": [],
"app": {"platform": "android"},
},
history_summary=[],
now=datetime(2026, 7, 16, tzinfo=timezone.utc),
)
assert "Device type: android" in prompt