feat(agent-runtime): add LLM-driven AI Planner with dual-provider tool calling

Replaces the stub Planner's fixed describe_screen/[] behavior with a real
decision-maker: AIPlanner uses native tool/function calling (Anthropic or
OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded
action per turn, with an explicit finish_task(success, reason) tool for
completion/failure instead of an ambiguous "no tool call" signal. Default
disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to
the existing stub Planner unchanged when disabled.

Amends CONSTITUTION.md's Perception Boundary with one narrow exception:
only the AI Planner may receive the current step's raw screenshot bytes
alongside Scene, for vision-grounded coordinate grounding. Also fixes a
latent gap in TaskRunner.run(): observe/plan exceptions are now caught per
iteration and turned into a failed task with a failure_reason, instead of
propagating uncaught.

openspec change: ai-planner-runtime.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-12 13:48:50 +08:00
co-authored by Claude Sonnet 5
parent b94abde92a
commit 61ff3b425d
19 changed files with 1977 additions and 19 deletions
+40 -18
View File
@@ -6,9 +6,11 @@ from dataclasses import dataclass, replace
from inspect import Parameter, signature
from core.models import Scene, Task, utc_now
from runtime.ai_planner import AIPlanner
from runtime.context import TaskContext
from runtime.executor import Executor
from runtime.planner import PlannedStep, Planner
from runtime.planner_config import PlannerConfig, load_config as load_planner_config
from semantic.models import SemanticScene
from skills_learning.config import (
SkillAuthoringConfig,
@@ -56,8 +58,10 @@ class TaskRunner:
skill_authoring_config: SkillAuthoringConfig | None = None,
skill_store: SkillStore | None = None,
skill_embedding_client: EmbeddingClient | None = None,
planner_config: PlannerConfig | None = None,
) -> None:
self.planner = planner or Planner()
self.planner_config = planner_config or load_planner_config()
self.planner = planner or self._default_planner()
self.executor = executor or Executor()
self.timeline = timeline
self.metadata_store = metadata_store
@@ -93,9 +97,20 @@ class TaskRunner:
self._update_task(task, status="running")
for _ in range(self.config.max_steps):
scene = self.observer(task.device_id)
context.add_scene(scene)
steps = self._plan(task.goal, scene, context)
try:
scene = self.observer(task.device_id)
context.add_scene(scene)
screenshot = self._planning_screenshot(task.device_id)
steps = self._plan(task.goal, scene, context, screenshot=screenshot)
except Exception as exc:
reason = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
self._update_task(
task,
status="failed",
completed=True,
failure_reason=reason,
)
return task
if not steps or self.planner.goal_reached(
goal=task.goal,
scene=scene,
@@ -195,31 +210,41 @@ class TaskRunner:
model_name=self.skill_authoring_config.embedding_model,
)
def _default_planner(self) -> Planner:
if self.planner_config.enabled:
return AIPlanner(config=self.planner_config)
return Planner()
def _plan(
self,
goal: str,
scene: Scene,
context: TaskContext,
screenshot: bytes | None = None,
) -> list[PlannedStep]:
if context.world is not None and self._planner_accepts_world():
return self.planner.plan(
goal=goal,
scene=scene,
context=context,
world=context.world,
)
return self.planner.plan(goal=goal, scene=scene, context=context)
kwargs: dict[str, object] = {}
if context.world is not None and self._planner_accepts("world"):
kwargs["world"] = context.world
if screenshot is not None and self._planner_accepts("screenshot"):
kwargs["screenshot"] = screenshot
return self.planner.plan(goal=goal, scene=scene, context=context, **kwargs)
def _planner_accepts_world(self) -> bool:
def _planner_accepts(self, name: str) -> bool:
try:
parameters = signature(self.planner.plan).parameters
except (TypeError, ValueError):
return True
return "world" in parameters or any(
return name in parameters or any(
parameter.kind is Parameter.VAR_KEYWORD
for parameter in parameters.values()
)
def _planning_screenshot(self, device_id: str) -> bytes | None:
try:
return self.screenshot_provider(device_id)
except Exception:
return None
def _update_world(
self,
world_handle: TaskWorldView | None,
@@ -255,10 +280,7 @@ class TaskRunner:
) -> None:
if not self.timeline:
return
try:
screenshot = self.screenshot_provider(task.device_id)
except Exception:
screenshot = None
screenshot = self._planning_screenshot(task.device_id)
self.timeline.append(
task_id=task.id,
scene=scene.to_dict(),