feat(agent-runtime): add LLM-driven AI Planner with dual-provider tool calling
Replaces the stub Planner's fixed describe_screen/[] behavior with a real decision-maker: AIPlanner uses native tool/function calling (Anthropic or OpenAI, pluggable via AI_PLANNER_PROVIDER) to select exactly one grounded action per turn, with an explicit finish_task(success, reason) tool for completion/failure instead of an ambiguous "no tool call" signal. Default disabled (AI_PLANNER_ENABLED=false) and additive; TaskRunner falls back to the existing stub Planner unchanged when disabled. Amends CONSTITUTION.md's Perception Boundary with one narrow exception: only the AI Planner may receive the current step's raw screenshot bytes alongside Scene, for vision-grounded coordinate grounding. Also fixes a latent gap in TaskRunner.run(): observe/plan exceptions are now caught per iteration and turned into a failed task with a failure_reason, instead of propagating uncaught. openspec change: ai-planner-runtime. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
+40
-18
@@ -6,9 +6,11 @@ from dataclasses import dataclass, replace
|
||||
from inspect import Parameter, signature
|
||||
|
||||
from core.models import Scene, Task, utc_now
|
||||
from runtime.ai_planner import AIPlanner
|
||||
from runtime.context import TaskContext
|
||||
from runtime.executor import Executor
|
||||
from runtime.planner import PlannedStep, Planner
|
||||
from runtime.planner_config import PlannerConfig, load_config as load_planner_config
|
||||
from semantic.models import SemanticScene
|
||||
from skills_learning.config import (
|
||||
SkillAuthoringConfig,
|
||||
@@ -56,8 +58,10 @@ class TaskRunner:
|
||||
skill_authoring_config: SkillAuthoringConfig | None = None,
|
||||
skill_store: SkillStore | None = None,
|
||||
skill_embedding_client: EmbeddingClient | None = None,
|
||||
planner_config: PlannerConfig | None = None,
|
||||
) -> None:
|
||||
self.planner = planner or Planner()
|
||||
self.planner_config = planner_config or load_planner_config()
|
||||
self.planner = planner or self._default_planner()
|
||||
self.executor = executor or Executor()
|
||||
self.timeline = timeline
|
||||
self.metadata_store = metadata_store
|
||||
@@ -93,9 +97,20 @@ class TaskRunner:
|
||||
self._update_task(task, status="running")
|
||||
|
||||
for _ in range(self.config.max_steps):
|
||||
scene = self.observer(task.device_id)
|
||||
context.add_scene(scene)
|
||||
steps = self._plan(task.goal, scene, context)
|
||||
try:
|
||||
scene = self.observer(task.device_id)
|
||||
context.add_scene(scene)
|
||||
screenshot = self._planning_screenshot(task.device_id)
|
||||
steps = self._plan(task.goal, scene, context, screenshot=screenshot)
|
||||
except Exception as exc:
|
||||
reason = f"{type(exc).__name__}: {exc}" if str(exc) else type(exc).__name__
|
||||
self._update_task(
|
||||
task,
|
||||
status="failed",
|
||||
completed=True,
|
||||
failure_reason=reason,
|
||||
)
|
||||
return task
|
||||
if not steps or self.planner.goal_reached(
|
||||
goal=task.goal,
|
||||
scene=scene,
|
||||
@@ -195,31 +210,41 @@ class TaskRunner:
|
||||
model_name=self.skill_authoring_config.embedding_model,
|
||||
)
|
||||
|
||||
def _default_planner(self) -> Planner:
|
||||
if self.planner_config.enabled:
|
||||
return AIPlanner(config=self.planner_config)
|
||||
return Planner()
|
||||
|
||||
def _plan(
|
||||
self,
|
||||
goal: str,
|
||||
scene: Scene,
|
||||
context: TaskContext,
|
||||
screenshot: bytes | None = None,
|
||||
) -> list[PlannedStep]:
|
||||
if context.world is not None and self._planner_accepts_world():
|
||||
return self.planner.plan(
|
||||
goal=goal,
|
||||
scene=scene,
|
||||
context=context,
|
||||
world=context.world,
|
||||
)
|
||||
return self.planner.plan(goal=goal, scene=scene, context=context)
|
||||
kwargs: dict[str, object] = {}
|
||||
if context.world is not None and self._planner_accepts("world"):
|
||||
kwargs["world"] = context.world
|
||||
if screenshot is not None and self._planner_accepts("screenshot"):
|
||||
kwargs["screenshot"] = screenshot
|
||||
return self.planner.plan(goal=goal, scene=scene, context=context, **kwargs)
|
||||
|
||||
def _planner_accepts_world(self) -> bool:
|
||||
def _planner_accepts(self, name: str) -> bool:
|
||||
try:
|
||||
parameters = signature(self.planner.plan).parameters
|
||||
except (TypeError, ValueError):
|
||||
return True
|
||||
return "world" in parameters or any(
|
||||
return name in parameters or any(
|
||||
parameter.kind is Parameter.VAR_KEYWORD
|
||||
for parameter in parameters.values()
|
||||
)
|
||||
|
||||
def _planning_screenshot(self, device_id: str) -> bytes | None:
|
||||
try:
|
||||
return self.screenshot_provider(device_id)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _update_world(
|
||||
self,
|
||||
world_handle: TaskWorldView | None,
|
||||
@@ -255,10 +280,7 @@ class TaskRunner:
|
||||
) -> None:
|
||||
if not self.timeline:
|
||||
return
|
||||
try:
|
||||
screenshot = self.screenshot_provider(task.device_id)
|
||||
except Exception:
|
||||
screenshot = None
|
||||
screenshot = self._planning_screenshot(task.device_id)
|
||||
self.timeline.append(
|
||||
task_id=task.id,
|
||||
scene=scene.to_dict(),
|
||||
|
||||
Reference in New Issue
Block a user