Implement Apex Agent MVP scaffold
This commit is contained in:
@@ -0,0 +1,2 @@
|
||||
"""Agent runtime loop, planner, executor, and in-run context."""
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from core.models import Scene
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from runtime.executor import StepResult
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskContext:
|
||||
task_id: str
|
||||
goal: str
|
||||
scenes: list[Scene] = field(default_factory=list)
|
||||
step_results: list["StepResult"] = field(default_factory=list)
|
||||
|
||||
def add_scene(self, scene: Scene) -> None:
|
||||
self.scenes.append(scene)
|
||||
|
||||
def add_step_result(self, result: "StepResult") -> None:
|
||||
self.step_results.append(result)
|
||||
|
||||
@property
|
||||
def latest_scene(self) -> Scene | None:
|
||||
if not self.scenes:
|
||||
return None
|
||||
return self.scenes[-1]
|
||||
|
||||
@@ -0,0 +1,148 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from time import sleep
|
||||
from typing import Any
|
||||
|
||||
from core.device_manager import DeviceManager
|
||||
from core.errors import ElementNotFoundError
|
||||
from runtime.context import TaskContext
|
||||
from runtime.planner import PlannedStep
|
||||
|
||||
ToolCallable = Callable[..., Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class StepResult:
|
||||
step: PlannedStep
|
||||
success: bool
|
||||
attempts: int
|
||||
result: Any = None
|
||||
error: str | None = None
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"step": {
|
||||
"action": self.step.action,
|
||||
"description": self.step.description,
|
||||
"args": dict(self.step.args),
|
||||
"expected_text": self.step.expected_text,
|
||||
},
|
||||
"success": self.success,
|
||||
"attempts": self.attempts,
|
||||
"result": self.result,
|
||||
"error": self.error,
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutorConfig:
|
||||
max_retries: int = 3
|
||||
backoff_seconds: float = 0.25
|
||||
|
||||
|
||||
class Executor:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
tools: dict[str, ToolCallable] | None = None,
|
||||
config: ExecutorConfig | None = None,
|
||||
) -> None:
|
||||
self.tools = tools or default_tool_registry()
|
||||
self.config = config or ExecutorConfig()
|
||||
|
||||
def execute(
|
||||
self,
|
||||
step: PlannedStep,
|
||||
*,
|
||||
context: TaskContext | None = None,
|
||||
) -> StepResult:
|
||||
if self.config.max_retries < 1:
|
||||
raise ValueError("max_retries must be at least 1")
|
||||
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, self.config.max_retries + 1):
|
||||
try:
|
||||
result = self._execute_once(step, context=context)
|
||||
return StepResult(
|
||||
step=step,
|
||||
success=True,
|
||||
attempts=attempt,
|
||||
result=result,
|
||||
)
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
if attempt < self.config.max_retries:
|
||||
sleep(self.config.backoff_seconds)
|
||||
|
||||
return StepResult(
|
||||
step=step,
|
||||
success=False,
|
||||
attempts=self.config.max_retries,
|
||||
error=str(last_error) if last_error else "step failed",
|
||||
)
|
||||
|
||||
def _execute_once(
|
||||
self,
|
||||
step: PlannedStep,
|
||||
*,
|
||||
context: TaskContext | None,
|
||||
) -> Any:
|
||||
if step.action == "wait_for_text":
|
||||
query = step.args["query"]
|
||||
scene = context.latest_scene if context else None
|
||||
if scene is None:
|
||||
raise ElementNotFoundError("element not found")
|
||||
result = self.tools["find_text"](scene=scene, query=query)
|
||||
if not result.get("found"):
|
||||
raise ElementNotFoundError("element not found")
|
||||
return result
|
||||
|
||||
tool = self.tools.get(step.action)
|
||||
if tool is None:
|
||||
raise KeyError(f"unknown tool action {step.action}")
|
||||
return tool(**step.args)
|
||||
|
||||
|
||||
def default_tool_registry(
|
||||
*,
|
||||
manager: DeviceManager | None = None,
|
||||
) -> dict[str, ToolCallable]:
|
||||
from tools.describe_screen import describe_screen
|
||||
from tools.find_icon import find_icon, find_icon_on_screen
|
||||
from tools.find_text import find_text, find_text_on_screen
|
||||
from tools.input_text import input_text
|
||||
from tools.launch_app import launch_app, terminate_app
|
||||
from tools.screenshot import take_screenshot
|
||||
from tools.swipe import swipe
|
||||
from tools.tap import tap
|
||||
from tools.ui_tree import get_ui_tree
|
||||
|
||||
return {
|
||||
"take_screenshot": _bind_manager(take_screenshot, manager),
|
||||
"screenshot": _bind_manager(take_screenshot, manager),
|
||||
"tap": _bind_manager(tap, manager),
|
||||
"swipe": _bind_manager(swipe, manager),
|
||||
"input_text": _bind_manager(input_text, manager),
|
||||
"launch_app": _bind_manager(launch_app, manager),
|
||||
"terminate_app": _bind_manager(terminate_app, manager),
|
||||
"get_ui_tree": _bind_manager(get_ui_tree, manager),
|
||||
"ui_tree": _bind_manager(get_ui_tree, manager),
|
||||
"describe_screen": _bind_manager(describe_screen, manager),
|
||||
"find_text": find_text,
|
||||
"find_text_on_screen": _bind_manager(find_text_on_screen, manager),
|
||||
"find_icon": find_icon,
|
||||
"find_icon_on_screen": _bind_manager(find_icon_on_screen, manager),
|
||||
}
|
||||
|
||||
|
||||
def _bind_manager(func: ToolCallable, manager: DeviceManager | None) -> ToolCallable:
|
||||
if manager is None:
|
||||
return func
|
||||
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
kwargs.setdefault("manager", manager)
|
||||
return func(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
@@ -0,0 +1,40 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from core.models import Scene
|
||||
from runtime.context import TaskContext
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PlannedStep:
|
||||
action: str
|
||||
description: str
|
||||
args: dict[str, Any] = field(default_factory=dict)
|
||||
expected_text: str | None = None
|
||||
|
||||
|
||||
class Planner:
|
||||
def plan(
|
||||
self,
|
||||
*,
|
||||
goal: str,
|
||||
scene: Scene,
|
||||
context: TaskContext,
|
||||
) -> list[PlannedStep]:
|
||||
if context.step_results:
|
||||
return []
|
||||
return [
|
||||
PlannedStep(
|
||||
action="describe_screen",
|
||||
description=f"Observe current screen for goal: {goal}",
|
||||
args={},
|
||||
)
|
||||
]
|
||||
|
||||
def goal_reached(self, *, goal: str, scene: Scene, context: TaskContext) -> bool:
|
||||
return bool(context.step_results) and all(
|
||||
result.success for result in context.step_results
|
||||
)
|
||||
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, replace
|
||||
|
||||
from core.models import Scene, Task, utc_now
|
||||
from runtime.context import TaskContext
|
||||
from runtime.executor import Executor
|
||||
from runtime.planner import PlannedStep, Planner
|
||||
from storage.task_metadata import TaskMetadataStore
|
||||
from storage.timeline import Timeline
|
||||
from tools.describe_screen import describe_screen
|
||||
from tools.screenshot import take_screenshot
|
||||
|
||||
|
||||
@dataclass
|
||||
class TaskRunnerConfig:
|
||||
max_steps: int = 20
|
||||
|
||||
|
||||
Observer = Callable[[str], Scene]
|
||||
ScreenshotProvider = Callable[[str], bytes]
|
||||
|
||||
|
||||
class TaskRunner:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
planner: Planner | None = None,
|
||||
executor: Executor | None = None,
|
||||
timeline: Timeline | None = None,
|
||||
metadata_store: TaskMetadataStore | None = None,
|
||||
config: TaskRunnerConfig | None = None,
|
||||
observer: Observer | None = None,
|
||||
screenshot_provider: ScreenshotProvider | None = None,
|
||||
) -> None:
|
||||
self.planner = planner or Planner()
|
||||
self.executor = executor or Executor()
|
||||
self.timeline = timeline
|
||||
self.metadata_store = metadata_store
|
||||
self.config = config or TaskRunnerConfig()
|
||||
self.observer = observer or (lambda device_id: describe_screen(device_id))
|
||||
self.screenshot_provider = screenshot_provider or (
|
||||
lambda device_id: take_screenshot(device_id)
|
||||
)
|
||||
|
||||
def run(self, task: Task) -> Task:
|
||||
context = TaskContext(task_id=task.id, goal=task.goal)
|
||||
self._update_task(task, status="running")
|
||||
|
||||
for _ in range(self.config.max_steps):
|
||||
scene = self.observer(task.device_id)
|
||||
context.add_scene(scene)
|
||||
steps = self.planner.plan(goal=task.goal, scene=scene, context=context)
|
||||
if not steps or self.planner.goal_reached(
|
||||
goal=task.goal,
|
||||
scene=scene,
|
||||
context=context,
|
||||
):
|
||||
self._update_task(task, status="completed", completed=True)
|
||||
return task
|
||||
|
||||
for step in steps:
|
||||
result = self.executor.execute(
|
||||
self._step_for_device(step, task.device_id),
|
||||
context=context,
|
||||
)
|
||||
context.add_step_result(result)
|
||||
self._append_timeline(task, scene, step, result)
|
||||
if not result.success:
|
||||
self._update_task(
|
||||
task,
|
||||
status="failed",
|
||||
completed=True,
|
||||
failure_reason=result.error or "step failed",
|
||||
)
|
||||
return task
|
||||
|
||||
self._update_task(
|
||||
task,
|
||||
status="failed",
|
||||
completed=True,
|
||||
failure_reason=f"max steps exceeded: {self.config.max_steps}",
|
||||
)
|
||||
return task
|
||||
|
||||
def _append_timeline(
|
||||
self,
|
||||
task: Task,
|
||||
scene: Scene,
|
||||
step: PlannedStep,
|
||||
result: object,
|
||||
) -> None:
|
||||
if not self.timeline:
|
||||
return
|
||||
try:
|
||||
screenshot = self.screenshot_provider(task.device_id)
|
||||
except Exception:
|
||||
screenshot = None
|
||||
self.timeline.append(
|
||||
task_id=task.id,
|
||||
scene=scene.to_dict(),
|
||||
prompt=task.goal,
|
||||
tool_call={
|
||||
"action": step.action,
|
||||
"description": step.description,
|
||||
"args": step.args,
|
||||
},
|
||||
result=result.to_dict() if hasattr(result, "to_dict") else {"result": result},
|
||||
screenshot=screenshot,
|
||||
)
|
||||
|
||||
def _step_for_device(self, step: PlannedStep, device_id: str) -> PlannedStep:
|
||||
device_scoped_actions = {
|
||||
"take_screenshot",
|
||||
"screenshot",
|
||||
"tap",
|
||||
"swipe",
|
||||
"input_text",
|
||||
"launch_app",
|
||||
"terminate_app",
|
||||
"get_ui_tree",
|
||||
"ui_tree",
|
||||
"describe_screen",
|
||||
"find_text_on_screen",
|
||||
"find_icon_on_screen",
|
||||
}
|
||||
if step.action not in device_scoped_actions or "device_id" in step.args:
|
||||
return step
|
||||
return replace(step, args={**step.args, "device_id": device_id})
|
||||
|
||||
def _update_task(
|
||||
self,
|
||||
task: Task,
|
||||
*,
|
||||
status: str,
|
||||
completed: bool = False,
|
||||
failure_reason: str | None = None,
|
||||
) -> None:
|
||||
task.status = status # type: ignore[assignment]
|
||||
task.updated_at = utc_now()
|
||||
if completed:
|
||||
task.completed_at = utc_now()
|
||||
task.failure_reason = failure_reason
|
||||
if self.metadata_store:
|
||||
self.metadata_store.update_task(
|
||||
task.id,
|
||||
status=task.status,
|
||||
completed=completed,
|
||||
failure_reason=failure_reason,
|
||||
)
|
||||
Reference in New Issue
Block a user