145 lines
5.5 KiB
Python
145 lines
5.5 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from dataclasses import dataclass
|
|
from typing import Any, Callable
|
|
|
|
from runtime.planner_config import PlannerConfig
|
|
from runtime.tool_specs import ACTION_TOOL_SPECS, ToolSpec
|
|
|
|
READ_TOOL_SPECS = [
|
|
ToolSpec("take_screenshot", "Capture the current device screen.", {"type": "object", "properties": {"device_id": {"type": "string"}}}),
|
|
ToolSpec("describe_screen", "Inspect the current screen and UI.", {"type": "object", "properties": {"device_id": {"type": "string"}}}),
|
|
ToolSpec("find_text", "Find visible text on the current screen.", {"type": "object", "required": ["query"], "properties": {"query": {"type": "string"}, "device_id": {"type": "string"}}}),
|
|
ToolSpec("get_ui_tree", "Read the current accessibility/UI tree.", {"type": "object", "properties": {"device_id": {"type": "string"}, "include_app_info": {"type": "boolean"}}}),
|
|
ToolSpec("list_devices", "List configured devices.", {"type": "object", "properties": {}}),
|
|
ToolSpec("device_status", "Read one device status.", {"type": "object", "required": ["device_id"], "properties": {"device_id": {"type": "string"}}}),
|
|
]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ChatResult:
|
|
content: str
|
|
tool_calls: int
|
|
|
|
|
|
class ConversationAgent:
|
|
"""Small OpenAI-compatible agent loop backed by Host Agent tools."""
|
|
|
|
def __init__(
|
|
self,
|
|
*,
|
|
config: PlannerConfig,
|
|
tools: dict[str, Callable[..., Any]],
|
|
max_rounds: int = 8,
|
|
) -> None:
|
|
self.config = config
|
|
self.tools = tools
|
|
self.max_rounds = max_rounds
|
|
self._client: Any | None = None
|
|
|
|
def chat(self, messages: list[dict[str, str]]) -> ChatResult:
|
|
if self.config.provider not in {"openai", "openai_compatible"}:
|
|
raise RuntimeError(
|
|
"conversation chat currently requires an OpenAI-compatible provider"
|
|
)
|
|
client = self._get_client()
|
|
history: list[dict[str, Any]] = [
|
|
{
|
|
"role": "system",
|
|
"content": (
|
|
"You are a mobile device assistant. Reply naturally when no "
|
|
"device action is needed. Call a tool when the user asks you "
|
|
"to inspect or operate a device. Never claim an action was "
|
|
"completed unless the tool result confirms it."
|
|
),
|
|
},
|
|
*messages,
|
|
]
|
|
calls = 0
|
|
for _ in range(self.max_rounds):
|
|
response = client.chat.completions.create(
|
|
model=self.config.resolved_model(),
|
|
messages=history,
|
|
tools=[_openai_tool(spec) for spec in [*ACTION_TOOL_SPECS, *READ_TOOL_SPECS]],
|
|
tool_choice="auto",
|
|
parallel_tool_calls=False,
|
|
timeout=self.config.timeout,
|
|
)
|
|
message = response.choices[0].message
|
|
text = getattr(message, "content", None)
|
|
tool_calls = getattr(message, "tool_calls", None) or []
|
|
if not tool_calls:
|
|
return ChatResult(content=str(text or ""), tool_calls=calls)
|
|
history.append(_message_dict(message))
|
|
for tool_call in tool_calls:
|
|
name = tool_call.function.name
|
|
try:
|
|
arguments = json.loads(tool_call.function.arguments or "{}")
|
|
if not isinstance(arguments, dict):
|
|
raise ValueError("tool arguments must be an object")
|
|
arguments.pop("purpose", None)
|
|
arguments.pop("expected_outcome", None)
|
|
result = self.tools[name](**arguments)
|
|
except Exception as exc:
|
|
result = {"ok": False, "error": str(exc)}
|
|
calls += 1
|
|
history.append(
|
|
{
|
|
"role": "tool",
|
|
"tool_call_id": tool_call.id,
|
|
"content": json.dumps(result, ensure_ascii=True, default=str),
|
|
}
|
|
)
|
|
raise RuntimeError("conversation exceeded the maximum tool-call rounds")
|
|
|
|
def _get_client(self) -> Any:
|
|
if self._client is None:
|
|
from openai import OpenAI
|
|
|
|
kwargs: dict[str, Any] = {}
|
|
if self.config.api_key:
|
|
kwargs["api_key"] = self.config.api_key
|
|
if self.config.base_url:
|
|
kwargs["base_url"] = self.config.base_url
|
|
self._client = OpenAI(**kwargs)
|
|
return self._client
|
|
|
|
|
|
def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
|
|
parameters = dict(spec.parameters)
|
|
properties = dict(parameters.get("properties") or {})
|
|
properties.setdefault(
|
|
"device_id",
|
|
{"type": "string", "description": "Target device ID when needed."},
|
|
)
|
|
parameters["properties"] = properties
|
|
return {
|
|
"type": "function",
|
|
"function": {
|
|
"name": spec.name,
|
|
"description": spec.description,
|
|
"parameters": parameters,
|
|
},
|
|
}
|
|
|
|
|
|
def _message_dict(message: Any) -> dict[str, Any]:
|
|
result: dict[str, Any] = {"role": "assistant"}
|
|
content = getattr(message, "content", None)
|
|
if content is not None:
|
|
result["content"] = content
|
|
calls = getattr(message, "tool_calls", None) or []
|
|
result["tool_calls"] = [
|
|
{
|
|
"id": call.id,
|
|
"type": "function",
|
|
"function": {
|
|
"name": call.function.name,
|
|
"arguments": call.function.arguments,
|
|
},
|
|
}
|
|
for call in calls
|
|
]
|
|
return result
|