Improve local agent chat and device health detection

2026-08-29 16:03:15 +08:00
parent 050d1329c4
commit a315c62f3a
11 changed files with 289 additions and 4 deletions
@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from runtime.planner_config import PlannerConfig
+from runtime.tool_specs import ACTION_TOOL_SPECS, ToolSpec
+
+READ_TOOL_SPECS = [
+    ToolSpec("take_screenshot", "Capture the current device screen.", {"type": "object", "properties": {"device_id": {"type": "string"}}}),
+    ToolSpec("describe_screen", "Inspect the current screen and UI.", {"type": "object", "properties": {"device_id": {"type": "string"}}}),
+    ToolSpec("find_text", "Find visible text on the current screen.", {"type": "object", "required": ["query"], "properties": {"query": {"type": "string"}, "device_id": {"type": "string"}}}),
+    ToolSpec("get_ui_tree", "Read the current accessibility/UI tree.", {"type": "object", "properties": {"device_id": {"type": "string"}, "include_app_info": {"type": "boolean"}}}),
+    ToolSpec("list_devices", "List configured devices.", {"type": "object", "properties": {}}),
+    ToolSpec("device_status", "Read one device status.", {"type": "object", "required": ["device_id"], "properties": {"device_id": {"type": "string"}}}),
+]
+
+
+@dataclass(frozen=True)
+class ChatResult:
+    content: str
+    tool_calls: int
+
+
+class ConversationAgent:
+    """Small OpenAI-compatible agent loop backed by Host Agent tools."""
+
+    def __init__(
+        self,
+        *,
+        config: PlannerConfig,
+        tools: dict[str, Callable[..., Any]],
+        max_rounds: int = 8,
+    ) -> None:
+        self.config = config
+        self.tools = tools
+        self.max_rounds = max_rounds
+        self._client: Any | None = None
+
+    def chat(self, messages: list[dict[str, str]]) -> ChatResult:
+        if self.config.provider not in {"openai", "openai_compatible"}:
+            raise RuntimeError(
+                "conversation chat currently requires an OpenAI-compatible provider"
+            )
+        client = self._get_client()
+        history: list[dict[str, Any]] = [
+            {
+                "role": "system",
+                "content": (
+                    "You are a mobile device assistant. Reply naturally when no "
+                    "device action is needed. Call a tool when the user asks you "
+                    "to inspect or operate a device. Never claim an action was "
+                    "completed unless the tool result confirms it."
+                ),
+            },
+            *messages,
+        ]
+        calls = 0
+        for _ in range(self.max_rounds):
+            response = client.chat.completions.create(
+                model=self.config.resolved_model(),
+                messages=history,
+                tools=[_openai_tool(spec) for spec in [*ACTION_TOOL_SPECS, *READ_TOOL_SPECS]],
+                tool_choice="auto",
+                parallel_tool_calls=False,
+                timeout=self.config.timeout,
+            )
+            message = response.choices[0].message
+            text = getattr(message, "content", None)
+            tool_calls = getattr(message, "tool_calls", None) or []
+            if not tool_calls:
+                return ChatResult(content=str(text or ""), tool_calls=calls)
+            history.append(_message_dict(message))
+            for tool_call in tool_calls:
+                name = tool_call.function.name
+                try:
+                    arguments = json.loads(tool_call.function.arguments or "{}")
+                    if not isinstance(arguments, dict):
+                        raise ValueError("tool arguments must be an object")
+                    arguments.pop("purpose", None)
+                    arguments.pop("expected_outcome", None)
+                    result = self.tools[name](**arguments)
+                except Exception as exc:
+                    result = {"ok": False, "error": str(exc)}
+                calls += 1
+                history.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": tool_call.id,
+                        "content": json.dumps(result, ensure_ascii=True, default=str),
+                    }
+                )
+        raise RuntimeError("conversation exceeded the maximum tool-call rounds")
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            from openai import OpenAI
+
+            kwargs: dict[str, Any] = {}
+            if self.config.api_key:
+                kwargs["api_key"] = self.config.api_key
+            if self.config.base_url:
+                kwargs["base_url"] = self.config.base_url
+            self._client = OpenAI(**kwargs)
+        return self._client
+
+
+def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
+    parameters = dict(spec.parameters)
+    properties = dict(parameters.get("properties") or {})
+    properties.setdefault(
+        "device_id",
+        {"type": "string", "description": "Target device ID when needed."},
+    )
+    parameters["properties"] = properties
+    return {
+        "type": "function",
+        "function": {
+            "name": spec.name,
+            "description": spec.description,
+            "parameters": parameters,
+        },
+    }
+
+
+def _message_dict(message: Any) -> dict[str, Any]:
+    result: dict[str, Any] = {"role": "assistant"}
+    content = getattr(message, "content", None)
+    if content is not None:
+        result["content"] = content
+    calls = getattr(message, "tool_calls", None) or []
+    result["tool_calls"] = [
+        {
+            "id": call.id,
+            "type": "function",
+            "function": {
+                "name": call.function.name,
+                "arguments": call.function.arguments,
+            },
+        }
+        for call in calls
+    ]
+    return result