feat: discover and add connected iOS devices

README.md

@@ -60,6 +60,83 @@ operator view for the tasks that actually execute on that Host, including
 per-step screenshots, OCR observations, and UI-tree results. The Cloud Console
 remains the fleet-level view for dispatch status and Cloud-proxy planner history.
 
+## Local-only Host Agent
+
+Run without a Cloud Control Plane by setting `HOST_AGENT_MODE=local`. Tasks
+submitted in the Host console are queued and executed in the same process:
+
+```bash
+export HOST_AGENT_MODE=local
+export AI_PLANNER_ENABLED=true
+export AI_PLANNER_PROVIDER=openai-compatible
+export AI_PLANNER_MODEL=qwen2.5
+export AI_PLANNER_API_KEY=local-key
+export AI_PLANNER_BASE_URL=http://127.0.0.1:11434/v1
+export AI_PLANNER_MULTIMODAL=true
+uv run --package device-host-agent device-host-agent setup
+uv run --package device-host-agent device-host-agent
+```
+
+`openai-compatible` works with Ollama, LM Studio, vLLM, or another server that
+implements OpenAI `/chat/completions`. Hosted `openai` and `anthropic` providers
+also accept `AI_PLANNER_API_KEY` and their conventional API key variables.
+
+The local Host console also exposes an authenticated conversational Agent API:
+
+```text
+POST http://127.0.0.1:8765/api/chat
+```
+
+Send a JSON body containing `device_id` and `messages` (`user`/`assistant` roles). The Agent can
+return ordinary assistant text or call the same device-operation tool contracts
+used by the Runtime; each tool result is fed back to the model before the final
+reply is returned. The endpoint uses the Host console session cookie, so it is
+not an unauthenticated device-control endpoint.
+
+Every chat request is bound to exactly one registered `device_id`. Keep a
+separate message history and client session for each phone; the Agent injects
+the bound device into device tools and rejects cross-device tool arguments.
+
+For vision-capable OpenAI models, a user message may contain standard OpenAI
+multimodal blocks:
+
+```json
+{
+  "messages": [{
+    "role": "user",
+    "content": [
+      {"type": "text", "text": "点击图片中的登录按钮"},
+      {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}
+    ]
+  }]
+}
+```
+
+The compact form `{ "role": "user", "text": "...", "image_base64": "..." }`
+is also accepted. The model can inspect the supplied image and then call a
+phone tool such as `tap` in the same conversation.
+
+Set `AI_PLANNER_MULTIMODAL=true` for a vision model. The Planner sends the
+screenshot and omits OCR-only elements and OCR metadata from the structured
+scene payload, avoiding duplicate OCR text.
+
+Local mode records LLM responses, reasoning fields, tool calls, tool results,
+and final replies in a local SQLite database. View them at
+`http://127.0.0.1:8765/conversations`; image bytes are excluded. Set
+`HOST_AGENT_CONVERSATION_LOG_PATH` to change the database path.
+
+The authenticated `Devices` page has an on-demand `Get screenshot` button for
+each connected device. Screenshots are captured only after the operator clicks
+the button; the page does not auto-refresh or capture screenshots as part of
+heartbeat synchronization.
+
+In local mode, Appium supervision is enabled by default. Host Agent probes
+`/status`, adopts a healthy existing Appium instance, starts Appium when no
+listener exists, restarts only processes it started if they crash, and stops
+those child processes on shutdown. Override `HOST_AGENT_APPIUM_*` or set
+`HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED=false` when an external process
+manager owns Appium.
+
 ## Project Direction
 
 The durable roadmap is in [docs/ROADMAP.md](docs/ROADMAP.md). The architecture

apps/device-host-agent/host_agent/app.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import asyncio
 import logging
 from contextlib import suppress
-from dataclasses import dataclass
+from dataclasses import dataclass, replace
 
 import uvicorn
 
@@ -12,6 +12,8 @@ from device.manager import DeviceManager
 from host_agent.assignment import AssignmentExecutor
 from host_agent.client import HostAgentClient, HostAgentEnrollmentClient
 from host_agent.config import HostAgentConfig, load_host_agent_config
+from host_agent.conversation import ConversationAgent
+from host_agent.conversation_log import ConversationLogStore
 from host_agent.dependency_supervisor import DependencySupervisor
 from host_agent.devices import register_local_device
 from host_agent.enrollment import resolve_host_identity
@@ -22,6 +24,7 @@ from host_agent.identity import HostIdentityStore
 from host_agent.instance_lock import InstanceLock
 from host_agent.lease import ActiveAssignmentRunner
 from host_agent.local_account import LocalAccountStore
+from host_agent.local_client import LocalHostAgentClient
 from host_agent.mcp_lock import McpBusyTracker
 from host_agent.mcp_token import McpTokenStore
 from host_agent.policy_cache import HostPolicyCacheStore
@@ -32,6 +35,8 @@ from host_agent.status import AgentStatusTracker
 from host_agent.web.app import create_console_app
 from host_agent.web.auth import SessionManager
 from host_agent.web.mcp import build_mcp_server
+from runtime.executor import default_tool_registry
+from runtime.planner_config import load_config as load_planner_config
 from storage.artifact_store import ArtifactStore
 from storage.device_config import DeviceConfigStore
 from storage.task_metadata import TaskMetadataStore
@@ -173,25 +178,37 @@ def create_application(
             startup_config.identity_path
         )
         owned_enrollment_client = enrollment_client is None
-        bootstrap_client = enrollment_client or HostAgentEnrollmentClient(
-            startup_config
-        )
+        bootstrap_client = enrollment_client or HostAgentEnrollmentClient(startup_config)
         try:
-            resolved_config = resolve_host_identity(
-                startup_config,
-                identity_store=resolved_identity_store,
-                client=bootstrap_client,
-            )
-            bootstrap_client.config = resolved_config
-            resolved_manager = manager or _configured_device_manager(
-                config_store,
-                config=resolved_config,
-                enrollment_client=bootstrap_client,
-            )
+            if startup_config.mode == "local":
+                resolved_config = replace(
+                    startup_config, control_plane_url="", host_id="local-host"
+                )
+                resolved_manager = manager or _configured_device_manager(
+                    config_store, config=resolved_config, enrollment_client=None
+                )
+            else:
+                resolved_config = resolve_host_identity(
+                    startup_config,
+                    identity_store=resolved_identity_store,
+                    client=bootstrap_client,
+                )
+                bootstrap_client.config = resolved_config
+                resolved_manager = manager or _configured_device_manager(
+                    config_store,
+                    config=resolved_config,
+                    enrollment_client=bootstrap_client,
+                )
         finally:
             if owned_enrollment_client:
                 bootstrap_client.close()
-        client = HostAgentClient(resolved_config)
+        if resolved_config.mode == "local":
+            client = LocalHostAgentClient(
+                host_id="local-host",
+                device_ids=lambda: [device.id for device in resolved_manager.list_devices()],
+            )
+        else:
+            client = HostAgentClient(resolved_config)
 
         history_store = ConsoleHistoryStore(
             resolved_config.identity_path.parent / "host_console_history.sqlite3",
@@ -214,12 +231,14 @@ def create_application(
                 "MCP token generated at %s", mcp_token_path
             )
         mcp_busy_tracker = McpBusyTracker(ttl_seconds=20.0)
+        conversation_log = ConversationLogStore(resolved_config.conversation_log_path) if resolved_config.mode == "local" else None
         executor = AssignmentExecutor(
             create_execution_factories(
                 resolved_manager,
                 metadata_store=metadata_store,
                 timeline=timeline,
                 host_agent_config=resolved_config,
+                conversation_log=conversation_log,
             ),
             mcp_busy_tracker=mcp_busy_tracker,
         )
@@ -228,6 +247,16 @@ def create_application(
             mcp_busy_tracker=mcp_busy_tracker,
             status_tracker=status_tracker,
         )
+        planner_config = load_planner_config()
+        conversation_agent = (
+            ConversationAgent(
+                config=planner_config,
+                tools=default_tool_registry(manager=resolved_manager),
+                event_logger=conversation_log.append if conversation_log else None,
+            )
+            if resolved_config.ai_planner_transport == "direct"
+            else None
+        )
         console_app = create_console_app(
             config=resolved_config,
             manager=resolved_manager,
@@ -247,6 +276,8 @@ def create_application(
             mcp_server=mcp_server,
             mcp_token_store=mcp_token_store,
             mcp_busy_tracker=mcp_busy_tracker,
+            conversation_agent=conversation_agent,
+            conversation_log=conversation_log,
         )
         console_server = _EmbeddedConsoleServer(
             uvicorn.Config(
@@ -352,7 +383,7 @@ def _configured_device_manager(
     config_store: DeviceConfigStore,
     *,
     config: HostAgentConfig,
-    enrollment_client: HostAgentEnrollmentClient,
+    enrollment_client: HostAgentEnrollmentClient | None,
 ) -> DeviceManager:
     manager = DeviceManager()
     for device in config_store.list():

apps/device-host-agent/host_agent/cli.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import argparse
 import getpass
+import os
 import sys
+from pathlib import Path
 from collections.abc import Sequence
 from dataclasses import replace
 
@@ -18,6 +20,7 @@ class LocalAccountSetupError(RuntimeError):
 
 
 def main(argv: Sequence[str] | None = None) -> None:
+    _load_dotenv()
     parser = argparse.ArgumentParser(description="Run the Device Host Agent")
     subparsers = parser.add_subparsers(dest="command")
     subparsers.add_parser("setup", help="Create the local operator account")
@@ -96,3 +99,19 @@ def _prompt_and_create(store: LocalAccountStore):
     if password != confirm:
         raise LocalAccountSetupError("passwords do not match")
     return store.create(username, password)
+
+
+def _load_dotenv() -> None:
+    """Load a simple repository-root .env without overriding shell values."""
+    path = Path.cwd() / ".env"
+    if not path.is_file():
+        return
+    for line in path.read_text(encoding="utf-8").splitlines():
+        line = line.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        name, value = line.split("=", 1)
+        name = name.strip()
+        value = value.strip()
+        if name and name not in os.environ:
+            os.environ[name] = value

apps/device-host-agent/host_agent/cloud_planner_client.py

@@ -62,11 +62,13 @@ class CloudProxyToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         payload: dict[str, Any] = {
             "host_id": self.config.host_id,
             "system_prompt": system_prompt,
             "user_prompt": user_prompt,
+            "history": history or [],
             "screenshot_base64": (
                 base64.b64encode(screenshot).decode("ascii")
                 if screenshot is not None

apps/device-host-agent/host_agent/config.py

@@ -13,6 +13,7 @@ class HostAgentConfigurationError(ValueError):
 
 _LOOPBACK_BIND_HOSTS = frozenset({"127.0.0.1", "localhost", "::1"})
 _AI_PLANNER_TRANSPORTS = frozenset({"direct", "cloud"})
+_HOST_AGENT_MODES = frozenset({"cloud", "local"})
 _REMOVED_RUNTIME_SUPERVISION_SETTINGS = (
     "HOST_AGENT_RUNTIME_SUPERVISED",
     "HOST_AGENT_RUNTIME_HOST",
@@ -22,7 +23,8 @@ _REMOVED_RUNTIME_SUPERVISION_SETTINGS = (
 
 @dataclass(frozen=True)
 class HostAgentConfig:
-    control_plane_url: str
+    control_plane_url: str = ""
+    mode: str = "cloud"
     host_id: str = ""
     token: str = field(default="", repr=False)
     identity_path: Path = Path("tasks/host_identity.json")
@@ -47,6 +49,7 @@ class HostAgentConfig:
     dependency_restart_max_attempts: int = 5
     task_progress_db_path: Path = Path("host_agent_data/task_progress.sqlite3")
     task_artifact_dir: Path = Path("host_agent_data/history")
+    conversation_log_path: Path = Path("host_agent_data/conversations.sqlite3")
     task_retention_max_count: int = 50
     task_retention_max_age_days: int = 7
     skill_sync_interval_seconds: float = 300.0
@@ -57,16 +60,19 @@ def load_host_agent_config(
 ) -> HostAgentConfig:
     values = os.environ if env is None else env
     _reject_removed_runtime_supervision_settings(values)
+    mode = values.get("HOST_AGENT_MODE", "cloud").strip().lower()
+    if mode not in _HOST_AGENT_MODES:
+        raise HostAgentConfigurationError("HOST_AGENT_MODE must be cloud or local")
     control_plane_url = (
         values.get(
             "HOST_AGENT_CONTROL_PLANE_URL",
-            "https://amcp.home.jerryyan.top",
+            "https://amcp.home.jerryyan.top" if mode == "cloud" else "",
         )
         .strip()
         .rstrip("/")
     )
     parsed_url = urlparse(control_plane_url)
-    if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
+    if mode == "cloud" and (parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc):
         raise HostAgentConfigurationError(
             "HOST_AGENT_CONTROL_PLANE_URL must be an HTTP(S) URL"
         )
@@ -82,9 +88,10 @@ def load_host_agent_config(
 
     config = HostAgentConfig(
         control_plane_url=control_plane_url,
+        mode=mode,
         identity_path=identity_path,
         local_account_path=local_account_path,
-        enrollment_managed=True,
+        enrollment_managed=mode == "cloud",
         display_name=values.get("HOST_AGENT_DISPLAY_NAME") or None,
         heartbeat_interval_seconds=_positive_float(
             values,
@@ -128,13 +135,15 @@ def load_host_agent_config(
             "HOST_AGENT_CONSOLE_HISTORY_LIMIT",
             200,
         ),
-        ai_planner_transport=_parse_ai_planner_transport(
-            values.get("AI_PLANNER_TRANSPORT")
-        ),
+        ai_planner_transport=("direct" if mode == "local" else _parse_ai_planner_transport(values.get("AI_PLANNER_TRANSPORT"))),
         dependency_supervisor_enabled=_truthy(
-            values, "HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED", False
+            values,
+            "HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED",
+            mode == "local",
+        ),
+        appium_supervised=_truthy(
+            values, "HOST_AGENT_APPIUM_SUPERVISED", mode == "local"
         ),
-        appium_supervised=_truthy(values, "HOST_AGENT_APPIUM_SUPERVISED", False),
         appium_host=values.get("HOST_AGENT_APPIUM_HOST", "127.0.0.1").strip(),
         appium_port=_positive_int(values, "HOST_AGENT_APPIUM_PORT", 4723),
         dependency_restart_max_attempts=_positive_int(
@@ -151,6 +160,7 @@ def load_host_agent_config(
                 "HOST_AGENT_TASK_ARTIFACT_DIR", "host_agent_data/history"
             ).strip()
         ),
+        conversation_log_path=Path(values.get("HOST_AGENT_CONVERSATION_LOG_PATH", "host_agent_data/conversations.sqlite3").strip()),
         task_retention_max_count=_positive_int(
             values, "HOST_AGENT_TASK_RETENTION_MAX_COUNT", 50
         ),

apps/device-host-agent/host_agent/conversation.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from runtime.planner_config import PlannerConfig
+from runtime.tool_specs import ACTION_TOOL_SPECS, ToolSpec
+
+READ_TOOL_SPECS = [
+    ToolSpec("take_screenshot", "Capture the current device screen.", {"type": "object", "properties": {"device_id": {"type": "string"}}}),
+    ToolSpec("describe_screen", "Inspect the current screen and UI.", {"type": "object", "properties": {"device_id": {"type": "string"}}}),
+    ToolSpec("find_text", "Find visible text on the current screen.", {"type": "object", "required": ["query"], "properties": {"query": {"type": "string"}, "device_id": {"type": "string"}}}),
+    ToolSpec("get_ui_tree", "Read the current accessibility/UI tree.", {"type": "object", "properties": {"device_id": {"type": "string"}, "include_app_info": {"type": "boolean"}}}),
+    ToolSpec("list_devices", "List configured devices.", {"type": "object", "properties": {}}),
+    ToolSpec("device_status", "Read one device status.", {"type": "object", "required": ["device_id"], "properties": {"device_id": {"type": "string"}}}),
+]
+
+
+@dataclass(frozen=True)
+class ChatResult:
+    content: str
+    tool_calls: int
+
+
+class ConversationAgent:
+    """Small OpenAI-compatible agent loop backed by Host Agent tools."""
+
+    def __init__(
+        self,
+        *,
+        config: PlannerConfig,
+        tools: dict[str, Callable[..., Any]],
+        max_rounds: int = 8,
+        event_logger: Callable[[dict[str, Any]], None] | None = None,
+    ) -> None:
+        self.config = config
+        self.tools = tools
+        self.max_rounds = max_rounds
+        self.event_logger = event_logger
+        self._client: Any | None = None
+
+    def chat(self, messages: list[dict[str, Any]], *, tools: dict[str, Callable[..., Any]] | None = None) -> ChatResult:
+        if self.config.provider == "anthropic":
+            return self._chat_anthropic(messages, tools=tools)
+        if self.config.provider not in {"openai", "openai_compatible"}:
+            raise RuntimeError("unsupported conversation provider")
+        client = self._get_client()
+        active_tools = tools or self.tools
+        history: list[dict[str, Any]] = [
+            {
+                "role": "system",
+                "content": (
+                    "You are a mobile device assistant. Reply naturally when no "
+                    "device action is needed. Call a tool when the user asks you "
+                    "to inspect or operate a device. Never claim an action was "
+                    "completed unless the tool result confirms it."
+                ),
+            },
+            *[_normalize_message(message) for message in messages],
+        ]
+        calls = 0
+        for _ in range(self.max_rounds):
+            response = client.chat.completions.create(
+                model=self.config.resolved_model(),
+                messages=history,
+                tools=[_openai_tool(spec) for spec in [*ACTION_TOOL_SPECS, *READ_TOOL_SPECS]],
+                tool_choice="auto",
+                parallel_tool_calls=False,
+                timeout=self.config.timeout,
+            )
+            message = response.choices[0].message
+            text = getattr(message, "content", None)
+            thinking = getattr(message, "reasoning_content", None) or getattr(message, "reasoning", None)
+            tool_calls = getattr(message, "tool_calls", None) or []
+            self._log({"type": "llm_response", "content": text, "thinking": thinking,
+                       "tool_calls": [{"id": c.id, "name": c.function.name, "arguments": c.function.arguments} for c in tool_calls]})
+            if not tool_calls:
+                self._log({"type": "final_reply", "content": str(text or "")})
+                return ChatResult(content=str(text or ""), tool_calls=calls)
+            history.append(_message_dict(message))
+            for tool_call in tool_calls:
+                name = tool_call.function.name
+                arguments: dict[str, Any] = {}
+                try:
+                    decoded_arguments = json.loads(tool_call.function.arguments or "{}")
+                    if not isinstance(decoded_arguments, dict):
+                        raise ValueError("tool arguments must be an object")
+                    arguments = decoded_arguments
+                    arguments.pop("purpose", None)
+                    arguments.pop("expected_outcome", None)
+                    result = active_tools[name](**arguments)
+                except Exception as exc:
+                    result = {"ok": False, "error": str(exc)}
+                calls += 1
+                self._log({"type": "tool_result", "tool_call_id": tool_call.id,
+                           "tool_name": name, "arguments": arguments, "result": result})
+                history.append(
+                    {
+                        "role": "tool",
+                        "tool_call_id": tool_call.id,
+                        "content": json.dumps(result, ensure_ascii=True, default=str),
+                    }
+                )
+        raise RuntimeError("conversation exceeded the maximum tool-call rounds")
+
+    def _chat_anthropic(self, messages: list[dict[str, Any]], *, tools: dict[str, Callable[..., Any]] | None = None) -> ChatResult:
+        import anthropic
+
+        kwargs: dict[str, Any] = {}
+        if self.config.api_key:
+            kwargs["api_key"] = self.config.api_key
+        if self.config.base_url:
+            kwargs["base_url"] = self.config.base_url
+        client = anthropic.Anthropic(**kwargs)
+        history = [_normalize_anthropic_message(message) for message in messages]
+        active_tools = tools or self.tools
+        calls = 0
+        for _ in range(self.max_rounds):
+            response = client.messages.create(
+                model=self.config.resolved_model(),
+                max_tokens=2048,
+                system="You are a mobile device assistant. Reply naturally, or call one device tool when needed. Never claim an action succeeded unless its tool result confirms it.",
+                messages=history,
+                tools=[_anthropic_tool(spec) for spec in [*ACTION_TOOL_SPECS, *READ_TOOL_SPECS]],
+                timeout=self.config.timeout,
+            )
+            blocks = getattr(response, "content", []) or []
+            text_parts = [getattr(block, "text", "") for block in blocks if getattr(block, "type", None) == "text"]
+            thinking_parts = [getattr(block, "thinking", "") for block in blocks if getattr(block, "type", None) == "thinking"]
+            uses = [block for block in blocks if getattr(block, "type", None) == "tool_use"]
+            self._log({"type": "llm_response", "content": "\n".join(x for x in text_parts if x), "thinking": "\n".join(x for x in thinking_parts if x), "tool_calls": [{"id": u.id, "name": u.name, "arguments": u.input} for u in uses]})
+            if not uses:
+                content = "\n".join(x for x in text_parts if x)
+                self._log({"type": "final_reply", "content": content})
+                return ChatResult(content=content, tool_calls=calls)
+            history.append({"role": "assistant", "content": [_anthropic_block_dict(block) for block in blocks if getattr(block, "type", None) != "thinking"]})
+            results = []
+            for use in uses:
+                arguments = dict(use.input) if isinstance(use.input, dict) else {}
+                try:
+                    result = active_tools[use.name](**arguments)
+                except Exception as exc:
+                    result = {"ok": False, "error": str(exc)}
+                calls += 1
+                self._log({"type": "tool_result", "tool_call_id": use.id, "tool_name": use.name, "arguments": arguments, "result": result})
+                results.append({"type": "tool_result", "tool_use_id": use.id, "content": json.dumps(result, ensure_ascii=True, default=str)})
+            history.append({"role": "user", "content": results})
+        raise RuntimeError("conversation exceeded the maximum tool-call rounds")
+
+    def chat_for_device(self, device_id: str, messages: list[dict[str, Any]]) -> ChatResult:
+        """Run a conversation bound to exactly one device."""
+        device_tools = {
+            name: _bind_device(tool, device_id, name=name)
+            for name, tool in self.tools.items()
+        }
+        return self.chat(messages, tools=device_tools)
+
+    def _log(self, event: dict[str, Any]) -> None:
+        if self.event_logger is not None:
+            try:
+                self.event_logger(event)
+            except Exception:
+                pass
+
+    def _get_client(self) -> Any:
+        if self._client is None:
+            from openai import OpenAI
+
+            kwargs: dict[str, Any] = {}
+            if self.config.api_key:
+                kwargs["api_key"] = self.config.api_key
+            if self.config.base_url:
+                kwargs["base_url"] = self.config.base_url
+            self._client = OpenAI(**kwargs)
+        return self._client
+
+
+def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
+    parameters = dict(spec.parameters)
+    properties = dict(parameters.get("properties") or {})
+    properties.setdefault(
+        "device_id",
+        {"type": "string", "description": "Target device ID when needed."},
+    )
+    parameters["properties"] = properties
+    return {
+        "type": "function",
+        "function": {
+            "name": spec.name,
+            "description": spec.description,
+            "parameters": parameters,
+        },
+    }
+
+
+def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
+    return {"name": spec.name, "description": spec.description, "input_schema": spec.parameters}
+
+
+def _anthropic_block_dict(block: Any) -> dict[str, Any]:
+    block_type = getattr(block, "type", "")
+    if block_type == "text":
+        return {"type": "text", "text": getattr(block, "text", "")}
+    if block_type == "thinking":
+        return {"type": "thinking", "thinking": getattr(block, "thinking", "")}
+    return {"type": "tool_use", "id": block.id, "name": block.name, "input": block.input}
+
+
+def _normalize_anthropic_message(message: dict[str, Any]) -> dict[str, Any]:
+    normalized = _normalize_message(message)
+    content = normalized["content"]
+    if isinstance(content, str):
+        return normalized
+    blocks = []
+    for block in content:
+        if block.get("type") == "text":
+            blocks.append(block)
+        elif block.get("type") == "image_url":
+            url = block.get("image_url", {}).get("url", "")
+            if isinstance(url, str) and url.startswith("data:"):
+                header, data = url.split(",", 1)
+                media_type = header[5:].split(";", 1)[0]
+                blocks.append({"type": "image", "source": {"type": "base64", "media_type": media_type, "data": data}})
+    return {"role": normalized["role"], "content": blocks}
+
+
+def _message_dict(message: Any) -> dict[str, Any]:
+    result: dict[str, Any] = {"role": "assistant"}
+    content = getattr(message, "content", None)
+    if content is not None:
+        result["content"] = content
+    calls = getattr(message, "tool_calls", None) or []
+    result["tool_calls"] = [
+        {
+            "id": call.id,
+            "type": "function",
+            "function": {
+                "name": call.function.name,
+                "arguments": call.function.arguments,
+            },
+        }
+        for call in calls
+    ]
+    return result
+
+
+def _normalize_message(message: dict[str, Any]) -> dict[str, Any]:
+    """Accept OpenAI text/image content blocks and a compact image_base64 form."""
+    role = message.get("role")
+    content = message.get("content")
+    if isinstance(content, str):
+        return {"role": role, "content": content}
+    if isinstance(content, list):
+        blocks: list[dict[str, Any]] = []
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "text" and isinstance(block.get("text"), str):
+                blocks.append({"type": "text", "text": block["text"]})
+            elif block.get("type") == "image_url":
+                image_url = block.get("image_url")
+                if isinstance(image_url, str):
+                    image_url = {"url": image_url}
+                if isinstance(image_url, dict) and isinstance(image_url.get("url"), str):
+                    blocks.append({"type": "image_url", "image_url": {"url": image_url["url"]}})
+        if blocks:
+            return {"role": role, "content": blocks}
+    image = message.get("image_base64")
+    if isinstance(image, str) and image:
+        mime = str(message.get("mime_type") or "image/png")
+        text = message.get("text")
+        blocks = []
+        if isinstance(text, str) and text:
+            blocks.append({"type": "text", "text": text})
+        blocks.append({"type": "image_url", "image_url": {"url": f"data:{mime};base64,{image}"}})
+        return {"role": role, "content": blocks}
+    raise ValueError("message content must be text, image blocks, or image_base64")
+
+
+def _bind_device(tool: Callable[..., Any], device_id: str, *, name: str) -> Callable[..., Any]:
+    def bound(**arguments: Any) -> Any:
+        if name == "list_devices":
+            return tool(**arguments)
+        requested = arguments.get("device_id")
+        if requested is not None and requested != device_id:
+            raise ValueError(f"conversation is bound to device {device_id}")
+        arguments["device_id"] = device_id
+        return tool(**arguments)
+
+    return bound

apps/device-host-agent/host_agent/conversation_log.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import json
+import sqlite3
+from datetime import UTC, datetime
+from pathlib import Path
+from threading import Lock
+from typing import Any
+
+
+class ConversationLogStore:
+    """SQLite-backed local audit log for chat and tool activity."""
+
+    def __init__(self, path: str | Path) -> None:
+        self.path = Path(path)
+        self._lock = Lock()
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        with self._connect() as connection:
+            connection.execute("CREATE TABLE IF NOT EXISTS conversation_events (id INTEGER PRIMARY KEY AUTOINCREMENT, occurred_at TEXT NOT NULL, event_type TEXT NOT NULL, payload_json TEXT NOT NULL)")
+
+    def append(self, event: dict[str, Any]) -> None:
+        with self._lock, self._connect() as connection:
+            connection.execute("INSERT INTO conversation_events (occurred_at, event_type, payload_json) VALUES (?, ?, ?)", (datetime.now(UTC).isoformat(), str(event.get("type") or "event"), json.dumps(_safe(event), ensure_ascii=True, default=str)))
+
+    def list_recent(self, *, limit: int = 200) -> list[dict[str, Any]]:
+        with self._connect() as connection:
+            rows = connection.execute("SELECT id, occurred_at, event_type, payload_json FROM conversation_events ORDER BY id DESC LIMIT ?", (max(1, min(limit, 1000)),)).fetchall()
+        result = []
+        for row in rows:
+            try:
+                payload = json.loads(row["payload_json"])
+            except (TypeError, ValueError):
+                payload = {}
+            result.append({"id": row["id"], "occurred_at": row["occurred_at"], "event_type": row["event_type"], **payload})
+        return result
+
+    def _connect(self) -> sqlite3.Connection:
+        connection = sqlite3.connect(self.path)
+        connection.row_factory = sqlite3.Row
+        return connection
+
+
+def _safe(value: Any) -> Any:
+    if isinstance(value, dict):
+        return {str(k): _safe(v) for k, v in value.items()}
+    if isinstance(value, list):
+        return [_safe(v) for v in value]
+    return value if isinstance(value, (str, int, float, bool)) or value is None else str(value)

apps/device-host-agent/host_agent/execution.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 from collections.abc import Callable
 from dataclasses import dataclass, replace
+from typing import Any
 
 from device.manager import DeviceManager
 from host_agent.cloud_planner_client import CloudProxyToolCallingClient
@@ -35,6 +36,7 @@ def create_execution_factories(
     metadata_store: TaskMetadataStore | None = None,
     timeline: Timeline | None = None,
     host_agent_config: HostAgentConfig | None = None,
+    conversation_log: Any | None = None,
 ) -> ExecutionFactories:
     shared_workflow_store = workflow_store or WorkflowStore()
     resolved_host_agent_config = host_agent_config
@@ -48,7 +50,10 @@ def create_execution_factories(
             ),
             metadata_store=metadata_store,
             timeline=timeline,
-            planner=_host_agent_planner(resolved_host_agent_config),
+            planner=_host_agent_planner(
+                resolved_host_agent_config,
+                event_logger=conversation_log.append if conversation_log is not None else None,
+            ),
             planner_config=_host_agent_planner_config(),
             device_platform_provider=lambda device_id: _device_platform(
                 manager, device_id
@@ -86,6 +91,8 @@ def _host_agent_planner_config() -> PlannerConfig:
 
 def _host_agent_planner(
     host_agent_config: HostAgentConfig | None,
+    *,
+    event_logger: Callable[[dict[str, Any]], None] | None = None,
 ) -> Planner | None:
     """Build the `AIPlanner` explicitly when the cloud-proxy transport is
     selected, so its `ToolCallingClient` is a `CloudProxyToolCallingClient`
@@ -100,11 +107,12 @@ def _host_agent_planner(
 
     resolved_config = host_agent_config or load_host_agent_config()
     if resolved_config.ai_planner_transport != "cloud":
-        return None
+        return AIPlanner(config=planner_config, event_logger=event_logger)
 
     return AIPlanner(
         client=CloudProxyToolCallingClient(resolved_config),
         config=planner_config,
+        event_logger=event_logger,
     )
 
 

apps/device-host-agent/host_agent/heartbeat.py

@@ -60,6 +60,8 @@ class HeartbeatSynchronizer:
             self.status_tracker.mark_host_policy(self.policy)
 
     async def sync_once(self) -> HeartbeatResponse:
+        self.probe_connected_devices()
+        self.connect_devices()
         snapshot = build_device_snapshot(self.manager)
         mcp_busy_ids = (
             self.mcp_busy_tracker.busy_device_ids()
@@ -108,9 +110,19 @@ class HeartbeatSynchronizer:
 
     def connect_devices(self) -> None:
         for device in self.manager.list_devices():
-            if device.status != "idle":
+            if device.status not in {"idle", "offline", "error"}:
                 continue
             try:
                 self.manager.connect(device.id)
             except DeviceRuntimeError:
                 continue
+
+    def probe_connected_devices(self) -> None:
+        active_id: str | None = None
+        if self.status_tracker is not None:
+            current = self.status_tracker.snapshot().get("current_assignment")
+            if isinstance(current, dict) and isinstance(current.get("device_id"), str):
+                active_id = current["device_id"]
+        for device in self.manager.list_devices():
+            if device.id != active_id and device.status == "busy":
+                self.manager.probe(device.id)

apps/device-host-agent/host_agent/ios_discovery.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import json
+import subprocess
+import tempfile
+from pathlib import Path
+from typing import Any
+
+
+class IOSDiscoveryError(RuntimeError):
+    pass
+
+
+def discover_connected_ios_devices(*, timeout_seconds: int = 10) -> list[dict[str, str]]:
+    """Return paired, currently connected physical iOS devices from CoreDevice."""
+    with tempfile.TemporaryDirectory(prefix="ios-device-discovery-") as temp_dir:
+        output_path = Path(temp_dir) / "devices.json"
+        try:
+            completed = subprocess.run(
+                [
+                    "xcrun",
+                    "devicectl",
+                    "list",
+                    "devices",
+                    "--json-output",
+                    str(output_path),
+                    "--timeout",
+                    str(timeout_seconds),
+                    "--quiet",
+                ],
+                capture_output=True,
+                text=True,
+                timeout=timeout_seconds + 2,
+                check=False,
+            )
+        except FileNotFoundError as exc:
+            raise IOSDiscoveryError("xcrun is unavailable; install Xcode command line tools") from exc
+        except subprocess.TimeoutExpired as exc:
+            raise IOSDiscoveryError("iOS device discovery timed out") from exc
+
+        if completed.returncode != 0:
+            detail = completed.stderr.strip() or completed.stdout.strip()
+            raise IOSDiscoveryError(detail or "devicectl failed to discover iOS devices")
+        try:
+            payload = json.loads(output_path.read_text(encoding="utf-8"))
+        except (OSError, ValueError) as exc:
+            raise IOSDiscoveryError("devicectl returned invalid device data") from exc
+
+    raw_devices = payload.get("result", {}).get("devices", [])
+    devices: list[dict[str, str]] = []
+    for item in raw_devices if isinstance(raw_devices, list) else []:
+        if not isinstance(item, dict):
+            continue
+        hardware = item.get("hardwareProperties", {})
+        properties = item.get("deviceProperties", {})
+        connection = item.get("connectionProperties", {})
+        if not all(isinstance(value, dict) for value in (hardware, properties, connection)):
+            continue
+        udid = hardware.get("udid")
+        if (
+            hardware.get("platform") != "iOS"
+            or hardware.get("reality") != "physical"
+            or connection.get("pairingState") != "paired"
+            or connection.get("tunnelState") != "connected"
+            or not isinstance(udid, str)
+            or not udid
+        ):
+            continue
+        devices.append(
+            {
+                "udid": udid,
+                "name": str(properties.get("name") or hardware.get("marketingName") or "iPhone"),
+                "model": str(hardware.get("marketingName") or hardware.get("productType") or "iPhone"),
+                "os_version": str(properties.get("osVersionNumber") or ""),
+                "transport": str(connection.get("transportType") or "unknown"),
+            }
+        )
+    return sorted(devices, key=lambda device: (device["name"], device["udid"]))
+

apps/device-host-agent/host_agent/local_client.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import asyncio
+import uuid
+from collections.abc import Callable
+from datetime import UTC, datetime, timedelta
+
+from cloud.internal_api.models import (
+    ClaimResponse,
+    HeartbeatResponse,
+    HostTaskCancellationResponse,
+    HostTaskSubmissionResponse,
+    LeaseRenewalResponse,
+    TerminalResultResponse,
+)
+from host_agent.progress import TaskProgressSnapshot
+
+
+class LocalHostAgentClient:
+    """In-process task broker used when the Host Agent runs without Cloud."""
+
+    def __init__(self, *, host_id: str, device_ids: Callable[[], list[str]]) -> None:
+        self.host_id = host_id
+        self._device_ids = device_ids
+        self._queue: asyncio.Queue[tuple[str, str, str | None]] = asyncio.Queue()
+        self._cancelled: set[str] = set()
+
+    async def submit_self_task(self, *, goal: str, device_id: str | None = None) -> HostTaskSubmissionResponse:
+        task_id = f"local-{uuid.uuid4().hex}"
+        await self._queue.put((task_id, goal, device_id))
+        return HostTaskSubmissionResponse(task_id=task_id)
+
+    async def claim(self):
+        task_id, goal, requested_device = await self._queue.get()
+        devices = self._device_ids()
+        device_id = requested_device or (devices[0] if devices else "")
+        if not device_id:
+            return None
+        from cloud.internal_api.models import AssignmentModel
+
+        return AssignmentModel(
+            task_id=task_id,
+            attempt=1,
+            lease_id=f"local-lease-{uuid.uuid4().hex}",
+            lease_expires_at=datetime.now(UTC) + timedelta(days=3650),
+            host_id=self.host_id,
+            device_id=device_id,
+            goal=goal,
+        )
+
+    async def heartbeat(self, *args, **kwargs) -> HeartbeatResponse:
+        return HeartbeatResponse(
+            host_id=self.host_id,
+            accepted_devices=len(self._device_ids()),
+            received_at=datetime.now(UTC),
+        )
+
+    async def renew(self, assignment, *, progress: TaskProgressSnapshot | None = None) -> LeaseRenewalResponse:
+        return LeaseRenewalResponse(
+            status="renewed",
+            lease_expires_at=datetime.now(UTC) + timedelta(days=3650),
+        )
+
+    async def report_result(self, assignment, *, status: str, failure_reason: str | None = None, result: dict | None = None) -> TerminalResultResponse:
+        return TerminalResultResponse(status="recorded")
+
+    async def cancel_task(self, task_id: str) -> HostTaskCancellationResponse:
+        self._cancelled.add(task_id)
+        return HostTaskCancellationResponse(task_id=task_id, status="cancel_requested")
+
+    async def aclose(self) -> None:
+        return None

apps/device-host-agent/host_agent/web/app.py

@@ -11,7 +11,9 @@ import jinja2
 from fastapi import Depends, FastAPI, HTTPException, Request
 from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse, Response
 
+from core.errors import DeviceNotFoundError, DeviceOfflineError, DeviceRuntimeError
 from device.manager import DeviceManager
+from driver.registry import build_driver_factory
 from host_agent.assignment import AssignmentExecutor
 from host_agent.client import (
     HostAgentClient,
@@ -20,9 +22,12 @@ from host_agent.client import (
     HostTaskSubmissionUnknownError,
 )
 from host_agent.config import HostAgentConfig
+from host_agent.conversation import ConversationAgent
+from host_agent.conversation_log import ConversationLogStore
 from host_agent.devices import register_local_device, unregister_local_device
 from host_agent.history import ConsoleHistoryStore
 from host_agent.identity import HostIdentityStore
+from host_agent.ios_discovery import IOSDiscoveryError, discover_connected_ios_devices
 from host_agent.local_account import LocalAccountStore
 from host_agent.mcp_lock import McpBusyTracker
 from host_agent.mcp_token import McpTokenStore
@@ -232,6 +237,8 @@ def create_console_app(
     mcp_server: FastMCP | None = None,
     mcp_token_store: McpTokenStore | None = None,
     mcp_busy_tracker: McpBusyTracker | None = None,
+    conversation_agent: ConversationAgent | None = None,
+    conversation_log: ConversationLogStore | None = None,
 ) -> FastAPI:
     app = FastAPI(title="Host Agent Console")
     cookie_secure = config.console_bind_host not in _LOOPBACK_BIND_HOSTS
@@ -415,6 +422,51 @@ def create_console_app(
             }
         )
 
+    @app.post("/api/chat")
+    async def api_chat(
+        request: Request,
+        session: SessionState = Depends(require_csrf),
+    ) -> JSONResponse:
+        if conversation_agent is None:
+            raise HTTPException(status_code=503, detail="chat agent is not configured")
+        payload = await request.json()
+        device_id = payload.get("device_id") if isinstance(payload, dict) else None
+        if not isinstance(device_id, str) or not device_id.strip():
+            raise HTTPException(status_code=400, detail="device_id is required")
+        if device_id not in {device.id for device in manager.list_devices()}:
+            raise HTTPException(status_code=404, detail="unknown device")
+        raw_messages = payload.get("messages") if isinstance(payload, dict) else None
+        if not isinstance(raw_messages, list) or not raw_messages:
+            raise HTTPException(status_code=400, detail="messages must be a non-empty list")
+        messages = [
+            {key: value for key, value in item.items() if key in {"role", "content", "image_base64", "mime_type", "text"}}
+            for item in raw_messages
+            if isinstance(item, dict)
+            and item.get("role") in {"user", "assistant"}
+            and (isinstance(item.get("content"), (str, list)) or isinstance(item.get("image_base64"), str))
+        ]
+        if not messages:
+            raise HTTPException(status_code=400, detail="messages are invalid")
+        if conversation_log is not None:
+            await asyncio.to_thread(
+                conversation_log.append,
+                {"type": "user_request", "device_id": device_id.strip(), "messages": messages},
+            )
+        try:
+            result = await asyncio.to_thread(
+                conversation_agent.chat_for_device, device_id.strip(), messages
+            )
+        except Exception as exc:
+            if conversation_log is not None:
+                await asyncio.to_thread(
+                    conversation_log.append,
+                    {"type": "agent_error", "device_id": device_id.strip(), "error": str(exc)},
+                )
+            raise HTTPException(status_code=502, detail=str(exc)) from exc
+        return JSONResponse(
+            {"content": result.content, "tool_calls": result.tool_calls}
+        )
+
     @app.get("/devices", response_class=HTMLResponse)
     async def devices_page(
         request: Request,
@@ -439,6 +491,124 @@ def create_console_app(
             error=None,
         )
 
+    @app.post("/api/devices/{device_id}/screenshot")
+    async def api_device_screenshot(
+        device_id: str,
+        session: SessionState = Depends(require_csrf),
+    ) -> Response:
+        """Capture one on-demand screenshot for a connected local device."""
+        try:
+            screenshot = await asyncio.to_thread(
+                lambda: manager.active_driver(device_id).screenshot()
+            )
+        except DeviceNotFoundError as exc:
+            raise HTTPException(status_code=404, detail=str(exc)) from exc
+        except DeviceOfflineError as exc:
+            raise HTTPException(status_code=503, detail=str(exc)) from exc
+        except DeviceRuntimeError as exc:
+            raise HTTPException(status_code=502, detail=str(exc)) from exc
+        except Exception as exc:
+            raise HTTPException(
+                status_code=502,
+                detail=str(exc) or "failed to capture device screenshot",
+            ) from exc
+
+        if not isinstance(screenshot, bytes) or not screenshot:
+            raise HTTPException(
+                status_code=502,
+                detail="device returned an empty screenshot",
+            )
+        return Response(
+            content=screenshot,
+            media_type="image/png",
+            headers={
+                "Cache-Control": "no-store",
+                "Pragma": "no-cache",
+                "X-Content-Type-Options": "nosniff",
+            },
+        )
+
+    @app.get("/api/devices/discover-ios")
+    async def api_discover_ios_devices(
+        session: SessionState = Depends(require_session),
+    ) -> JSONResponse:
+        try:
+            discovered = await asyncio.to_thread(discover_connected_ios_devices)
+        except IOSDiscoveryError as exc:
+            raise HTTPException(status_code=502, detail=str(exc)) from exc
+
+        configured = await asyncio.to_thread(config_store.list)
+        configured_udids = {
+            str(record["connection_info"].get("udid"))
+            for record in configured
+            if record["driver_type"] == "wda"
+        }
+        used_wda_ports = {
+            record["connection_info"].get("wda_local_port") for record in configured
+        }
+        used_mjpeg_ports = {
+            record["connection_info"].get("mjpegServerPort") for record in configured
+        }
+        next_wda_port = 8100
+        next_mjpeg_port = 9100
+        result = []
+        for device in discovered:
+            while next_wda_port in used_wda_ports:
+                next_wda_port += 1
+            while next_mjpeg_port in used_mjpeg_ports:
+                next_mjpeg_port += 1
+            result.append(
+                {
+                    **device,
+                    "configured": device["udid"] in configured_udids,
+                    "suggested_wda_port": next_wda_port,
+                    "suggested_mjpeg_port": next_mjpeg_port,
+                }
+            )
+            used_wda_ports.add(next_wda_port)
+            used_mjpeg_ports.add(next_mjpeg_port)
+            next_wda_port += 1
+            next_mjpeg_port += 1
+        return JSONResponse({"devices": result})
+
+    @app.post("/api/devices/test-connection")
+    async def api_device_test_connection(
+        request: Request,
+        session: SessionState = Depends(require_csrf),
+    ) -> JSONResponse:
+        payload = await request.json()
+        if not isinstance(payload, dict):
+            raise HTTPException(status_code=400, detail="Request must be an object.")
+        driver_type = str(payload.get("driver_type", "")).strip()
+        connection_info = payload.get("connection_info")
+        if not driver_type or not isinstance(connection_info, dict):
+            raise HTTPException(
+                status_code=400,
+                detail="Driver type and connection info are required.",
+            )
+
+        driver = None
+        connected = False
+        try:
+            driver = build_driver_factory(driver_type, connection_info)()
+            await asyncio.to_thread(driver.connect)
+            connected = True
+            await asyncio.to_thread(driver.health_check)
+        except ValueError as exc:
+            raise HTTPException(status_code=400, detail=str(exc)) from exc
+        except Exception as exc:
+            raise HTTPException(
+                status_code=502,
+                detail=str(exc) or "device connection test failed",
+            ) from exc
+        finally:
+            if connected and driver is not None:
+                try:
+                    await asyncio.to_thread(driver.disconnect)
+                except Exception:
+                    pass
+        return JSONResponse({"ok": True, "driver_type": driver_type})
+
     @app.post("/devices/save")
     async def devices_save(
         request: Request,
@@ -465,6 +635,26 @@ def create_console_app(
                 else:
                     connection_info = parsed
 
+        if error is None:
+            # The console exposes the common Appium settings as regular form
+            # fields. Advanced JSON remains available for uncommon capabilities.
+            field_map = {
+                "server_url": "server_url",
+                "udid": "udid",
+                "device_name": "device_name",
+            }
+            for form_key, config_key in field_map.items():
+                value = str(form.get(form_key, "")).strip()
+                if value:
+                    connection_info[config_key] = value
+            port_key = "wda_local_port" if driver_type == "wda" else "system_port"
+            port_value = str(form.get(port_key, "")).strip()
+            if port_value:
+                try:
+                    connection_info[port_key] = int(port_value)
+                except ValueError:
+                    error = f"{port_key} must be an integer."
+
         if error is None:
             try:
                 await asyncio.to_thread(
@@ -578,6 +768,15 @@ def create_console_app(
             entries=entries,
         )
 
+    @app.get("/conversations", response_class=HTMLResponse)
+    async def conversations_page(
+        session: SessionState = Depends(require_session),
+    ) -> HTMLResponse:
+        events = await asyncio.to_thread(
+            conversation_log.list_recent if conversation_log is not None else (lambda: [])
+        )
+        return _render("conversations.html", title="Conversations", session=session, events=events)
+
     def _tasks_list_context(
         session: SessionState,
         *,

apps/device-host-agent/host_agent/web/templates/base.html

@@ -26,6 +26,7 @@ form.inline { display: inline; margin: 0; }
       <a href="/tasks">Tasks</a>
       <a href="/account">Account</a>
       <a href="/history">History</a>
+      <a href="/conversations">Conversations</a>
       <form class="inline" method="post" action="/logout">
         <input type="hidden" name="csrf_token" value="{{ session.csrf_token }}">
         <button type="submit">Logout</button>

apps/device-host-agent/host_agent/web/templates/conversations.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+{% block body %}
+<h1>Conversations</h1>
+{% if not events %}<p>No conversation activity recorded yet.</p>{% endif %}
+{% for event in events %}
+<article>
+  <h2>{{ event["event_type"] }} <small>{{ event["occurred_at"] }}</small></h2>
+  {% if event.get("content") %}<pre>{{ event["content"] }}</pre>{% endif %}
+  {% if event.get("thinking") %}<details><summary>LLM reasoning</summary><pre>{{ event["thinking"] }}</pre></details>{% endif %}
+  {% if event.get("tool_calls") %}<details open><summary>Tool calls</summary><pre>{{ event["tool_calls"] | tojson(indent=2) }}</pre></details>{% endif %}
+  {% if event.get("tool_name") %}<p><strong>{{ event["tool_name"] }}</strong></p><pre>{{ event.get("arguments") | tojson(indent=2) }}</pre><pre>{{ event.get("result") | tojson(indent=2) }}</pre>{% endif %}
+</article>
+{% endfor %}
+{% endblock %}

apps/device-host-agent/host_agent/web/templates/devices.html

@@ -5,13 +5,20 @@
     <p class="error">{{ error }}</p>
     {% endif %}
     <table>
-      <thead><tr><th>ID</th><th>Name</th><th>Driver</th><th>Cloud ID</th><th></th></tr></thead>
+      <thead><tr><th>ID</th><th>Name</th><th>Driver</th><th>Cloud ID</th><th>Screenshot</th><th></th></tr></thead>
       <tbody>{% for device in devices %}
         <tr>
           <td>{{ device["device_id"] }}</td>
           <td>{{ device["name"] or "" }}</td>
           <td>{{ device["driver_type"] }}</td>
           <td>{{ device["cloud_device_id"] or "" }}</td>
+          <td>
+            <button type="button" class="screenshot-button" data-device-id="{{ device["device_id"] }}">Get screenshot</button>
+            <div class="screenshot-preview" data-screenshot-preview hidden>
+              <p class="screenshot-status" data-screenshot-status></p>
+              <img alt="Current screen for {{ device["device_id"] }}" data-screenshot-image hidden>
+            </div>
+          </td>
           <td>
             <a href="/devices?edit={{ device["device_id"] }}">Edit</a>
             <form class="inline" method="post" action="/devices/remove">
@@ -24,14 +31,173 @@
         {% endfor %}</tbody>
     </table>
     <h2>{{ "Edit device" if edit_record else "Add device" }}</h2>
+    <button type="button" id="discover-ios">Scan connected iPhones</button>
+    <p id="discovery-status" class="screenshot-status"></p>
+    <div id="discovered-devices"></div>
     <form method="post" action="/devices/save">
       <input type="hidden" name="csrf_token" value="{{ csrf_token }}">
       <label>Device ID <input type="text" name="device_id" value="{{ edit_record["device_id"] if edit_record else "" }}" required></label><br>
       <label>Name <input type="text" name="name" value="{{ edit_record["name"] if edit_record else "" }}"></label><br>
-      <label>Driver type <input type="text" name="driver_type" value="{{ edit_record["driver_type"] if edit_record else "wda" }}" required></label><br>
-      <label>Connection info (JSON)<br>
-        <textarea name="connection_info" rows="3" cols="50">{{ connection_info_json }}</textarea>
+      <label>Platform and protocol
+        <select name="driver_type" id="driver-type" required>
+          <option value="wda"{% if not edit_record or edit_record["driver_type"] == "wda" %} selected{% endif %}>iOS - Appium / XCUITest (WDA)</option>
+          <option value="uiautomator2"{% if edit_record and edit_record["driver_type"] == "uiautomator2" %} selected{% endif %}>Android - Appium / UiAutomator2</option>
+        </select>
       </label><br>
+      <label>Appium server URL <input type="url" name="server_url" value="{{ edit_record["connection_info"].get("server_url", "http://127.0.0.1:4723") if edit_record else "http://127.0.0.1:4723" }}" required></label><br>
+      <label>Device UDID <input type="text" name="udid" value="{{ edit_record["connection_info"].get("udid", "") if edit_record else "" }}" required></label><br>
+      <label>Device name <input type="text" name="device_name" value="{{ edit_record["connection_info"].get("device_name", "") if edit_record else "" }}" placeholder="iPhone"></label><br>
+      <label class="ios-setting">WDA local port <input type="number" min="1" max="65535" name="wda_local_port" value="{{ edit_record["connection_info"].get("wda_local_port", "") if edit_record else "" }}" placeholder="8100"></label>
+      <label class="android-setting">UiAutomator2 system port <input type="number" min="1" max="65535" name="system_port" value="{{ edit_record["connection_info"].get("system_port", "") if edit_record else "" }}" placeholder="8200"></label><br>
+      <details>
+        <summary>Advanced connection capabilities (JSON)</summary>
+        <textarea name="connection_info" rows="4" cols="60">{{ connection_info_json }}</textarea>
+      </details>
+      <p id="connection-test-status" class="screenshot-status"></p>
+      <button type="button" id="test-connection">Test connection</button>
       <button type="submit">Save</button>
     </form>
+    <style>
+      .screenshot-preview { margin-top: 0.5rem; max-width: 260px; }
+      .screenshot-preview img { display: block; width: 100%; height: auto; border: 1px solid #c8d0d6; }
+      .screenshot-status { margin: 0 0 0.35rem; color: #5e6b73; }
+      .screenshot-status.error { color: #b00020; }
+    </style>
+    <script>
+      (() => {
+        const driverType = document.getElementById("driver-type");
+        const syncPlatformFields = () => {
+          const ios = driverType.value === "wda";
+          document.querySelectorAll(".ios-setting").forEach((el) => { el.hidden = !ios; });
+          document.querySelectorAll(".android-setting").forEach((el) => { el.hidden = ios; });
+        };
+        driverType.addEventListener("change", syncPlatformFields);
+        syncPlatformFields();
+        const csrfInput = document.querySelector('input[name="csrf_token"]');
+        const csrfToken = csrfInput ? csrfInput.value : "";
+        const form = document.querySelector('form[action="/devices/save"]');
+        const discoverButton = document.getElementById("discover-ios");
+        const discoveryStatus = document.getElementById("discovery-status");
+        const discoveredDevices = document.getElementById("discovered-devices");
+        discoverButton.addEventListener("click", async () => {
+          discoverButton.disabled = true;
+          discoveryStatus.classList.remove("error");
+          discoveryStatus.textContent = "Scanning...";
+          discoveredDevices.replaceChildren();
+          try {
+            const response = await fetch("/api/devices/discover-ios");
+            const payload = await response.json();
+            if (!response.ok) throw new Error(payload.detail || "Discovery failed.");
+            discoveryStatus.textContent = payload.devices.length
+              ? `Found ${payload.devices.length} connected iPhone(s).`
+              : "No connected, paired iPhones found.";
+            payload.devices.forEach((device, index) => {
+              const row = document.createElement("p");
+              const description = document.createElement("span");
+              description.textContent = `${device.name} - ${device.model} - iOS ${device.os_version} (${device.transport}) `;
+              const select = document.createElement("button");
+              select.type = "button";
+              select.textContent = device.configured ? "Already added" : "Use this iPhone";
+              select.disabled = device.configured;
+              select.addEventListener("click", () => {
+                form.elements.driver_type.value = "wda";
+                form.elements.udid.value = device.udid;
+                form.elements.device_name.value = device.name;
+                form.elements.wda_local_port.value = device.suggested_wda_port;
+                form.elements.device_id.value ||= `ios-phone-${index + 1}`;
+                form.elements.name.value ||= device.name;
+                const advanced = JSON.parse(form.elements.connection_info.value || "{}");
+                advanced.mjpegServerPort = device.suggested_mjpeg_port;
+                advanced.derivedDataPath = `/tmp/wda-${device.udid}`;
+                form.elements.connection_info.value = JSON.stringify(advanced, null, 2);
+                syncPlatformFields();
+                form.scrollIntoView({ behavior: "smooth", block: "start" });
+              });
+              row.append(description, select);
+              discoveredDevices.append(row);
+            });
+          } catch (error) {
+            discoveryStatus.classList.add("error");
+            discoveryStatus.textContent = error.message || "Discovery failed.";
+          } finally {
+            discoverButton.disabled = false;
+          }
+        });
+        const testButton = document.getElementById("test-connection");
+        const testStatus = document.getElementById("connection-test-status");
+        const connectionInfo = () => {
+          const data = new FormData(form);
+          let info = {};
+          const advanced = String(data.get("connection_info") || "{}");
+          info = JSON.parse(advanced);
+          ["server_url", "udid", "device_name"].forEach((key) => {
+            const value = String(data.get(key) || "").trim();
+            if (value) info[key] = value;
+          });
+          const portKey = data.get("driver_type") === "wda" ? "wda_local_port" : "system_port";
+          const port = String(data.get(portKey) || "").trim();
+          if (port) info[portKey] = Number(port);
+          return { driver_type: data.get("driver_type"), connection_info: info };
+        };
+        testButton.addEventListener("click", async () => {
+          testButton.disabled = true;
+          testStatus.classList.remove("error");
+          testStatus.textContent = "Testing connection...";
+          try {
+            const response = await fetch("/api/devices/test-connection", {
+              method: "POST",
+              headers: { "Content-Type": "application/json", "X-CSRF-Token": csrfToken },
+              body: JSON.stringify(connectionInfo()),
+            });
+            const payload = await response.json();
+            if (!response.ok) throw new Error(payload.detail || "Connection test failed.");
+            testStatus.textContent = "Connection successful.";
+          } catch (error) {
+            testStatus.classList.add("error");
+            testStatus.textContent = error.message || "Connection test failed.";
+          } finally {
+            testButton.disabled = false;
+          }
+        });
+        document.querySelectorAll(".screenshot-button").forEach((button) => {
+          button.addEventListener("click", async () => {
+            const deviceId = button.dataset.deviceId;
+            const preview = button.parentElement.querySelector("[data-screenshot-preview]");
+            const status = preview.querySelector("[data-screenshot-status]");
+            const image = preview.querySelector("[data-screenshot-image]");
+            const previousUrl = image.dataset.objectUrl;
+            if (previousUrl) URL.revokeObjectURL(previousUrl);
+            button.disabled = true;
+            preview.hidden = false;
+            image.hidden = true;
+            status.classList.remove("error");
+            status.textContent = "Capturing...";
+            try {
+              const response = await fetch(
+                "/api/devices/" + encodeURIComponent(deviceId) + "/screenshot",
+                { method: "POST", headers: { "X-CSRF-Token": csrfToken } }
+              );
+              if (!response.ok) {
+                let detail = "Screenshot failed.";
+                try {
+                  const payload = await response.json();
+                  if (payload.detail) detail = payload.detail;
+                } catch (_) {}
+                throw new Error(detail);
+              }
+              const objectUrl = URL.createObjectURL(await response.blob());
+              image.src = objectUrl;
+              image.dataset.objectUrl = objectUrl;
+              image.hidden = false;
+              status.textContent = "Captured.";
+            } catch (error) {
+              status.classList.add("error");
+              status.textContent = error.message || "Screenshot failed.";
+            } finally {
+              button.disabled = false;
+            }
+          });
+        });
+      })();
+    </script>
 {% endblock %}

apps/device-host-agent/tests/host_agent/web/test_templates.py

@@ -47,6 +47,7 @@ def test_devices_renders(env, sample_session) -> None:
         **make_devices_context(sample_session)
     )
     assert '<form method="post" action="/devices/save">' in html
+    assert 'class="screenshot-button"' in html
 
 
 def test_account_renders(env, sample_session) -> None:

apps/device-host-agent/tests/test_cloud_planner_client.py

@@ -96,6 +96,36 @@ def test_decide_base64_encodes_screenshot() -> None:
     assert body["screenshot_base64"] == "aGVsbG8="
 
 
+def test_decide_forwards_planner_history() -> None:
+    seen_requests: list[httpx.Request] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        seen_requests.append(request)
+        return httpx.Response(200, json={"tool_name": "tap", "arguments": {}})
+
+    client = _client(handler)
+    history = [
+        {
+            "user_prompt": "first screen",
+            "tool_name": "tap",
+            "arguments": {"x": 1, "y": 2},
+            "rationale": "Open it.",
+            "tool_result": {"success": True},
+        }
+    ]
+
+    client.decide(
+        system_prompt="sp",
+        user_prompt="next screen",
+        screenshot=None,
+        tools=_TOOLS,
+        timeout=10.0,
+        history=history,
+    )
+
+    assert json.loads(seen_requests[0].content)["history"] == history
+
+
 def test_decide_clamps_legacy_timeout_and_waits_for_cloud_profile_timeout() -> None:
     seen_requests: list[httpx.Request] = []
 

apps/device-host-agent/tests/test_config.py

@@ -27,6 +27,17 @@ def test_load_host_agent_config_allows_explicit_direct_planner_transport() -> No
     assert config.ai_planner_transport == "direct"
 
 
+def test_load_host_agent_config_supports_local_mode() -> None:
+    config = load_host_agent_config({"HOST_AGENT_MODE": "local"})
+
+    assert config.mode == "local"
+    assert config.control_plane_url == ""
+    assert config.enrollment_managed is False
+    assert config.ai_planner_transport == "direct"
+    assert config.dependency_supervisor_enabled is True
+    assert config.appium_supervised is True
+
+
 def test_load_host_agent_config_parses_poll_and_retry_values() -> None:
     config = load_host_agent_config(
         {

apps/device-host-agent/tests/test_heartbeat.py

@@ -17,6 +17,9 @@ class ConnectableDriver:
     def connect(self) -> None:
         return None
 
+    def screenshot(self) -> bytes:
+        return b"ok"
+
 
 def _config() -> HostAgentConfig:
     return HostAgentConfig(
@@ -89,6 +92,33 @@ def test_heartbeat_synchronizer_runs_at_configured_interval_until_stopped() -> N
     assert manager.status("device-a") == "busy"
 
 
+def test_offline_device_is_retried_on_next_heartbeat_cycle() -> None:
+    class FlakyDriver(ConnectableDriver):
+        def __init__(self, fail: bool) -> None:
+            self.fail = fail
+
+        def screenshot(self) -> bytes:
+            if self.fail:
+                raise RuntimeError("WDA disconnected")
+            return b"ok"
+
+    instances: list[FlakyDriver] = []
+
+    def factory() -> FlakyDriver:
+        driver = FlakyDriver(not instances)
+        instances.append(driver)
+        return driver
+
+    manager = DeviceManager()
+    manager.register_device("device-a", factory)  # type: ignore[arg-type]
+    manager.connect("device-a")
+    sync = HeartbeatSynchronizer(manager, object(), _config())  # type: ignore[arg-type]
+    sync.probe_connected_devices()
+    assert manager.status("device-a") == "offline"
+    sync.connect_devices()
+    assert manager.status("device-a") == "busy"
+
+
 def test_sync_once_notifies_status_tracker_and_on_sync_with_device_count() -> None:
     manager = DeviceManager()
     manager.register_device(

apps/device-host-agent/tests/test_web_app.py

@@ -242,6 +242,147 @@ def test_add_device_appears_in_devices_page_and_manager(tmp_path) -> None:
     assert [device.id for device in context["manager"].list_devices()] == ["device-a"]
 
 
+def test_devices_page_captures_screenshot_only_when_button_endpoint_is_called(
+    tmp_path,
+) -> None:
+    class ScreenshotDriver:
+        def __init__(self) -> None:
+            self.capture_count = 0
+
+        def connect(self) -> None:
+            pass
+
+        def disconnect(self) -> None:
+            pass
+
+        def screenshot(self) -> bytes:
+            self.capture_count += 1
+            return b"fake-png"
+
+    driver = ScreenshotDriver()
+    client, context = _build_client(tmp_path)
+    context["config_store"].add(
+        device_id="device-a",
+        name="Lab iPhone",
+        driver_type="wda",
+        connection_info={},
+    )
+    context["manager"].register_device("device-a", lambda: driver)
+    context["manager"].connect("device-a")
+    csrf_token = _login(client)
+
+    page = client.get("/devices")
+    assert page.status_code == 200
+    assert 'class="screenshot-button"' in page.text
+    assert 'data-device-id="device-a"' in page.text
+    assert driver.capture_count == 0
+
+    response = client.post(
+        "/api/devices/device-a/screenshot",
+        headers={"X-CSRF-Token": csrf_token},
+    )
+
+    assert response.status_code == 200
+    assert response.content == b"fake-png"
+    assert response.headers["content-type"] == "image/png"
+    assert response.headers["cache-control"] == "no-store"
+    assert driver.capture_count == 1
+
+
+def test_device_screenshot_requires_csrf_and_connected_device(tmp_path) -> None:
+    class ScreenshotDriver:
+        def connect(self) -> None:
+            pass
+
+        def disconnect(self) -> None:
+            pass
+
+        def screenshot(self) -> bytes:
+            return b"fake-png"
+
+    client, context = _build_client(tmp_path)
+    context["manager"].register_device("device-a", ScreenshotDriver)
+    csrf_token = _login(client)
+
+    missing_csrf = client.post("/api/devices/device-a/screenshot")
+    assert missing_csrf.status_code == 403
+
+    offline = client.post(
+        "/api/devices/device-a/screenshot",
+        headers={"X-CSRF-Token": csrf_token},
+    )
+    assert offline.status_code == 503
+
+
+def test_device_connection_test_connects_checks_health_and_disconnects(
+    tmp_path, monkeypatch
+) -> None:
+    events: list[str] = []
+
+    class ProbeDriver:
+        def connect(self) -> None:
+            events.append("connect")
+
+        def health_check(self) -> None:
+            events.append("health")
+
+        def disconnect(self) -> None:
+            events.append("disconnect")
+
+    def factory(driver_type, connection_info):
+        assert driver_type == "wda"
+        assert connection_info["udid"] == "ios-udid"
+        return ProbeDriver
+
+    monkeypatch.setattr("host_agent.web.app.build_driver_factory", factory)
+    client, context = _build_client(tmp_path)
+    csrf_token = _login(client)
+
+    response = client.post(
+        "/api/devices/test-connection",
+        json={"driver_type": "wda", "connection_info": {"udid": "ios-udid"}},
+        headers={"X-CSRF-Token": csrf_token},
+    )
+
+    assert response.status_code == 200
+    assert response.json() == {"ok": True, "driver_type": "wda"}
+    assert events == ["connect", "health", "disconnect"]
+    assert context["config_store"].list() == []
+
+
+def test_ios_discovery_returns_connected_devices_and_unique_ports(
+    tmp_path, monkeypatch
+) -> None:
+    monkeypatch.setattr(
+        "host_agent.web.app.discover_connected_ios_devices",
+        lambda: [
+            {
+                "udid": "ios-new",
+                "name": "New iPhone",
+                "model": "iPhone 15",
+                "os_version": "18.0",
+                "transport": "wired",
+            }
+        ],
+    )
+    client, context = _build_client(tmp_path)
+    context["config_store"].add(
+        device_id="existing",
+        driver_type="wda",
+        connection_info={"udid": "ios-old", "wda_local_port": 8100, "mjpegServerPort": 9100},
+    )
+    _login(client)
+
+    response = client.get("/api/devices/discover-ios")
+
+    assert response.status_code == 200
+    device = response.json()["devices"][0]
+    assert device["udid"] == "ios-new"
+    assert device["configured"] is False
+    assert device["suggested_wda_port"] == 8101
+    assert device["suggested_mjpeg_port"] == 9101
+
+
 def test_remove_device_unregisters_from_manager(tmp_path) -> None:
     client, context = _build_client(tmp_path)
     csrf_token = _login(client)

device/manager.py

@@ -122,6 +122,26 @@ class DeviceManager:
             self._drivers.pop(device_id, None)
             self._set_status(device_id, "offline" if offline else "error")
 
+    def probe(self, device_id: str) -> bool:
+        """Check whether an established driver is still reachable."""
+        with self._lock:
+            self._device(device_id)
+            driver = self._drivers.get(device_id)
+            if driver is None:
+                return False
+        try:
+            health_check = getattr(driver, "health_check", None)
+            if callable(health_check):
+                health_check()
+            else:
+                # Compatibility for drivers implemented before health_check
+                # existed. Built-in drivers use the non-screen health check.
+                driver.screenshot()
+        except Exception:
+            self.mark_error(device_id, offline=True)
+            return False
+        return True
+
     def active_driver(self, device_id: str | None = None) -> Driver:
         with self._lock:
             if device_id is None:

driver/android_driver.py

@@ -81,6 +81,13 @@ class AndroidDriver(Driver):
         except Exception as exc:
             raise DriverError("screenshot failed") from exc
 
+    def health_check(self) -> None:
+        client = self._require_client()
+        try:
+            client.get_status()
+        except Exception as exc:
+            raise DriverError("health check failed") from exc
+
     def tap(self, x: float, y: float) -> None:
         client = self._require_client()
         try:

driver/base.py

@@ -25,6 +25,15 @@ class Driver(ABC):
     def screenshot(self) -> bytes:
         """Return the current screen as image bytes."""
 
+    def health_check(self) -> None:
+        """Verify the live session without reading the device screen.
+
+        Drivers with a transport-level status endpoint should override this
+        method. The default is a no-op for legacy drivers that do not expose
+        a separate health check.
+        """
+        return None
+
     @abstractmethod
     def tap(self, x: float, y: float) -> None:
         """Tap the screen at the given coordinates."""

driver/wda_driver.py

@@ -71,6 +71,13 @@ class WDADriver(Driver):
         except Exception as exc:
             raise DriverError("screenshot failed") from exc
 
+    def health_check(self) -> None:
+        client = self._require_client()
+        try:
+            client.get_status()
+        except Exception as exc:
+            raise DriverError("health check failed") from exc
+
     def tap(self, x: float, y: float) -> None:
         client = self._require_client()
         try:

packages/cloud-platform/cloud/internal_api/api.py

@@ -6,6 +6,7 @@ import json
 import logging
 from collections.abc import Awaitable, Callable
 from datetime import timedelta
+from inspect import Parameter, signature
 from time import monotonic
 from typing import TYPE_CHECKING
 from uuid import uuid4
@@ -505,13 +506,21 @@ def create_internal_router(
 
         started_at = monotonic()
         try:
-            decision = client.decide(
-                system_prompt=payload.system_prompt,
-                user_prompt=payload.user_prompt,
-                screenshot=screenshot,
-                tools=tools,
-                timeout=planner_timeout,
-            )
+            decision_kwargs = {
+                "system_prompt": payload.system_prompt,
+                "user_prompt": payload.user_prompt,
+                "screenshot": screenshot,
+                "tools": tools,
+                "timeout": planner_timeout,
+            }
+            parameters = signature(client.decide).parameters.values()
+            if any(
+                parameter.name == "history"
+                or parameter.kind == Parameter.VAR_KEYWORD
+                for parameter in parameters
+            ):
+                decision_kwargs["history"] = payload.history
+            decision = client.decide(**decision_kwargs)
         except ToolCallUnavailable as exc:
             logger.info(
                 "planner-decision request failed",

packages/cloud-platform/cloud/internal_api/models.py

@@ -143,6 +143,7 @@ class PlannerDecisionRequest(BaseModel):
     host_id: str = Field(min_length=1)
     system_prompt: str
     user_prompt: str
+    history: list[dict[str, Any]] = Field(default_factory=list)
     screenshot_base64: str | None = None
     tools: list[PlannerToolSpecModel] = Field(default_factory=list)
     timeout_seconds: float = Field(default=30.0, gt=0, le=120)

runtime/ai_planner.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from collections.abc import Callable
+from inspect import Parameter, signature
 from typing import TYPE_CHECKING, Any
 
 from core.errors import TaskFailedError
@@ -23,9 +25,11 @@ class AIPlanner(Planner):
         *,
         client: ToolCallingClient | None = None,
         config: PlannerConfig | None = None,
+        event_logger: Callable[[dict[str, Any]], None] | None = None,
     ) -> None:
         self.config = config or load_config()
         self.client = client or build_client(self.config)
+        self.event_logger = event_logger
 
     def plan(
         self,
@@ -36,25 +40,58 @@ class AIPlanner(Planner):
         world: "WorldState | None" = None,
         screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
+        _sync_tool_results(context)
+        scene_json = _without_ocr(scene.to_dict()) if self.config.multimodal else scene.to_dict()
         user_prompt = planner_user_prompt(
             goal=goal,
-            scene_json=scene.to_dict(),
-            history_summary=_history_summary(world),
+            scene_json=scene_json,
             device_platform=context.device_platform,
         )
-        decision = self.client.decide(
-            system_prompt=PLANNER_SYSTEM_PROMPT,
-            user_prompt=user_prompt,
-            screenshot=screenshot,
-            tools=ALL_TOOL_SPECS,
-            timeout=self.config.timeout,
-        )
+        if context.step_results and not context.step_results[-1].success:
+            user_prompt += (
+                "\n\nThe previous tool call failed. Diagnose the failure and choose a "
+                "corrected action or finish the task if it cannot proceed.\n"
+                f"Previous failure: {context.step_results[-1].error or 'unknown error'}"
+            )
+        self._log({"type": "llm_request", "task_id": context.task_id, "goal": goal,
+                   "system_prompt": PLANNER_SYSTEM_PROMPT, "user_prompt": user_prompt,
+                   "has_screenshot": screenshot is not None})
+        try:
+            kwargs = {
+                "system_prompt": PLANNER_SYSTEM_PROMPT,
+                "user_prompt": user_prompt,
+                "screenshot": screenshot,
+                "tools": ALL_TOOL_SPECS,
+                "timeout": self.config.timeout,
+            }
+            if _accepts_history(self.client.decide):
+                kwargs["history"] = context.planner_history[
+                    -self.config.history_max_turns :
+                ]
+            decision = self.client.decide(**kwargs)
+        except Exception as exc:
+            self._log({"type": "agent_error", "task_id": context.task_id, "error": str(exc)})
+            raise
+        self._log({"type": "llm_response", "task_id": context.task_id,
+                   "content": decision.text_output, "thinking": decision.thinking,
+                   "tool_name": decision.tool_name, "arguments": decision.arguments,
+                   "purpose": decision.purpose, "expected_outcome": decision.expected_outcome})
 
         if decision.tool_name == FINISH_TASK_TOOL:
             if decision.arguments.get("success"):
                 return []
             raise TaskFailedError(decision.arguments.get("reason") or "task failed")
 
+        context.planner_history.append(
+            {
+                "user_prompt": user_prompt,
+                "tool_name": decision.tool_name,
+                "arguments": _conversation_arguments(decision),
+                "rationale": decision.text_output,
+                "tool_result": None,
+            }
+        )
+
         return [
             PlannedStep(
                 action=decision.tool_name,
@@ -78,19 +115,45 @@ class AIPlanner(Planner):
         # (mapped to an empty plan above), never via this hook.
         return False
 
+    def _log(self, event: dict[str, Any]) -> None:
+        if self.event_logger is not None:
+            try:
+                self.event_logger(event)
+            except Exception:
+                pass
 
-def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]:
-    if world is None:
-        return []
-    return [
-        {
-            "page": event.page,
-            "action": event.action,
-            "arguments": dict(event.arguments),
-            "rationale": event.rationale,
-            "purpose": event.purpose,
-            "expected_outcome": event.expected_outcome,
-            "success": event.success,
-        }
-        for event in world.history
-    ]
+
+def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
+    cleaned = dict(scene_json)
+    elements = cleaned.get("elements")
+    if isinstance(elements, list):
+        cleaned["elements"] = [
+            {key: value for key, value in element.items() if key not in {"source", "confidence", "foreground_color", "background_color"}}
+            for element in elements
+            if isinstance(element, dict) and element.get("source") != "ocr"
+        ]
+    cleaned.pop("ocr_elements", None)
+    return cleaned
+
+
+def _sync_tool_results(context: TaskContext) -> None:
+    for turn, result in zip(context.planner_history, context.step_results, strict=False):
+        if turn.get("tool_result") is None:
+            turn["tool_result"] = result.to_dict()
+
+
+def _conversation_arguments(decision: Any) -> dict[str, Any]:
+    arguments = dict(decision.arguments)
+    if decision.purpose is not None:
+        arguments["purpose"] = decision.purpose
+    if decision.expected_outcome is not None:
+        arguments["expected_outcome"] = decision.expected_outcome
+    return arguments
+
+
+def _accepts_history(method: Any) -> bool:
+    parameters = signature(method).parameters.values()
+    return any(
+        parameter.name == "history" or parameter.kind == Parameter.VAR_KEYWORD
+        for parameter in parameters
+    )

runtime/context.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 from core.models import Scene
 
@@ -18,6 +18,7 @@ class TaskContext:
     scenes: list[Scene] = field(default_factory=list)
     step_results: list["StepResult"] = field(default_factory=list)
     world: "WorldState | None" = None
+    planner_history: list[dict[str, Any]] = field(default_factory=list)
 
     def add_scene(self, scene: Scene) -> None:
         self.scenes.append(scene)

runtime/executor.py

@@ -124,7 +124,7 @@ def default_tool_registry(
     from tools.tap import tap
     from tools.ui_tree import get_ui_tree
 
-    return {
+    registry = {
         "take_screenshot": _bind_manager(take_screenshot, manager),
         "screenshot": _bind_manager(take_screenshot, manager),
         "tap": _bind_manager(tap, manager),
@@ -143,6 +143,10 @@ def default_tool_registry(
         "find_icon": find_icon,
         "find_icon_on_screen": _bind_manager(find_icon_on_screen, manager),
     }
+    if manager is not None:
+        registry["list_devices"] = lambda: [device.to_dict() for device in manager.list_devices()]
+        registry["device_status"] = lambda device_id: {"device_id": device_id, "status": manager.status(device_id)}
+    return registry
 
 
 def _bind_manager(func: ToolCallable, manager: DeviceManager | None) -> ToolCallable:

runtime/planner_config.py

@@ -8,14 +8,20 @@ DEFAULT_PROVIDER = "anthropic"
 DEFAULT_MODEL_BY_PROVIDER = {
     "anthropic": "claude-sonnet-5",
     "openai": "gpt-5.6",
+    "openai_compatible": "local-model",
 }
 DEFAULT_TIMEOUT_SECONDS = 30.0
+DEFAULT_HISTORY_MAX_TURNS = 20
 
 ENABLED_ENV = "AI_PLANNER_ENABLED"
 PROVIDER_ENV = "AI_PLANNER_PROVIDER"
 MODEL_ENV = "AI_PLANNER_MODEL"
 TIMEOUT_ENV = "AI_PLANNER_TIMEOUT_SECONDS"
 THINKING_BUDGET_ENV = "AI_PLANNER_THINKING_BUDGET_TOKENS"
+API_KEY_ENV = "AI_PLANNER_API_KEY"
+BASE_URL_ENV = "AI_PLANNER_BASE_URL"
+MULTIMODAL_ENV = "AI_PLANNER_MULTIMODAL"
+HISTORY_MAX_TURNS_ENV = "AI_PLANNER_HISTORY_MAX_TURNS"
 
 SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODEL_BY_PROVIDER)
 
@@ -27,6 +33,10 @@ class PlannerConfig:
     model: str = ""
     timeout: float = DEFAULT_TIMEOUT_SECONDS
     thinking_budget_tokens: int | None = None
+    api_key: str | None = None
+    base_url: str | None = None
+    multimodal: bool = False
+    history_max_turns: int = DEFAULT_HISTORY_MAX_TURNS
 
     def resolved_model(self) -> str:
         return self.model or DEFAULT_MODEL_BY_PROVIDER[self.provider]
@@ -40,6 +50,13 @@ def load_config(env: Mapping[str, str] | None = None) -> PlannerConfig:
         model=values.get(MODEL_ENV) or "",
         timeout=_parse_timeout(values.get(TIMEOUT_ENV)),
         thinking_budget_tokens=_parse_thinking_budget(values.get(THINKING_BUDGET_ENV)),
+        api_key=values.get(API_KEY_ENV) or _provider_key(values),
+        base_url=values.get(BASE_URL_ENV) or None,
+        multimodal=_parse_bool(values.get(MULTIMODAL_ENV), default=False),
+        history_max_turns=_parse_positive_int(
+            values.get(HISTORY_MAX_TURNS_ENV),
+            default=DEFAULT_HISTORY_MAX_TURNS,
+        ),
     )
 
 
@@ -53,6 +70,8 @@ def _parse_provider(value: str | None) -> str:
     if value is None:
         return DEFAULT_PROVIDER
     provider = value.strip().lower()
+    if provider in {"openai-compatible", "openai_compatible", "local"}:
+        return "openai_compatible"
     return provider if provider in SUPPORTED_PROVIDERS else DEFAULT_PROVIDER
 
 
@@ -74,3 +93,22 @@ def _parse_thinking_budget(value: str | None) -> int | None:
     except ValueError:
         return None
     return budget if budget > 0 else None
+
+
+def _parse_positive_int(value: str | None, *, default: int) -> int:
+    if value is None:
+        return default
+    try:
+        parsed = int(value)
+    except ValueError:
+        return default
+    return parsed if parsed > 0 else default
+
+
+def _provider_key(values: Mapping[str, str]) -> str | None:
+    provider = (values.get(PROVIDER_ENV) or DEFAULT_PROVIDER).strip().lower()
+    if provider == "openai":
+        return values.get("OPENAI_API_KEY") or None
+    if provider == "anthropic":
+        return values.get("ANTHROPIC_API_KEY") or None
+    return None

runtime/planner_prompts.py

@@ -66,7 +66,6 @@ def planner_user_prompt(
     *,
     goal: str,
     scene_json: dict[str, Any],
-    history_summary: list[dict[str, Any]],
     device_platform: str | None = None,
     now: datetime | None = None,
 ) -> str:
@@ -83,8 +82,6 @@ def planner_user_prompt(
         f"{goal}\n\n"
         "Current Scene (JSON):\n"
         f"{json.dumps(scene_json, ensure_ascii=False, sort_keys=True)}\n\n"
-        "Recent history, oldest first (JSON):\n"
-        f"{json.dumps(history_summary, ensure_ascii=False, sort_keys=True)}\n\n"
         "Call exactly one tool for this turn."
     )
 

runtime/task.py

@@ -187,13 +187,18 @@ class TaskRunner:
                         "failed",
                         result.error or "step failed",
                     )
-                    self._update_task(
-                        task,
-                        status="failed",
-                        completed=True,
-                        failure_reason=result.error or "step failed",
-                    )
-                    return task
+                    # AI planners can use the structured failure feedback to
+                    # correct malformed arguments or choose another action.
+                    # Deterministic planners retain their fail-fast behavior.
+                    if self.planner.__class__.__name__ != "AIPlanner":
+                        self._update_task(
+                            task,
+                            status="failed",
+                            completed=True,
+                            failure_reason=result.error or "step failed",
+                        )
+                        return task
+                    break
 
         self._emit_step_progress(
             len(context.step_results),

runtime/tool_calling_client.py

@@ -51,6 +51,7 @@ class ToolCallingClient(Protocol):
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision: ...
 
 
@@ -80,6 +81,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_message(
@@ -87,6 +89,7 @@ class AnthropicToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -103,6 +106,7 @@ class AnthropicToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -123,6 +127,7 @@ class AnthropicToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -146,10 +151,8 @@ class AnthropicToolCallingClient:
                 }
             ],
             "messages": [
-                {
-                    "role": "user",
-                    "content": _anthropic_content(user_prompt, screenshot),
-                }
+                *_anthropic_history(history or []),
+                {"role": "user", "content": _anthropic_content(user_prompt, screenshot)},
             ],
             "tools": [_anthropic_tool(spec) for spec in tools],
             "tool_choice": {
@@ -207,6 +210,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         try:
             response = self._create_completion(
@@ -214,6 +218,7 @@ class OpenAIToolCallingClient:
                 user_prompt,
                 screenshot,
                 tools,
+                history=history,
                 timeout=timeout,
                 forced=False,
             )
@@ -227,6 +232,7 @@ class OpenAIToolCallingClient:
                     user_prompt,
                     screenshot,
                     tools,
+                    history=history,
                     timeout=timeout,
                     forced=True,
                 )
@@ -247,6 +253,7 @@ class OpenAIToolCallingClient:
         screenshot: bytes | None,
         tools: list[ToolSpec],
         *,
+        history: list[dict[str, Any]] | None,
         timeout: float,
         forced: bool,
     ) -> Any:
@@ -257,6 +264,7 @@ class OpenAIToolCallingClient:
             "timeout": timeout,
             "messages": [
                 {"role": "system", "content": system_prompt},
+                *_openai_history(history or []),
                 {"role": "user", "content": _openai_content(user_prompt, screenshot)},
             ],
             "tools": [_openai_tool(spec) for spec in tools],
@@ -288,10 +296,14 @@ class OpenAIToolCallingClient:
 
 def build_client(config: PlannerConfig) -> ToolCallingClient:
     model = config.resolved_model()
-    if config.provider == "openai":
-        return OpenAIToolCallingClient(model=model)
+    if config.provider in {"openai", "openai_compatible"}:
+        return OpenAIToolCallingClient(
+            model=model, api_key=config.api_key, base_url=config.base_url
+        )
     return AnthropicToolCallingClient(
         model=model,
+        api_key=config.api_key,
+        base_url=config.base_url,
         thinking_budget_tokens=config.thinking_budget_tokens,
     )
 
@@ -316,6 +328,44 @@ def _anthropic_content(
     return content
 
 
+def _anthropic_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_use_id = f"planner-turn-{index}"
+        assistant_content: list[dict[str, Any]] = []
+        rationale = turn.get("rationale")
+        if isinstance(rationale, str) and rationale:
+            assistant_content.append({"type": "text", "text": rationale})
+        assistant_content.append(
+            {
+                "type": "tool_use",
+                "id": tool_use_id,
+                "name": turn["tool_name"],
+                "input": dict(turn.get("arguments") or {}),
+            }
+        )
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {"role": "assistant", "content": assistant_content},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "tool_result",
+                            "tool_use_id": tool_use_id,
+                            "content": json.dumps(result, ensure_ascii=False, default=str),
+                        }
+                    ],
+                },
+            ]
+        )
+    return messages
+
+
 def _anthropic_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "name": spec.name,
@@ -389,6 +439,43 @@ def _openai_content(
     ]
 
 
+def _openai_history(history: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    messages: list[dict[str, Any]] = []
+    for index, turn in enumerate(history):
+        result = turn.get("tool_result")
+        if result is None:
+            continue
+        tool_call_id = f"planner-turn-{index}"
+        messages.extend(
+            [
+                {"role": "user", "content": turn["user_prompt"]},
+                {
+                    "role": "assistant",
+                    "content": turn.get("rationale"),
+                    "tool_calls": [
+                        {
+                            "id": tool_call_id,
+                            "type": "function",
+                            "function": {
+                                "name": turn["tool_name"],
+                                "arguments": json.dumps(
+                                    turn.get("arguments") or {},
+                                    ensure_ascii=False,
+                                ),
+                            },
+                        }
+                    ],
+                },
+                {
+                    "role": "tool",
+                    "tool_call_id": tool_call_id,
+                    "content": json.dumps(result, ensure_ascii=False, default=str),
+                },
+            ]
+        )
+    return messages
+
+
 def _openai_tool(spec: ToolSpec) -> dict[str, Any]:
     return {
         "type": "function",

storage/task_metadata.py

@@ -133,4 +133,24 @@ class TaskMetadataStore:
     def _connect(self) -> sqlite3.Connection:
         connection = sqlite3.connect(self.db_path)
         connection.row_factory = sqlite3.Row
+        # A previous interrupted startup can leave a zero-byte SQLite file
+        # behind before ``__init__`` reaches ``_ensure_schema``. Ensure the
+        # table exists on every connection so the console can recover without
+        # manual deletion or database repair.
+        connection.execute(
+            """
+            create table if not exists tasks (
+                id text primary key,
+                goal text not null,
+                device_id text not null,
+                status text not null,
+                created_at text not null,
+                updated_at text not null,
+                completed_at text,
+                failure_reason text,
+                source_task_id text,
+                source_attempt integer
+            )
+            """
+        )
         return connection

tests/test_ai_planner.py

@@ -8,6 +8,7 @@ from core.errors import TaskFailedError
 from core.models import Bounds, Scene, SceneElement
 from runtime.ai_planner import AIPlanner
 from runtime.context import TaskContext
+from runtime.executor import StepResult
 from runtime.planner_config import PlannerConfig
 from runtime.tool_calling_client import ToolCallDecision
 from runtime.tool_specs import ALL_TOOL_SPECS
@@ -26,6 +27,7 @@ class FakeToolCallingClient:
         screenshot: bytes | None,
         tools: list[Any],
         timeout: float,
+        history: list[dict[str, Any]] | None = None,
     ) -> ToolCallDecision:
         self.calls.append(
             {
@@ -34,6 +36,7 @@ class FakeToolCallingClient:
                 "screenshot": screenshot,
                 "tools": tools,
                 "timeout": timeout,
+                "history": list(history) if history is not None else None,
             }
         )
         return self.decision
@@ -242,63 +245,64 @@ def test_ai_planner_propagates_rationale_and_thinking_to_planned_step() -> None:
     assert steps[0].expected_outcome == "The account settings page is visible."
 
 
-def test_history_summary_returns_compact_format() -> None:
-    from collections import deque
-    from runtime.ai_planner import _history_summary
-    from world.models import WorldEvent, WorldState
-
-    state = WorldState(
-        history=deque(
-            [
-                WorldEvent(
-                    action="tap",
-                    success=True,
-                    rationale="Opened settings.",
-                    arguments={"x": 1, "y": 2},
-                    purpose="Open settings.",
-                    expected_outcome="Settings is visible.",
-                    page="Home",
-                ),
-                WorldEvent(
-                    action="swipe",
-                    success=False,
-                    rationale=None,
-                    arguments={"start_y": 700, "end_y": 200},
-                    page="Settings",
-                ),
-            ]
+def test_ai_planner_carries_completed_turn_into_the_next_llm_call() -> None:
+    client = FakeToolCallingClient(
+        ToolCallDecision(
+            tool_name="tap",
+            arguments={"x": 1, "y": 2},
+            text_output="Opening the send control.",
+            purpose="Open the send control.",
+            expected_outcome="The composer is focused.",
         )
     )
+    planner = AIPlanner(client=client)
+    context = _context()
 
-    summary = _history_summary(state)
+    first_step = planner.plan(goal=context.goal, scene=_scene(), context=context)[0]
+    context.add_step_result(
+        StepResult(
+            step=first_step,
+            success=True,
+            attempts=1,
+            result={"ok": True},
+        )
+    )
+    planner.plan(goal=context.goal, scene=_scene(), context=context)
 
-    assert summary == [
+    assert client.calls[0]["history"] == []
+    history = client.calls[1]["history"]
+    assert history is not None
+    assert history[0]["tool_name"] == "tap"
+    assert history[0]["arguments"] == {
+        "x": 1,
+        "y": 2,
+        "purpose": "Open the send control.",
+        "expected_outcome": "The composer is focused.",
+    }
+    assert history[0]["tool_result"]["success"] is True
+    assert history[0]["tool_result"]["result"] == {"ok": True}
+
+
+def test_ai_planner_limits_history_sent_to_the_llm() -> None:
+    client = FakeToolCallingClient(
+        ToolCallDecision(tool_name="tap", arguments={"x": 1, "y": 2})
+    )
+    planner = AIPlanner(client=client, config=PlannerConfig(history_max_turns=2))
+    context = _context()
+    context.planner_history.extend(
         {
-            "page": "Home",
-            "action": "tap",
-            "arguments": {"x": 1, "y": 2},
-            "rationale": "Opened settings.",
-            "purpose": "Open settings.",
-            "expected_outcome": "Settings is visible.",
-            "success": True,
-        },
-        {
-            "page": "Settings",
-            "action": "swipe",
-            "arguments": {"start_y": 700, "end_y": 200},
+            "user_prompt": f"turn-{index}",
+            "tool_name": "tap",
+            "arguments": {},
             "rationale": None,
-            "purpose": None,
-            "expected_outcome": None,
-            "success": False,
-        },
+            "tool_result": {"success": True},
+        }
+        for index in range(3)
+    )
+
+    planner.plan(goal=context.goal, scene=_scene(), context=context)
+
+    assert [turn["user_prompt"] for turn in client.calls[0]["history"]] == [
+        "turn-1",
+        "turn-2",
     ]
-    # Must not contain scene element data
-    for entry in summary:
-        assert "scene_summary" not in entry
-        assert "elements" not in entry
-
-
-def test_history_summary_returns_empty_for_none_world() -> None:
-    from runtime.ai_planner import _history_summary
-
-    assert _history_summary(None) == []

tests/test_device_manager.py

@@ -30,3 +30,45 @@ def test_device_manager_marks_unreachable_device_offline() -> None:
         manager.connect("iphone-1", max_retries=2, retry_backoff_seconds=0)
 
     assert manager.status("iphone-1") == "offline"
+
+
+def test_probe_marks_connected_device_offline_when_driver_is_unreachable() -> None:
+    class BrokenDriver:
+        def connect(self) -> None:
+            return None
+
+        def screenshot(self) -> bytes:
+            raise RuntimeError("WDA disconnected")
+
+    manager = DeviceManager()
+    manager.register_device("iphone-1", lambda: BrokenDriver())  # type: ignore[arg-type]
+    manager.connect("iphone-1")
+
+    assert manager.probe("iphone-1") is False
+    assert manager.status("iphone-1") == "offline"
+
+
+def test_probe_uses_health_check_without_capturing_screen() -> None:
+    class HealthCheckedDriver:
+        def __init__(self) -> None:
+            self.health_checks = 0
+            self.screenshots = 0
+
+        def connect(self) -> None:
+            return None
+
+        def screenshot(self) -> bytes:
+            self.screenshots += 1
+            return b"screen"
+
+        def health_check(self) -> None:
+            self.health_checks += 1
+
+    driver = HealthCheckedDriver()
+    manager = DeviceManager()
+    manager.register_device("iphone-1", lambda: driver)  # type: ignore[arg-type]
+    manager.connect("iphone-1")
+
+    assert manager.probe("iphone-1") is True
+    assert driver.health_checks == 1
+    assert driver.screenshots == 0

tests/test_planner_prompts.py

@@ -9,7 +9,6 @@ def test_planner_user_prompt_includes_time_zone_and_configured_device_type() ->
     prompt = planner_user_prompt(
         goal="open settings",
         scene_json={"screen": {"width": 1, "height": 1}, "elements": []},
-        history_summary=[],
         device_platform="ios",
         now=datetime(
             2026,
@@ -35,8 +34,16 @@ def test_planner_user_prompt_uses_scene_platform_when_context_is_unavailable() -
             "elements": [],
             "app": {"platform": "android"},
         },
-        history_summary=[],
         now=datetime(2026, 7, 16, tzinfo=timezone.utc),
     )
 
     assert "Device type: android" in prompt
+
+
+def test_planner_user_prompt_does_not_duplicate_conversation_history() -> None:
+    prompt = planner_user_prompt(
+        goal="open settings",
+        scene_json={"screen": {"width": 1, "height": 1}, "elements": []},
+    )
+
+    assert "Recent history" not in prompt

tests/test_task_metadata.py

@@ -0,0 +1,10 @@
+from storage.task_metadata import TaskMetadataStore
+
+
+def test_store_recovers_from_preexisting_empty_database(tmp_path) -> None:
+    path = tmp_path / "task_progress.sqlite3"
+    path.touch()
+
+    store = TaskMetadataStore(path)
+
+    assert store.list_tasks() == []