From 71fd182f50f13d168a1aa1218cb7661f55e2f018 Mon Sep 17 00:00:00 2001
From: showtan001 <240788545@qq.com>
Date: Sat, 29 Aug 2026 21:17:20 +0800
Subject: [PATCH] Support multimodal images in local chat agent

---
 README.md                                     | 19 ++++++++++
 .../host_agent/conversation.py                | 37 ++++++++++++++++++-
 apps/device-host-agent/host_agent/web/app.py  |  4 +-
 3 files changed, 56 insertions(+), 4 deletions(-)

diff --git a/README.md b/README.md
index 34cd7cc..ec4e8d5 100644
--- a/README.md
+++ b/README.md
@@ -92,6 +92,25 @@ used by the Runtime; each tool result is fed back to the model before the final
 reply is returned. The endpoint uses the Host console session cookie, so it is
 not an unauthenticated device-control endpoint.
 
+For vision-capable OpenAI models, a user message may contain standard OpenAI
+multimodal blocks:
+
+```json
+{
+  "messages": [{
+    "role": "user",
+    "content": [
+      {"type": "text", "text": "点击图片中的登录按钮"},
+      {"type": "image_url", "image_url": {"url": "data:image/png;base64,..."}}
+    ]
+  }]
+}
+```
+
+The compact form `{ "role": "user", "text": "...", "image_base64": "..." }`
+is also accepted. The model can inspect the supplied image and then call a
+phone tool such as `tap` in the same conversation.
+
 In local mode, Appium supervision is enabled by default. Host Agent probes
 `/status`, adopts a healthy existing Appium instance, starts Appium when no
 listener exists, restarts only processes it started if they crash, and stops
diff --git a/apps/device-host-agent/host_agent/conversation.py b/apps/device-host-agent/host_agent/conversation.py
index 873f361..7cbc865 100644
--- a/apps/device-host-agent/host_agent/conversation.py
+++ b/apps/device-host-agent/host_agent/conversation.py
@@ -38,7 +38,7 @@ class ConversationAgent:
         self.max_rounds = max_rounds
         self._client: Any | None = None
 
-    def chat(self, messages: list[dict[str, str]]) -> ChatResult:
+    def chat(self, messages: list[dict[str, Any]]) -> ChatResult:
         if self.config.provider not in {"openai", "openai_compatible"}:
             raise RuntimeError(
                 "conversation chat currently requires an OpenAI-compatible provider"
@@ -54,7 +54,7 @@ class ConversationAgent:
                     "completed unless the tool result confirms it."
                 ),
             },
-            *messages,
+            *[_normalize_message(message) for message in messages],
         ]
         calls = 0
         for _ in range(self.max_rounds):
@@ -142,3 +142,36 @@ def _message_dict(message: Any) -> dict[str, Any]:
         for call in calls
     ]
     return result
+
+
+def _normalize_message(message: dict[str, Any]) -> dict[str, Any]:
+    """Accept OpenAI text/image content blocks and a compact image_base64 form."""
+    role = message.get("role")
+    content = message.get("content")
+    if isinstance(content, str):
+        return {"role": role, "content": content}
+    if isinstance(content, list):
+        blocks: list[dict[str, Any]] = []
+        for block in content:
+            if not isinstance(block, dict):
+                continue
+            if block.get("type") == "text" and isinstance(block.get("text"), str):
+                blocks.append({"type": "text", "text": block["text"]})
+            elif block.get("type") == "image_url":
+                image_url = block.get("image_url")
+                if isinstance(image_url, str):
+                    image_url = {"url": image_url}
+                if isinstance(image_url, dict) and isinstance(image_url.get("url"), str):
+                    blocks.append({"type": "image_url", "image_url": {"url": image_url["url"]}})
+        if blocks:
+            return {"role": role, "content": blocks}
+    image = message.get("image_base64")
+    if isinstance(image, str) and image:
+        mime = str(message.get("mime_type") or "image/png")
+        text = message.get("text")
+        blocks = []
+        if isinstance(text, str) and text:
+            blocks.append({"type": "text", "text": text})
+        blocks.append({"type": "image_url", "image_url": {"url": f"data:{mime};base64,{image}"}})
+        return {"role": role, "content": blocks}
+    raise ValueError("message content must be text, image blocks, or image_base64")
diff --git a/apps/device-host-agent/host_agent/web/app.py b/apps/device-host-agent/host_agent/web/app.py
index 39ef6b5..8544a54 100644
--- a/apps/device-host-agent/host_agent/web/app.py
+++ b/apps/device-host-agent/host_agent/web/app.py
@@ -429,11 +429,11 @@ def create_console_app(
         if not isinstance(raw_messages, list) or not raw_messages:
             raise HTTPException(status_code=400, detail="messages must be a non-empty list")
         messages = [
-            {"role": str(item["role"]), "content": str(item["content"])}
+            {key: value for key, value in item.items() if key in {"role", "content", "image_base64", "mime_type", "text"}}
             for item in raw_messages
             if isinstance(item, dict)
             and item.get("role") in {"user", "assistant"}
-            and isinstance(item.get("content"), str)
+            and (isinstance(item.get("content"), (str, list)) or isinstance(item.get("image_base64"), str))
         ]
         if not messages:
             raise HTTPException(status_code=400, detail="messages are invalid")