From 44e1a6651ad3dbf83fbbbbcf1bacd2464aaa6177 Mon Sep 17 00:00:00 2001
From: showtan001 <240788545@qq.com>
Date: Sun, 30 Aug 2026 21:56:11 +0800
Subject: [PATCH] Support multimodal planner scenes without OCR

---
 README.md                 |  5 +++++
 runtime/ai_planner.py     | 16 +++++++++++++++-
 runtime/planner_config.py |  3 +++
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 309f1f7..62026e5 100644
--- a/README.md
+++ b/README.md
@@ -72,6 +72,7 @@ export AI_PLANNER_PROVIDER=openai-compatible
 export AI_PLANNER_MODEL=qwen2.5
 export AI_PLANNER_API_KEY=local-key
 export AI_PLANNER_BASE_URL=http://127.0.0.1:11434/v1
+export AI_PLANNER_MULTIMODAL=true
 uv run --package device-host-agent device-host-agent setup
 uv run --package device-host-agent device-host-agent
 ```
@@ -115,6 +116,10 @@ The compact form `{ "role": "user", "text": "...", "image_base64": "..." }`
 is also accepted. The model can inspect the supplied image and then call a
 phone tool such as `tap` in the same conversation.
 
+Set `AI_PLANNER_MULTIMODAL=true` for a vision model. The Planner sends the
+screenshot and omits OCR-only elements and OCR metadata from the structured
+scene payload, avoiding duplicate OCR text.
+
 Local mode records LLM responses, reasoning fields, tool calls, tool results,
 and final replies in a local SQLite database. View them at
 `http://127.0.0.1:8765/conversations`; image bytes are excluded. Set
diff --git a/runtime/ai_planner.py b/runtime/ai_planner.py
index 856d16f..09d8706 100644
--- a/runtime/ai_planner.py
+++ b/runtime/ai_planner.py
@@ -36,9 +36,10 @@ class AIPlanner(Planner):
         world: "WorldState | None" = None,
         screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
+        scene_json = _without_ocr(scene.to_dict()) if self.config.multimodal else scene.to_dict()
         user_prompt = planner_user_prompt(
             goal=goal,
-            scene_json=scene.to_dict(),
+            scene_json=scene_json,
             history_summary=_history_summary(world),
             device_platform=context.device_platform,
         )
@@ -94,3 +95,16 @@ def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]:
         }
         for event in world.history
     ]
+
+
+def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
+    cleaned = dict(scene_json)
+    elements = cleaned.get("elements")
+    if isinstance(elements, list):
+        cleaned["elements"] = [
+            {key: value for key, value in element.items() if key not in {"source", "confidence", "foreground_color", "background_color"}}
+            for element in elements
+            if isinstance(element, dict) and element.get("source") != "ocr"
+        ]
+    cleaned.pop("ocr_elements", None)
+    return cleaned
diff --git a/runtime/planner_config.py b/runtime/planner_config.py
index 24f9c99..872263a 100644
--- a/runtime/planner_config.py
+++ b/runtime/planner_config.py
@@ -19,6 +19,7 @@ TIMEOUT_ENV = "AI_PLANNER_TIMEOUT_SECONDS"
 THINKING_BUDGET_ENV = "AI_PLANNER_THINKING_BUDGET_TOKENS"
 API_KEY_ENV = "AI_PLANNER_API_KEY"
 BASE_URL_ENV = "AI_PLANNER_BASE_URL"
+MULTIMODAL_ENV = "AI_PLANNER_MULTIMODAL"
 
 SUPPORTED_PROVIDERS = frozenset(DEFAULT_MODEL_BY_PROVIDER)
 
@@ -32,6 +33,7 @@ class PlannerConfig:
     thinking_budget_tokens: int | None = None
     api_key: str | None = None
     base_url: str | None = None
+    multimodal: bool = False
 
     def resolved_model(self) -> str:
         return self.model or DEFAULT_MODEL_BY_PROVIDER[self.provider]
@@ -47,6 +49,7 @@ def load_config(env: Mapping[str, str] | None = None) -> PlannerConfig:
         thinking_budget_tokens=_parse_thinking_budget(values.get(THINKING_BUDGET_ENV)),
         api_key=values.get(API_KEY_ENV) or _provider_key(values),
         base_url=values.get(BASE_URL_ENV) or None,
+        multimodal=_parse_bool(values.get(MULTIMODAL_ENV), default=False),
     )