Support multimodal planner scenes without OCR

runtime/ai_planner.py

@@ -36,9 +36,10 @@ class AIPlanner(Planner):
         world: "WorldState | None" = None,
         screenshot: bytes | None = None,
     ) -> list[PlannedStep]:
+        scene_json = _without_ocr(scene.to_dict()) if self.config.multimodal else scene.to_dict()
         user_prompt = planner_user_prompt(
             goal=goal,
-            scene_json=scene.to_dict(),
+            scene_json=scene_json,
             history_summary=_history_summary(world),
             device_platform=context.device_platform,
         )
@@ -94,3 +95,16 @@ def _history_summary(world: "WorldState | None") -> list[dict[str, Any]]:
         }
         for event in world.history
     ]
+
+
+def _without_ocr(scene_json: dict[str, Any]) -> dict[str, Any]:
+    cleaned = dict(scene_json)
+    elements = cleaned.get("elements")
+    if isinstance(elements, list):
+        cleaned["elements"] = [
+            {key: value for key, value in element.items() if key not in {"source", "confidence", "foreground_color", "background_color"}}
+            for element in elements
+            if isinstance(element, dict) and element.get("source") != "ocr"
+        ]
+    cleaned.pop("ocr_elements", None)
+    return cleaned