fix(perception): degrade to OCR-only when UI tree is unavailable
Tests / Test passed: 858

driver.tree() failures (WDA/Appium session errors) previously raised
uncaught, killing describe_screen() before OCR ever ran. Malformed
tree content (invalid XML) had the same problem inside
parse_ui_tree(). Both are now caught and logged, falling back to an
empty ui_elements list so the scene degrades to OCR-only, mirroring
the existing OCR-failure fallback in run_ocr().
This commit is contained in:
2026-07-15 10:03:30 +08:00
parent 778af2da53
commit 8d5b02e37f
4 changed files with 126 additions and 9 deletions
+11 -1
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import logging
from abc import ABC, abstractmethod
from typing import Any
@@ -9,6 +10,8 @@ from perception.scene_builder import build_scene as build_fused_scene
from perception.scene_builder import infer_png_size
from perception.ui_parser import parse_ui_tree
logger = logging.getLogger(__name__)
class PerceptionProvider(ABC):
@abstractmethod
@@ -22,10 +25,17 @@ class DefaultPerceptionProvider(PerceptionProvider):
def build_scene(self, screenshot: bytes, tree: Any) -> Scene:
width, height = infer_png_size(screenshot)
try:
ui_elements = parse_ui_tree(tree)
except Exception:
logger.warning(
"UI tree parsing failed; continuing without UI tree", exc_info=True
)
ui_elements = []
return build_fused_scene(
screen_width=width,
screen_height=height,
ui_elements=parse_ui_tree(tree),
ui_elements=ui_elements,
ocr_elements=run_ocr(screenshot, engine=self._ocr_engine),
)