feat: checkpoint device agent runtime milestones

tests/fakes.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import Any
 
-from core.driver import Driver
+from driver.base import Driver
 
 PNG_10X20 = (
     b"\x89PNG\r\n\x1a\n"
@@ -82,4 +82,3 @@ class FakeDriver(Driver):
 
     def unlock(self) -> None:
         self.calls.append(("unlock", ()))
-

tests/test_console_api.py

@@ -0,0 +1,194 @@
+from __future__ import annotations
+
+import base64
+from datetime import UTC, datetime
+
+import pytest
+
+from core.models import Task
+from device.manager import DeviceManager
+from runtime.task import TaskRunner, TaskRunnerConfig
+from storage.artifact_store import ArtifactStore
+from storage.device_config import DeviceConfigStore
+from storage.task_metadata import TaskMetadataStore
+from storage.timeline import Timeline
+from tests.fakes import PNG_10X20, FakeDriver
+
+
+def _client(tmp_path, *, manager=None, runner=None, config_store=None, timeline=None):
+    pytest.importorskip("fastapi")
+    from fastapi.testclient import TestClient
+
+    from api.rest import create_app
+
+    metadata_store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
+    app = create_app(
+        manager=manager or DeviceManager(),
+        metadata_store=metadata_store,
+        task_runner=runner,
+        device_config_store=config_store
+        or DeviceConfigStore(tmp_path / "device_config.sqlite3"),
+        timeline=timeline or Timeline(ArtifactStore(tmp_path / "history")),
+    )
+    return TestClient(app), metadata_store
+
+
+def test_console_status_endpoints_cover_empty_and_populated_states(tmp_path) -> None:
+    manager = DeviceManager()
+    client, metadata_store = _client(tmp_path, manager=manager)
+
+    assert client.get("/console/devices").json() == []
+    assert client.get("/console/tasks").json() == []
+
+    manager.register_device(
+        "iphone-1",
+        lambda: FakeDriver(),
+        name="Desk iPhone",
+        driver_type="wda",
+    )
+    older = Task(
+        id="task-old",
+        goal="open settings",
+        device_id="iphone-1",
+        created_at=datetime(2026, 1, 1, tzinfo=UTC),
+        updated_at=datetime(2026, 1, 1, tzinfo=UTC),
+    )
+    newer = Task(
+        id="task-new",
+        goal="search",
+        device_id="iphone-2",
+        status="running",
+        created_at=datetime(2026, 1, 2, tzinfo=UTC),
+        updated_at=datetime(2026, 1, 2, tzinfo=UTC),
+    )
+    metadata_store.create_task(older)
+    metadata_store.create_task(newer)
+
+    devices = client.get("/console/devices").json()
+    assert devices == [
+        {
+            "id": "iphone-1",
+            "name": "Desk iPhone",
+            "status": "idle",
+            "driver_type": "wda",
+            "connection_info": {},
+        }
+    ]
+    assert [task["id"] for task in client.get("/console/tasks").json()] == [
+        "task-new",
+        "task-old",
+    ]
+    assert [
+        task["id"]
+        for task in client.get("/console/tasks?device_id=iphone-1").json()
+    ] == ["task-old"]
+    assert [task["id"] for task in client.get("/console/tasks?status=running").json()] == [
+        "task-new"
+    ]
+    assert client.get("/console/tasks/task-old").json()["goal"] == "open settings"
+    assert client.get("/console/tasks/missing").status_code == 404
+
+
+def test_console_timeline_inlines_screenshot_and_handles_empty_history(tmp_path) -> None:
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    client, metadata_store = _client(tmp_path, timeline=timeline)
+    task = Task(id="task-1", goal="tap search", device_id="iphone-1")
+    metadata_store.create_task(task)
+
+    assert client.get("/console/tasks/task-1/timeline").json() == []
+
+    timeline.append(
+        task_id="task-1",
+        scene={"screen": {"width": 10, "height": 20}, "elements": []},
+        prompt="tap search",
+        tool_call={"action": "tap", "args": {"x": 1, "y": 2}},
+        result={"ok": True},
+        screenshot=PNG_10X20,
+    )
+
+    records = client.get("/console/tasks/task-1/timeline").json()
+    assert records[0]["index"] == 1
+    assert records[0]["image_base64"] == base64.b64encode(PNG_10X20).decode("ascii")
+    assert client.get("/console/tasks/missing/timeline").status_code == 404
+
+
+def test_console_device_registration_and_unregistration(tmp_path) -> None:
+    config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+    manager = DeviceManager()
+    client, _ = _client(tmp_path, manager=manager, config_store=config_store)
+
+    rejected = client.post(
+        "/console/devices",
+        json={"driver_type": "android", "connection_info": {}},
+    )
+    assert rejected.status_code == 400
+    assert config_store.list() == []
+    assert manager.list_devices() == []
+
+    response = client.post(
+        "/console/devices",
+        json={
+            "driver_type": "wda",
+            "name": "Desk iPhone",
+            "connection_info": {
+                "server_url": "http://127.0.0.1:4723",
+                "udid": "abc123",
+            },
+        },
+    )
+    assert response.status_code == 201
+    device_id = response.json()["id"]
+    assert response.json()["status"] == "idle"
+    assert config_store.get(device_id)["connection_info"]["udid"] == "abc123"
+    assert [device.id for device in manager.list_devices()] == [device_id]
+
+    delete_response = client.delete(f"/console/devices/{device_id}")
+    assert delete_response.status_code == 204
+    assert config_store.get(device_id) is None
+    assert manager.list_devices() == []
+    assert client.delete("/console/devices/missing").status_code == 404
+
+
+def test_console_config_get_update_and_validation(tmp_path) -> None:
+    config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+    config_store.set_setting("max_steps", 7)
+    runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
+    client, _ = _client(tmp_path, runner=runner, config_store=config_store)
+
+    assert runner.config.max_steps == 7
+    assert client.get("/console/config").json() == {"max_steps": 7}
+
+    response = client.put("/console/config", json={"max_steps": 30})
+    assert response.status_code == 200
+    assert response.json() == {"max_steps": 30}
+    assert runner.config.max_steps == 30
+    assert config_store.get_setting("max_steps") == "30"
+
+    rejected = client.put("/console/config", json={"max_steps": 0})
+    assert rejected.status_code == 400
+    assert runner.config.max_steps == 30
+    assert config_store.get_setting("max_steps") == "30"
+
+
+def test_console_startup_reloads_persisted_devices_and_settings(tmp_path) -> None:
+    config_store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+    config_store.add(
+        device_id="persisted-1",
+        name="Persisted iPhone",
+        driver_type="wda",
+        connection_info={"udid": "abc123"},
+    )
+    config_store.set_setting("max_steps", 31)
+    manager = DeviceManager()
+    runner = TaskRunner(config=TaskRunnerConfig(max_steps=1))
+
+    client, _ = _client(
+        tmp_path,
+        manager=manager,
+        runner=runner,
+        config_store=config_store,
+    )
+
+    assert runner.config.max_steps == 31
+    assert [device.id for device in manager.list_devices()] == ["persisted-1"]
+    assert client.get("/console/devices").json()[0]["name"] == "Persisted iPhone"

tests/test_describe_screen_semantic.py

@@ -0,0 +1,121 @@
+from __future__ import annotations
+
+from typing import Any
+
+from core.models import Bounds, Scene, SceneElement
+from device.manager import DeviceManager
+from semantic.config import SemanticConfig
+from semantic.llm_client import EnrichmentUnavailable
+from tools.describe_screen import describe_screen
+from tools.describe_screen_semantic import describe_screen_semantic
+from tests.fakes import FakeDriver
+
+
+class FakePerceptionProvider:
+    def __init__(self, scene: Scene) -> None:
+        self.scene = scene
+        self.calls: list[tuple[bytes, Any]] = []
+
+    def build_scene(self, screenshot: bytes, raw_tree: Any) -> Scene:
+        self.calls.append((screenshot, raw_tree))
+        return self.scene
+
+
+class FakeClient:
+    def __init__(
+        self,
+        *,
+        response: dict[str, Any] | None = None,
+        error: Exception | None = None,
+    ) -> None:
+        self.response = response
+        self.error = error
+
+    def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
+        if self.error:
+            raise self.error
+        assert self.response is not None
+        return self.response
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="send",
+                type="button",
+                text="Send",
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def _connected_manager(driver: FakeDriver) -> DeviceManager:
+    manager = DeviceManager()
+    manager.register_device("phone", lambda: driver)
+    manager.connect("phone", max_retries=1)
+    return manager
+
+
+def test_describe_screen_existing_behavior_is_unchanged() -> None:
+    scene = _scene()
+    provider = FakePerceptionProvider(scene)
+    driver = FakeDriver()
+    manager = _connected_manager(driver)
+
+    result = describe_screen(
+        "phone",
+        manager=manager,
+        perception_provider=provider,
+    )
+
+    assert result is scene
+    assert isinstance(result, Scene)
+    assert len(provider.calls) == 1
+    assert ("screenshot", ()) in driver.calls
+    assert ("tree", ()) in driver.calls
+
+
+def test_describe_screen_semantic_returns_scene_and_semantic_scene() -> None:
+    scene = _scene()
+    provider = FakePerceptionProvider(scene)
+    manager = _connected_manager(FakeDriver())
+    client = FakeClient(
+        response={
+            "page": "Chat",
+            "intents": ["send a message"],
+            "widgets": [{"element_id": "send", "purpose": "send message"}],
+        }
+    )
+
+    result = describe_screen_semantic(
+        "phone",
+        manager=manager,
+        perception_provider=provider,
+        client=client,
+        semantic_config=SemanticConfig(enabled=True),
+    )
+
+    assert result["scene"] is scene
+    assert result["semantic_scene"] is not None
+    assert result["semantic_scene"].to_dict()["page"] == "Chat"  # type: ignore[union-attr]
+
+
+def test_describe_screen_semantic_degrades_semantic_scene_to_none() -> None:
+    scene = _scene()
+    provider = FakePerceptionProvider(scene)
+    manager = _connected_manager(FakeDriver())
+
+    result = describe_screen_semantic(
+        "phone",
+        manager=manager,
+        perception_provider=provider,
+        client=FakeClient(error=EnrichmentUnavailable("timeout")),
+        semantic_config=SemanticConfig(enabled=True),
+    )
+
+    assert result["scene"] is scene
+    assert result["semantic_scene"] is None

tests/test_device_config.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+from storage.device_config import DeviceConfigStore
+
+
+def test_device_config_store_add_remove_list_and_get(tmp_path) -> None:
+    store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+
+    store.add(
+        device_id="iphone-1",
+        name="Desk iPhone",
+        driver_type="wda",
+        connection_info={"server_url": "http://127.0.0.1:4723", "udid": "abc123"},
+    )
+    store.add(
+        device_id="iphone-2",
+        name=None,
+        driver_type="wda",
+        connection_info={"wda_local_port": 8101},
+    )
+
+    assert store.get("iphone-1") == {
+        "device_id": "iphone-1",
+        "name": "Desk iPhone",
+        "driver_type": "wda",
+        "connection_info": {
+            "server_url": "http://127.0.0.1:4723",
+            "udid": "abc123",
+        },
+    }
+    assert [config["device_id"] for config in store.list()] == ["iphone-1", "iphone-2"]
+
+    store.remove("iphone-1")
+
+    assert store.get("iphone-1") is None
+    assert [config["device_id"] for config in store.list()] == ["iphone-2"]
+
+
+def test_device_config_store_settings_get_set_and_defaults(tmp_path) -> None:
+    store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+
+    assert store.get_setting("max_steps") == "20"
+    assert store.get_setting("missing") is None
+
+    store.set_setting("max_steps", 30)
+    store.set_setting("feature_flag", "enabled")
+
+    reopened = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+    assert reopened.get_setting("max_steps") == "30"
+    assert reopened.get_setting("feature_flag") == "enabled"
+
+
+def test_device_config_store_unknown_device_remove_is_noop(tmp_path) -> None:
+    store = DeviceConfigStore(tmp_path / "device_config.sqlite3")
+
+    store.remove("missing")
+
+    assert store.get("missing") is None

tests/test_device_manager.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import pytest
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from core.errors import DeviceOfflineError
 from tests.fakes import FakeDriver
 
@@ -30,4 +30,3 @@ def test_device_manager_marks_unreachable_device_offline() -> None:
         manager.connect("iphone-1", max_retries=2, retry_backoff_seconds=0)
 
     assert manager.status("iphone-1") == "offline"
-

tests/test_mcp.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from api.mcp import tool_handlers
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from tests.fakes import FakeDriver
 
 

tests/test_rest_api.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import pytest
 
-from core.device_manager import DeviceManager
+from device.manager import DeviceManager
 from core.models import Task
 from storage.task_metadata import TaskMetadataStore
 from tests.fakes import FakeDriver
@@ -41,4 +41,3 @@ def test_rest_start_task_and_poll_until_complete(tmp_path) -> None:
     status_response = client.get(f"/task/{task_id}")
     assert status_response.status_code == 200
     assert status_response.json()["status"] == "completed"
-

tests/test_scene_builder.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from core.models import Bounds, SceneElement
-from vision.scene_builder import bbox_iou, build_scene
+from perception.scene_builder import bbox_iou, build_scene
 
 
 def test_scene_builder_merges_overlapping_ocr_into_ui_element() -> None:
@@ -42,4 +42,3 @@ def test_scene_builder_merges_overlapping_ocr_into_ui_element() -> None:
     assert scene.elements[0].text == "Search"
     assert scene.elements[1].text == "Footer"
     assert bbox_iou(ui_button.bounds, ocr_label.bounds) > 0.5
-

tests/test_semantic_enricher.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from typing import Any
+
+from core.models import Bounds, Scene, SceneElement
+from semantic.config import SemanticConfig, load_config
+from semantic.enricher import enrich_scene
+from semantic.llm_client import EnrichmentUnavailable
+
+
+class FakeClient:
+    def __init__(
+        self,
+        *,
+        response: dict[str, Any] | None = None,
+        error: Exception | None = None,
+    ) -> None:
+        self.response = response
+        self.error = error
+        self.calls: list[tuple[dict[str, Any], float]] = []
+
+    def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
+        self.calls.append((scene_json, timeout))
+        if self.error:
+            raise self.error
+        assert self.response is not None
+        return self.response
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="input",
+                type="input",
+                bounds=Bounds(1, 2, 3, 4),
+                text="Message",
+            ),
+            SceneElement(
+                id="send",
+                type="button",
+                bounds=Bounds(5, 2, 3, 4),
+                text="Send",
+            ),
+        ],
+    )
+
+
+def _semantic_payload() -> dict[str, Any]:
+    return {
+        "page": "Chat",
+        "intents": ["send a message"],
+        "widgets": [
+            {"element_id": "input", "purpose": "message text input"},
+            {"element_id": "send", "purpose": "send message"},
+        ],
+    }
+
+
+def test_enrichment_config_is_disabled_by_default() -> None:
+    assert load_config({}).enabled is False
+
+
+def test_enrich_scene_returns_semantic_scene_when_enabled() -> None:
+    client = FakeClient(response=_semantic_payload())
+
+    result = enrich_scene(
+        _scene(),
+        client=client,
+        config=SemanticConfig(enabled=True, timeout=2.5),
+    )
+
+    assert result is not None
+    assert result.to_dict() == _semantic_payload()
+    assert len(client.calls) == 1
+    assert client.calls[0][1] == 2.5
+
+
+def test_enrich_scene_disabled_short_circuits_without_client_call() -> None:
+    client = FakeClient(response=_semantic_payload())
+
+    result = enrich_scene(
+        _scene(),
+        client=client,
+        config=SemanticConfig(enabled=False),
+    )
+
+    assert result is None
+    assert client.calls == []
+
+
+def test_enrich_scene_degrades_to_none_on_client_failure() -> None:
+    client = FakeClient(error=EnrichmentUnavailable("rate limited"))
+
+    result = enrich_scene(
+        _scene(),
+        client=client,
+        config=SemanticConfig(enabled=True),
+    )
+
+    assert result is None
+
+
+def test_enrich_scene_filters_dangling_element_ids() -> None:
+    payload = _semantic_payload()
+    payload["widgets"] = [
+        {"element_id": "send", "purpose": "send message"},
+        {"element_id": "missing", "purpose": "unknown control"},
+    ]
+    client = FakeClient(response=payload)
+
+    result = enrich_scene(
+        _scene(),
+        client=client,
+        config=SemanticConfig(enabled=True),
+    )
+
+    assert result is not None
+    assert result.to_dict()["widgets"] == [
+        {"element_id": "send", "purpose": "send message"}
+    ]
+
+
+def test_enrich_scene_returns_none_when_all_widgets_are_dangling() -> None:
+    client = FakeClient(
+        response={
+            "page": "Chat",
+            "intents": ["send a message"],
+            "widgets": [{"element_id": "missing", "purpose": "unknown control"}],
+        }
+    )
+
+    result = enrich_scene(
+        _scene(),
+        client=client,
+        config=SemanticConfig(enabled=True),
+    )
+
+    assert result is None

tests/test_semantic_integration.py

@@ -0,0 +1,48 @@
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from core.models import Bounds, Scene, SceneElement
+from semantic.config import SemanticConfig
+from semantic.enricher import enrich_scene
+
+
+@pytest.mark.integration
+def test_real_anthropic_semantic_enrichment_returns_schema_valid_scene() -> None:
+    if not os.environ.get("ANTHROPIC_API_KEY"):
+        pytest.skip("ANTHROPIC_API_KEY is required for semantic integration test")
+    try:
+        import anthropic  # noqa: F401
+    except ImportError:
+        pytest.skip("anthropic SDK is not installed")
+
+    scene = Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="input",
+                type="input",
+                text="Message",
+                bounds=Bounds(1, 2, 3, 4),
+            ),
+            SceneElement(
+                id="send",
+                type="button",
+                text="Send",
+                bounds=Bounds(5, 2, 3, 4),
+            ),
+        ],
+    )
+
+    result = enrich_scene(
+        scene,
+        config=SemanticConfig(enabled=True, timeout=10.0),
+    )
+
+    assert result is not None
+    assert result.page
+    assert result.intents
+    assert all(widget.element_id in {"input", "send"} for widget in result.widgets)

tests/test_semantic_llm_client.py

@@ -0,0 +1,110 @@
+from __future__ import annotations
+
+import pytest
+
+from semantic.llm_client import AnthropicSemanticClient, EnrichmentUnavailable
+
+RateLimitError = type("RateLimitError", (Exception,), {})
+
+
+class FakeMessages:
+    def __init__(self, *, response: object | None = None, error: Exception | None = None):
+        self.response = response
+        self.error = error
+        self.calls: list[dict[str, object]] = []
+
+    def create(self, **kwargs: object) -> object:
+        self.calls.append(kwargs)
+        if self.error:
+            raise self.error
+        return self.response
+
+
+class FakeTransport:
+    def __init__(self, messages: FakeMessages) -> None:
+        self.messages = messages
+
+
+def _payload() -> dict[str, object]:
+    return {
+        "page": "Chat",
+        "intents": ["send a message"],
+        "widgets": [{"element_id": "send", "purpose": "send message"}],
+    }
+
+
+def test_anthropic_semantic_client_requests_schema_output_and_prompt_cache() -> None:
+    messages = FakeMessages(response={"content": [{"parsed": _payload()}]})
+    client = AnthropicSemanticClient(model="test-model", transport=FakeTransport(messages))
+
+    result = client.enrich({"screen": {"width": 10, "height": 20}}, timeout=1.5)
+
+    assert result == _payload()
+    assert len(messages.calls) == 1
+    call = messages.calls[0]
+    assert call["model"] == "test-model"
+    assert call["timeout"] == 1.5
+    assert call["output_config"] == {
+        "format": {
+            "type": "json_schema",
+            "schema": {
+                "type": "object",
+                "additionalProperties": False,
+                "required": ["page", "intents", "widgets"],
+                "properties": {
+                    "page": {"type": "string", "minLength": 1},
+                    "intents": {
+                        "type": "array",
+                        "items": {"type": "string", "minLength": 1},
+                    },
+                    "widgets": {
+                        "type": "array",
+                        "items": {
+                            "type": "object",
+                            "additionalProperties": False,
+                            "required": ["element_id", "purpose"],
+                            "properties": {
+                                "element_id": {"type": "string", "minLength": 1},
+                                "purpose": {"type": "string", "minLength": 1},
+                            },
+                        },
+                    },
+                },
+            },
+        }
+    }
+    assert call["system"][0]["cache_control"] == {"type": "ephemeral"}  # type: ignore[index]
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        TimeoutError("timed out"),
+        RateLimitError("rate limited"),
+    ],
+)
+def test_anthropic_semantic_client_maps_transport_failures(
+    error: Exception,
+) -> None:
+    messages = FakeMessages(error=error)
+    client = AnthropicSemanticClient(transport=FakeTransport(messages))
+
+    with pytest.raises(EnrichmentUnavailable):
+        client.enrich({"screen": {"width": 10, "height": 20}}, timeout=1)
+
+
+@pytest.mark.parametrize(
+    "response",
+    [
+        {"content": [{"text": "not json"}]},
+        {"page": "Chat", "intents": ["send a message"]},
+    ],
+)
+def test_anthropic_semantic_client_maps_malformed_or_schema_invalid_responses(
+    response: object,
+) -> None:
+    messages = FakeMessages(response=response)
+    client = AnthropicSemanticClient(transport=FakeTransport(messages))
+
+    with pytest.raises(EnrichmentUnavailable):
+        client.enrich({"screen": {"width": 10, "height": 20}}, timeout=1)

tests/test_semantic_models.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from semantic.models import SemanticScene, SemanticWidget
+
+
+def test_semantic_scene_round_trips_through_dict() -> None:
+    scene = SemanticScene(
+        page="Chat",
+        intents=["send a message", "open attachments"],
+        widgets=[
+            SemanticWidget(element_id="input", purpose="message text input"),
+            SemanticWidget(element_id="send", purpose="send message"),
+        ],
+    )
+
+    restored = SemanticScene.from_dict(scene.to_dict())
+
+    assert restored == scene
+    assert restored.to_dict() == {
+        "page": "Chat",
+        "intents": ["send a message", "open attachments"],
+        "widgets": [
+            {"element_id": "input", "purpose": "message text input"},
+            {"element_id": "send", "purpose": "send message"},
+        ],
+    }

tests/test_semantic_task_loop.py

@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pytest
+
+from core.models import Bounds, Scene, SceneElement, Task
+from device.manager import DeviceManager
+from runtime.executor import Executor, ExecutorConfig
+from runtime.planner import PlannedStep, Planner
+from runtime.task import TaskRunner, TaskRunnerConfig
+from semantic.config import SemanticConfig
+from semantic.llm_client import EnrichmentUnavailable
+from tools.describe_screen_semantic import describe_screen_semantic
+from tests.fakes import FakeDriver, PNG_10X20
+
+
+class FakePerceptionProvider:
+    def __init__(self, scene: Scene) -> None:
+        self.scene = scene
+
+    def build_scene(self, screenshot: bytes, raw_tree: Any) -> Scene:
+        return self.scene
+
+
+class FakeClient:
+    def __init__(
+        self,
+        *,
+        response: dict[str, Any] | None = None,
+        error: Exception | None = None,
+    ) -> None:
+        self.response = response
+        self.error = error
+
+    def enrich(self, scene_json: dict[str, Any], *, timeout: float) -> dict[str, Any]:
+        if self.error:
+            raise self.error
+        assert self.response is not None
+        return self.response
+
+
+class ObserveSemanticPlanner(Planner):
+    def plan(self, *, goal: str, scene: Scene, context: object) -> list[PlannedStep]:
+        if context.step_results:  # type: ignore[attr-defined]
+            return []
+        return [
+            PlannedStep(
+                action="describe_screen_semantic",
+                description=f"Observe current screen for goal: {goal}",
+            )
+        ]
+
+    def goal_reached(self, *, goal: str, scene: Scene, context: object) -> bool:
+        return bool(context.step_results) and all(  # type: ignore[attr-defined]
+            result.success for result in context.step_results  # type: ignore[attr-defined]
+        )
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="send",
+                type="button",
+                text="Send",
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def _connected_manager() -> DeviceManager:
+    manager = DeviceManager()
+    manager.register_device("phone", FakeDriver)
+    manager.connect("phone", max_retries=1)
+    return manager
+
+
+@pytest.mark.parametrize(
+    "client,expected_semantic",
+    [
+        (
+            FakeClient(
+                response={
+                    "page": "Chat",
+                    "intents": ["send a message"],
+                    "widgets": [{"element_id": "send", "purpose": "send message"}],
+                }
+            ),
+            True,
+        ),
+        (FakeClient(error=EnrichmentUnavailable("forced failure")), False),
+    ],
+)
+def test_task_loop_completes_when_describe_screen_semantic_succeeds_or_degrades(
+    client: FakeClient,
+    expected_semantic: bool,
+) -> None:
+    scene = _scene()
+    manager = _connected_manager()
+    provider = FakePerceptionProvider(scene)
+    tool_results: list[dict[str, object]] = []
+
+    def semantic_tool(device_id: str | None = None) -> dict[str, object]:
+        result = describe_screen_semantic(
+            device_id,
+            manager=manager,
+            perception_provider=provider,
+            client=client,
+            semantic_config=SemanticConfig(enabled=True),
+        )
+        tool_results.append(result)
+        return result
+
+    runner = TaskRunner(
+        planner=ObserveSemanticPlanner(),
+        executor=Executor(
+            tools={"describe_screen_semantic": semantic_tool},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        config=TaskRunnerConfig(max_steps=3),
+        observer=lambda device_id: scene,
+        screenshot_provider=lambda device_id: PNG_10X20,
+    )
+    task = Task(goal="inspect current screen", device_id="phone")
+
+    result = runner.run(task)
+
+    assert result.status == "completed"
+    assert len(tool_results) == 1
+    assert (tool_results[0]["semantic_scene"] is not None) is expected_semantic

tests/test_smoke.py

@@ -4,6 +4,16 @@ import importlib
 
 
 def test_imports_new_packages() -> None:
-    for package in ("core", "tools", "vision", "runtime", "api", "storage"):
+    for package in (
+        "api",
+        "core",
+        "device",
+        "driver",
+        "perception",
+        "runtime",
+        "semantic",
+        "storage",
+        "tools",
+        "world",
+    ):
         importlib.import_module(package)
-

tests/test_wda_integration.py

@@ -4,7 +4,7 @@ import os
 
 import pytest
 
-from core.wda_driver import WDADriver, WDADriverConfig
+from driver.wda_driver import WDADriver, WDADriverConfig
 
 
 @pytest.mark.integration
@@ -25,4 +25,3 @@ def test_wda_driver_screenshot_against_real_device() -> None:
         assert driver.screenshot()
     finally:
         driver.disconnect()
-

tests/test_world_model.py

@@ -0,0 +1,144 @@
+from __future__ import annotations
+
+from core.models import Bounds, Scene, SceneElement
+from runtime.executor import StepResult
+from runtime.planner import PlannedStep
+from semantic.models import SemanticScene, SemanticWidget
+from world.config import WorldConfig, load_config
+from world.model import WorldModel
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="send",
+                type="button",
+                text="Send",
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def _semantic_scene(page: str = "Chat") -> SemanticScene:
+    return SemanticScene(
+        page=page,
+        intents=["send a message"],
+        widgets=[SemanticWidget(element_id="send", purpose="send message")],
+    )
+
+
+def _result(step: PlannedStep, *, success: bool = True) -> StepResult:
+    return StepResult(step=step, success=success, attempts=1)
+
+
+def test_world_config_defaults_to_enabled_with_history_size_bound() -> None:
+    config = load_config({})
+
+    assert config.enabled is True
+    assert config.history_size == 10
+
+
+def test_observe_updates_current_page_from_semantic_scene() -> None:
+    step = PlannedStep(action="tap", description="tap send")
+    model = WorldModel(config=WorldConfig(history_size=2))
+
+    model.observe(_scene(), _semantic_scene("Chat"), step, _result(step))
+
+    assert model.state.current_page == "Chat"
+
+
+def test_observe_leaves_current_page_unchanged_without_semantic_scene() -> None:
+    step = PlannedStep(action="tap", description="tap send")
+    model = WorldModel(config=WorldConfig(history_size=2))
+    model.state.current_page = "Chat"
+
+    model.observe(_scene(), None, step, _result(step))
+
+    assert model.state.current_page == "Chat"
+
+
+def test_observe_updates_current_app_for_successful_launch_and_clear_for_terminate() -> None:
+    model = WorldModel(config=WorldConfig(history_size=2))
+    launch = PlannedStep(
+        action="launch_app",
+        description="launch chat",
+        args={"app_id": "com.example.chat"},
+    )
+    terminate = PlannedStep(
+        action="terminate_app",
+        description="close chat",
+        args={"app_id": "com.example.chat"},
+    )
+
+    model.observe(_scene(), None, launch, _result(launch))
+    assert model.state.current_app == "com.example.chat"
+
+    model.observe(_scene(), None, terminate, _result(terminate))
+    assert model.state.current_app is None
+
+
+def test_observe_does_not_update_current_app_on_failed_or_unrelated_step() -> None:
+    model = WorldModel(config=WorldConfig(history_size=3))
+    model.state.current_app = "com.example.chat"
+    failed_launch = PlannedStep(
+        action="launch_app",
+        description="launch other",
+        args={"app_id": "com.example.other"},
+    )
+    tap = PlannedStep(action="tap", description="tap", args={"x": 1, "y": 2})
+
+    model.observe(_scene(), None, failed_launch, _result(failed_launch, success=False))
+    model.observe(_scene(), None, tap, _result(tap))
+
+    assert model.state.current_app == "com.example.chat"
+
+
+def test_observe_merges_explicit_remember_variables_only() -> None:
+    model = WorldModel(config=WorldConfig(history_size=3))
+    remember_step = PlannedStep(
+        action="tap",
+        description="select chat",
+        args={"remember": {"contact": "Zhang San"}},
+    )
+    plain_step = PlannedStep(action="tap", description="tap send", args={})
+
+    model.observe(_scene(), None, remember_step, _result(remember_step))
+    model.observe(_scene(), None, plain_step, _result(plain_step))
+
+    assert model.state.variables == {"contact": "Zhang San"}
+
+
+def test_observe_skips_malformed_fields_without_raising() -> None:
+    model = WorldModel(config=WorldConfig(history_size=2))
+    missing_app = PlannedStep(action="launch_app", description="launch", args={})
+    bad_remember = PlannedStep(
+        action="tap",
+        description="tap",
+        args={"remember": "not-a-dict"},
+    )
+
+    model.observe(_scene(), None, missing_app, _result(missing_app))
+    model.observe(_scene(), None, bad_remember, _result(bad_remember))
+
+    assert model.state.current_app is None
+    assert model.state.variables == {}
+    assert len(model.state.history) == 2
+
+
+def test_observe_appends_semantic_or_raw_scene_history_with_eviction() -> None:
+    model = WorldModel(config=WorldConfig(history_size=2))
+    first = PlannedStep(action="first", description="first")
+    second = PlannedStep(action="second", description="second")
+    third = PlannedStep(action="third", description="third")
+
+    model.observe(_scene(), _semantic_scene("Chat"), first, _result(first))
+    model.observe(_scene(), None, second, _result(second))
+    model.observe(_scene(), None, third, _result(third))
+
+    assert [event.action for event in model.state.history] == ["second", "third"]
+    assert model.state.history[0].scene_summary.to_dict() == _scene().to_dict()
+    assert model.state.history.maxlen == 2

tests/test_world_models.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from core.models import Bounds, Scene, SceneElement
+from semantic.models import SemanticScene, SemanticWidget
+from world.models import WorldEvent, WorldState
+
+
+def _scene() -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="send",
+                type="button",
+                text="Send",
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def test_world_event_and_state_to_dict() -> None:
+    semantic_scene = SemanticScene(
+        page="Chat",
+        intents=["send a message"],
+        widgets=[SemanticWidget(element_id="send", purpose="send message")],
+    )
+    event = WorldEvent(
+        scene_summary=semantic_scene,
+        action="tap",
+        success=True,
+    )
+    state = WorldState.with_history_bound(2)
+    state.current_app = "com.example.chat"
+    state.current_page = "Chat"
+    state.variables["contact"] = "Zhang San"
+    state.history.append(event)
+
+    data = state.to_dict()
+
+    assert data["current_app"] == "com.example.chat"
+    assert data["current_page"] == "Chat"
+    assert data["variables"] == {"contact": "Zhang San"}
+    assert data["history"][0]["scene_summary"] == semantic_scene.to_dict()
+    assert data["history"][0]["action"] == "tap"
+    assert data["history"][0]["success"] is True
+    assert data["history"][0]["timestamp"]
+
+
+def test_world_state_history_evicts_oldest_entry_at_bound() -> None:
+    state = WorldState.with_history_bound(2)
+
+    state.history.append(WorldEvent(_scene(), "first", True))
+    state.history.append(WorldEvent(_scene(), "second", True))
+    state.history.append(WorldEvent(_scene(), "third", True))
+
+    assert len(state.history) == 2
+    assert [event.action for event in state.history] == ["second", "third"]
+    assert state.history.maxlen == 2

tests/test_world_task_runner.py

@@ -0,0 +1,275 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from typing import Any
+
+from core.models import Bounds, Scene, SceneElement, Task
+from runtime.context import TaskContext
+from runtime.executor import Executor, ExecutorConfig
+from runtime.planner import PlannedStep, Planner
+from runtime.task import TaskRunner, TaskRunnerConfig
+from semantic.models import SemanticScene, SemanticWidget
+from world.config import WorldConfig
+from world.model import WorldModel
+
+
+class RecordingPlanner(Planner):
+    def __init__(self, steps: list[PlannedStep]) -> None:
+        self.steps = steps
+        self.worlds: list[Any] = []
+        self.world_snapshots: list[dict[str, Any] | None] = []
+
+    def plan(self, *, goal, scene, context, world=None):
+        self.worlds.append(world)
+        self.world_snapshots.append(world.to_dict() if world is not None else None)
+        if len(context.step_results) >= len(self.steps):
+            return []
+        return [self.steps[len(context.step_results)]]
+
+    def goal_reached(self, *, goal, scene, context):
+        return len(context.step_results) >= len(self.steps) and all(
+            result.success for result in context.step_results
+        )
+
+
+class KwargRecordingPlanner(Planner):
+    def __init__(self) -> None:
+        self.received_world_kwarg: list[bool] = []
+
+    def plan(self, *, goal, scene, context, **kwargs):
+        self.received_world_kwarg.append("world" in kwargs)
+        if context.step_results:
+            return []
+        return [PlannedStep(action="tap", description="tap")]
+
+    def goal_reached(self, *, goal, scene, context):
+        return bool(context.step_results)
+
+
+class NoWorldPlanner(Planner):
+    def __init__(self) -> None:
+        self.calls = 0
+
+    def plan(self, *, goal, scene, context):
+        self.calls += 1
+        if context.step_results:
+            return []
+        return [PlannedStep(action="tap", description="tap")]
+
+    def goal_reached(self, *, goal, scene, context):
+        return bool(context.step_results)
+
+
+def _scene(text: str = "Send") -> Scene:
+    return Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="send",
+                type="button",
+                text=text,
+                bounds=Bounds(1, 2, 3, 4),
+            )
+        ],
+    )
+
+
+def _semantic_scene(page: str) -> SemanticScene:
+    return SemanticScene(
+        page=page,
+        intents=["send a message"],
+        widgets=[SemanticWidget(element_id="send", purpose="send message")],
+    )
+
+
+def _runner(
+    *,
+    planner: Planner,
+    executor: Executor,
+    scenes: list[Scene] | None = None,
+    world_model: WorldModel | None = None,
+    world_config: WorldConfig | None = None,
+) -> TaskRunner:
+    scene_iter: Iterator[Scene] = iter(scenes or [_scene(), _scene(), _scene()])
+
+    def observer(device_id: str) -> Scene:
+        return next(scene_iter, _scene())
+
+    return TaskRunner(
+        planner=planner,
+        executor=executor,
+        config=TaskRunnerConfig(max_steps=5),
+        observer=observer,
+        world_model=world_model,
+        world_config=world_config,
+    )
+
+
+def test_planner_plan_accepts_calls_with_and_without_world() -> None:
+    scene = _scene()
+    planner = Planner()
+    context = TaskContext(
+        task_id="task",
+        goal="inspect",
+    )
+    world_model = WorldModel(config=WorldConfig(history_size=2))
+
+    without_world = planner.plan(goal="inspect", scene=scene, context=context)
+    with_world = planner.plan(
+        goal="inspect",
+        scene=scene,
+        context=context,
+        world=world_model.state,
+    )
+
+    assert with_world == without_world
+
+
+def test_task_runner_populates_world_after_step_when_enabled_by_default() -> None:
+    planner = RecordingPlanner([PlannedStep(action="tap", description="tap")])
+    runner = _runner(
+        planner=planner,
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+    )
+
+    result = runner.run(Task(goal="inspect", device_id="phone"))
+
+    assert result.status == "completed"
+    assert planner.worlds[0] is not None
+    assert planner.world_snapshots[1] is not None
+    assert len(planner.world_snapshots[1]["history"]) == 1
+
+
+def test_task_runner_keeps_existing_planner_subclasses_without_world_working() -> None:
+    planner = NoWorldPlanner()
+    runner = _runner(
+        planner=planner,
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+    )
+
+    result = runner.run(Task(goal="inspect", device_id="phone"))
+
+    assert result.status == "completed"
+    assert planner.calls == 2
+
+
+def test_task_runner_disabled_world_config_does_not_pass_world_kwarg() -> None:
+    planner = KwargRecordingPlanner()
+    runner = _runner(
+        planner=planner,
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        world_config=WorldConfig(enabled=False),
+    )
+
+    result = runner.run(Task(goal="inspect", device_id="phone"))
+
+    assert result.status == "completed"
+    assert planner.received_world_kwarg == [False, False]
+
+
+def test_task_runner_passes_current_world_to_planner_by_second_step() -> None:
+    planner = RecordingPlanner(
+        [
+            PlannedStep(action="tap", description="first"),
+            PlannedStep(action="tap", description="second"),
+        ]
+    )
+    runner = _runner(
+        planner=planner,
+        executor=Executor(
+            tools={"tap": lambda **kwargs: {"ok": True}},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+    )
+
+    runner.run(Task(goal="inspect", device_id="phone"))
+
+    assert planner.worlds[1] is not None
+    assert planner.world_snapshots[1] is not None
+    assert planner.world_snapshots[2] is not None
+    assert len(planner.world_snapshots[1]["history"]) == 1
+    assert len(planner.world_snapshots[2]["history"]) == 2
+
+
+def test_task_runner_world_tracks_app_page_and_history_across_steps() -> None:
+    world_model = WorldModel(config=WorldConfig(history_size=3))
+    planner = RecordingPlanner(
+        [
+            PlannedStep(
+                action="launch_app",
+                description="launch chat",
+                args={"app_id": "com.example.chat"},
+            ),
+            PlannedStep(action="tap", description="open chat"),
+        ]
+    )
+
+    def launch_app(**kwargs):
+        return {"ok": True, "semantic_scene": _semantic_scene("Home")}
+
+    def tap(**kwargs):
+        return {"ok": True, "semantic_scene": _semantic_scene("Chat")}
+
+    runner = _runner(
+        planner=planner,
+        executor=Executor(
+            tools={"launch_app": launch_app, "tap": tap},
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        world_model=world_model,
+    )
+
+    result = runner.run(Task(goal="open chat", device_id="phone"))
+
+    assert result.status == "completed"
+    assert world_model.state.current_app == "com.example.chat"
+    assert world_model.state.current_page == "Chat"
+    assert [event.action for event in world_model.state.history] == [
+        "launch_app",
+        "tap",
+    ]
+    assert planner.world_snapshots[1]["current_page"] == "Home"
+    assert planner.world_snapshots[2]["current_page"] == "Chat"
+
+
+def test_task_runner_world_tracking_failures_do_not_interrupt_loop() -> None:
+    world_model = WorldModel(config=WorldConfig(history_size=3))
+    planner = RecordingPlanner(
+        [
+            PlannedStep(action="launch_app", description="missing app args"),
+            PlannedStep(
+                action="tap",
+                description="bad remember",
+                args={"remember": "not-a-dict"},
+            ),
+        ]
+    )
+    runner = _runner(
+        planner=planner,
+        executor=Executor(
+            tools={
+                "launch_app": lambda **kwargs: {"ok": True},
+                "tap": lambda **kwargs: {"ok": True},
+            },
+            config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+        ),
+        world_model=world_model,
+    )
+
+    result = runner.run(Task(goal="inspect", device_id="phone"))
+
+    assert result.status == "completed"
+    assert world_model.state.current_app is None
+    assert world_model.state.current_page is None
+    assert world_model.state.variables == {}
+    assert len(world_model.state.history) == 2