Implement Apex Agent MVP scaffold

tests/__init__.py

@@ -0,0 +1,2 @@
+"""Test package for Apex Agent."""
+

tests/fakes.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+from typing import Any
+
+from core.driver import Driver
+
+PNG_10X20 = (
+    b"\x89PNG\r\n\x1a\n"
+    b"\x00\x00\x00\r"
+    b"IHDR"
+    b"\x00\x00\x00\x0a"
+    b"\x00\x00\x00\x14"
+)
+
+TREE_XML = """
+<AppiumAUT x="0" y="0" width="10" height="20">
+  <XCUIElementTypeButton name="Search" label="Search" x="1" y="2" width="4" height="4" />
+  <XCUIElementTypeImage name="Settings" label="Settings" x="6" y="2" width="3" height="3" />
+</AppiumAUT>
+"""
+
+
+class FakeDriver(Driver):
+    def __init__(
+        self,
+        *,
+        fail_connect: bool = False,
+        tree: Any = TREE_XML,
+        screenshot: bytes = PNG_10X20,
+    ) -> None:
+        self.fail_connect = fail_connect
+        self.connected = False
+        self.calls: list[tuple[str, tuple[Any, ...]]] = []
+        self._tree = tree
+        self._screenshot = screenshot
+
+    def connect(self) -> None:
+        self.calls.append(("connect", ()))
+        if self.fail_connect:
+            raise RuntimeError("connection refused")
+        self.connected = True
+
+    def disconnect(self) -> None:
+        self.calls.append(("disconnect", ()))
+        self.connected = False
+
+    def screenshot(self) -> bytes:
+        self.calls.append(("screenshot", ()))
+        return self._screenshot
+
+    def tap(self, x: float, y: float) -> None:
+        self.calls.append(("tap", (x, y)))
+
+    def swipe(
+        self,
+        start_x: float,
+        start_y: float,
+        end_x: float,
+        end_y: float,
+        duration_ms: int = 500,
+    ) -> None:
+        self.calls.append(("swipe", (start_x, start_y, end_x, end_y, duration_ms)))
+
+    def input(self, text: str) -> None:
+        self.calls.append(("input", (text,)))
+
+    def launch(self, app_id: str) -> None:
+        self.calls.append(("launch", (app_id,)))
+
+    def terminate(self, app_id: str) -> None:
+        self.calls.append(("terminate", (app_id,)))
+
+    def tree(self) -> Any:
+        self.calls.append(("tree", ()))
+        return self._tree
+
+    def home(self) -> None:
+        self.calls.append(("home", ()))
+
+    def lock(self) -> None:
+        self.calls.append(("lock", ()))
+
+    def unlock(self) -> None:
+        self.calls.append(("unlock", ()))
+

tests/test_device_manager.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import pytest
+
+from core.device_manager import DeviceManager
+from core.errors import DeviceOfflineError
+from tests.fakes import FakeDriver
+
+
+def test_device_manager_connect_disconnect_transitions() -> None:
+    driver = FakeDriver()
+    manager = DeviceManager()
+    manager.register_device("iphone-1", lambda: driver, connection_info={"wda_port": 8100})
+
+    assert manager.list_devices()[0].status == "idle"
+    assert manager.connect("iphone-1", max_retries=1) is driver
+    assert manager.status("iphone-1") == "busy"
+
+    manager.disconnect("iphone-1")
+    assert manager.status("iphone-1") == "idle"
+    assert driver.calls[0] == ("connect", ())
+    assert driver.calls[-1] == ("disconnect", ())
+
+
+def test_device_manager_marks_unreachable_device_offline() -> None:
+    manager = DeviceManager()
+    manager.register_device("iphone-1", lambda: FakeDriver(fail_connect=True))
+
+    with pytest.raises(DeviceOfflineError):
+        manager.connect("iphone-1", max_retries=2, retry_backoff_seconds=0)
+
+    assert manager.status("iphone-1") == "offline"
+

tests/test_executor.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from runtime.executor import Executor, ExecutorConfig
+from runtime.planner import PlannedStep
+
+
+def test_executor_retries_until_transient_tool_succeeds() -> None:
+    calls = {"count": 0}
+
+    def flaky_tool() -> dict[str, bool]:
+        calls["count"] += 1
+        if calls["count"] < 3:
+            raise RuntimeError("transient")
+        return {"ok": True}
+
+    executor = Executor(
+        tools={"flaky": flaky_tool},
+        config=ExecutorConfig(max_retries=3, backoff_seconds=0),
+    )
+
+    result = executor.execute(PlannedStep(action="flaky", description="retry"))
+
+    assert result.success is True
+    assert result.attempts == 3
+    assert result.result == {"ok": True}
+

tests/test_find_helpers.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from core.models import Bounds, Scene, SceneElement
+from tools.find_icon import find_icon
+from tools.find_text import find_text
+
+
+def test_find_text_found_and_not_found() -> None:
+    scene = Scene(
+        width=100,
+        height=100,
+        elements=[
+            SceneElement(
+                id="button",
+                type="button",
+                text="Search",
+                bounds=Bounds(10, 20, 40, 20),
+            )
+        ],
+    )
+
+    assert find_text(scene, "sea")["found"] is True
+    assert find_text(scene, "missing") == {
+        "found": False,
+        "query": "missing",
+        "reason": "not found",
+    }
+
+
+def test_find_icon_found_and_not_found() -> None:
+    scene = Scene(
+        width=100,
+        height=100,
+        elements=[
+            SceneElement(
+                id="settings-icon",
+                type="image",
+                text="Settings",
+                bounds=Bounds(50, 50, 20, 20),
+            )
+        ],
+    )
+
+    assert find_icon(scene, "settings")["found"] is True
+    assert find_icon(scene, "profile")["reason"] == "not found"
+

tests/test_mcp.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from api.mcp import tool_handlers
+from core.device_manager import DeviceManager
+from tests.fakes import FakeDriver
+
+
+def test_mcp_handlers_are_semantic_and_hide_driver_terms() -> None:
+    manager = DeviceManager()
+    manager.register_device("iphone-1", lambda: FakeDriver())
+    manager.connect("iphone-1", max_retries=1)
+    handlers = tool_handlers(manager=manager)
+
+    responses = [
+        handlers["take_screenshot"](device_id="iphone-1"),
+        handlers["tap"](x=1, y=2, device_id="iphone-1"),
+        handlers["swipe"](
+            start_x=1,
+            start_y=2,
+            end_x=3,
+            end_y=4,
+            device_id="iphone-1",
+        ),
+        handlers["input_text"](text="hello", device_id="iphone-1"),
+        handlers["launch_app"](app_id="com.example.app", device_id="iphone-1"),
+        handlers["find_text"](query="Search", device_id="iphone-1"),
+        handlers["find_icon"](name="Settings", device_id="iphone-1"),
+        handlers["get_ui_tree"](device_id="iphone-1"),
+        handlers["describe_screen"](device_id="iphone-1"),
+        handlers["list_devices"](),
+        handlers["device_status"](device_id="iphone-1"),
+    ]
+
+    serialized = repr(responses)
+    assert "WDA" not in serialized
+    assert "Appium" not in serialized
+    assert "XCUI" not in serialized
+    assert all(response is not None for response in responses)

tests/test_rest_api.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import pytest
+
+from core.device_manager import DeviceManager
+from core.models import Task
+from storage.task_metadata import TaskMetadataStore
+from tests.fakes import FakeDriver
+
+
+def test_rest_start_task_and_poll_until_complete(tmp_path) -> None:
+    pytest.importorskip("fastapi")
+    from fastapi.testclient import TestClient
+
+    from api.rest import create_app
+
+    driver = FakeDriver()
+    manager = DeviceManager()
+    manager.register_device("iphone-1", lambda: driver)
+    manager.connect("iphone-1", max_retries=1)
+    store = TaskMetadataStore(tmp_path / "tasks.sqlite3")
+
+    class CompletingRunner:
+        def run(self, task: Task) -> None:
+            store.update_task(task.id, status="completed", completed=True)
+
+    app = create_app(
+        manager=manager,
+        metadata_store=store,
+        task_runner=CompletingRunner(),
+    )
+    client = TestClient(app)
+
+    response = client.post(
+        "/agent/task",
+        json={"goal": "search", "device_id": "iphone-1"},
+    )
+    assert response.status_code == 200
+    task_id = response.json()["task_id"]
+
+    status_response = client.get(f"/task/{task_id}")
+    assert status_response.status_code == 200
+    assert status_response.json()["status"] == "completed"
+

tests/test_scene_builder.py

@@ -0,0 +1,45 @@
+from __future__ import annotations
+
+from core.models import Bounds, SceneElement
+from vision.scene_builder import bbox_iou, build_scene
+
+
+def test_scene_builder_merges_overlapping_ocr_into_ui_element() -> None:
+    ui_button = SceneElement(
+        id="ui-button",
+        type="button",
+        text=None,
+        bounds=Bounds(10, 10, 100, 40),
+        confidence=1.0,
+        source="ui",
+    )
+    ocr_label = SceneElement(
+        id="ocr-label",
+        type="text",
+        text="Search",
+        bounds=Bounds(12, 12, 96, 36),
+        confidence=0.92,
+        source="ocr",
+    )
+    ocr_only = SceneElement(
+        id="ocr-only",
+        type="text",
+        text="Footer",
+        bounds=Bounds(0, 100, 40, 20),
+        confidence=0.8,
+        source="ocr",
+    )
+
+    scene = build_scene(
+        screen_width=120,
+        screen_height=140,
+        ui_elements=[ui_button],
+        ocr_elements=[ocr_label, ocr_only],
+    )
+
+    assert len(scene.elements) == 2
+    assert scene.elements[0].type == "button"
+    assert scene.elements[0].text == "Search"
+    assert scene.elements[1].text == "Footer"
+    assert bbox_iou(ui_button.bounds, ocr_label.bounds) > 0.5
+

tests/test_smoke.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+import importlib
+
+
+def test_imports_new_packages() -> None:
+    for package in ("core", "tools", "vision", "runtime", "api", "storage"):
+        importlib.import_module(package)
+

tests/test_task_loop.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from core.models import Bounds, Scene, SceneElement, Task
+from runtime.executor import Executor, ExecutorConfig
+from runtime.planner import PlannedStep, Planner
+from runtime.task import TaskRunner, TaskRunnerConfig
+from storage.artifact_store import ArtifactStore
+from storage.task_metadata import TaskMetadataStore
+from storage.timeline import Timeline
+from tests.fakes import PNG_10X20
+
+
+class ScriptedPlanner(Planner):
+    def __init__(self, steps: list[PlannedStep]) -> None:
+        self.steps = steps
+
+    def plan(self, *, goal, scene, context):
+        if len(context.step_results) >= len(self.steps):
+            return []
+        return [self.steps[len(context.step_results)]]
+
+    def goal_reached(self, *, goal, scene, context):
+        return len(context.step_results) >= len(self.steps) and all(
+            result.success for result in context.step_results
+        )
+
+
+def test_task_runner_executes_loop_and_writes_timeline(tmp_path) -> None:
+    scene = Scene(
+        width=10,
+        height=20,
+        elements=[
+            SceneElement(
+                id="search",
+                type="input",
+                text="Search",
+                bounds=Bounds(1, 2, 4, 4),
+            )
+        ],
+    )
+    planner = ScriptedPlanner(
+        [
+            PlannedStep(action="tap", description="tap search", args={"x": 3, "y": 4}),
+            PlannedStep(
+                action="input_text",
+                description="type query",
+                args={"text": "Mac mini"},
+            ),
+        ]
+    )
+    executor = Executor(
+        tools={
+            "tap": lambda **kwargs: {"ok": True, **kwargs},
+            "input_text": lambda **kwargs: {"ok": True, **kwargs},
+        },
+        config=ExecutorConfig(max_retries=1, backoff_seconds=0),
+    )
+    metadata = TaskMetadataStore(tmp_path / "tasks.sqlite3")
+    timeline = Timeline(ArtifactStore(tmp_path / "history"))
+    task = Task(goal="open app and search", device_id="iphone-1")
+    metadata.create_task(task)
+
+    runner = TaskRunner(
+        planner=planner,
+        executor=executor,
+        metadata_store=metadata,
+        timeline=timeline,
+        config=TaskRunnerConfig(max_steps=5),
+        observer=lambda device_id: scene,
+        screenshot_provider=lambda device_id: PNG_10X20,
+    )
+
+    result = runner.run(task)
+
+    assert result.status == "completed"
+    assert len(timeline.read(task.id)) == 2
+    assert metadata.get_task(task.id)["status"] == "completed"
+

tests/test_timeline.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from storage.artifact_store import ArtifactStore
+from storage.timeline import Timeline
+from tests.fakes import PNG_10X20
+
+
+def test_timeline_records_survive_reopening_store(tmp_path) -> None:
+    store = ArtifactStore(tmp_path / "history")
+    timeline = Timeline(store)
+    timeline.append(
+        task_id="task-1",
+        scene={"screen": {"width": 1, "height": 1}, "elements": []},
+        prompt="goal",
+        tool_call={"action": "tap"},
+        result={"ok": True},
+        screenshot=PNG_10X20,
+    )
+    timeline.append(
+        task_id="task-1",
+        scene={"screen": {"width": 1, "height": 1}, "elements": []},
+        prompt="goal",
+        tool_call={"action": "input_text"},
+        result={"ok": True},
+        screenshot=PNG_10X20,
+    )
+
+    reopened = Timeline(ArtifactStore(tmp_path / "history"))
+    records = reopened.read("task-1")
+
+    assert [record["index"] for record in records] == [1, 2]
+    assert records[0]["screenshot_path"].endswith("001.png")
+

tests/test_wda_integration.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from core.wda_driver import WDADriver, WDADriverConfig
+
+
+@pytest.mark.integration
+def test_wda_driver_screenshot_against_real_device() -> None:
+    server_url = os.getenv("APEX_WDA_SERVER_URL")
+    if not server_url:
+        pytest.skip("set APEX_WDA_SERVER_URL to run WDA hardware integration")
+
+    driver = WDADriver(
+        WDADriverConfig(
+            server_url=server_url,
+            udid=os.getenv("APEX_WDA_UDID") or None,
+            device_name=os.getenv("APEX_WDA_DEVICE_NAME") or None,
+        )
+    )
+    driver.connect()
+    try:
+        assert driver.screenshot()
+    finally:
+        driver.disconnect()
+