feat(host-agent): add optional Appium/Runtime supervisor

Adds an opt-in dependency supervisor inside the Host Agent that probes, spawns, and restarts the two local processes the macOS single-machine real-device workflow depends on: the Appium server (gates Driver.connect()) and the local Runtime API (local inspection). Default-off; gated by HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED plus per-dependency *_SUPERVISED flags. Mitigates the live-incident failure mode where forgetting to start Appium silently keeps devices offline and tasks queued forever with no error surfaced in Host Agent logs. Behavior (per openspec change): - Adopt-don't-fight: probe (TCP + dependency-specific HTTP health check) before spawn. Healthy listener → adopted (never killed/restarted). Unhealthy listener → port-conflict error, skip. No listener → spawn. - Only supervisor-spawned processes are restarted on crash, with capped exponential backoff (1s/2s/4s/8s, capped at 30s) and a per-process-lifetime attempt ceiling (HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS, default 5). - Spawn failures (e.g. missing executable) logged distinctly from crashes. - Graceful stop terminates only spawned children; adopted processes untouched. - Supervisor starts before the heartbeat loop's first connect_devices() pass and stops alongside existing heartbeat/console teardown. Validation: ruff check + format clean, compileall clean, openspec validate --strict valid. Non-integration suite 503 passed / 44 deselected / 2 failed (both failures pre-existing from unrelated 03c7c30 LLM_PROVIDER_ENC_KEY; verified by stashing this change). macOS real-device manual verification (task 6.4) deferred to a macOS host. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-07-14 09:28:57 +08:00
parent 03c7c30067
commit 75879c8a52
12 changed files with 1475 additions and 0 deletions
@@ -0,0 +1,554 @@
+from __future__ import annotations
+
+import asyncio
+import io
+import logging
+import subprocess
+from collections.abc import Sequence
+
+import httpx
+import pytest
+
+from host_agent.dependency_supervisor import (
+    DependencySupervisor,
+    ProbeResult,
+    SupervisedDependency,
+    _SupervisorKnobs,
+    appium_argv_factory,
+    probe_appium,
+    probe_runtime,
+    runtime_argv_factory,
+)
+from host_agent.config import HostAgentConfig
+
+
+# ---------------------------------------------------------------------------
+# Test doubles
+# ---------------------------------------------------------------------------
+
+
+class _FakePopen:
+    """Minimal subprocess.Popen stand-in for supervisor tests."""
+
+    instances: list[_FakePopen] = []
+
+    def __init__(
+        self,
+        argv: Sequence[str],
+        *,
+        stdout_lines: Sequence[str] = (),
+        pid: int = 0,
+        **_kwargs: object,
+    ) -> None:
+        self.argv = list(argv)
+        self.pid = pid or (100 + len(_FakePopen.instances))
+        self.returncode: int | None = None
+        self.stdout = io.StringIO("".join(line + "\n" for line in stdout_lines))
+        self.terminate_calls = 0
+        self.kill_calls = 0
+        self.wait_calls = 0
+        _FakePopen.instances.append(self)
+
+    def poll(self) -> int | None:
+        return self.returncode
+
+    def terminate(self) -> None:
+        self.terminate_calls += 1
+        self.returncode = -15
+
+    def kill(self) -> None:
+        self.kill_calls += 1
+        self.returncode = -9
+
+    def wait(self, timeout: float | None = None) -> int:
+        self.wait_calls += 1
+        if self.returncode is None:
+            raise subprocess.TimeoutExpired(cmd=self.argv, timeout=timeout or 0)
+        return self.returncode
+
+
+@pytest.fixture(autouse=True)
+def _reset_fake_popen() -> None:
+    _FakePopen.instances.clear()
+    yield
+    _FakePopen.instances.clear()
+
+
+def _dep(
+    name: str = "appium",
+    *,
+    host: str = "127.0.0.1",
+    port: int = 4723,
+    probe_responses: Sequence[ProbeResult] = (ProbeResult.NO_LISTENER,),
+    argv_factory=None,
+) -> tuple[SupervisedDependency, list[ProbeResult]]:
+    """Build a SupervisedDependency whose probe returns scripted responses."""
+    call_log: list[ProbeResult] = []
+    responses = list(probe_responses)
+    fallback = probe_responses[-1] if probe_responses else ProbeResult.NO_LISTENER
+
+    def probe(_host: str, _port: int) -> ProbeResult:
+        if responses:
+            result = responses.pop(0)
+        else:
+            result = fallback
+        call_log.append(result)
+        return result
+
+    if argv_factory is None:
+
+        def argv_factory(h: str, p: int) -> list[str]:
+            return ["echo", name]
+
+    dep = SupervisedDependency(
+        name=name,
+        host=host,
+        port=port,
+        argv_factory=argv_factory,
+        probe=probe,
+    )
+    return dep, call_log
+
+
+class _FakeSleep:
+    """Records sleep durations so tests can assert backoff progression."""
+
+    def __init__(self) -> None:
+        self.calls: list[float] = []
+
+    async def __call__(self, seconds: float) -> None:
+        self.calls.append(seconds)
+
+
+def _build_supervisor(
+    deps: list[SupervisedDependency],
+    *,
+    max_attempts: int = 5,
+    knobs: _SupervisorKnobs | None = None,
+    sleep: _FakeSleep | None = None,
+    popen_factory=None,
+) -> tuple[DependencySupervisor, _FakeSleep]:
+    fake_sleep = sleep or _FakeSleep()
+    sup = DependencySupervisor(
+        list(deps),
+        max_attempts=max_attempts,
+        knobs=knobs
+        or _SupervisorKnobs(
+            startup_timeout_seconds=1.0,
+            readiness_poll_interval_seconds=0.01,
+            crash_poll_interval_seconds=0.01,
+            initial_backoff_seconds=1.0,
+            max_backoff_seconds=30.0,
+            terminate_grace_period_seconds=1.0,
+        ),
+        sleep=fake_sleep,
+        popen_factory=popen_factory or _FakePopen,
+        logger=logging.getLogger("test"),
+    )
+    return sup, fake_sleep
+
+
+class _DummySocket:
+    def __enter__(self) -> "_DummySocket":
+        return self
+
+    def __exit__(self, *exc) -> None:
+        return None
+
+
+def _raise_connection_refused(*_args, **_kwargs):
+    raise ConnectionRefusedError("no listener")
+
+
+def _ok_connection(*_args, **_kwargs):
+    return _DummySocket()
+
+
+# ---------------------------------------------------------------------------
+# Probe tests (tasks 2.1, 2.3)
+# ---------------------------------------------------------------------------
+
+
+def test_probe_returns_no_listener_when_port_is_closed(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.socket.create_connection",
+        _raise_connection_refused,
+    )
+    assert probe_appium("127.0.0.1", 4723) is ProbeResult.NO_LISTENER
+    assert probe_runtime("127.0.0.1", 8000) is ProbeResult.NO_LISTENER
+
+
+def test_probe_returns_healthy_on_appium_status_endpoint(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.socket.create_connection",
+        _ok_connection,
+    )
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.httpx.get",
+        lambda url, timeout=2.0: httpx.Response(200, json={"ready": True}),
+    )
+    assert probe_appium("127.0.0.1", 4723) is ProbeResult.HEALTHY
+
+
+def test_probe_returns_healthy_on_runtime_devices_endpoint(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.socket.create_connection",
+        _ok_connection,
+    )
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.httpx.get",
+        lambda url, timeout=2.0: httpx.Response(200, json=[]),
+    )
+    assert probe_runtime("127.0.0.1", 8000) is ProbeResult.HEALTHY
+
+
+def test_probe_returns_unhealthy_when_listener_returns_non_200(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.socket.create_connection",
+        _ok_connection,
+    )
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.httpx.get",
+        lambda url, timeout=2.0: httpx.Response(500, text="boom"),
+    )
+    assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
+
+
+def test_probe_returns_unhealthy_when_listener_returns_non_json(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.socket.create_connection",
+        _ok_connection,
+    )
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.httpx.get",
+        lambda url, timeout=2.0: httpx.Response(200, text="not json"),
+    )
+    assert probe_runtime("127.0.0.1", 8000) is ProbeResult.UNHEALTHY_LISTENER
+
+
+def test_probe_returns_unhealthy_on_http_transport_error(monkeypatch) -> None:
+    monkeypatch.setattr(
+        "host_agent.dependency_supervisor.socket.create_connection",
+        _ok_connection,
+    )
+
+    def raise_http_error(url, timeout=2.0):
+        raise httpx.ConnectError("reset")
+
+    monkeypatch.setattr("host_agent.dependency_supervisor.httpx.get", raise_http_error)
+    assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
+
+
+# ---------------------------------------------------------------------------
+# Adoption logic tests (tasks 2.2, 2.3)
+# ---------------------------------------------------------------------------
+
+
+def test_start_adopts_existing_healthy_instance() -> None:
+    async def scenario() -> None:
+        dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,))
+        sup, _ = _build_supervisor([dep])
+
+        await sup.start()
+
+        assert dep.adopted is True
+        assert dep.ready is True
+        assert dep.process is None
+
+    asyncio.run(scenario())
+
+
+def test_start_skips_when_port_has_unhealthy_listener() -> None:
+    async def scenario() -> None:
+        dep, _ = _dep(probe_responses=(ProbeResult.UNHEALTHY_LISTENER,))
+        sup, _ = _build_supervisor([dep])
+
+        await sup.start()
+
+        assert dep.adopted is False
+        assert dep.process is None
+        assert dep.given_up is True
+
+    asyncio.run(scenario())
+
+
+def test_start_spawns_when_nothing_is_listening() -> None:
+    async def scenario() -> None:
+        # NO_LISTENER for initial probe → spawn path.
+        # HEALTHY on first readiness probe.
+        dep, _ = _dep(
+            probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
+        )
+        sup, _ = _build_supervisor([dep])
+
+        await sup.start()
+
+        assert dep.process is not None
+        assert dep.adopted is False
+        assert dep.ready is True
+        assert dep.given_up is False
+
+    asyncio.run(scenario())
+
+
+# ---------------------------------------------------------------------------
+# Supervisor core tests (task 3.6)
+# ---------------------------------------------------------------------------
+
+
+def test_spawn_failure_marks_dependency_given_up() -> None:
+    async def scenario() -> None:
+        dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,))
+
+        def raising_popen(argv, **_kwargs):
+            raise FileNotFoundError("appium: not found")
+
+        sup, _ = _build_supervisor([dep], popen_factory=raising_popen)
+        await sup.start()
+
+        assert dep.process is None
+        assert dep.given_up is True
+        assert dep.ready is False
+
+    asyncio.run(scenario())
+
+
+def test_spawn_uses_appium_argv_factory() -> None:
+    async def scenario() -> None:
+        captured: list[list[str]] = []
+
+        def recording_popen(argv, **kwargs):
+            captured.append(list(argv))
+            return _FakePopen(argv, stdout_lines=("hello",))
+
+        dep, _ = _dep(
+            name="appium",
+            port=4723,
+            probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
+            argv_factory=appium_argv_factory,
+        )
+        sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
+        await sup.start()
+
+        assert captured == [["appium", "--address", "127.0.0.1", "--port", "4723"]]
+
+    asyncio.run(scenario())
+
+
+def test_spawn_uses_runtime_argv_factory() -> None:
+    async def scenario() -> None:
+        captured: list[list[str]] = []
+
+        def recording_popen(argv, **kwargs):
+            captured.append(list(argv))
+            return _FakePopen(argv)
+
+        dep, _ = _dep(
+            name="runtime",
+            port=8000,
+            probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
+            argv_factory=runtime_argv_factory,
+        )
+        sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
+        await sup.start()
+
+        assert captured == [
+            [
+                "uvicorn",
+                "api.rest:create_app",
+                "--factory",
+                "--host",
+                "127.0.0.1",
+                "--port",
+                "8000",
+            ]
+        ]
+
+    asyncio.run(scenario())
+
+
+def test_readiness_timeout_leaves_process_running_without_restart_loop() -> None:
+    async def scenario() -> None:
+        # NO_LISTENER for initial probe; probe never becomes HEALTHY → startup
+        # timeout path (1.0s in test knobs).
+        dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,))
+        sup, _ = _build_supervisor([dep])
+
+        await sup.start()
+
+        assert dep.process is not None  # still running
+        assert dep.ready is False
+        assert dep.given_up is False  # not given up; crash-restart owns crashes
+
+    asyncio.run(scenario())
+
+
+def test_crash_triggers_restart_with_backoff() -> None:
+    async def scenario() -> None:
+        # Probe responses:
+        #   1. start._start_one → NO_LISTENER → spawn path
+        #   2. first readiness probe → HEALTHY → ready
+        #   3. after restart, readiness probe → HEALTHY → ready again
+        dep, _ = _dep(
+            probe_responses=(
+                ProbeResult.NO_LISTENER,
+                ProbeResult.HEALTHY,
+                ProbeResult.HEALTHY,
+            ),
+            argv_factory=appium_argv_factory,
+        )
+
+        spawn_count = {"n": 0}
+
+        def popen(argv, **kwargs):
+            spawn_count["n"] += 1
+            return _FakePopen(argv)
+
+        sup, sleep = _build_supervisor([dep], popen_factory=popen)
+        await sup.start()
+        assert dep.ready is True
+        crashes_before = dep.restart_attempts
+
+        # Crash the spawned process now (after readiness succeeded). The
+        # supervisor's run loop will detect this via poll().
+        first_proc = dep.process
+        assert isinstance(first_proc, _FakePopen)
+        first_proc.returncode = 1
+
+        stop = asyncio.Event()
+        task = asyncio.create_task(sup.run(stop))
+        # Give the loop a chance to detect the crash, sleep backoff, respawn.
+        await asyncio.sleep(0.05)
+        stop.set()
+        await asyncio.wait_for(task, timeout=1.0)
+
+        assert dep.restart_attempts == crashes_before + 1
+        assert dep.ready is True
+        assert sleep.calls == [1.0]  # exponential backoff base for attempt 1
+        assert spawn_count["n"] == 2
+
+    asyncio.run(scenario())
+
+
+def test_restart_exhaustion_gives_up() -> None:
+    async def scenario() -> None:
+        # Process crashes after every spawn; limit max_attempts to 2.
+        dep, _ = _dep(
+            probe_responses=(ProbeResult.NO_LISTENER,) + (ProbeResult.HEALTHY,) * 10,
+        )
+        spawn_count = {"n": 0}
+
+        def popen(argv, **kwargs):
+            spawn_count["n"] += 1
+            proc = _FakePopen(argv)
+            proc.returncode = 1
+            return proc
+
+        sup, sleep = _build_supervisor([dep], popen_factory=popen, max_attempts=2)
+        await sup.start()
+
+        stop = asyncio.Event()
+        task = asyncio.create_task(sup.run(stop))
+        deadline = asyncio.get_running_loop().time() + 2.0
+        while not dep.given_up and asyncio.get_running_loop().time() < deadline:
+            await asyncio.sleep(0.01)
+        stop.set()
+        await asyncio.wait_for(task, timeout=2.0)
+
+        assert dep.given_up is True
+        # Initial spawn + 2 successful restarts before the 3rd crash gives up.
+        assert dep.restart_attempts == 3
+        # Backoffs: 1.0 (attempt 1), 2.0 (attempt 2). No third restart.
+        assert sleep.calls == [1.0, 2.0]
+
+    asyncio.run(scenario())
+
+
+def test_adopted_process_is_never_restarted_or_killed() -> None:
+    async def scenario() -> None:
+        dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,))
+        sup, _ = _build_supervisor([dep])
+        await sup.start()
+
+        stop = asyncio.Event()
+        task = asyncio.create_task(sup.run(stop))
+        await asyncio.sleep(0.02)
+        stop.set()
+        await asyncio.wait_for(task, timeout=1.0)
+
+        await sup.stop()
+
+        assert dep.adopted is True
+        assert dep.process is None
+        assert dep.restart_attempts == 0
+
+    asyncio.run(scenario())
+
+
+def test_stop_terminates_only_spawned_children() -> None:
+    async def scenario() -> None:
+        spawned_dep, _ = _dep(
+            name="spawned",
+            port=4723,
+            probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
+        )
+        adopted_dep, _ = _dep(
+            name="adopted",
+            port=8000,
+            probe_responses=(ProbeResult.HEALTHY,),
+        )
+        sup, _ = _build_supervisor([spawned_dep, adopted_dep])
+        await sup.start()
+
+        assert spawned_dep.process is not None
+        spawned_proc = spawned_dep.process
+        assert adopted_dep.adopted is True
+
+        await sup.stop()
+
+        assert isinstance(spawned_proc, _FakePopen)
+        assert spawned_proc.terminate_calls == 1
+        assert spawned_dep.process is None
+
+    asyncio.run(scenario())
+
+
+# ---------------------------------------------------------------------------
+# from_host_agent_config factory (covers wiring helper for task 4.1)
+# ---------------------------------------------------------------------------
+
+
+def test_from_host_agent_config_builds_empty_supervisor_when_no_dep_selected() -> None:
+    config = HostAgentConfig(
+        control_plane_url="https://control.example",
+        host_id="host-a",
+        token="t",
+        dependency_supervisor_enabled=True,
+    )
+    sup = DependencySupervisor.from_host_agent_config(config)
+    assert sup.dependencies == []
+
+
+def test_from_host_agent_config_includes_appium_and_runtime_when_selected() -> None:
+    config = HostAgentConfig(
+        control_plane_url="https://control.example",
+        host_id="host-a",
+        token="t",
+        dependency_supervisor_enabled=True,
+        appium_supervised=True,
+        appium_host="10.0.0.5",
+        appium_port=4724,
+        runtime_supervised=True,
+        runtime_host="10.0.0.5",
+        runtime_port=8001,
+        dependency_restart_max_attempts=7,
+    )
+    sup = DependencySupervisor.from_host_agent_config(config)
+    names = [dep.name for dep in sup.dependencies]
+    assert names == ["appium", "runtime"]
+    appium = sup.dependencies[0]
+    assert appium.host == "10.0.0.5"
+    assert appium.port == 4724
+    runtime = sup.dependencies[1]
+    assert runtime.port == 8001
+    assert sup._max_attempts == 7