feat(host-agent): add optional Appium/Runtime supervisor

Adds an opt-in dependency supervisor inside the Host Agent that probes,
spawns, and restarts the two local processes the macOS single-machine
real-device workflow depends on: the Appium server (gates Driver.connect())
and the local Runtime API (local inspection). Default-off; gated by
HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED plus per-dependency *_SUPERVISED
flags.

Mitigates the live-incident failure mode where forgetting to start Appium
silently keeps devices offline and tasks queued forever with no error
surfaced in Host Agent logs.

Behavior (per openspec change):
- Adopt-don't-fight: probe (TCP + dependency-specific HTTP health check)
  before spawn. Healthy listener → adopted (never killed/restarted).
  Unhealthy listener → port-conflict error, skip. No listener → spawn.
- Only supervisor-spawned processes are restarted on crash, with capped
  exponential backoff (1s/2s/4s/8s, capped at 30s) and a per-process-lifetime
  attempt ceiling (HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS, default 5).
- Spawn failures (e.g. missing executable) logged distinctly from crashes.
- Graceful stop terminates only spawned children; adopted processes untouched.
- Supervisor starts before the heartbeat loop's first connect_devices() pass
  and stops alongside existing heartbeat/console teardown.

Validation: ruff check + format clean, compileall clean, openspec validate
--strict valid. Non-integration suite 503 passed / 44 deselected / 2 failed
(both failures pre-existing from unrelated 03c7c30 LLM_PROVIDER_ENC_KEY;
verified by stashing this change). macOS real-device manual verification
(task 6.4) deferred to a macOS host.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

apps/device-host-agent/host_agent/dependency_supervisor.py

@@ -0,0 +1,438 @@
+"""Optional supervisor for the two local external processes the Host Agent
+depends on for the macOS single-machine real-device workflow: the Appium
+server (which gates real ``Driver.connect()``) and the local Runtime API
+(used for local inspection).
+
+Lives in ``host_agent`` because it spawns and monitors host-level processes
+alongside the heartbeat/claim loop. Off by default; see ``HostAgentConfig``.
+
+Design (``openspec/changes/host-agent-dependency-supervisor/design.md``):
+- Default off. Each dependency also has its own opt-in flag.
+- Before spawn: TCP connect + dependency-specific HTTP health probe. A healthy
+  listener is *adopted* (never killed or restarted). An unhealthy listener is
+  a port conflict — logged and skipped. No listener → spawn.
+- Only supervisor-spawned processes are restarted on unexpected exit, with
+  capped exponential backoff and a per-process-lifetime attempt ceiling.
+- Adopted processes are never touched by ``stop()``.
+- Spawns use stdlib ``subprocess.Popen`` (no new dependency). Child stdout/stderr
+  is forwarded into this module's logger, tagged per dependency.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import socket
+import subprocess
+import threading
+from collections.abc import Awaitable, Callable, Sequence
+from dataclasses import dataclass
+from enum import Enum
+from typing import TYPE_CHECKING
+
+import httpx
+
+if TYPE_CHECKING:
+    from host_agent.config import HostAgentConfig
+
+_LOGGER = logging.getLogger("host_agent.dependency_supervisor")
+
+_PROBE_TCP_TIMEOUT_SECONDS = 1.0
+_PROBE_HTTP_TIMEOUT_SECONDS = 2.0
+
+
+class ProbeResult(str, Enum):
+    """Outcome of probing a dependency's configured address."""
+
+    HEALTHY = "healthy"
+    UNHEALTHY_LISTENER = "unhealthy_listener"
+    NO_LISTENER = "no_listener"
+
+
+def probe_appium(host: str, port: int) -> ProbeResult:
+    """Probe Appium at ``host:port``. Healthy iff ``GET /status`` returns 200
+    with a JSON body (the documented shape per ``docs/MACOS_IPHONE_SETUP.md``
+    §6 — contains ``ready``/build info)."""
+    return _probe_http(host, port, path="/status")
+
+
+def probe_runtime(host: str, port: int) -> ProbeResult:
+    """Probe the local Runtime API at ``host:port``. Healthy iff ``GET /devices``
+    returns 200 with a JSON body.
+
+    ``api.rest.create_app`` does not expose a dedicated ``/health`` endpoint;
+    ``/devices`` is the stable read-only GET that proves the FastAPI app is
+    mounted and the device manager is reachable. Per design.md Decision 2 this
+    is the "equivalent existing endpoint" used for the readiness check.
+    """
+    return _probe_http(host, port, path="/devices")
+
+
+def _probe_http(host: str, port: int, *, path: str) -> ProbeResult:
+    # Step 1: plain TCP connect — distinguish "nothing listening" (→ spawn)
+    # from "something is there but wrong" (→ port conflict, skip).
+    try:
+        with socket.create_connection((host, port), timeout=_PROBE_TCP_TIMEOUT_SECONDS):
+            pass
+    except OSError:
+        return ProbeResult.NO_LISTENER
+
+    # Step 2: dependency-specific HTTP health check.
+    try:
+        response = httpx.get(
+            f"http://{host}:{port}{path}",
+            timeout=_PROBE_HTTP_TIMEOUT_SECONDS,
+        )
+    except httpx.HTTPError:
+        return ProbeResult.UNHEALTHY_LISTENER
+    if response.status_code != 200:
+        return ProbeResult.UNHEALTHY_LISTENER
+    try:
+        response.json()
+    except ValueError:
+        return ProbeResult.UNHEALTHY_LISTENER
+    return ProbeResult.HEALTHY
+
+
+def appium_argv_factory(host: str, port: int) -> list[str]:
+    return ["appium", "--address", host, "--port", str(port)]
+
+
+def runtime_argv_factory(host: str, port: int) -> list[str]:
+    return [
+        "uvicorn",
+        "api.rest:create_app",
+        "--factory",
+        "--host",
+        host,
+        "--port",
+        str(port),
+    ]
+
+
+@dataclass
+class SupervisedDependency:
+    """Config + mutable runtime state for one supervised external process."""
+
+    name: str
+    host: str
+    port: int
+    argv_factory: Callable[[str, int], Sequence[str]]
+    probe: Callable[[str, int], ProbeResult]
+    # Mutable state:
+    process: subprocess.Popen | None = None
+    adopted: bool = False
+    given_up: bool = False
+    restart_attempts: int = 0
+    ready: bool = False
+
+
+@dataclass
+class _SupervisorKnobs:
+    """Tunables exposed for tests; production uses the defaults."""
+
+    startup_timeout_seconds: float = 30.0
+    readiness_poll_interval_seconds: float = 0.5
+    crash_poll_interval_seconds: float = 1.0
+    initial_backoff_seconds: float = 1.0
+    max_backoff_seconds: float = 30.0
+    terminate_grace_period_seconds: float = 5.0
+
+
+class DependencySupervisor:
+    """Manage zero or more supervised local dependencies.
+
+    Each dependency is either *adopted* (an existing healthy instance was
+    detected — never killed or restarted) or *spawned* (a child ``Popen``
+    handle the supervisor owns — restart-on-crash with capped exponential
+    backoff up to ``max_attempts`` per process lifetime).
+    """
+
+    def __init__(
+        self,
+        dependencies: list[SupervisedDependency],
+        *,
+        max_attempts: int,
+        knobs: _SupervisorKnobs | None = None,
+        sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+        popen_factory: Callable[[Sequence[str]], subprocess.Popen] = subprocess.Popen,
+        logger: logging.Logger = _LOGGER,
+    ) -> None:
+        self._dependencies = dependencies
+        self._max_attempts = max_attempts
+        self._knobs = knobs or _SupervisorKnobs()
+        self._sleep = sleep
+        self._popen_factory = popen_factory
+        self._logger = logger
+        self._reader_threads: list[threading.Thread] = []
+
+    @classmethod
+    def from_host_agent_config(
+        cls,
+        ha_config: HostAgentConfig,
+        **kwargs: object,
+    ) -> DependencySupervisor:
+        """Build a supervisor reflecting ``HostAgentConfig`` flags.
+
+        Caller is responsible for only invoking this when
+        ``dependency_supervisor_enabled`` is true; the resulting supervisor
+        will contain only the dependencies whose individual ``*_supervised``
+        flag is also true (possibly an empty list).
+        """
+        deps: list[SupervisedDependency] = []
+        if ha_config.appium_supervised:
+            deps.append(
+                SupervisedDependency(
+                    name="appium",
+                    host=ha_config.appium_host,
+                    port=ha_config.appium_port,
+                    argv_factory=appium_argv_factory,
+                    probe=probe_appium,
+                )
+            )
+        if ha_config.runtime_supervised:
+            deps.append(
+                SupervisedDependency(
+                    name="runtime",
+                    host=ha_config.runtime_host,
+                    port=ha_config.runtime_port,
+                    argv_factory=runtime_argv_factory,
+                    probe=probe_runtime,
+                )
+            )
+        return cls(
+            deps,
+            max_attempts=ha_config.dependency_restart_max_attempts,
+            **kwargs,  # type: ignore[arg-type]
+        )
+
+    @property
+    def dependencies(self) -> list[SupervisedDependency]:
+        return list(self._dependencies)
+
+    async def start(self) -> None:
+        """Probe + spawn/adopt + readiness wait.
+
+        Call before the heartbeat loop's first ``connect_devices()`` pass so
+        a supervised Appium is up before any ``Driver.connect()`` attempt.
+        """
+        for dep in self._dependencies:
+            await self._start_one(dep)
+
+    async def _start_one(self, dep: SupervisedDependency) -> None:
+        result = await asyncio.to_thread(dep.probe, dep.host, dep.port)
+        if result is ProbeResult.HEALTHY:
+            dep.adopted = True
+            dep.ready = True
+            self._logger.info(
+                "dependency-supervisor: %s adopted existing instance at %s:%s",
+                dep.name,
+                dep.host,
+                dep.port,
+            )
+            return
+        if result is ProbeResult.UNHEALTHY_LISTENER:
+            self._logger.error(
+                "dependency-supervisor: %s port %s occupied by an unhealthy "
+                "listener; leaving un-started to avoid a port conflict",
+                dep.name,
+                dep.port,
+            )
+            dep.given_up = True
+            return
+        await self._spawn_with_readiness_wait(dep)
+
+    async def _spawn_with_readiness_wait(
+        self,
+        dep: SupervisedDependency,
+    ) -> bool:
+        """Spawn ``dep`` and wait for it to become healthy.
+
+        Returns True on readiness, False otherwise. On spawn failure (missing
+        executable) or readiness timeout while the process is still running,
+        marks the dependency as ``given_up`` (no crash-restart loop entered).
+        If the process exits during startup, returns False without giving up
+        — the crash-restart loop in ``run()`` handles subsequent restarts.
+        """
+        argv = list(dep.argv_factory(dep.host, dep.port))
+        try:
+            proc = self._popen_factory(
+                argv,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.STDOUT,
+                bufsize=1,
+                text=True,
+            )
+        except FileNotFoundError:
+            self._logger.error(
+                "dependency-supervisor: %s spawn failed — executable not "
+                "found on PATH (tried: %s)",
+                dep.name,
+                argv[0],
+            )
+            dep.given_up = True
+            return False
+        except OSError as exc:
+            self._logger.error(
+                "dependency-supervisor: %s spawn failed: %s",
+                dep.name,
+                exc,
+            )
+            dep.given_up = True
+            return False
+
+        dep.process = proc
+        dep.adopted = False
+        dep.ready = False
+        self._start_reader_thread(dep, proc)
+        self._logger.info(
+            "dependency-supervisor: %s spawned (pid %s) at %s:%s",
+            dep.name,
+            proc.pid,
+            dep.host,
+            dep.port,
+        )
+
+        loop = asyncio.get_running_loop()
+        deadline = loop.time() + self._knobs.startup_timeout_seconds
+        while True:
+            if proc.poll() is not None:
+                self._logger.error(
+                    "dependency-supervisor: %s exited during startup with code %s",
+                    dep.name,
+                    proc.returncode,
+                )
+                dep.ready = False
+                return False
+            probe_result = await asyncio.to_thread(dep.probe, dep.host, dep.port)
+            if probe_result is ProbeResult.HEALTHY:
+                dep.ready = True
+                self._logger.info(
+                    "dependency-supervisor: %s ready at %s:%s",
+                    dep.name,
+                    dep.host,
+                    dep.port,
+                )
+                return True
+            if loop.time() >= deadline:
+                self._logger.error(
+                    "dependency-supervisor: %s started but did not become "
+                    "healthy within %ss; process still running, not entering "
+                    "crash-restart loop",
+                    dep.name,
+                    self._knobs.startup_timeout_seconds,
+                )
+                dep.ready = False
+                return False
+            await self._sleep(self._knobs.readiness_poll_interval_seconds)
+
+    def _start_reader_thread(
+        self,
+        dep: SupervisedDependency,
+        proc: subprocess.Popen,
+    ) -> None:
+        def reader_loop() -> None:
+            stdout = proc.stdout
+            if stdout is None:
+                return
+            for raw_line in stdout:
+                line = raw_line.rstrip()
+                if line:
+                    self._logger.info("%s: %s", dep.name, line)
+
+        thread = threading.Thread(
+            target=reader_loop,
+            name=f"dep-sup-{dep.name}",
+            daemon=True,
+        )
+        thread.start()
+        self._reader_threads.append(thread)
+
+    async def run(self, stop: asyncio.Event) -> None:
+        """Background monitor: detect crashes and apply backoff restart.
+
+        Returns when ``stop`` is set. Crashes are only detected for
+        spawned processes (not adopted ones).
+        """
+        if not self._dependencies:
+            await stop.wait()
+            return
+        while not stop.is_set():
+            for dep in self._dependencies:
+                await self._check_one(dep)
+            try:
+                await asyncio.wait_for(
+                    stop.wait(),
+                    timeout=self._knobs.crash_poll_interval_seconds,
+                )
+            except TimeoutError:
+                continue
+
+    async def _check_one(self, dep: SupervisedDependency) -> None:
+        if dep.given_up or dep.adopted or dep.process is None:
+            return
+        if dep.process.poll() is None:
+            return
+
+        self._logger.warning(
+            "dependency-supervisor: %s exited unexpectedly with code %s",
+            dep.name,
+            dep.process.returncode,
+        )
+        dep.process = None
+        dep.ready = False
+        dep.restart_attempts += 1
+        if dep.restart_attempts > self._max_attempts:
+            self._logger.error(
+                "dependency-supervisor: %s restart attempts exhausted "
+                "(crashes: %s, limit: %s); giving up for the rest of this "
+                "process lifetime",
+                dep.name,
+                dep.restart_attempts,
+                self._max_attempts,
+            )
+            dep.given_up = True
+            return
+
+        backoff = min(
+            self._knobs.initial_backoff_seconds * (2 ** (dep.restart_attempts - 1)),
+            self._knobs.max_backoff_seconds,
+        )
+        self._logger.info(
+            "dependency-supervisor: %s restarting in %.1fs (crash %s/%s)",
+            dep.name,
+            backoff,
+            dep.restart_attempts,
+            self._max_attempts,
+        )
+        await self._sleep(backoff)
+        await self._spawn_with_readiness_wait(dep)
+
+    async def stop(self) -> None:
+        """Terminate spawned children. Adopted processes are left untouched."""
+        for dep in self._dependencies:
+            if dep.adopted or dep.process is None:
+                continue
+            proc = dep.process
+            self._logger.info(
+                "dependency-supervisor: stopping %s (pid %s)",
+                dep.name,
+                proc.pid,
+            )
+            proc.terminate()
+            try:
+                await asyncio.to_thread(
+                    proc.wait, self._knobs.terminate_grace_period_seconds
+                )
+            except subprocess.TimeoutExpired:
+                self._logger.warning(
+                    "dependency-supervisor: %s did not exit within %ss, killing",
+                    dep.name,
+                    self._knobs.terminate_grace_period_seconds,
+                )
+                proc.kill()
+                await asyncio.to_thread(proc.wait)
+            except OSError:
+                pass
+            finally:
+                dep.process = None