feat(host-agent): add optional Appium/Runtime supervisor
Tests / Test failed: 2, passed: 691

Adds an opt-in dependency supervisor inside the Host Agent that probes,
spawns, and restarts the two local processes the macOS single-machine
real-device workflow depends on: the Appium server (gates Driver.connect())
and the local Runtime API (local inspection). Default-off; gated by
HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED plus per-dependency *_SUPERVISED
flags.

Mitigates the live-incident failure mode where forgetting to start Appium
silently keeps devices offline and tasks queued forever with no error
surfaced in Host Agent logs.

Behavior (per openspec change):
- Adopt-don't-fight: probe (TCP + dependency-specific HTTP health check)
  before spawn. Healthy listener → adopted (never killed/restarted).
  Unhealthy listener → port-conflict error, skip. No listener → spawn.
- Only supervisor-spawned processes are restarted on crash, with capped
  exponential backoff (1s/2s/4s/8s, capped at 30s) and a per-process-lifetime
  attempt ceiling (HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS, default 5).
- Spawn failures (e.g. missing executable) logged distinctly from crashes.
- Graceful stop terminates only spawned children; adopted processes untouched.
- Supervisor starts before the heartbeat loop's first connect_devices() pass
  and stops alongside existing heartbeat/console teardown.

Validation: ruff check + format clean, compileall clean, openspec validate
--strict valid. Non-integration suite 503 passed / 44 deselected / 2 failed
(both failures pre-existing from unrelated 03c7c30 LLM_PROVIDER_ENC_KEY;
verified by stashing this change). macOS real-device manual verification
(task 6.4) deferred to a macOS host.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-07-14 09:28:57 +08:00
co-authored by Claude Opus 4.6
parent 03c7c30067
commit 75879c8a52
12 changed files with 1475 additions and 0 deletions
+23
View File
@@ -11,6 +11,7 @@ from device.manager import DeviceManager
from host_agent.assignment import AssignmentExecutor
from host_agent.client import HostAgentClient, HostAgentEnrollmentClient
from host_agent.config import HostAgentConfig, load_host_agent_config
from host_agent.dependency_supervisor import DependencySupervisor
from host_agent.devices import register_local_device
from host_agent.enrollment import resolve_host_identity
from host_agent.execution import create_execution_factories
@@ -34,12 +35,23 @@ class HostAgentApplication:
processor: AssignmentProcessor
console_server: uvicorn.Server | None = None
console_enrollment_client: HostAgentEnrollmentClient | None = None
dependency_supervisor: DependencySupervisor | None = None
def run(self) -> None:
asyncio.run(self.run_async())
async def run_async(self, stop: asyncio.Event | None = None) -> None:
stop_requested = stop or asyncio.Event()
supervisor_stop = asyncio.Event()
supervisor_task: asyncio.Task[None] | None = None
if self.dependency_supervisor is not None:
# Bring up supervised dependencies (probe + spawn/adopt + readiness
# wait) before the heartbeat loop's first connect_devices() pass,
# so a supervised Appium is ready before any Driver.connect().
await self.dependency_supervisor.start()
supervisor_task = asyncio.create_task(
self.dependency_supervisor.run(supervisor_stop)
)
heartbeat_stop = asyncio.Event()
heartbeat_task = asyncio.create_task(self.heartbeat.run(heartbeat_stop))
console_task = (
@@ -75,12 +87,17 @@ class HostAgentApplication:
with suppress(Exception):
await asyncio.shield(active_processing)
heartbeat_stop.set()
supervisor_stop.set()
if self.console_server is not None:
self.console_server.should_exit = True
try:
await asyncio.gather(heartbeat_task, return_exceptions=True)
with suppress(Exception):
await self.heartbeat.sync_once()
if supervisor_task is not None:
await asyncio.gather(supervisor_task, return_exceptions=True)
with suppress(Exception):
await self.dependency_supervisor.stop()
if console_task is not None:
with suppress(asyncio.CancelledError):
await asyncio.gather(console_task, return_exceptions=True)
@@ -213,12 +230,18 @@ def create_application(
history_store, assignment, result
),
)
dependency_supervisor: DependencySupervisor | None = None
if resolved_config.dependency_supervisor_enabled:
dependency_supervisor = DependencySupervisor.from_host_agent_config(
resolved_config
)
return HostAgentApplication(
client=client,
heartbeat=heartbeat,
processor=processor,
console_server=console_server,
console_enrollment_client=console_enrollment_client,
dependency_supervisor=dependency_supervisor,
)
@@ -35,6 +35,14 @@ class HostAgentConfig:
console_session_ttl_seconds: float = 43200.0
console_history_limit: int = 200
ai_planner_transport: str = "direct"
dependency_supervisor_enabled: bool = False
appium_supervised: bool = False
appium_host: str = "127.0.0.1"
appium_port: int = 4723
runtime_supervised: bool = False
runtime_host: str = "127.0.0.1"
runtime_port: int = 8000
dependency_restart_max_attempts: int = 5
def load_host_agent_config(
@@ -115,6 +123,18 @@ def load_host_agent_config(
ai_planner_transport=_parse_ai_planner_transport(
values.get("AI_PLANNER_TRANSPORT")
),
dependency_supervisor_enabled=_truthy(
values, "HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED", False
),
appium_supervised=_truthy(values, "HOST_AGENT_APPIUM_SUPERVISED", False),
appium_host=values.get("HOST_AGENT_APPIUM_HOST", "127.0.0.1").strip(),
appium_port=_positive_int(values, "HOST_AGENT_APPIUM_PORT", 4723),
runtime_supervised=_truthy(values, "HOST_AGENT_RUNTIME_SUPERVISED", False),
runtime_host=values.get("HOST_AGENT_RUNTIME_HOST", "127.0.0.1").strip(),
runtime_port=_positive_int(values, "HOST_AGENT_RUNTIME_PORT", 8000),
dependency_restart_max_attempts=_positive_int(
values, "HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS", 5
),
)
if config.max_retry_backoff_seconds < config.retry_backoff_seconds:
raise HostAgentConfigurationError(
@@ -0,0 +1,438 @@
"""Optional supervisor for the two local external processes the Host Agent
depends on for the macOS single-machine real-device workflow: the Appium
server (which gates real ``Driver.connect()``) and the local Runtime API
(used for local inspection).
Lives in ``host_agent`` because it spawns and monitors host-level processes
alongside the heartbeat/claim loop. Off by default; see ``HostAgentConfig``.
Design (``openspec/changes/host-agent-dependency-supervisor/design.md``):
- Default off. Each dependency also has its own opt-in flag.
- Before spawn: TCP connect + dependency-specific HTTP health probe. A healthy
listener is *adopted* (never killed or restarted). An unhealthy listener is
a port conflict — logged and skipped. No listener → spawn.
- Only supervisor-spawned processes are restarted on unexpected exit, with
capped exponential backoff and a per-process-lifetime attempt ceiling.
- Adopted processes are never touched by ``stop()``.
- Spawns use stdlib ``subprocess.Popen`` (no new dependency). Child stdout/stderr
is forwarded into this module's logger, tagged per dependency.
"""
from __future__ import annotations
import asyncio
import logging
import socket
import subprocess
import threading
from collections.abc import Awaitable, Callable, Sequence
from dataclasses import dataclass
from enum import Enum
from typing import TYPE_CHECKING
import httpx
if TYPE_CHECKING:
from host_agent.config import HostAgentConfig
_LOGGER = logging.getLogger("host_agent.dependency_supervisor")
_PROBE_TCP_TIMEOUT_SECONDS = 1.0
_PROBE_HTTP_TIMEOUT_SECONDS = 2.0
class ProbeResult(str, Enum):
"""Outcome of probing a dependency's configured address."""
HEALTHY = "healthy"
UNHEALTHY_LISTENER = "unhealthy_listener"
NO_LISTENER = "no_listener"
def probe_appium(host: str, port: int) -> ProbeResult:
"""Probe Appium at ``host:port``. Healthy iff ``GET /status`` returns 200
with a JSON body (the documented shape per ``docs/MACOS_IPHONE_SETUP.md``
§6 — contains ``ready``/build info)."""
return _probe_http(host, port, path="/status")
def probe_runtime(host: str, port: int) -> ProbeResult:
"""Probe the local Runtime API at ``host:port``. Healthy iff ``GET /devices``
returns 200 with a JSON body.
``api.rest.create_app`` does not expose a dedicated ``/health`` endpoint;
``/devices`` is the stable read-only GET that proves the FastAPI app is
mounted and the device manager is reachable. Per design.md Decision 2 this
is the "equivalent existing endpoint" used for the readiness check.
"""
return _probe_http(host, port, path="/devices")
def _probe_http(host: str, port: int, *, path: str) -> ProbeResult:
# Step 1: plain TCP connect — distinguish "nothing listening" (→ spawn)
# from "something is there but wrong" (→ port conflict, skip).
try:
with socket.create_connection((host, port), timeout=_PROBE_TCP_TIMEOUT_SECONDS):
pass
except OSError:
return ProbeResult.NO_LISTENER
# Step 2: dependency-specific HTTP health check.
try:
response = httpx.get(
f"http://{host}:{port}{path}",
timeout=_PROBE_HTTP_TIMEOUT_SECONDS,
)
except httpx.HTTPError:
return ProbeResult.UNHEALTHY_LISTENER
if response.status_code != 200:
return ProbeResult.UNHEALTHY_LISTENER
try:
response.json()
except ValueError:
return ProbeResult.UNHEALTHY_LISTENER
return ProbeResult.HEALTHY
def appium_argv_factory(host: str, port: int) -> list[str]:
return ["appium", "--address", host, "--port", str(port)]
def runtime_argv_factory(host: str, port: int) -> list[str]:
return [
"uvicorn",
"api.rest:create_app",
"--factory",
"--host",
host,
"--port",
str(port),
]
@dataclass
class SupervisedDependency:
"""Config + mutable runtime state for one supervised external process."""
name: str
host: str
port: int
argv_factory: Callable[[str, int], Sequence[str]]
probe: Callable[[str, int], ProbeResult]
# Mutable state:
process: subprocess.Popen | None = None
adopted: bool = False
given_up: bool = False
restart_attempts: int = 0
ready: bool = False
@dataclass
class _SupervisorKnobs:
"""Tunables exposed for tests; production uses the defaults."""
startup_timeout_seconds: float = 30.0
readiness_poll_interval_seconds: float = 0.5
crash_poll_interval_seconds: float = 1.0
initial_backoff_seconds: float = 1.0
max_backoff_seconds: float = 30.0
terminate_grace_period_seconds: float = 5.0
class DependencySupervisor:
"""Manage zero or more supervised local dependencies.
Each dependency is either *adopted* (an existing healthy instance was
detected — never killed or restarted) or *spawned* (a child ``Popen``
handle the supervisor owns — restart-on-crash with capped exponential
backoff up to ``max_attempts`` per process lifetime).
"""
def __init__(
self,
dependencies: list[SupervisedDependency],
*,
max_attempts: int,
knobs: _SupervisorKnobs | None = None,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
popen_factory: Callable[[Sequence[str]], subprocess.Popen] = subprocess.Popen,
logger: logging.Logger = _LOGGER,
) -> None:
self._dependencies = dependencies
self._max_attempts = max_attempts
self._knobs = knobs or _SupervisorKnobs()
self._sleep = sleep
self._popen_factory = popen_factory
self._logger = logger
self._reader_threads: list[threading.Thread] = []
@classmethod
def from_host_agent_config(
cls,
ha_config: HostAgentConfig,
**kwargs: object,
) -> DependencySupervisor:
"""Build a supervisor reflecting ``HostAgentConfig`` flags.
Caller is responsible for only invoking this when
``dependency_supervisor_enabled`` is true; the resulting supervisor
will contain only the dependencies whose individual ``*_supervised``
flag is also true (possibly an empty list).
"""
deps: list[SupervisedDependency] = []
if ha_config.appium_supervised:
deps.append(
SupervisedDependency(
name="appium",
host=ha_config.appium_host,
port=ha_config.appium_port,
argv_factory=appium_argv_factory,
probe=probe_appium,
)
)
if ha_config.runtime_supervised:
deps.append(
SupervisedDependency(
name="runtime",
host=ha_config.runtime_host,
port=ha_config.runtime_port,
argv_factory=runtime_argv_factory,
probe=probe_runtime,
)
)
return cls(
deps,
max_attempts=ha_config.dependency_restart_max_attempts,
**kwargs, # type: ignore[arg-type]
)
@property
def dependencies(self) -> list[SupervisedDependency]:
return list(self._dependencies)
async def start(self) -> None:
"""Probe + spawn/adopt + readiness wait.
Call before the heartbeat loop's first ``connect_devices()`` pass so
a supervised Appium is up before any ``Driver.connect()`` attempt.
"""
for dep in self._dependencies:
await self._start_one(dep)
async def _start_one(self, dep: SupervisedDependency) -> None:
result = await asyncio.to_thread(dep.probe, dep.host, dep.port)
if result is ProbeResult.HEALTHY:
dep.adopted = True
dep.ready = True
self._logger.info(
"dependency-supervisor: %s adopted existing instance at %s:%s",
dep.name,
dep.host,
dep.port,
)
return
if result is ProbeResult.UNHEALTHY_LISTENER:
self._logger.error(
"dependency-supervisor: %s port %s occupied by an unhealthy "
"listener; leaving un-started to avoid a port conflict",
dep.name,
dep.port,
)
dep.given_up = True
return
await self._spawn_with_readiness_wait(dep)
async def _spawn_with_readiness_wait(
self,
dep: SupervisedDependency,
) -> bool:
"""Spawn ``dep`` and wait for it to become healthy.
Returns True on readiness, False otherwise. On spawn failure (missing
executable) or readiness timeout while the process is still running,
marks the dependency as ``given_up`` (no crash-restart loop entered).
If the process exits during startup, returns False without giving up
— the crash-restart loop in ``run()`` handles subsequent restarts.
"""
argv = list(dep.argv_factory(dep.host, dep.port))
try:
proc = self._popen_factory(
argv,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=1,
text=True,
)
except FileNotFoundError:
self._logger.error(
"dependency-supervisor: %s spawn failed — executable not "
"found on PATH (tried: %s)",
dep.name,
argv[0],
)
dep.given_up = True
return False
except OSError as exc:
self._logger.error(
"dependency-supervisor: %s spawn failed: %s",
dep.name,
exc,
)
dep.given_up = True
return False
dep.process = proc
dep.adopted = False
dep.ready = False
self._start_reader_thread(dep, proc)
self._logger.info(
"dependency-supervisor: %s spawned (pid %s) at %s:%s",
dep.name,
proc.pid,
dep.host,
dep.port,
)
loop = asyncio.get_running_loop()
deadline = loop.time() + self._knobs.startup_timeout_seconds
while True:
if proc.poll() is not None:
self._logger.error(
"dependency-supervisor: %s exited during startup with code %s",
dep.name,
proc.returncode,
)
dep.ready = False
return False
probe_result = await asyncio.to_thread(dep.probe, dep.host, dep.port)
if probe_result is ProbeResult.HEALTHY:
dep.ready = True
self._logger.info(
"dependency-supervisor: %s ready at %s:%s",
dep.name,
dep.host,
dep.port,
)
return True
if loop.time() >= deadline:
self._logger.error(
"dependency-supervisor: %s started but did not become "
"healthy within %ss; process still running, not entering "
"crash-restart loop",
dep.name,
self._knobs.startup_timeout_seconds,
)
dep.ready = False
return False
await self._sleep(self._knobs.readiness_poll_interval_seconds)
def _start_reader_thread(
self,
dep: SupervisedDependency,
proc: subprocess.Popen,
) -> None:
def reader_loop() -> None:
stdout = proc.stdout
if stdout is None:
return
for raw_line in stdout:
line = raw_line.rstrip()
if line:
self._logger.info("%s: %s", dep.name, line)
thread = threading.Thread(
target=reader_loop,
name=f"dep-sup-{dep.name}",
daemon=True,
)
thread.start()
self._reader_threads.append(thread)
async def run(self, stop: asyncio.Event) -> None:
"""Background monitor: detect crashes and apply backoff restart.
Returns when ``stop`` is set. Crashes are only detected for
spawned processes (not adopted ones).
"""
if not self._dependencies:
await stop.wait()
return
while not stop.is_set():
for dep in self._dependencies:
await self._check_one(dep)
try:
await asyncio.wait_for(
stop.wait(),
timeout=self._knobs.crash_poll_interval_seconds,
)
except TimeoutError:
continue
async def _check_one(self, dep: SupervisedDependency) -> None:
if dep.given_up or dep.adopted or dep.process is None:
return
if dep.process.poll() is None:
return
self._logger.warning(
"dependency-supervisor: %s exited unexpectedly with code %s",
dep.name,
dep.process.returncode,
)
dep.process = None
dep.ready = False
dep.restart_attempts += 1
if dep.restart_attempts > self._max_attempts:
self._logger.error(
"dependency-supervisor: %s restart attempts exhausted "
"(crashes: %s, limit: %s); giving up for the rest of this "
"process lifetime",
dep.name,
dep.restart_attempts,
self._max_attempts,
)
dep.given_up = True
return
backoff = min(
self._knobs.initial_backoff_seconds * (2 ** (dep.restart_attempts - 1)),
self._knobs.max_backoff_seconds,
)
self._logger.info(
"dependency-supervisor: %s restarting in %.1fs (crash %s/%s)",
dep.name,
backoff,
dep.restart_attempts,
self._max_attempts,
)
await self._sleep(backoff)
await self._spawn_with_readiness_wait(dep)
async def stop(self) -> None:
"""Terminate spawned children. Adopted processes are left untouched."""
for dep in self._dependencies:
if dep.adopted or dep.process is None:
continue
proc = dep.process
self._logger.info(
"dependency-supervisor: stopping %s (pid %s)",
dep.name,
proc.pid,
)
proc.terminate()
try:
await asyncio.to_thread(
proc.wait, self._knobs.terminate_grace_period_seconds
)
except subprocess.TimeoutExpired:
self._logger.warning(
"dependency-supervisor: %s did not exit within %ss, killing",
dep.name,
self._knobs.terminate_grace_period_seconds,
)
proc.kill()
await asyncio.to_thread(proc.wait)
except OSError:
pass
finally:
dep.process = None
+122
View File
@@ -454,3 +454,125 @@ def test_console_is_created_by_default(tmp_path, monkeypatch) -> None:
assert application.console_server is not None
asyncio.run(application.client.aclose())
def test_dependency_supervisor_is_none_when_disabled(tmp_path, monkeypatch) -> None:
monkeypatch.chdir(tmp_path)
application = create_application(config=_config(), manager=DeviceManager())
assert application.dependency_supervisor is None
asyncio.run(application.client.aclose())
def test_dependency_supervisor_constructed_when_enabled_with_no_deps(
tmp_path, monkeypatch
) -> None:
monkeypatch.chdir(tmp_path)
config = HostAgentConfig(
control_plane_url="https://control.example",
host_id="host-a",
token="secret",
dependency_supervisor_enabled=True,
)
application = create_application(config=config, manager=DeviceManager())
assert application.dependency_supervisor is not None
assert application.dependency_supervisor.dependencies == []
asyncio.run(application.client.aclose())
def test_dependency_supervisor_constructed_when_enabled_with_appium_only(
tmp_path, monkeypatch
) -> None:
monkeypatch.chdir(tmp_path)
config = HostAgentConfig(
control_plane_url="https://control.example",
host_id="host-a",
token="secret",
dependency_supervisor_enabled=True,
appium_supervised=True,
appium_host="127.0.0.1",
appium_port=4723,
)
application = create_application(config=config, manager=DeviceManager())
assert application.dependency_supervisor is not None
deps = application.dependency_supervisor.dependencies
assert [dep.name for dep in deps] == ["appium"]
asyncio.run(application.client.aclose())
def test_run_async_starts_supervisor_before_first_heartbeat_connect() -> None:
async def scenario() -> None:
events: list[str] = []
class SupervisedNoOp:
def __init__(self) -> None:
self.started = False
self.stopped = False
async def start(self) -> None:
self.started = True
events.append("supervisor-start")
async def run(self, stop: asyncio.Event) -> None:
events.append("supervisor-run-entered")
await stop.wait()
async def stop(self) -> None:
self.stopped = True
events.append("supervisor-stop")
class BlockingClient:
async def claim(self):
await asyncio.Event().wait()
async def aclose(self):
events.append("closed")
class RecordingHeartbeat:
def __init__(self) -> None:
self.connect_called = False
def connect_devices(self) -> None:
self.connect_called = True
events.append("connect-devices")
async def run(self, stop: asyncio.Event) -> None:
# Mirror HeartbeatSynchronizer.run which calls connect_devices()
# at the very top — supervisor must have started already.
self.connect_devices()
await stop.wait()
async def sync_once(self):
events.append("final-heartbeat")
class IdleProcessor:
async def process(self, assignment):
raise AssertionError("no assignment expected")
def request_stop(self):
events.append("stop-work")
supervisor = SupervisedNoOp()
application = HostAgentApplication(
client=BlockingClient(), # type: ignore[arg-type]
heartbeat=RecordingHeartbeat(), # type: ignore[arg-type]
processor=IdleProcessor(), # type: ignore[arg-type]
dependency_supervisor=supervisor, # type: ignore[arg-type]
)
stop = asyncio.Event()
running = asyncio.create_task(application.run_async(stop))
# Yield long enough for startup sequencing to land.
await asyncio.sleep(0.05)
stop.set()
await asyncio.wait_for(running, timeout=1.0)
assert supervisor.started is True
assert supervisor.stopped is True
# Supervisor startup must precede the heartbeat's connect_devices().
assert events.index("supervisor-start") < events.index("connect-devices")
# Supervisor stop must run before client close.
assert events.index("supervisor-stop") < events.index("closed")
asyncio.run(scenario())
@@ -173,3 +173,55 @@ def test_console_numeric_fields_reject_invalid_values(
) -> None:
with pytest.raises(HostAgentConfigurationError):
load_host_agent_config(overrides)
def test_dependency_supervisor_defaults_to_disabled() -> None:
config = load_host_agent_config({})
assert config.dependency_supervisor_enabled is False
assert config.appium_supervised is False
assert config.appium_host == "127.0.0.1"
assert config.appium_port == 4723
assert config.runtime_supervised is False
assert config.runtime_host == "127.0.0.1"
assert config.runtime_port == 8000
assert config.dependency_restart_max_attempts == 5
def test_dependency_supervisor_env_vars_parse_bool_and_numeric_fields() -> None:
config = load_host_agent_config(
{
"HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED": "true",
"HOST_AGENT_APPIUM_SUPERVISED": "1",
"HOST_AGENT_APPIUM_HOST": "0.0.0.0",
"HOST_AGENT_APPIUM_PORT": "4724",
"HOST_AGENT_RUNTIME_SUPERVISED": "true",
"HOST_AGENT_RUNTIME_HOST": "localhost",
"HOST_AGENT_RUNTIME_PORT": "8001",
"HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS": "8",
}
)
assert config.dependency_supervisor_enabled is True
assert config.appium_supervised is True
assert config.appium_host == "0.0.0.0"
assert config.appium_port == 4724
assert config.runtime_supervised is True
assert config.runtime_host == "localhost"
assert config.runtime_port == 8001
assert config.dependency_restart_max_attempts == 8
@pytest.mark.parametrize(
"overrides",
[
{"HOST_AGENT_APPIUM_PORT": "0"},
{"HOST_AGENT_RUNTIME_PORT": "not-a-number"},
{"HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS": "-1"},
],
)
def test_dependency_supervisor_numeric_fields_reject_invalid_values(
overrides: dict[str, str],
) -> None:
with pytest.raises(HostAgentConfigurationError):
load_host_agent_config(overrides)
@@ -0,0 +1,554 @@
from __future__ import annotations
import asyncio
import io
import logging
import subprocess
from collections.abc import Sequence
import httpx
import pytest
from host_agent.dependency_supervisor import (
DependencySupervisor,
ProbeResult,
SupervisedDependency,
_SupervisorKnobs,
appium_argv_factory,
probe_appium,
probe_runtime,
runtime_argv_factory,
)
from host_agent.config import HostAgentConfig
# ---------------------------------------------------------------------------
# Test doubles
# ---------------------------------------------------------------------------
class _FakePopen:
"""Minimal subprocess.Popen stand-in for supervisor tests."""
instances: list[_FakePopen] = []
def __init__(
self,
argv: Sequence[str],
*,
stdout_lines: Sequence[str] = (),
pid: int = 0,
**_kwargs: object,
) -> None:
self.argv = list(argv)
self.pid = pid or (100 + len(_FakePopen.instances))
self.returncode: int | None = None
self.stdout = io.StringIO("".join(line + "\n" for line in stdout_lines))
self.terminate_calls = 0
self.kill_calls = 0
self.wait_calls = 0
_FakePopen.instances.append(self)
def poll(self) -> int | None:
return self.returncode
def terminate(self) -> None:
self.terminate_calls += 1
self.returncode = -15
def kill(self) -> None:
self.kill_calls += 1
self.returncode = -9
def wait(self, timeout: float | None = None) -> int:
self.wait_calls += 1
if self.returncode is None:
raise subprocess.TimeoutExpired(cmd=self.argv, timeout=timeout or 0)
return self.returncode
@pytest.fixture(autouse=True)
def _reset_fake_popen() -> None:
_FakePopen.instances.clear()
yield
_FakePopen.instances.clear()
def _dep(
name: str = "appium",
*,
host: str = "127.0.0.1",
port: int = 4723,
probe_responses: Sequence[ProbeResult] = (ProbeResult.NO_LISTENER,),
argv_factory=None,
) -> tuple[SupervisedDependency, list[ProbeResult]]:
"""Build a SupervisedDependency whose probe returns scripted responses."""
call_log: list[ProbeResult] = []
responses = list(probe_responses)
fallback = probe_responses[-1] if probe_responses else ProbeResult.NO_LISTENER
def probe(_host: str, _port: int) -> ProbeResult:
if responses:
result = responses.pop(0)
else:
result = fallback
call_log.append(result)
return result
if argv_factory is None:
def argv_factory(h: str, p: int) -> list[str]:
return ["echo", name]
dep = SupervisedDependency(
name=name,
host=host,
port=port,
argv_factory=argv_factory,
probe=probe,
)
return dep, call_log
class _FakeSleep:
"""Records sleep durations so tests can assert backoff progression."""
def __init__(self) -> None:
self.calls: list[float] = []
async def __call__(self, seconds: float) -> None:
self.calls.append(seconds)
def _build_supervisor(
deps: list[SupervisedDependency],
*,
max_attempts: int = 5,
knobs: _SupervisorKnobs | None = None,
sleep: _FakeSleep | None = None,
popen_factory=None,
) -> tuple[DependencySupervisor, _FakeSleep]:
fake_sleep = sleep or _FakeSleep()
sup = DependencySupervisor(
list(deps),
max_attempts=max_attempts,
knobs=knobs
or _SupervisorKnobs(
startup_timeout_seconds=1.0,
readiness_poll_interval_seconds=0.01,
crash_poll_interval_seconds=0.01,
initial_backoff_seconds=1.0,
max_backoff_seconds=30.0,
terminate_grace_period_seconds=1.0,
),
sleep=fake_sleep,
popen_factory=popen_factory or _FakePopen,
logger=logging.getLogger("test"),
)
return sup, fake_sleep
class _DummySocket:
def __enter__(self) -> "_DummySocket":
return self
def __exit__(self, *exc) -> None:
return None
def _raise_connection_refused(*_args, **_kwargs):
raise ConnectionRefusedError("no listener")
def _ok_connection(*_args, **_kwargs):
return _DummySocket()
# ---------------------------------------------------------------------------
# Probe tests (tasks 2.1, 2.3)
# ---------------------------------------------------------------------------
def test_probe_returns_no_listener_when_port_is_closed(monkeypatch) -> None:
monkeypatch.setattr(
"host_agent.dependency_supervisor.socket.create_connection",
_raise_connection_refused,
)
assert probe_appium("127.0.0.1", 4723) is ProbeResult.NO_LISTENER
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.NO_LISTENER
def test_probe_returns_healthy_on_appium_status_endpoint(monkeypatch) -> None:
monkeypatch.setattr(
"host_agent.dependency_supervisor.socket.create_connection",
_ok_connection,
)
monkeypatch.setattr(
"host_agent.dependency_supervisor.httpx.get",
lambda url, timeout=2.0: httpx.Response(200, json={"ready": True}),
)
assert probe_appium("127.0.0.1", 4723) is ProbeResult.HEALTHY
def test_probe_returns_healthy_on_runtime_devices_endpoint(monkeypatch) -> None:
monkeypatch.setattr(
"host_agent.dependency_supervisor.socket.create_connection",
_ok_connection,
)
monkeypatch.setattr(
"host_agent.dependency_supervisor.httpx.get",
lambda url, timeout=2.0: httpx.Response(200, json=[]),
)
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.HEALTHY
def test_probe_returns_unhealthy_when_listener_returns_non_200(monkeypatch) -> None:
monkeypatch.setattr(
"host_agent.dependency_supervisor.socket.create_connection",
_ok_connection,
)
monkeypatch.setattr(
"host_agent.dependency_supervisor.httpx.get",
lambda url, timeout=2.0: httpx.Response(500, text="boom"),
)
assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
def test_probe_returns_unhealthy_when_listener_returns_non_json(monkeypatch) -> None:
monkeypatch.setattr(
"host_agent.dependency_supervisor.socket.create_connection",
_ok_connection,
)
monkeypatch.setattr(
"host_agent.dependency_supervisor.httpx.get",
lambda url, timeout=2.0: httpx.Response(200, text="not json"),
)
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.UNHEALTHY_LISTENER
def test_probe_returns_unhealthy_on_http_transport_error(monkeypatch) -> None:
monkeypatch.setattr(
"host_agent.dependency_supervisor.socket.create_connection",
_ok_connection,
)
def raise_http_error(url, timeout=2.0):
raise httpx.ConnectError("reset")
monkeypatch.setattr("host_agent.dependency_supervisor.httpx.get", raise_http_error)
assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
# ---------------------------------------------------------------------------
# Adoption logic tests (tasks 2.2, 2.3)
# ---------------------------------------------------------------------------
def test_start_adopts_existing_healthy_instance() -> None:
async def scenario() -> None:
dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,))
sup, _ = _build_supervisor([dep])
await sup.start()
assert dep.adopted is True
assert dep.ready is True
assert dep.process is None
asyncio.run(scenario())
def test_start_skips_when_port_has_unhealthy_listener() -> None:
async def scenario() -> None:
dep, _ = _dep(probe_responses=(ProbeResult.UNHEALTHY_LISTENER,))
sup, _ = _build_supervisor([dep])
await sup.start()
assert dep.adopted is False
assert dep.process is None
assert dep.given_up is True
asyncio.run(scenario())
def test_start_spawns_when_nothing_is_listening() -> None:
async def scenario() -> None:
# NO_LISTENER for initial probe → spawn path.
# HEALTHY on first readiness probe.
dep, _ = _dep(
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
)
sup, _ = _build_supervisor([dep])
await sup.start()
assert dep.process is not None
assert dep.adopted is False
assert dep.ready is True
assert dep.given_up is False
asyncio.run(scenario())
# ---------------------------------------------------------------------------
# Supervisor core tests (task 3.6)
# ---------------------------------------------------------------------------
def test_spawn_failure_marks_dependency_given_up() -> None:
async def scenario() -> None:
dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,))
def raising_popen(argv, **_kwargs):
raise FileNotFoundError("appium: not found")
sup, _ = _build_supervisor([dep], popen_factory=raising_popen)
await sup.start()
assert dep.process is None
assert dep.given_up is True
assert dep.ready is False
asyncio.run(scenario())
def test_spawn_uses_appium_argv_factory() -> None:
async def scenario() -> None:
captured: list[list[str]] = []
def recording_popen(argv, **kwargs):
captured.append(list(argv))
return _FakePopen(argv, stdout_lines=("hello",))
dep, _ = _dep(
name="appium",
port=4723,
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
argv_factory=appium_argv_factory,
)
sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
await sup.start()
assert captured == [["appium", "--address", "127.0.0.1", "--port", "4723"]]
asyncio.run(scenario())
def test_spawn_uses_runtime_argv_factory() -> None:
async def scenario() -> None:
captured: list[list[str]] = []
def recording_popen(argv, **kwargs):
captured.append(list(argv))
return _FakePopen(argv)
dep, _ = _dep(
name="runtime",
port=8000,
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
argv_factory=runtime_argv_factory,
)
sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
await sup.start()
assert captured == [
[
"uvicorn",
"api.rest:create_app",
"--factory",
"--host",
"127.0.0.1",
"--port",
"8000",
]
]
asyncio.run(scenario())
def test_readiness_timeout_leaves_process_running_without_restart_loop() -> None:
async def scenario() -> None:
# NO_LISTENER for initial probe; probe never becomes HEALTHY → startup
# timeout path (1.0s in test knobs).
dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,))
sup, _ = _build_supervisor([dep])
await sup.start()
assert dep.process is not None # still running
assert dep.ready is False
assert dep.given_up is False # not given up; crash-restart owns crashes
asyncio.run(scenario())
def test_crash_triggers_restart_with_backoff() -> None:
async def scenario() -> None:
# Probe responses:
# 1. start._start_one → NO_LISTENER → spawn path
# 2. first readiness probe → HEALTHY → ready
# 3. after restart, readiness probe → HEALTHY → ready again
dep, _ = _dep(
probe_responses=(
ProbeResult.NO_LISTENER,
ProbeResult.HEALTHY,
ProbeResult.HEALTHY,
),
argv_factory=appium_argv_factory,
)
spawn_count = {"n": 0}
def popen(argv, **kwargs):
spawn_count["n"] += 1
return _FakePopen(argv)
sup, sleep = _build_supervisor([dep], popen_factory=popen)
await sup.start()
assert dep.ready is True
crashes_before = dep.restart_attempts
# Crash the spawned process now (after readiness succeeded). The
# supervisor's run loop will detect this via poll().
first_proc = dep.process
assert isinstance(first_proc, _FakePopen)
first_proc.returncode = 1
stop = asyncio.Event()
task = asyncio.create_task(sup.run(stop))
# Give the loop a chance to detect the crash, sleep backoff, respawn.
await asyncio.sleep(0.05)
stop.set()
await asyncio.wait_for(task, timeout=1.0)
assert dep.restart_attempts == crashes_before + 1
assert dep.ready is True
assert sleep.calls == [1.0] # exponential backoff base for attempt 1
assert spawn_count["n"] == 2
asyncio.run(scenario())
def test_restart_exhaustion_gives_up() -> None:
async def scenario() -> None:
# Process crashes after every spawn; limit max_attempts to 2.
dep, _ = _dep(
probe_responses=(ProbeResult.NO_LISTENER,) + (ProbeResult.HEALTHY,) * 10,
)
spawn_count = {"n": 0}
def popen(argv, **kwargs):
spawn_count["n"] += 1
proc = _FakePopen(argv)
proc.returncode = 1
return proc
sup, sleep = _build_supervisor([dep], popen_factory=popen, max_attempts=2)
await sup.start()
stop = asyncio.Event()
task = asyncio.create_task(sup.run(stop))
deadline = asyncio.get_running_loop().time() + 2.0
while not dep.given_up and asyncio.get_running_loop().time() < deadline:
await asyncio.sleep(0.01)
stop.set()
await asyncio.wait_for(task, timeout=2.0)
assert dep.given_up is True
# Initial spawn + 2 successful restarts before the 3rd crash gives up.
assert dep.restart_attempts == 3
# Backoffs: 1.0 (attempt 1), 2.0 (attempt 2). No third restart.
assert sleep.calls == [1.0, 2.0]
asyncio.run(scenario())
def test_adopted_process_is_never_restarted_or_killed() -> None:
async def scenario() -> None:
dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,))
sup, _ = _build_supervisor([dep])
await sup.start()
stop = asyncio.Event()
task = asyncio.create_task(sup.run(stop))
await asyncio.sleep(0.02)
stop.set()
await asyncio.wait_for(task, timeout=1.0)
await sup.stop()
assert dep.adopted is True
assert dep.process is None
assert dep.restart_attempts == 0
asyncio.run(scenario())
def test_stop_terminates_only_spawned_children() -> None:
async def scenario() -> None:
spawned_dep, _ = _dep(
name="spawned",
port=4723,
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
)
adopted_dep, _ = _dep(
name="adopted",
port=8000,
probe_responses=(ProbeResult.HEALTHY,),
)
sup, _ = _build_supervisor([spawned_dep, adopted_dep])
await sup.start()
assert spawned_dep.process is not None
spawned_proc = spawned_dep.process
assert adopted_dep.adopted is True
await sup.stop()
assert isinstance(spawned_proc, _FakePopen)
assert spawned_proc.terminate_calls == 1
assert spawned_dep.process is None
asyncio.run(scenario())
# ---------------------------------------------------------------------------
# from_host_agent_config factory (covers wiring helper for task 4.1)
# ---------------------------------------------------------------------------
def test_from_host_agent_config_builds_empty_supervisor_when_no_dep_selected() -> None:
config = HostAgentConfig(
control_plane_url="https://control.example",
host_id="host-a",
token="t",
dependency_supervisor_enabled=True,
)
sup = DependencySupervisor.from_host_agent_config(config)
assert sup.dependencies == []
def test_from_host_agent_config_includes_appium_and_runtime_when_selected() -> None:
config = HostAgentConfig(
control_plane_url="https://control.example",
host_id="host-a",
token="t",
dependency_supervisor_enabled=True,
appium_supervised=True,
appium_host="10.0.0.5",
appium_port=4724,
runtime_supervised=True,
runtime_host="10.0.0.5",
runtime_port=8001,
dependency_restart_max_attempts=7,
)
sup = DependencySupervisor.from_host_agent_config(config)
names = [dep.name for dep in sup.dependencies]
assert names == ["appium", "runtime"]
appium = sup.dependencies[0]
assert appium.host == "10.0.0.5"
assert appium.port == 4724
runtime = sup.dependencies[1]
assert runtime.port == 8001
assert sup._max_attempts == 7