Adds an opt-in dependency supervisor inside the Host Agent that probes,
spawns, and restarts the two local processes the macOS single-machine
real-device workflow depends on: the Appium server (gates Driver.connect())
and the local Runtime API (local inspection). Default-off; gated by
HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED plus per-dependency *_SUPERVISED
flags.
Mitigates the live-incident failure mode where forgetting to start Appium
silently keeps devices offline and tasks queued forever with no error
surfaced in Host Agent logs.
Behavior (per openspec change):
- Adopt-don't-fight: probe (TCP + dependency-specific HTTP health check)
before spawn. Healthy listener → adopted (never killed/restarted).
Unhealthy listener → port-conflict error, skip. No listener → spawn.
- Only supervisor-spawned processes are restarted on crash, with capped
exponential backoff (1s/2s/4s/8s, capped at 30s) and a per-process-lifetime
attempt ceiling (HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS, default 5).
- Spawn failures (e.g. missing executable) logged distinctly from crashes.
- Graceful stop terminates only spawned children; adopted processes untouched.
- Supervisor starts before the heartbeat loop's first connect_devices() pass
and stops alongside existing heartbeat/console teardown.
Validation: ruff check + format clean, compileall clean, openspec validate
--strict valid. Non-integration suite 503 passed / 44 deselected / 2 failed
(both failures pre-existing from unrelated 03c7c30 LLM_PROVIDER_ENC_KEY;
verified by stashing this change). macOS real-device manual verification
(task 6.4) deferred to a macOS host.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -454,3 +454,125 @@ def test_console_is_created_by_default(tmp_path, monkeypatch) -> None:
|
||||
|
||||
assert application.console_server is not None
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_dependency_supervisor_is_none_when_disabled(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
application = create_application(config=_config(), manager=DeviceManager())
|
||||
|
||||
assert application.dependency_supervisor is None
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_dependency_supervisor_constructed_when_enabled_with_no_deps(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="secret",
|
||||
dependency_supervisor_enabled=True,
|
||||
)
|
||||
application = create_application(config=config, manager=DeviceManager())
|
||||
|
||||
assert application.dependency_supervisor is not None
|
||||
assert application.dependency_supervisor.dependencies == []
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_dependency_supervisor_constructed_when_enabled_with_appium_only(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="secret",
|
||||
dependency_supervisor_enabled=True,
|
||||
appium_supervised=True,
|
||||
appium_host="127.0.0.1",
|
||||
appium_port=4723,
|
||||
)
|
||||
application = create_application(config=config, manager=DeviceManager())
|
||||
|
||||
assert application.dependency_supervisor is not None
|
||||
deps = application.dependency_supervisor.dependencies
|
||||
assert [dep.name for dep in deps] == ["appium"]
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_run_async_starts_supervisor_before_first_heartbeat_connect() -> None:
|
||||
async def scenario() -> None:
|
||||
events: list[str] = []
|
||||
|
||||
class SupervisedNoOp:
|
||||
def __init__(self) -> None:
|
||||
self.started = False
|
||||
self.stopped = False
|
||||
|
||||
async def start(self) -> None:
|
||||
self.started = True
|
||||
events.append("supervisor-start")
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
events.append("supervisor-run-entered")
|
||||
await stop.wait()
|
||||
|
||||
async def stop(self) -> None:
|
||||
self.stopped = True
|
||||
events.append("supervisor-stop")
|
||||
|
||||
class BlockingClient:
|
||||
async def claim(self):
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def aclose(self):
|
||||
events.append("closed")
|
||||
|
||||
class RecordingHeartbeat:
|
||||
def __init__(self) -> None:
|
||||
self.connect_called = False
|
||||
|
||||
def connect_devices(self) -> None:
|
||||
self.connect_called = True
|
||||
events.append("connect-devices")
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
# Mirror HeartbeatSynchronizer.run which calls connect_devices()
|
||||
# at the very top — supervisor must have started already.
|
||||
self.connect_devices()
|
||||
await stop.wait()
|
||||
|
||||
async def sync_once(self):
|
||||
events.append("final-heartbeat")
|
||||
|
||||
class IdleProcessor:
|
||||
async def process(self, assignment):
|
||||
raise AssertionError("no assignment expected")
|
||||
|
||||
def request_stop(self):
|
||||
events.append("stop-work")
|
||||
|
||||
supervisor = SupervisedNoOp()
|
||||
application = HostAgentApplication(
|
||||
client=BlockingClient(), # type: ignore[arg-type]
|
||||
heartbeat=RecordingHeartbeat(), # type: ignore[arg-type]
|
||||
processor=IdleProcessor(), # type: ignore[arg-type]
|
||||
dependency_supervisor=supervisor, # type: ignore[arg-type]
|
||||
)
|
||||
stop = asyncio.Event()
|
||||
running = asyncio.create_task(application.run_async(stop))
|
||||
# Yield long enough for startup sequencing to land.
|
||||
await asyncio.sleep(0.05)
|
||||
stop.set()
|
||||
await asyncio.wait_for(running, timeout=1.0)
|
||||
|
||||
assert supervisor.started is True
|
||||
assert supervisor.stopped is True
|
||||
# Supervisor startup must precede the heartbeat's connect_devices().
|
||||
assert events.index("supervisor-start") < events.index("connect-devices")
|
||||
# Supervisor stop must run before client close.
|
||||
assert events.index("supervisor-stop") < events.index("closed")
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
Reference in New Issue
Block a user