from __future__ import annotations import asyncio import io import logging import subprocess from collections.abc import Sequence import httpx import pytest from host_agent.dependency_supervisor import ( DependencySupervisor, ProbeResult, SupervisedDependency, _SupervisorKnobs, appium_argv_factory, probe_appium, ) from host_agent.config import HostAgentConfig # --------------------------------------------------------------------------- # Test doubles # --------------------------------------------------------------------------- class _FakePopen: """Minimal subprocess.Popen stand-in for supervisor tests.""" instances: list[_FakePopen] = [] def __init__( self, argv: Sequence[str], *, stdout_lines: Sequence[str] = (), pid: int = 0, **_kwargs: object, ) -> None: self.argv = list(argv) self.pid = pid or (100 + len(_FakePopen.instances)) self.returncode: int | None = None self.stdout = io.StringIO("".join(line + "\n" for line in stdout_lines)) self.terminate_calls = 0 self.kill_calls = 0 self.wait_calls = 0 _FakePopen.instances.append(self) def poll(self) -> int | None: return self.returncode def terminate(self) -> None: self.terminate_calls += 1 self.returncode = -15 def kill(self) -> None: self.kill_calls += 1 self.returncode = -9 def wait(self, timeout: float | None = None) -> int: self.wait_calls += 1 if self.returncode is None: raise subprocess.TimeoutExpired(cmd=self.argv, timeout=timeout or 0) return self.returncode @pytest.fixture(autouse=True) def _reset_fake_popen() -> None: _FakePopen.instances.clear() yield _FakePopen.instances.clear() def _dep( name: str = "appium", *, host: str = "127.0.0.1", port: int = 4723, probe_responses: Sequence[ProbeResult] = (ProbeResult.NO_LISTENER,), argv_factory=None, ) -> tuple[SupervisedDependency, list[ProbeResult]]: """Build a SupervisedDependency whose probe returns scripted responses.""" call_log: list[ProbeResult] = [] responses = list(probe_responses) fallback = probe_responses[-1] if probe_responses else ProbeResult.NO_LISTENER def probe(_host: str, _port: int) -> ProbeResult: if responses: result = responses.pop(0) else: result = fallback call_log.append(result) return result if argv_factory is None: def argv_factory(h: str, p: int) -> list[str]: return ["echo", name] dep = SupervisedDependency( name=name, host=host, port=port, argv_factory=argv_factory, probe=probe, ) return dep, call_log class _FakeSleep: """Records sleep durations so tests can assert backoff progression.""" def __init__(self) -> None: self.calls: list[float] = [] async def __call__(self, seconds: float) -> None: self.calls.append(seconds) def _build_supervisor( deps: list[SupervisedDependency], *, max_attempts: int = 5, knobs: _SupervisorKnobs | None = None, sleep: _FakeSleep | None = None, popen_factory=None, ) -> tuple[DependencySupervisor, _FakeSleep]: fake_sleep = sleep or _FakeSleep() sup = DependencySupervisor( list(deps), max_attempts=max_attempts, knobs=knobs or _SupervisorKnobs( startup_timeout_seconds=1.0, readiness_poll_interval_seconds=0.01, crash_poll_interval_seconds=0.01, initial_backoff_seconds=1.0, max_backoff_seconds=30.0, terminate_grace_period_seconds=1.0, ), sleep=fake_sleep, popen_factory=popen_factory or _FakePopen, logger=logging.getLogger("test"), ) return sup, fake_sleep class _DummySocket: def __enter__(self) -> "_DummySocket": return self def __exit__(self, *exc) -> None: return None def _raise_connection_refused(*_args, **_kwargs): raise ConnectionRefusedError("no listener") def _ok_connection(*_args, **_kwargs): return _DummySocket() # --------------------------------------------------------------------------- # Probe tests (tasks 2.1, 2.3) # --------------------------------------------------------------------------- def test_probe_returns_no_listener_when_port_is_closed(monkeypatch) -> None: monkeypatch.setattr( "host_agent.dependency_supervisor.socket.create_connection", _raise_connection_refused, ) assert probe_appium("127.0.0.1", 4723) is ProbeResult.NO_LISTENER def test_probe_returns_healthy_on_appium_status_endpoint(monkeypatch) -> None: monkeypatch.setattr( "host_agent.dependency_supervisor.socket.create_connection", _ok_connection, ) monkeypatch.setattr( "host_agent.dependency_supervisor.httpx.get", lambda url, timeout=2.0: httpx.Response(200, json={"ready": True}), ) assert probe_appium("127.0.0.1", 4723) is ProbeResult.HEALTHY def test_probe_returns_unhealthy_when_listener_returns_non_200(monkeypatch) -> None: monkeypatch.setattr( "host_agent.dependency_supervisor.socket.create_connection", _ok_connection, ) monkeypatch.setattr( "host_agent.dependency_supervisor.httpx.get", lambda url, timeout=2.0: httpx.Response(500, text="boom"), ) assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER def test_probe_returns_unhealthy_when_listener_returns_non_json(monkeypatch) -> None: monkeypatch.setattr( "host_agent.dependency_supervisor.socket.create_connection", _ok_connection, ) monkeypatch.setattr( "host_agent.dependency_supervisor.httpx.get", lambda url, timeout=2.0: httpx.Response(200, text="not json"), ) assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER def test_probe_returns_unhealthy_on_http_transport_error(monkeypatch) -> None: monkeypatch.setattr( "host_agent.dependency_supervisor.socket.create_connection", _ok_connection, ) def raise_http_error(url, timeout=2.0): raise httpx.ConnectError("reset") monkeypatch.setattr("host_agent.dependency_supervisor.httpx.get", raise_http_error) assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER # --------------------------------------------------------------------------- # Adoption logic tests (tasks 2.2, 2.3) # --------------------------------------------------------------------------- def test_start_adopts_existing_healthy_instance() -> None: async def scenario() -> None: dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,)) sup, _ = _build_supervisor([dep]) await sup.start() assert dep.adopted is True assert dep.ready is True assert dep.process is None asyncio.run(scenario()) def test_start_skips_when_port_has_unhealthy_listener() -> None: async def scenario() -> None: dep, _ = _dep(probe_responses=(ProbeResult.UNHEALTHY_LISTENER,)) sup, _ = _build_supervisor([dep]) await sup.start() assert dep.adopted is False assert dep.process is None assert dep.given_up is True asyncio.run(scenario()) def test_start_spawns_when_nothing_is_listening() -> None: async def scenario() -> None: # NO_LISTENER for initial probe → spawn path. # HEALTHY on first readiness probe. dep, _ = _dep( probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY), ) sup, _ = _build_supervisor([dep]) await sup.start() assert dep.process is not None assert dep.adopted is False assert dep.ready is True assert dep.given_up is False asyncio.run(scenario()) # --------------------------------------------------------------------------- # Supervisor core tests (task 3.6) # --------------------------------------------------------------------------- def test_spawn_failure_marks_dependency_given_up() -> None: async def scenario() -> None: dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,)) def raising_popen(argv, **_kwargs): raise FileNotFoundError("appium: not found") sup, _ = _build_supervisor([dep], popen_factory=raising_popen) await sup.start() assert dep.process is None assert dep.given_up is True assert dep.ready is False asyncio.run(scenario()) def test_spawn_uses_appium_argv_factory() -> None: async def scenario() -> None: captured: list[list[str]] = [] def recording_popen(argv, **kwargs): captured.append(list(argv)) return _FakePopen(argv, stdout_lines=("hello",)) dep, _ = _dep( name="appium", port=4723, probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY), argv_factory=appium_argv_factory, ) sup, _ = _build_supervisor([dep], popen_factory=recording_popen) await sup.start() assert captured == [["appium", "--address", "127.0.0.1", "--port", "4723"]] asyncio.run(scenario()) def test_readiness_timeout_leaves_process_running_without_restart_loop() -> None: async def scenario() -> None: # NO_LISTENER for initial probe; probe never becomes HEALTHY → startup # timeout path (1.0s in test knobs). dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,)) sup, _ = _build_supervisor([dep]) await sup.start() assert dep.process is not None # still running assert dep.ready is False assert dep.given_up is False # not given up; crash-restart owns crashes asyncio.run(scenario()) def test_crash_triggers_restart_with_backoff() -> None: async def scenario() -> None: # Probe responses: # 1. start._start_one → NO_LISTENER → spawn path # 2. first readiness probe → HEALTHY → ready # 3. after restart, readiness probe → HEALTHY → ready again dep, _ = _dep( probe_responses=( ProbeResult.NO_LISTENER, ProbeResult.HEALTHY, ProbeResult.HEALTHY, ), argv_factory=appium_argv_factory, ) spawn_count = {"n": 0} def popen(argv, **kwargs): spawn_count["n"] += 1 return _FakePopen(argv) sup, sleep = _build_supervisor([dep], popen_factory=popen) await sup.start() assert dep.ready is True crashes_before = dep.restart_attempts # Crash the spawned process now (after readiness succeeded). The # supervisor's run loop will detect this via poll(). first_proc = dep.process assert isinstance(first_proc, _FakePopen) first_proc.returncode = 1 stop = asyncio.Event() task = asyncio.create_task(sup.run(stop)) # Give the loop a chance to detect the crash, sleep backoff, respawn. await asyncio.sleep(0.05) stop.set() await asyncio.wait_for(task, timeout=1.0) assert dep.restart_attempts == crashes_before + 1 assert dep.ready is True assert sleep.calls == [1.0] # exponential backoff base for attempt 1 assert spawn_count["n"] == 2 asyncio.run(scenario()) def test_restart_exhaustion_gives_up() -> None: async def scenario() -> None: # Process crashes after every spawn; limit max_attempts to 2. dep, _ = _dep( probe_responses=(ProbeResult.NO_LISTENER,) + (ProbeResult.HEALTHY,) * 10, ) spawn_count = {"n": 0} def popen(argv, **kwargs): spawn_count["n"] += 1 proc = _FakePopen(argv) proc.returncode = 1 return proc sup, sleep = _build_supervisor([dep], popen_factory=popen, max_attempts=2) await sup.start() stop = asyncio.Event() task = asyncio.create_task(sup.run(stop)) deadline = asyncio.get_running_loop().time() + 2.0 while not dep.given_up and asyncio.get_running_loop().time() < deadline: await asyncio.sleep(0.01) stop.set() await asyncio.wait_for(task, timeout=2.0) assert dep.given_up is True # Initial spawn + 2 successful restarts before the 3rd crash gives up. assert dep.restart_attempts == 3 # Backoffs: 1.0 (attempt 1), 2.0 (attempt 2). No third restart. assert sleep.calls == [1.0, 2.0] asyncio.run(scenario()) def test_adopted_process_is_never_restarted_or_killed() -> None: async def scenario() -> None: dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,)) sup, _ = _build_supervisor([dep]) await sup.start() stop = asyncio.Event() task = asyncio.create_task(sup.run(stop)) await asyncio.sleep(0.02) stop.set() await asyncio.wait_for(task, timeout=1.0) await sup.stop() assert dep.adopted is True assert dep.process is None assert dep.restart_attempts == 0 asyncio.run(scenario()) def test_stop_terminates_only_spawned_children() -> None: async def scenario() -> None: spawned_dep, _ = _dep( name="spawned", port=4723, probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY), ) adopted_dep, _ = _dep( name="adopted", port=8000, probe_responses=(ProbeResult.HEALTHY,), ) sup, _ = _build_supervisor([spawned_dep, adopted_dep]) await sup.start() assert spawned_dep.process is not None spawned_proc = spawned_dep.process assert adopted_dep.adopted is True await sup.stop() assert isinstance(spawned_proc, _FakePopen) assert spawned_proc.terminate_calls == 1 assert spawned_dep.process is None asyncio.run(scenario()) # --------------------------------------------------------------------------- # from_host_agent_config factory (covers wiring helper for task 4.1) # --------------------------------------------------------------------------- def test_from_host_agent_config_builds_empty_supervisor_when_no_dep_selected() -> None: config = HostAgentConfig( control_plane_url="https://control.example", host_id="host-a", token="t", dependency_supervisor_enabled=True, ) sup = DependencySupervisor.from_host_agent_config(config) assert sup.dependencies == [] def test_from_host_agent_config_includes_appium_when_selected() -> None: config = HostAgentConfig( control_plane_url="https://control.example", host_id="host-a", token="t", dependency_supervisor_enabled=True, appium_supervised=True, appium_host="10.0.0.5", appium_port=4724, dependency_restart_max_attempts=7, ) sup = DependencySupervisor.from_host_agent_config(config) names = [dep.name for dep in sup.dependencies] assert names == ["appium"] appium = sup.dependencies[0] assert appium.host == "10.0.0.5" assert appium.port == 4724 assert sup._max_attempts == 7