Adds an opt-in dependency supervisor inside the Host Agent that probes,
spawns, and restarts the two local processes the macOS single-machine
real-device workflow depends on: the Appium server (gates Driver.connect())
and the local Runtime API (local inspection). Default-off; gated by
HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED plus per-dependency *_SUPERVISED
flags.
Mitigates the live-incident failure mode where forgetting to start Appium
silently keeps devices offline and tasks queued forever with no error
surfaced in Host Agent logs.
Behavior (per openspec change):
- Adopt-don't-fight: probe (TCP + dependency-specific HTTP health check)
before spawn. Healthy listener → adopted (never killed/restarted).
Unhealthy listener → port-conflict error, skip. No listener → spawn.
- Only supervisor-spawned processes are restarted on crash, with capped
exponential backoff (1s/2s/4s/8s, capped at 30s) and a per-process-lifetime
attempt ceiling (HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS, default 5).
- Spawn failures (e.g. missing executable) logged distinctly from crashes.
- Graceful stop terminates only spawned children; adopted processes untouched.
- Supervisor starts before the heartbeat loop's first connect_devices() pass
and stops alongside existing heartbeat/console teardown.
Validation: ruff check + format clean, compileall clean, openspec validate
--strict valid. Non-integration suite 503 passed / 44 deselected / 2 failed
(both failures pre-existing from unrelated 03c7c30 LLM_PROVIDER_ENC_KEY;
verified by stashing this change). macOS real-device manual verification
(task 6.4) deferred to a macOS host.
Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -11,6 +11,7 @@ from device.manager import DeviceManager
|
||||
from host_agent.assignment import AssignmentExecutor
|
||||
from host_agent.client import HostAgentClient, HostAgentEnrollmentClient
|
||||
from host_agent.config import HostAgentConfig, load_host_agent_config
|
||||
from host_agent.dependency_supervisor import DependencySupervisor
|
||||
from host_agent.devices import register_local_device
|
||||
from host_agent.enrollment import resolve_host_identity
|
||||
from host_agent.execution import create_execution_factories
|
||||
@@ -34,12 +35,23 @@ class HostAgentApplication:
|
||||
processor: AssignmentProcessor
|
||||
console_server: uvicorn.Server | None = None
|
||||
console_enrollment_client: HostAgentEnrollmentClient | None = None
|
||||
dependency_supervisor: DependencySupervisor | None = None
|
||||
|
||||
def run(self) -> None:
|
||||
asyncio.run(self.run_async())
|
||||
|
||||
async def run_async(self, stop: asyncio.Event | None = None) -> None:
|
||||
stop_requested = stop or asyncio.Event()
|
||||
supervisor_stop = asyncio.Event()
|
||||
supervisor_task: asyncio.Task[None] | None = None
|
||||
if self.dependency_supervisor is not None:
|
||||
# Bring up supervised dependencies (probe + spawn/adopt + readiness
|
||||
# wait) before the heartbeat loop's first connect_devices() pass,
|
||||
# so a supervised Appium is ready before any Driver.connect().
|
||||
await self.dependency_supervisor.start()
|
||||
supervisor_task = asyncio.create_task(
|
||||
self.dependency_supervisor.run(supervisor_stop)
|
||||
)
|
||||
heartbeat_stop = asyncio.Event()
|
||||
heartbeat_task = asyncio.create_task(self.heartbeat.run(heartbeat_stop))
|
||||
console_task = (
|
||||
@@ -75,12 +87,17 @@ class HostAgentApplication:
|
||||
with suppress(Exception):
|
||||
await asyncio.shield(active_processing)
|
||||
heartbeat_stop.set()
|
||||
supervisor_stop.set()
|
||||
if self.console_server is not None:
|
||||
self.console_server.should_exit = True
|
||||
try:
|
||||
await asyncio.gather(heartbeat_task, return_exceptions=True)
|
||||
with suppress(Exception):
|
||||
await self.heartbeat.sync_once()
|
||||
if supervisor_task is not None:
|
||||
await asyncio.gather(supervisor_task, return_exceptions=True)
|
||||
with suppress(Exception):
|
||||
await self.dependency_supervisor.stop()
|
||||
if console_task is not None:
|
||||
with suppress(asyncio.CancelledError):
|
||||
await asyncio.gather(console_task, return_exceptions=True)
|
||||
@@ -213,12 +230,18 @@ def create_application(
|
||||
history_store, assignment, result
|
||||
),
|
||||
)
|
||||
dependency_supervisor: DependencySupervisor | None = None
|
||||
if resolved_config.dependency_supervisor_enabled:
|
||||
dependency_supervisor = DependencySupervisor.from_host_agent_config(
|
||||
resolved_config
|
||||
)
|
||||
return HostAgentApplication(
|
||||
client=client,
|
||||
heartbeat=heartbeat,
|
||||
processor=processor,
|
||||
console_server=console_server,
|
||||
console_enrollment_client=console_enrollment_client,
|
||||
dependency_supervisor=dependency_supervisor,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -35,6 +35,14 @@ class HostAgentConfig:
|
||||
console_session_ttl_seconds: float = 43200.0
|
||||
console_history_limit: int = 200
|
||||
ai_planner_transport: str = "direct"
|
||||
dependency_supervisor_enabled: bool = False
|
||||
appium_supervised: bool = False
|
||||
appium_host: str = "127.0.0.1"
|
||||
appium_port: int = 4723
|
||||
runtime_supervised: bool = False
|
||||
runtime_host: str = "127.0.0.1"
|
||||
runtime_port: int = 8000
|
||||
dependency_restart_max_attempts: int = 5
|
||||
|
||||
|
||||
def load_host_agent_config(
|
||||
@@ -115,6 +123,18 @@ def load_host_agent_config(
|
||||
ai_planner_transport=_parse_ai_planner_transport(
|
||||
values.get("AI_PLANNER_TRANSPORT")
|
||||
),
|
||||
dependency_supervisor_enabled=_truthy(
|
||||
values, "HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED", False
|
||||
),
|
||||
appium_supervised=_truthy(values, "HOST_AGENT_APPIUM_SUPERVISED", False),
|
||||
appium_host=values.get("HOST_AGENT_APPIUM_HOST", "127.0.0.1").strip(),
|
||||
appium_port=_positive_int(values, "HOST_AGENT_APPIUM_PORT", 4723),
|
||||
runtime_supervised=_truthy(values, "HOST_AGENT_RUNTIME_SUPERVISED", False),
|
||||
runtime_host=values.get("HOST_AGENT_RUNTIME_HOST", "127.0.0.1").strip(),
|
||||
runtime_port=_positive_int(values, "HOST_AGENT_RUNTIME_PORT", 8000),
|
||||
dependency_restart_max_attempts=_positive_int(
|
||||
values, "HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS", 5
|
||||
),
|
||||
)
|
||||
if config.max_retry_backoff_seconds < config.retry_backoff_seconds:
|
||||
raise HostAgentConfigurationError(
|
||||
|
||||
@@ -0,0 +1,438 @@
|
||||
"""Optional supervisor for the two local external processes the Host Agent
|
||||
depends on for the macOS single-machine real-device workflow: the Appium
|
||||
server (which gates real ``Driver.connect()``) and the local Runtime API
|
||||
(used for local inspection).
|
||||
|
||||
Lives in ``host_agent`` because it spawns and monitors host-level processes
|
||||
alongside the heartbeat/claim loop. Off by default; see ``HostAgentConfig``.
|
||||
|
||||
Design (``openspec/changes/host-agent-dependency-supervisor/design.md``):
|
||||
- Default off. Each dependency also has its own opt-in flag.
|
||||
- Before spawn: TCP connect + dependency-specific HTTP health probe. A healthy
|
||||
listener is *adopted* (never killed or restarted). An unhealthy listener is
|
||||
a port conflict — logged and skipped. No listener → spawn.
|
||||
- Only supervisor-spawned processes are restarted on unexpected exit, with
|
||||
capped exponential backoff and a per-process-lifetime attempt ceiling.
|
||||
- Adopted processes are never touched by ``stop()``.
|
||||
- Spawns use stdlib ``subprocess.Popen`` (no new dependency). Child stdout/stderr
|
||||
is forwarded into this module's logger, tagged per dependency.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
import subprocess
|
||||
import threading
|
||||
from collections.abc import Awaitable, Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import httpx
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from host_agent.config import HostAgentConfig
|
||||
|
||||
_LOGGER = logging.getLogger("host_agent.dependency_supervisor")
|
||||
|
||||
_PROBE_TCP_TIMEOUT_SECONDS = 1.0
|
||||
_PROBE_HTTP_TIMEOUT_SECONDS = 2.0
|
||||
|
||||
|
||||
class ProbeResult(str, Enum):
|
||||
"""Outcome of probing a dependency's configured address."""
|
||||
|
||||
HEALTHY = "healthy"
|
||||
UNHEALTHY_LISTENER = "unhealthy_listener"
|
||||
NO_LISTENER = "no_listener"
|
||||
|
||||
|
||||
def probe_appium(host: str, port: int) -> ProbeResult:
|
||||
"""Probe Appium at ``host:port``. Healthy iff ``GET /status`` returns 200
|
||||
with a JSON body (the documented shape per ``docs/MACOS_IPHONE_SETUP.md``
|
||||
§6 — contains ``ready``/build info)."""
|
||||
return _probe_http(host, port, path="/status")
|
||||
|
||||
|
||||
def probe_runtime(host: str, port: int) -> ProbeResult:
|
||||
"""Probe the local Runtime API at ``host:port``. Healthy iff ``GET /devices``
|
||||
returns 200 with a JSON body.
|
||||
|
||||
``api.rest.create_app`` does not expose a dedicated ``/health`` endpoint;
|
||||
``/devices`` is the stable read-only GET that proves the FastAPI app is
|
||||
mounted and the device manager is reachable. Per design.md Decision 2 this
|
||||
is the "equivalent existing endpoint" used for the readiness check.
|
||||
"""
|
||||
return _probe_http(host, port, path="/devices")
|
||||
|
||||
|
||||
def _probe_http(host: str, port: int, *, path: str) -> ProbeResult:
|
||||
# Step 1: plain TCP connect — distinguish "nothing listening" (→ spawn)
|
||||
# from "something is there but wrong" (→ port conflict, skip).
|
||||
try:
|
||||
with socket.create_connection((host, port), timeout=_PROBE_TCP_TIMEOUT_SECONDS):
|
||||
pass
|
||||
except OSError:
|
||||
return ProbeResult.NO_LISTENER
|
||||
|
||||
# Step 2: dependency-specific HTTP health check.
|
||||
try:
|
||||
response = httpx.get(
|
||||
f"http://{host}:{port}{path}",
|
||||
timeout=_PROBE_HTTP_TIMEOUT_SECONDS,
|
||||
)
|
||||
except httpx.HTTPError:
|
||||
return ProbeResult.UNHEALTHY_LISTENER
|
||||
if response.status_code != 200:
|
||||
return ProbeResult.UNHEALTHY_LISTENER
|
||||
try:
|
||||
response.json()
|
||||
except ValueError:
|
||||
return ProbeResult.UNHEALTHY_LISTENER
|
||||
return ProbeResult.HEALTHY
|
||||
|
||||
|
||||
def appium_argv_factory(host: str, port: int) -> list[str]:
|
||||
return ["appium", "--address", host, "--port", str(port)]
|
||||
|
||||
|
||||
def runtime_argv_factory(host: str, port: int) -> list[str]:
|
||||
return [
|
||||
"uvicorn",
|
||||
"api.rest:create_app",
|
||||
"--factory",
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class SupervisedDependency:
|
||||
"""Config + mutable runtime state for one supervised external process."""
|
||||
|
||||
name: str
|
||||
host: str
|
||||
port: int
|
||||
argv_factory: Callable[[str, int], Sequence[str]]
|
||||
probe: Callable[[str, int], ProbeResult]
|
||||
# Mutable state:
|
||||
process: subprocess.Popen | None = None
|
||||
adopted: bool = False
|
||||
given_up: bool = False
|
||||
restart_attempts: int = 0
|
||||
ready: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class _SupervisorKnobs:
|
||||
"""Tunables exposed for tests; production uses the defaults."""
|
||||
|
||||
startup_timeout_seconds: float = 30.0
|
||||
readiness_poll_interval_seconds: float = 0.5
|
||||
crash_poll_interval_seconds: float = 1.0
|
||||
initial_backoff_seconds: float = 1.0
|
||||
max_backoff_seconds: float = 30.0
|
||||
terminate_grace_period_seconds: float = 5.0
|
||||
|
||||
|
||||
class DependencySupervisor:
|
||||
"""Manage zero or more supervised local dependencies.
|
||||
|
||||
Each dependency is either *adopted* (an existing healthy instance was
|
||||
detected — never killed or restarted) or *spawned* (a child ``Popen``
|
||||
handle the supervisor owns — restart-on-crash with capped exponential
|
||||
backoff up to ``max_attempts`` per process lifetime).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dependencies: list[SupervisedDependency],
|
||||
*,
|
||||
max_attempts: int,
|
||||
knobs: _SupervisorKnobs | None = None,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
popen_factory: Callable[[Sequence[str]], subprocess.Popen] = subprocess.Popen,
|
||||
logger: logging.Logger = _LOGGER,
|
||||
) -> None:
|
||||
self._dependencies = dependencies
|
||||
self._max_attempts = max_attempts
|
||||
self._knobs = knobs or _SupervisorKnobs()
|
||||
self._sleep = sleep
|
||||
self._popen_factory = popen_factory
|
||||
self._logger = logger
|
||||
self._reader_threads: list[threading.Thread] = []
|
||||
|
||||
@classmethod
|
||||
def from_host_agent_config(
|
||||
cls,
|
||||
ha_config: HostAgentConfig,
|
||||
**kwargs: object,
|
||||
) -> DependencySupervisor:
|
||||
"""Build a supervisor reflecting ``HostAgentConfig`` flags.
|
||||
|
||||
Caller is responsible for only invoking this when
|
||||
``dependency_supervisor_enabled`` is true; the resulting supervisor
|
||||
will contain only the dependencies whose individual ``*_supervised``
|
||||
flag is also true (possibly an empty list).
|
||||
"""
|
||||
deps: list[SupervisedDependency] = []
|
||||
if ha_config.appium_supervised:
|
||||
deps.append(
|
||||
SupervisedDependency(
|
||||
name="appium",
|
||||
host=ha_config.appium_host,
|
||||
port=ha_config.appium_port,
|
||||
argv_factory=appium_argv_factory,
|
||||
probe=probe_appium,
|
||||
)
|
||||
)
|
||||
if ha_config.runtime_supervised:
|
||||
deps.append(
|
||||
SupervisedDependency(
|
||||
name="runtime",
|
||||
host=ha_config.runtime_host,
|
||||
port=ha_config.runtime_port,
|
||||
argv_factory=runtime_argv_factory,
|
||||
probe=probe_runtime,
|
||||
)
|
||||
)
|
||||
return cls(
|
||||
deps,
|
||||
max_attempts=ha_config.dependency_restart_max_attempts,
|
||||
**kwargs, # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
@property
|
||||
def dependencies(self) -> list[SupervisedDependency]:
|
||||
return list(self._dependencies)
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Probe + spawn/adopt + readiness wait.
|
||||
|
||||
Call before the heartbeat loop's first ``connect_devices()`` pass so
|
||||
a supervised Appium is up before any ``Driver.connect()`` attempt.
|
||||
"""
|
||||
for dep in self._dependencies:
|
||||
await self._start_one(dep)
|
||||
|
||||
async def _start_one(self, dep: SupervisedDependency) -> None:
|
||||
result = await asyncio.to_thread(dep.probe, dep.host, dep.port)
|
||||
if result is ProbeResult.HEALTHY:
|
||||
dep.adopted = True
|
||||
dep.ready = True
|
||||
self._logger.info(
|
||||
"dependency-supervisor: %s adopted existing instance at %s:%s",
|
||||
dep.name,
|
||||
dep.host,
|
||||
dep.port,
|
||||
)
|
||||
return
|
||||
if result is ProbeResult.UNHEALTHY_LISTENER:
|
||||
self._logger.error(
|
||||
"dependency-supervisor: %s port %s occupied by an unhealthy "
|
||||
"listener; leaving un-started to avoid a port conflict",
|
||||
dep.name,
|
||||
dep.port,
|
||||
)
|
||||
dep.given_up = True
|
||||
return
|
||||
await self._spawn_with_readiness_wait(dep)
|
||||
|
||||
async def _spawn_with_readiness_wait(
|
||||
self,
|
||||
dep: SupervisedDependency,
|
||||
) -> bool:
|
||||
"""Spawn ``dep`` and wait for it to become healthy.
|
||||
|
||||
Returns True on readiness, False otherwise. On spawn failure (missing
|
||||
executable) or readiness timeout while the process is still running,
|
||||
marks the dependency as ``given_up`` (no crash-restart loop entered).
|
||||
If the process exits during startup, returns False without giving up
|
||||
— the crash-restart loop in ``run()`` handles subsequent restarts.
|
||||
"""
|
||||
argv = list(dep.argv_factory(dep.host, dep.port))
|
||||
try:
|
||||
proc = self._popen_factory(
|
||||
argv,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
bufsize=1,
|
||||
text=True,
|
||||
)
|
||||
except FileNotFoundError:
|
||||
self._logger.error(
|
||||
"dependency-supervisor: %s spawn failed — executable not "
|
||||
"found on PATH (tried: %s)",
|
||||
dep.name,
|
||||
argv[0],
|
||||
)
|
||||
dep.given_up = True
|
||||
return False
|
||||
except OSError as exc:
|
||||
self._logger.error(
|
||||
"dependency-supervisor: %s spawn failed: %s",
|
||||
dep.name,
|
||||
exc,
|
||||
)
|
||||
dep.given_up = True
|
||||
return False
|
||||
|
||||
dep.process = proc
|
||||
dep.adopted = False
|
||||
dep.ready = False
|
||||
self._start_reader_thread(dep, proc)
|
||||
self._logger.info(
|
||||
"dependency-supervisor: %s spawned (pid %s) at %s:%s",
|
||||
dep.name,
|
||||
proc.pid,
|
||||
dep.host,
|
||||
dep.port,
|
||||
)
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
deadline = loop.time() + self._knobs.startup_timeout_seconds
|
||||
while True:
|
||||
if proc.poll() is not None:
|
||||
self._logger.error(
|
||||
"dependency-supervisor: %s exited during startup with code %s",
|
||||
dep.name,
|
||||
proc.returncode,
|
||||
)
|
||||
dep.ready = False
|
||||
return False
|
||||
probe_result = await asyncio.to_thread(dep.probe, dep.host, dep.port)
|
||||
if probe_result is ProbeResult.HEALTHY:
|
||||
dep.ready = True
|
||||
self._logger.info(
|
||||
"dependency-supervisor: %s ready at %s:%s",
|
||||
dep.name,
|
||||
dep.host,
|
||||
dep.port,
|
||||
)
|
||||
return True
|
||||
if loop.time() >= deadline:
|
||||
self._logger.error(
|
||||
"dependency-supervisor: %s started but did not become "
|
||||
"healthy within %ss; process still running, not entering "
|
||||
"crash-restart loop",
|
||||
dep.name,
|
||||
self._knobs.startup_timeout_seconds,
|
||||
)
|
||||
dep.ready = False
|
||||
return False
|
||||
await self._sleep(self._knobs.readiness_poll_interval_seconds)
|
||||
|
||||
def _start_reader_thread(
|
||||
self,
|
||||
dep: SupervisedDependency,
|
||||
proc: subprocess.Popen,
|
||||
) -> None:
|
||||
def reader_loop() -> None:
|
||||
stdout = proc.stdout
|
||||
if stdout is None:
|
||||
return
|
||||
for raw_line in stdout:
|
||||
line = raw_line.rstrip()
|
||||
if line:
|
||||
self._logger.info("%s: %s", dep.name, line)
|
||||
|
||||
thread = threading.Thread(
|
||||
target=reader_loop,
|
||||
name=f"dep-sup-{dep.name}",
|
||||
daemon=True,
|
||||
)
|
||||
thread.start()
|
||||
self._reader_threads.append(thread)
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
"""Background monitor: detect crashes and apply backoff restart.
|
||||
|
||||
Returns when ``stop`` is set. Crashes are only detected for
|
||||
spawned processes (not adopted ones).
|
||||
"""
|
||||
if not self._dependencies:
|
||||
await stop.wait()
|
||||
return
|
||||
while not stop.is_set():
|
||||
for dep in self._dependencies:
|
||||
await self._check_one(dep)
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
stop.wait(),
|
||||
timeout=self._knobs.crash_poll_interval_seconds,
|
||||
)
|
||||
except TimeoutError:
|
||||
continue
|
||||
|
||||
async def _check_one(self, dep: SupervisedDependency) -> None:
|
||||
if dep.given_up or dep.adopted or dep.process is None:
|
||||
return
|
||||
if dep.process.poll() is None:
|
||||
return
|
||||
|
||||
self._logger.warning(
|
||||
"dependency-supervisor: %s exited unexpectedly with code %s",
|
||||
dep.name,
|
||||
dep.process.returncode,
|
||||
)
|
||||
dep.process = None
|
||||
dep.ready = False
|
||||
dep.restart_attempts += 1
|
||||
if dep.restart_attempts > self._max_attempts:
|
||||
self._logger.error(
|
||||
"dependency-supervisor: %s restart attempts exhausted "
|
||||
"(crashes: %s, limit: %s); giving up for the rest of this "
|
||||
"process lifetime",
|
||||
dep.name,
|
||||
dep.restart_attempts,
|
||||
self._max_attempts,
|
||||
)
|
||||
dep.given_up = True
|
||||
return
|
||||
|
||||
backoff = min(
|
||||
self._knobs.initial_backoff_seconds * (2 ** (dep.restart_attempts - 1)),
|
||||
self._knobs.max_backoff_seconds,
|
||||
)
|
||||
self._logger.info(
|
||||
"dependency-supervisor: %s restarting in %.1fs (crash %s/%s)",
|
||||
dep.name,
|
||||
backoff,
|
||||
dep.restart_attempts,
|
||||
self._max_attempts,
|
||||
)
|
||||
await self._sleep(backoff)
|
||||
await self._spawn_with_readiness_wait(dep)
|
||||
|
||||
async def stop(self) -> None:
|
||||
"""Terminate spawned children. Adopted processes are left untouched."""
|
||||
for dep in self._dependencies:
|
||||
if dep.adopted or dep.process is None:
|
||||
continue
|
||||
proc = dep.process
|
||||
self._logger.info(
|
||||
"dependency-supervisor: stopping %s (pid %s)",
|
||||
dep.name,
|
||||
proc.pid,
|
||||
)
|
||||
proc.terminate()
|
||||
try:
|
||||
await asyncio.to_thread(
|
||||
proc.wait, self._knobs.terminate_grace_period_seconds
|
||||
)
|
||||
except subprocess.TimeoutExpired:
|
||||
self._logger.warning(
|
||||
"dependency-supervisor: %s did not exit within %ss, killing",
|
||||
dep.name,
|
||||
self._knobs.terminate_grace_period_seconds,
|
||||
)
|
||||
proc.kill()
|
||||
await asyncio.to_thread(proc.wait)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
dep.process = None
|
||||
@@ -454,3 +454,125 @@ def test_console_is_created_by_default(tmp_path, monkeypatch) -> None:
|
||||
|
||||
assert application.console_server is not None
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_dependency_supervisor_is_none_when_disabled(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
application = create_application(config=_config(), manager=DeviceManager())
|
||||
|
||||
assert application.dependency_supervisor is None
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_dependency_supervisor_constructed_when_enabled_with_no_deps(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="secret",
|
||||
dependency_supervisor_enabled=True,
|
||||
)
|
||||
application = create_application(config=config, manager=DeviceManager())
|
||||
|
||||
assert application.dependency_supervisor is not None
|
||||
assert application.dependency_supervisor.dependencies == []
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_dependency_supervisor_constructed_when_enabled_with_appium_only(
|
||||
tmp_path, monkeypatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path)
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="secret",
|
||||
dependency_supervisor_enabled=True,
|
||||
appium_supervised=True,
|
||||
appium_host="127.0.0.1",
|
||||
appium_port=4723,
|
||||
)
|
||||
application = create_application(config=config, manager=DeviceManager())
|
||||
|
||||
assert application.dependency_supervisor is not None
|
||||
deps = application.dependency_supervisor.dependencies
|
||||
assert [dep.name for dep in deps] == ["appium"]
|
||||
asyncio.run(application.client.aclose())
|
||||
|
||||
|
||||
def test_run_async_starts_supervisor_before_first_heartbeat_connect() -> None:
|
||||
async def scenario() -> None:
|
||||
events: list[str] = []
|
||||
|
||||
class SupervisedNoOp:
|
||||
def __init__(self) -> None:
|
||||
self.started = False
|
||||
self.stopped = False
|
||||
|
||||
async def start(self) -> None:
|
||||
self.started = True
|
||||
events.append("supervisor-start")
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
events.append("supervisor-run-entered")
|
||||
await stop.wait()
|
||||
|
||||
async def stop(self) -> None:
|
||||
self.stopped = True
|
||||
events.append("supervisor-stop")
|
||||
|
||||
class BlockingClient:
|
||||
async def claim(self):
|
||||
await asyncio.Event().wait()
|
||||
|
||||
async def aclose(self):
|
||||
events.append("closed")
|
||||
|
||||
class RecordingHeartbeat:
|
||||
def __init__(self) -> None:
|
||||
self.connect_called = False
|
||||
|
||||
def connect_devices(self) -> None:
|
||||
self.connect_called = True
|
||||
events.append("connect-devices")
|
||||
|
||||
async def run(self, stop: asyncio.Event) -> None:
|
||||
# Mirror HeartbeatSynchronizer.run which calls connect_devices()
|
||||
# at the very top — supervisor must have started already.
|
||||
self.connect_devices()
|
||||
await stop.wait()
|
||||
|
||||
async def sync_once(self):
|
||||
events.append("final-heartbeat")
|
||||
|
||||
class IdleProcessor:
|
||||
async def process(self, assignment):
|
||||
raise AssertionError("no assignment expected")
|
||||
|
||||
def request_stop(self):
|
||||
events.append("stop-work")
|
||||
|
||||
supervisor = SupervisedNoOp()
|
||||
application = HostAgentApplication(
|
||||
client=BlockingClient(), # type: ignore[arg-type]
|
||||
heartbeat=RecordingHeartbeat(), # type: ignore[arg-type]
|
||||
processor=IdleProcessor(), # type: ignore[arg-type]
|
||||
dependency_supervisor=supervisor, # type: ignore[arg-type]
|
||||
)
|
||||
stop = asyncio.Event()
|
||||
running = asyncio.create_task(application.run_async(stop))
|
||||
# Yield long enough for startup sequencing to land.
|
||||
await asyncio.sleep(0.05)
|
||||
stop.set()
|
||||
await asyncio.wait_for(running, timeout=1.0)
|
||||
|
||||
assert supervisor.started is True
|
||||
assert supervisor.stopped is True
|
||||
# Supervisor startup must precede the heartbeat's connect_devices().
|
||||
assert events.index("supervisor-start") < events.index("connect-devices")
|
||||
# Supervisor stop must run before client close.
|
||||
assert events.index("supervisor-stop") < events.index("closed")
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
@@ -173,3 +173,55 @@ def test_console_numeric_fields_reject_invalid_values(
|
||||
) -> None:
|
||||
with pytest.raises(HostAgentConfigurationError):
|
||||
load_host_agent_config(overrides)
|
||||
|
||||
|
||||
def test_dependency_supervisor_defaults_to_disabled() -> None:
|
||||
config = load_host_agent_config({})
|
||||
|
||||
assert config.dependency_supervisor_enabled is False
|
||||
assert config.appium_supervised is False
|
||||
assert config.appium_host == "127.0.0.1"
|
||||
assert config.appium_port == 4723
|
||||
assert config.runtime_supervised is False
|
||||
assert config.runtime_host == "127.0.0.1"
|
||||
assert config.runtime_port == 8000
|
||||
assert config.dependency_restart_max_attempts == 5
|
||||
|
||||
|
||||
def test_dependency_supervisor_env_vars_parse_bool_and_numeric_fields() -> None:
|
||||
config = load_host_agent_config(
|
||||
{
|
||||
"HOST_AGENT_DEPENDENCY_SUPERVISOR_ENABLED": "true",
|
||||
"HOST_AGENT_APPIUM_SUPERVISED": "1",
|
||||
"HOST_AGENT_APPIUM_HOST": "0.0.0.0",
|
||||
"HOST_AGENT_APPIUM_PORT": "4724",
|
||||
"HOST_AGENT_RUNTIME_SUPERVISED": "true",
|
||||
"HOST_AGENT_RUNTIME_HOST": "localhost",
|
||||
"HOST_AGENT_RUNTIME_PORT": "8001",
|
||||
"HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS": "8",
|
||||
}
|
||||
)
|
||||
|
||||
assert config.dependency_supervisor_enabled is True
|
||||
assert config.appium_supervised is True
|
||||
assert config.appium_host == "0.0.0.0"
|
||||
assert config.appium_port == 4724
|
||||
assert config.runtime_supervised is True
|
||||
assert config.runtime_host == "localhost"
|
||||
assert config.runtime_port == 8001
|
||||
assert config.dependency_restart_max_attempts == 8
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"HOST_AGENT_APPIUM_PORT": "0"},
|
||||
{"HOST_AGENT_RUNTIME_PORT": "not-a-number"},
|
||||
{"HOST_AGENT_DEPENDENCY_RESTART_MAX_ATTEMPTS": "-1"},
|
||||
],
|
||||
)
|
||||
def test_dependency_supervisor_numeric_fields_reject_invalid_values(
|
||||
overrides: dict[str, str],
|
||||
) -> None:
|
||||
with pytest.raises(HostAgentConfigurationError):
|
||||
load_host_agent_config(overrides)
|
||||
|
||||
@@ -0,0 +1,554 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import io
|
||||
import logging
|
||||
import subprocess
|
||||
from collections.abc import Sequence
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from host_agent.dependency_supervisor import (
|
||||
DependencySupervisor,
|
||||
ProbeResult,
|
||||
SupervisedDependency,
|
||||
_SupervisorKnobs,
|
||||
appium_argv_factory,
|
||||
probe_appium,
|
||||
probe_runtime,
|
||||
runtime_argv_factory,
|
||||
)
|
||||
from host_agent.config import HostAgentConfig
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Test doubles
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakePopen:
|
||||
"""Minimal subprocess.Popen stand-in for supervisor tests."""
|
||||
|
||||
instances: list[_FakePopen] = []
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
argv: Sequence[str],
|
||||
*,
|
||||
stdout_lines: Sequence[str] = (),
|
||||
pid: int = 0,
|
||||
**_kwargs: object,
|
||||
) -> None:
|
||||
self.argv = list(argv)
|
||||
self.pid = pid or (100 + len(_FakePopen.instances))
|
||||
self.returncode: int | None = None
|
||||
self.stdout = io.StringIO("".join(line + "\n" for line in stdout_lines))
|
||||
self.terminate_calls = 0
|
||||
self.kill_calls = 0
|
||||
self.wait_calls = 0
|
||||
_FakePopen.instances.append(self)
|
||||
|
||||
def poll(self) -> int | None:
|
||||
return self.returncode
|
||||
|
||||
def terminate(self) -> None:
|
||||
self.terminate_calls += 1
|
||||
self.returncode = -15
|
||||
|
||||
def kill(self) -> None:
|
||||
self.kill_calls += 1
|
||||
self.returncode = -9
|
||||
|
||||
def wait(self, timeout: float | None = None) -> int:
|
||||
self.wait_calls += 1
|
||||
if self.returncode is None:
|
||||
raise subprocess.TimeoutExpired(cmd=self.argv, timeout=timeout or 0)
|
||||
return self.returncode
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_fake_popen() -> None:
|
||||
_FakePopen.instances.clear()
|
||||
yield
|
||||
_FakePopen.instances.clear()
|
||||
|
||||
|
||||
def _dep(
|
||||
name: str = "appium",
|
||||
*,
|
||||
host: str = "127.0.0.1",
|
||||
port: int = 4723,
|
||||
probe_responses: Sequence[ProbeResult] = (ProbeResult.NO_LISTENER,),
|
||||
argv_factory=None,
|
||||
) -> tuple[SupervisedDependency, list[ProbeResult]]:
|
||||
"""Build a SupervisedDependency whose probe returns scripted responses."""
|
||||
call_log: list[ProbeResult] = []
|
||||
responses = list(probe_responses)
|
||||
fallback = probe_responses[-1] if probe_responses else ProbeResult.NO_LISTENER
|
||||
|
||||
def probe(_host: str, _port: int) -> ProbeResult:
|
||||
if responses:
|
||||
result = responses.pop(0)
|
||||
else:
|
||||
result = fallback
|
||||
call_log.append(result)
|
||||
return result
|
||||
|
||||
if argv_factory is None:
|
||||
|
||||
def argv_factory(h: str, p: int) -> list[str]:
|
||||
return ["echo", name]
|
||||
|
||||
dep = SupervisedDependency(
|
||||
name=name,
|
||||
host=host,
|
||||
port=port,
|
||||
argv_factory=argv_factory,
|
||||
probe=probe,
|
||||
)
|
||||
return dep, call_log
|
||||
|
||||
|
||||
class _FakeSleep:
|
||||
"""Records sleep durations so tests can assert backoff progression."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.calls: list[float] = []
|
||||
|
||||
async def __call__(self, seconds: float) -> None:
|
||||
self.calls.append(seconds)
|
||||
|
||||
|
||||
def _build_supervisor(
|
||||
deps: list[SupervisedDependency],
|
||||
*,
|
||||
max_attempts: int = 5,
|
||||
knobs: _SupervisorKnobs | None = None,
|
||||
sleep: _FakeSleep | None = None,
|
||||
popen_factory=None,
|
||||
) -> tuple[DependencySupervisor, _FakeSleep]:
|
||||
fake_sleep = sleep or _FakeSleep()
|
||||
sup = DependencySupervisor(
|
||||
list(deps),
|
||||
max_attempts=max_attempts,
|
||||
knobs=knobs
|
||||
or _SupervisorKnobs(
|
||||
startup_timeout_seconds=1.0,
|
||||
readiness_poll_interval_seconds=0.01,
|
||||
crash_poll_interval_seconds=0.01,
|
||||
initial_backoff_seconds=1.0,
|
||||
max_backoff_seconds=30.0,
|
||||
terminate_grace_period_seconds=1.0,
|
||||
),
|
||||
sleep=fake_sleep,
|
||||
popen_factory=popen_factory or _FakePopen,
|
||||
logger=logging.getLogger("test"),
|
||||
)
|
||||
return sup, fake_sleep
|
||||
|
||||
|
||||
class _DummySocket:
|
||||
def __enter__(self) -> "_DummySocket":
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _raise_connection_refused(*_args, **_kwargs):
|
||||
raise ConnectionRefusedError("no listener")
|
||||
|
||||
|
||||
def _ok_connection(*_args, **_kwargs):
|
||||
return _DummySocket()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Probe tests (tasks 2.1, 2.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_probe_returns_no_listener_when_port_is_closed(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_raise_connection_refused,
|
||||
)
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.NO_LISTENER
|
||||
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.NO_LISTENER
|
||||
|
||||
|
||||
def test_probe_returns_healthy_on_appium_status_endpoint(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_ok_connection,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.httpx.get",
|
||||
lambda url, timeout=2.0: httpx.Response(200, json={"ready": True}),
|
||||
)
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.HEALTHY
|
||||
|
||||
|
||||
def test_probe_returns_healthy_on_runtime_devices_endpoint(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_ok_connection,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.httpx.get",
|
||||
lambda url, timeout=2.0: httpx.Response(200, json=[]),
|
||||
)
|
||||
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.HEALTHY
|
||||
|
||||
|
||||
def test_probe_returns_unhealthy_when_listener_returns_non_200(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_ok_connection,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.httpx.get",
|
||||
lambda url, timeout=2.0: httpx.Response(500, text="boom"),
|
||||
)
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
|
||||
|
||||
|
||||
def test_probe_returns_unhealthy_when_listener_returns_non_json(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_ok_connection,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.httpx.get",
|
||||
lambda url, timeout=2.0: httpx.Response(200, text="not json"),
|
||||
)
|
||||
assert probe_runtime("127.0.0.1", 8000) is ProbeResult.UNHEALTHY_LISTENER
|
||||
|
||||
|
||||
def test_probe_returns_unhealthy_on_http_transport_error(monkeypatch) -> None:
|
||||
monkeypatch.setattr(
|
||||
"host_agent.dependency_supervisor.socket.create_connection",
|
||||
_ok_connection,
|
||||
)
|
||||
|
||||
def raise_http_error(url, timeout=2.0):
|
||||
raise httpx.ConnectError("reset")
|
||||
|
||||
monkeypatch.setattr("host_agent.dependency_supervisor.httpx.get", raise_http_error)
|
||||
assert probe_appium("127.0.0.1", 4723) is ProbeResult.UNHEALTHY_LISTENER
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Adoption logic tests (tasks 2.2, 2.3)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_start_adopts_existing_healthy_instance() -> None:
|
||||
async def scenario() -> None:
|
||||
dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,))
|
||||
sup, _ = _build_supervisor([dep])
|
||||
|
||||
await sup.start()
|
||||
|
||||
assert dep.adopted is True
|
||||
assert dep.ready is True
|
||||
assert dep.process is None
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_start_skips_when_port_has_unhealthy_listener() -> None:
|
||||
async def scenario() -> None:
|
||||
dep, _ = _dep(probe_responses=(ProbeResult.UNHEALTHY_LISTENER,))
|
||||
sup, _ = _build_supervisor([dep])
|
||||
|
||||
await sup.start()
|
||||
|
||||
assert dep.adopted is False
|
||||
assert dep.process is None
|
||||
assert dep.given_up is True
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_start_spawns_when_nothing_is_listening() -> None:
|
||||
async def scenario() -> None:
|
||||
# NO_LISTENER for initial probe → spawn path.
|
||||
# HEALTHY on first readiness probe.
|
||||
dep, _ = _dep(
|
||||
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
|
||||
)
|
||||
sup, _ = _build_supervisor([dep])
|
||||
|
||||
await sup.start()
|
||||
|
||||
assert dep.process is not None
|
||||
assert dep.adopted is False
|
||||
assert dep.ready is True
|
||||
assert dep.given_up is False
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Supervisor core tests (task 3.6)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_spawn_failure_marks_dependency_given_up() -> None:
|
||||
async def scenario() -> None:
|
||||
dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,))
|
||||
|
||||
def raising_popen(argv, **_kwargs):
|
||||
raise FileNotFoundError("appium: not found")
|
||||
|
||||
sup, _ = _build_supervisor([dep], popen_factory=raising_popen)
|
||||
await sup.start()
|
||||
|
||||
assert dep.process is None
|
||||
assert dep.given_up is True
|
||||
assert dep.ready is False
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_spawn_uses_appium_argv_factory() -> None:
|
||||
async def scenario() -> None:
|
||||
captured: list[list[str]] = []
|
||||
|
||||
def recording_popen(argv, **kwargs):
|
||||
captured.append(list(argv))
|
||||
return _FakePopen(argv, stdout_lines=("hello",))
|
||||
|
||||
dep, _ = _dep(
|
||||
name="appium",
|
||||
port=4723,
|
||||
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
|
||||
argv_factory=appium_argv_factory,
|
||||
)
|
||||
sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
|
||||
await sup.start()
|
||||
|
||||
assert captured == [["appium", "--address", "127.0.0.1", "--port", "4723"]]
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_spawn_uses_runtime_argv_factory() -> None:
|
||||
async def scenario() -> None:
|
||||
captured: list[list[str]] = []
|
||||
|
||||
def recording_popen(argv, **kwargs):
|
||||
captured.append(list(argv))
|
||||
return _FakePopen(argv)
|
||||
|
||||
dep, _ = _dep(
|
||||
name="runtime",
|
||||
port=8000,
|
||||
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
|
||||
argv_factory=runtime_argv_factory,
|
||||
)
|
||||
sup, _ = _build_supervisor([dep], popen_factory=recording_popen)
|
||||
await sup.start()
|
||||
|
||||
assert captured == [
|
||||
[
|
||||
"uvicorn",
|
||||
"api.rest:create_app",
|
||||
"--factory",
|
||||
"--host",
|
||||
"127.0.0.1",
|
||||
"--port",
|
||||
"8000",
|
||||
]
|
||||
]
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_readiness_timeout_leaves_process_running_without_restart_loop() -> None:
|
||||
async def scenario() -> None:
|
||||
# NO_LISTENER for initial probe; probe never becomes HEALTHY → startup
|
||||
# timeout path (1.0s in test knobs).
|
||||
dep, _ = _dep(probe_responses=(ProbeResult.NO_LISTENER,))
|
||||
sup, _ = _build_supervisor([dep])
|
||||
|
||||
await sup.start()
|
||||
|
||||
assert dep.process is not None # still running
|
||||
assert dep.ready is False
|
||||
assert dep.given_up is False # not given up; crash-restart owns crashes
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_crash_triggers_restart_with_backoff() -> None:
|
||||
async def scenario() -> None:
|
||||
# Probe responses:
|
||||
# 1. start._start_one → NO_LISTENER → spawn path
|
||||
# 2. first readiness probe → HEALTHY → ready
|
||||
# 3. after restart, readiness probe → HEALTHY → ready again
|
||||
dep, _ = _dep(
|
||||
probe_responses=(
|
||||
ProbeResult.NO_LISTENER,
|
||||
ProbeResult.HEALTHY,
|
||||
ProbeResult.HEALTHY,
|
||||
),
|
||||
argv_factory=appium_argv_factory,
|
||||
)
|
||||
|
||||
spawn_count = {"n": 0}
|
||||
|
||||
def popen(argv, **kwargs):
|
||||
spawn_count["n"] += 1
|
||||
return _FakePopen(argv)
|
||||
|
||||
sup, sleep = _build_supervisor([dep], popen_factory=popen)
|
||||
await sup.start()
|
||||
assert dep.ready is True
|
||||
crashes_before = dep.restart_attempts
|
||||
|
||||
# Crash the spawned process now (after readiness succeeded). The
|
||||
# supervisor's run loop will detect this via poll().
|
||||
first_proc = dep.process
|
||||
assert isinstance(first_proc, _FakePopen)
|
||||
first_proc.returncode = 1
|
||||
|
||||
stop = asyncio.Event()
|
||||
task = asyncio.create_task(sup.run(stop))
|
||||
# Give the loop a chance to detect the crash, sleep backoff, respawn.
|
||||
await asyncio.sleep(0.05)
|
||||
stop.set()
|
||||
await asyncio.wait_for(task, timeout=1.0)
|
||||
|
||||
assert dep.restart_attempts == crashes_before + 1
|
||||
assert dep.ready is True
|
||||
assert sleep.calls == [1.0] # exponential backoff base for attempt 1
|
||||
assert spawn_count["n"] == 2
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_restart_exhaustion_gives_up() -> None:
|
||||
async def scenario() -> None:
|
||||
# Process crashes after every spawn; limit max_attempts to 2.
|
||||
dep, _ = _dep(
|
||||
probe_responses=(ProbeResult.NO_LISTENER,) + (ProbeResult.HEALTHY,) * 10,
|
||||
)
|
||||
spawn_count = {"n": 0}
|
||||
|
||||
def popen(argv, **kwargs):
|
||||
spawn_count["n"] += 1
|
||||
proc = _FakePopen(argv)
|
||||
proc.returncode = 1
|
||||
return proc
|
||||
|
||||
sup, sleep = _build_supervisor([dep], popen_factory=popen, max_attempts=2)
|
||||
await sup.start()
|
||||
|
||||
stop = asyncio.Event()
|
||||
task = asyncio.create_task(sup.run(stop))
|
||||
deadline = asyncio.get_running_loop().time() + 2.0
|
||||
while not dep.given_up and asyncio.get_running_loop().time() < deadline:
|
||||
await asyncio.sleep(0.01)
|
||||
stop.set()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
assert dep.given_up is True
|
||||
# Initial spawn + 2 successful restarts before the 3rd crash gives up.
|
||||
assert dep.restart_attempts == 3
|
||||
# Backoffs: 1.0 (attempt 1), 2.0 (attempt 2). No third restart.
|
||||
assert sleep.calls == [1.0, 2.0]
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_adopted_process_is_never_restarted_or_killed() -> None:
|
||||
async def scenario() -> None:
|
||||
dep, _ = _dep(probe_responses=(ProbeResult.HEALTHY,))
|
||||
sup, _ = _build_supervisor([dep])
|
||||
await sup.start()
|
||||
|
||||
stop = asyncio.Event()
|
||||
task = asyncio.create_task(sup.run(stop))
|
||||
await asyncio.sleep(0.02)
|
||||
stop.set()
|
||||
await asyncio.wait_for(task, timeout=1.0)
|
||||
|
||||
await sup.stop()
|
||||
|
||||
assert dep.adopted is True
|
||||
assert dep.process is None
|
||||
assert dep.restart_attempts == 0
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
def test_stop_terminates_only_spawned_children() -> None:
|
||||
async def scenario() -> None:
|
||||
spawned_dep, _ = _dep(
|
||||
name="spawned",
|
||||
port=4723,
|
||||
probe_responses=(ProbeResult.NO_LISTENER, ProbeResult.HEALTHY),
|
||||
)
|
||||
adopted_dep, _ = _dep(
|
||||
name="adopted",
|
||||
port=8000,
|
||||
probe_responses=(ProbeResult.HEALTHY,),
|
||||
)
|
||||
sup, _ = _build_supervisor([spawned_dep, adopted_dep])
|
||||
await sup.start()
|
||||
|
||||
assert spawned_dep.process is not None
|
||||
spawned_proc = spawned_dep.process
|
||||
assert adopted_dep.adopted is True
|
||||
|
||||
await sup.stop()
|
||||
|
||||
assert isinstance(spawned_proc, _FakePopen)
|
||||
assert spawned_proc.terminate_calls == 1
|
||||
assert spawned_dep.process is None
|
||||
|
||||
asyncio.run(scenario())
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# from_host_agent_config factory (covers wiring helper for task 4.1)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_from_host_agent_config_builds_empty_supervisor_when_no_dep_selected() -> None:
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="t",
|
||||
dependency_supervisor_enabled=True,
|
||||
)
|
||||
sup = DependencySupervisor.from_host_agent_config(config)
|
||||
assert sup.dependencies == []
|
||||
|
||||
|
||||
def test_from_host_agent_config_includes_appium_and_runtime_when_selected() -> None:
|
||||
config = HostAgentConfig(
|
||||
control_plane_url="https://control.example",
|
||||
host_id="host-a",
|
||||
token="t",
|
||||
dependency_supervisor_enabled=True,
|
||||
appium_supervised=True,
|
||||
appium_host="10.0.0.5",
|
||||
appium_port=4724,
|
||||
runtime_supervised=True,
|
||||
runtime_host="10.0.0.5",
|
||||
runtime_port=8001,
|
||||
dependency_restart_max_attempts=7,
|
||||
)
|
||||
sup = DependencySupervisor.from_host_agent_config(config)
|
||||
names = [dep.name for dep in sup.dependencies]
|
||||
assert names == ["appium", "runtime"]
|
||||
appium = sup.dependencies[0]
|
||||
assert appium.host == "10.0.0.5"
|
||||
assert appium.port == 4724
|
||||
runtime = sup.dependencies[1]
|
||||
assert runtime.port == 8001
|
||||
assert sup._max_attempts == 7
|
||||
Reference in New Issue
Block a user