feat(host-agent): add single-instance lock to prevent duplicate-process dispatch races
Acquire an exclusive, non-blocking filelock on the identity state directory as the first action of create_application(), before resolve_host_identity() or any enrollment/heartbeat side effect. A second process against the same identity_path exits immediately with InstanceAlreadyRunningError naming the lock path; the lock releases automatically on any process exit (including SIGKILL) via OS-level advisory locking, and explicitly during run_async()'s shutdown finally block. filelock is promoted from transitive to direct dependency (version unchanged at 3.29.7). Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@ from host_agent.execution import create_execution_factories
|
||||
from host_agent.heartbeat import HeartbeatSynchronizer
|
||||
from host_agent.history import ConsoleHistoryStore
|
||||
from host_agent.identity import HostIdentityStore
|
||||
from host_agent.instance_lock import InstanceLock
|
||||
from host_agent.lease import ActiveAssignmentRunner
|
||||
from host_agent.local_account import LocalAccountStore
|
||||
from host_agent.policy_cache import HostPolicyCacheStore
|
||||
@@ -40,6 +41,7 @@ class HostAgentApplication:
|
||||
console_server: uvicorn.Server | None = None
|
||||
console_enrollment_client: HostAgentEnrollmentClient | None = None
|
||||
dependency_supervisor: DependencySupervisor | None = None
|
||||
instance_lock: InstanceLock | None = None
|
||||
|
||||
def run(self) -> None:
|
||||
asyncio.run(self.run_async())
|
||||
@@ -109,6 +111,8 @@ class HostAgentApplication:
|
||||
if self.console_enrollment_client is not None:
|
||||
self.console_enrollment_client.close()
|
||||
await self.client.aclose()
|
||||
if self.instance_lock is not None:
|
||||
self.instance_lock.release()
|
||||
|
||||
async def _claim_until_stopped(
|
||||
self,
|
||||
@@ -151,114 +155,125 @@ def create_application(
|
||||
enrollment_client: HostAgentEnrollmentClient | None = None,
|
||||
) -> HostAgentApplication:
|
||||
startup_config = config or load_host_agent_config()
|
||||
config_store = device_config_store or DeviceConfigStore()
|
||||
resolved_identity_store = identity_store or HostIdentityStore(
|
||||
startup_config.identity_path
|
||||
)
|
||||
owned_enrollment_client = enrollment_client is None
|
||||
bootstrap_client = enrollment_client or HostAgentEnrollmentClient(startup_config)
|
||||
instance_lock = InstanceLock(startup_config.identity_path.parent)
|
||||
instance_lock.acquire()
|
||||
try:
|
||||
resolved_config = resolve_host_identity(
|
||||
startup_config,
|
||||
identity_store=resolved_identity_store,
|
||||
client=bootstrap_client,
|
||||
config_store = device_config_store or DeviceConfigStore()
|
||||
resolved_identity_store = identity_store or HostIdentityStore(
|
||||
startup_config.identity_path
|
||||
)
|
||||
bootstrap_client.config = resolved_config
|
||||
resolved_manager = manager or _configured_device_manager(
|
||||
config_store,
|
||||
config=resolved_config,
|
||||
enrollment_client=bootstrap_client,
|
||||
owned_enrollment_client = enrollment_client is None
|
||||
bootstrap_client = enrollment_client or HostAgentEnrollmentClient(
|
||||
startup_config
|
||||
)
|
||||
finally:
|
||||
if owned_enrollment_client:
|
||||
bootstrap_client.close()
|
||||
client = HostAgentClient(resolved_config)
|
||||
try:
|
||||
resolved_config = resolve_host_identity(
|
||||
startup_config,
|
||||
identity_store=resolved_identity_store,
|
||||
client=bootstrap_client,
|
||||
)
|
||||
bootstrap_client.config = resolved_config
|
||||
resolved_manager = manager or _configured_device_manager(
|
||||
config_store,
|
||||
config=resolved_config,
|
||||
enrollment_client=bootstrap_client,
|
||||
)
|
||||
finally:
|
||||
if owned_enrollment_client:
|
||||
bootstrap_client.close()
|
||||
client = HostAgentClient(resolved_config)
|
||||
|
||||
history_store = ConsoleHistoryStore(
|
||||
resolved_config.identity_path.parent / "host_console_history.sqlite3",
|
||||
limit=resolved_config.console_history_limit,
|
||||
)
|
||||
status_tracker = AgentStatusTracker()
|
||||
console_enrollment_client: HostAgentEnrollmentClient | None = None
|
||||
if resolved_config.enrollment_managed:
|
||||
console_enrollment_client = HostAgentEnrollmentClient(resolved_config)
|
||||
metadata_store = TaskMetadataStore(db_path=resolved_config.task_progress_db_path)
|
||||
timeline = Timeline(ArtifactStore(root=resolved_config.task_artifact_dir))
|
||||
executor = AssignmentExecutor(
|
||||
create_execution_factories(
|
||||
resolved_manager,
|
||||
history_store = ConsoleHistoryStore(
|
||||
resolved_config.identity_path.parent / "host_console_history.sqlite3",
|
||||
limit=resolved_config.console_history_limit,
|
||||
)
|
||||
status_tracker = AgentStatusTracker()
|
||||
console_enrollment_client: HostAgentEnrollmentClient | None = None
|
||||
if resolved_config.enrollment_managed:
|
||||
console_enrollment_client = HostAgentEnrollmentClient(resolved_config)
|
||||
metadata_store = TaskMetadataStore(
|
||||
db_path=resolved_config.task_progress_db_path
|
||||
)
|
||||
timeline = Timeline(ArtifactStore(root=resolved_config.task_artifact_dir))
|
||||
executor = AssignmentExecutor(
|
||||
create_execution_factories(
|
||||
resolved_manager,
|
||||
metadata_store=metadata_store,
|
||||
timeline=timeline,
|
||||
host_agent_config=resolved_config,
|
||||
)
|
||||
)
|
||||
console_app = create_console_app(
|
||||
config=resolved_config,
|
||||
manager=resolved_manager,
|
||||
config_store=config_store,
|
||||
local_account_store=LocalAccountStore(resolved_config.local_account_path),
|
||||
identity_store=resolved_identity_store,
|
||||
history_store=history_store,
|
||||
status_tracker=status_tracker,
|
||||
session_manager=SessionManager(
|
||||
ttl_seconds=resolved_config.console_session_ttl_seconds
|
||||
),
|
||||
enrollment_client=console_enrollment_client,
|
||||
metadata_store=metadata_store,
|
||||
timeline=timeline,
|
||||
host_agent_config=resolved_config,
|
||||
executor=executor,
|
||||
)
|
||||
)
|
||||
console_app = create_console_app(
|
||||
config=resolved_config,
|
||||
manager=resolved_manager,
|
||||
config_store=config_store,
|
||||
local_account_store=LocalAccountStore(resolved_config.local_account_path),
|
||||
identity_store=resolved_identity_store,
|
||||
history_store=history_store,
|
||||
status_tracker=status_tracker,
|
||||
session_manager=SessionManager(
|
||||
ttl_seconds=resolved_config.console_session_ttl_seconds
|
||||
),
|
||||
enrollment_client=console_enrollment_client,
|
||||
metadata_store=metadata_store,
|
||||
timeline=timeline,
|
||||
executor=executor,
|
||||
)
|
||||
console_server = _EmbeddedConsoleServer(
|
||||
uvicorn.Config(
|
||||
console_app,
|
||||
host=resolved_config.console_bind_host,
|
||||
port=resolved_config.console_port,
|
||||
log_level="warning",
|
||||
console_server = _EmbeddedConsoleServer(
|
||||
uvicorn.Config(
|
||||
console_app,
|
||||
host=resolved_config.console_bind_host,
|
||||
port=resolved_config.console_port,
|
||||
log_level="warning",
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
heartbeat = HeartbeatSynchronizer(
|
||||
resolved_manager,
|
||||
client,
|
||||
resolved_config,
|
||||
status_tracker=status_tracker,
|
||||
on_sync=lambda device_count: history_store.record_heartbeat(
|
||||
device_count=device_count
|
||||
),
|
||||
policy_cache=HostPolicyCacheStore(
|
||||
resolved_config.identity_path.parent / "host_governance_policy.json"
|
||||
),
|
||||
on_policy_sync=lambda revision: history_store.record_policy_sync(
|
||||
revision=revision
|
||||
),
|
||||
)
|
||||
active_runner = ActiveAssignmentRunner(client, executor)
|
||||
processor = AssignmentProcessor(
|
||||
client,
|
||||
active_runner,
|
||||
status_tracker=status_tracker,
|
||||
on_result=lambda assignment, result: _on_assignment_finished(
|
||||
history_store,
|
||||
metadata_store,
|
||||
timeline,
|
||||
heartbeat = HeartbeatSynchronizer(
|
||||
resolved_manager,
|
||||
client,
|
||||
resolved_config,
|
||||
assignment,
|
||||
result,
|
||||
),
|
||||
)
|
||||
dependency_supervisor: DependencySupervisor | None = None
|
||||
if resolved_config.dependency_supervisor_enabled:
|
||||
dependency_supervisor = DependencySupervisor.from_host_agent_config(
|
||||
resolved_config
|
||||
status_tracker=status_tracker,
|
||||
on_sync=lambda device_count: history_store.record_heartbeat(
|
||||
device_count=device_count
|
||||
),
|
||||
policy_cache=HostPolicyCacheStore(
|
||||
resolved_config.identity_path.parent / "host_governance_policy.json"
|
||||
),
|
||||
on_policy_sync=lambda revision: history_store.record_policy_sync(
|
||||
revision=revision
|
||||
),
|
||||
)
|
||||
return HostAgentApplication(
|
||||
client=client,
|
||||
heartbeat=heartbeat,
|
||||
processor=processor,
|
||||
console_server=console_server,
|
||||
console_enrollment_client=console_enrollment_client,
|
||||
dependency_supervisor=dependency_supervisor,
|
||||
)
|
||||
active_runner = ActiveAssignmentRunner(client, executor)
|
||||
processor = AssignmentProcessor(
|
||||
client,
|
||||
active_runner,
|
||||
status_tracker=status_tracker,
|
||||
on_result=lambda assignment, result: _on_assignment_finished(
|
||||
history_store,
|
||||
metadata_store,
|
||||
timeline,
|
||||
resolved_config,
|
||||
assignment,
|
||||
result,
|
||||
),
|
||||
)
|
||||
dependency_supervisor: DependencySupervisor | None = None
|
||||
if resolved_config.dependency_supervisor_enabled:
|
||||
dependency_supervisor = DependencySupervisor.from_host_agent_config(
|
||||
resolved_config
|
||||
)
|
||||
return HostAgentApplication(
|
||||
client=client,
|
||||
heartbeat=heartbeat,
|
||||
processor=processor,
|
||||
console_server=console_server,
|
||||
console_enrollment_client=console_enrollment_client,
|
||||
dependency_supervisor=dependency_supervisor,
|
||||
instance_lock=instance_lock,
|
||||
)
|
||||
except BaseException:
|
||||
instance_lock.release()
|
||||
raise
|
||||
|
||||
|
||||
def _on_assignment_finished(
|
||||
|
||||
@@ -8,6 +8,7 @@ from dataclasses import replace
|
||||
|
||||
from host_agent.app import create_application
|
||||
from host_agent.config import load_host_agent_config
|
||||
from host_agent.instance_lock import InstanceAlreadyRunningError
|
||||
from host_agent.local_account import LocalAccountStore
|
||||
|
||||
|
||||
@@ -30,16 +31,22 @@ def main(argv: Sequence[str] | None = None) -> None:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from exc
|
||||
|
||||
create_application(config=config).run()
|
||||
try:
|
||||
create_application(config=config).run()
|
||||
except InstanceAlreadyRunningError as exc:
|
||||
print(f"error: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1) from exc
|
||||
|
||||
|
||||
def _run_setup() -> None:
|
||||
config = load_host_agent_config()
|
||||
store = LocalAccountStore(config.local_account_path)
|
||||
if store.load() is not None:
|
||||
confirm = input(
|
||||
"A local account already exists. Overwrite it? [y/N] "
|
||||
).strip().lower()
|
||||
confirm = (
|
||||
input("A local account already exists. Overwrite it? [y/N] ")
|
||||
.strip()
|
||||
.lower()
|
||||
)
|
||||
if confirm != "y":
|
||||
print("Setup cancelled; existing account left unchanged.")
|
||||
return
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Per-installation exclusive-execution lock for the Host Agent process.
|
||||
|
||||
Backed by ``filelock.FileLock`` (OS-level advisory file locking) so that
|
||||
the lock is released automatically when the holding process exits for
|
||||
any reason — clean shutdown, unhandled exception, or ``SIGKILL`` — without
|
||||
requiring any stale-lock cleanup step. See
|
||||
``openspec/changes/host-agent-single-instance-lock/design.md`` for the
|
||||
rationale behind choosing ``filelock`` over hand-rolled ``fcntl``/``msvcrt``
|
||||
branching.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from filelock import FileLock, Timeout
|
||||
|
||||
_LOCK_FILENAME = "host_agent.lock"
|
||||
|
||||
|
||||
class InstanceAlreadyRunningError(RuntimeError):
|
||||
"""Raised when another live process already holds the instance lock."""
|
||||
|
||||
def __init__(self, lock_path: Path) -> None:
|
||||
self.lock_path = lock_path
|
||||
super().__init__(
|
||||
"another Host Agent instance is already running for this "
|
||||
f"identity state directory (lock held at {lock_path}); "
|
||||
"stop the other process before starting a new one"
|
||||
)
|
||||
|
||||
|
||||
class InstanceLock:
|
||||
"""Exclusive, non-blocking lock scoped to a Host Agent state directory.
|
||||
|
||||
The state directory (typically ``identity_path.parent``) is created on
|
||||
construction to match the colocated-file convention already used for
|
||||
``host_console_history.sqlite3`` and ``host_governance_policy.json``.
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: Path) -> None:
|
||||
state_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._lock_path = state_dir / _LOCK_FILENAME
|
||||
self._lock = FileLock(str(self._lock_path), timeout=0)
|
||||
self._acquired = False
|
||||
|
||||
@property
|
||||
def lock_path(self) -> Path:
|
||||
return self._lock_path
|
||||
|
||||
def acquire(self) -> None:
|
||||
try:
|
||||
self._lock.acquire()
|
||||
except Timeout as exc:
|
||||
raise InstanceAlreadyRunningError(self._lock_path) from exc
|
||||
self._acquired = True
|
||||
|
||||
def release(self) -> None:
|
||||
if not self._acquired:
|
||||
return
|
||||
try:
|
||||
self._lock.release()
|
||||
finally:
|
||||
self._acquired = False
|
||||
|
||||
def __enter__(self) -> InstanceLock:
|
||||
self.acquire()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc_info: object) -> None:
|
||||
self.release()
|
||||
Reference in New Issue
Block a user