from __future__ import annotations import asyncio from collections.abc import Awaitable, Callable from datetime import timedelta from time import monotonic from typing import TYPE_CHECKING from uuid import uuid4 from fastapi import APIRouter, HTTPException, Request, status from fastapi.responses import JSONResponse from cloud.auth import ( AuthProvider, ConfiguredEnrollmentTokenProvider, HostAuthorizationError, digest_token, ) from cloud.internal_api.models import ( AssignmentModel, ClaimRequest, ClaimResponse, DeviceEnrollmentRequest, DeviceEnrollmentResponse, HeartbeatRequest, HeartbeatResponse, HostEnrollmentRequest, HostEnrollmentResponse, LeaseRenewalRequest, LeaseRenewalResponse, StaleLeaseConflict, TerminalResultRequest, TerminalResultResponse, ) from cloud.repository import ( DeviceEnrollmentConflictError, EnrollmentTokenConflictError, HostEnrollmentConflictError, ) from core.models import Device, utc_now if TYPE_CHECKING: from cloud.pool import DevicePool def create_internal_router( *, pool: DevicePool, auth_provider: AuthProvider, enrollment_auth_provider: ConfiguredEnrollmentTokenProvider | None = None, version_prefix: str = "/internal/v1", claim_poll_interval_seconds: float = 0.1, lease_duration_seconds: float = 60.0, sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, ) -> APIRouter: if claim_poll_interval_seconds <= 0: raise ValueError("claim_poll_interval_seconds must be greater than zero") if lease_duration_seconds <= 0: raise ValueError("lease_duration_seconds must be greater than zero") router = APIRouter(prefix=version_prefix, tags=["host-agent"]) enrollment_auth = enrollment_auth_provider or ConfiguredEnrollmentTokenProvider(()) def authorize_host(request: Request, host_id: str) -> None: principal = auth_provider.authenticate(request) if principal is None: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized", headers={"WWW-Authenticate": "Bearer"}, ) try: principal.require_host(host_id) except HostAuthorizationError as exc: raise HTTPException( status_code=status.HTTP_403_FORBIDDEN, detail=str(exc), ) from exc @router.post( "/enrollments", response_model=HostEnrollmentResponse, status_code=status.HTTP_201_CREATED, ) def enroll_host( payload: HostEnrollmentRequest, request: Request, ) -> HostEnrollmentResponse: enrollment_principal = enrollment_auth.authenticate(request) if enrollment_principal is None: raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="unauthorized", headers={"WWW-Authenticate": "Bearer"}, ) try: enrollment = pool.store.enroll_host( host_id=f"host-{uuid4().hex}", agent_instance_id=payload.agent_instance_id, credential_digest=digest_token(payload.host_token), enrollment_token_digest=enrollment_principal.token_digest, display_name=payload.display_name, enrolled_at=utc_now(), ) except (EnrollmentTokenConflictError, HostEnrollmentConflictError) as exc: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=str(exc), ) from exc return HostEnrollmentResponse(host_id=enrollment.host_id) @router.post( "/hosts/{host_id}/devices/enroll", response_model=DeviceEnrollmentResponse, status_code=status.HTTP_201_CREATED, ) def enroll_device( host_id: str, payload: DeviceEnrollmentRequest, request: Request, ) -> DeviceEnrollmentResponse: authorize_host(request, host_id) try: enrollment = pool.store.enroll_device( device_id=f"device-{uuid4().hex}", host_id=host_id, local_device_id=payload.local_device_id, driver_type=payload.driver_type, name=payload.name, capability_tags=list(payload.capability_tags), enrolled_at=utc_now(), ) except DeviceEnrollmentConflictError as exc: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=str(exc), ) from exc return DeviceEnrollmentResponse(device_id=enrollment.device_id) @router.put( "/hosts/{host_id}/heartbeat", response_model=HeartbeatResponse, ) def heartbeat( host_id: str, payload: HeartbeatRequest, request: Request, ) -> HeartbeatResponse: authorize_host(request, host_id) allow_device_takeover = _validate_snapshot( pool, host_id=host_id, payload=payload, ) devices = [ Device( id=device.device_id, driver_type=device.driver_type, status=device.status, capability_tags=list(device.capability_tags), ) for device in payload.devices ] pool.sync_host_devices( host_id, devices, address=payload.address, allow_device_takeover=allow_device_takeover, ) return HeartbeatResponse( host_id=host_id, accepted_devices=len(devices), received_at=utc_now(), ) @router.post( "/hosts/{host_id}/assignments/claim", response_model=ClaimResponse, ) async def claim_assignment( host_id: str, payload: ClaimRequest, request: Request, ) -> ClaimResponse: authorize_host(request, host_id) if payload.host_id != host_id: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="claim host_id must match the request path", ) deadline = monotonic() + payload.timeout_seconds while True: assignment = pool.store.claim_assignment(host_id=host_id, now=utc_now()) if assignment is not None: return ClaimResponse( assignment=AssignmentModel( task_id=assignment.task_id, attempt=assignment.attempt, lease_id=assignment.lease_id, lease_expires_at=assignment.lease_expires_at, host_id=assignment.host_id, device_id=assignment.device_id, goal=assignment.goal, workflow_definition_id=assignment.workflow_definition_id, ) ) remaining = deadline - monotonic() if remaining <= 0: return ClaimResponse(timed_out=True) await sleep(min(claim_poll_interval_seconds, remaining)) @router.post( "/hosts/{host_id}/assignments/{task_id}/renew", response_model=LeaseRenewalResponse, responses={status.HTTP_409_CONFLICT: {"model": StaleLeaseConflict}}, ) def renew_assignment( host_id: str, task_id: str, payload: LeaseRenewalRequest, request: Request, ): authorize_host(request, host_id) _validate_assignment_identity( host_id=host_id, task_id=task_id, payload_host_id=payload.host_id, payload_task_id=payload.task_id, ) now = utc_now() lease_expires_at = now + timedelta(seconds=lease_duration_seconds) renewal_status = pool.store.renew_lease( task_id=task_id, attempt=payload.attempt, lease_id=payload.lease_id, host_id=host_id, lease_expires_at=lease_expires_at, now=now, ) if renewal_status == "not_found": raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail="assignment not found", ) if renewal_status != "renewed": return _stale_lease_conflict("assignment lease is stale or expired") return LeaseRenewalResponse( status="renewed", lease_expires_at=lease_expires_at, ) @router.post( "/hosts/{host_id}/assignments/{task_id}/result", response_model=TerminalResultResponse, responses={status.HTTP_409_CONFLICT: {"model": StaleLeaseConflict}}, ) def report_result( host_id: str, task_id: str, payload: TerminalResultRequest, request: Request, ): authorize_host(request, host_id) _validate_assignment_identity( host_id=host_id, task_id=task_id, payload_host_id=payload.host_id, payload_task_id=payload.task_id, ) result_status = pool.store.record_task_result( task_id=task_id, attempt=payload.attempt, lease_id=payload.lease_id, host_id=host_id, status=payload.status, failure_reason=payload.failure_reason, terminal_result=payload.result, completed_at=utc_now(), ) if result_status == "conflict": return _stale_lease_conflict("assignment lease is stale or superseded") return TerminalResultResponse(status=result_status) return router def _validate_assignment_identity( *, host_id: str, task_id: str, payload_host_id: str, payload_task_id: str, ) -> None: if payload_host_id != host_id or payload_task_id != task_id: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="assignment identity must match the request path", ) def _stale_lease_conflict(detail: str) -> JSONResponse: return JSONResponse( status_code=status.HTTP_409_CONFLICT, content=StaleLeaseConflict(detail=detail).model_dump(), ) def _validate_snapshot( pool: DevicePool, *, host_id: str, payload: HeartbeatRequest, ) -> bool: if payload.host_id != host_id: raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="heartbeat host_id must match the request path", ) device_ids = [device.device_id for device in payload.devices] if len(device_ids) != len(set(device_ids)): raise HTTPException( status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, detail="heartbeat snapshot contains duplicate device ids", ) if pool.store.is_enrollment_managed_host(host_id): enrollments = { enrollment.device_id: enrollment for enrollment in pool.store.list_device_enrollments(host_id) if enrollment.revoked_at is None } invalid = [ device.device_id for device in payload.devices if device.device_id not in enrollments or enrollments[device.device_id].driver_type != device.driver_type ] if invalid: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=f"device enrollment conflict: {sorted(set(invalid))}", ) return False now = utc_now() hosts = {host.host_id: host for host in pool.store.list_hosts()} conflicts: list[str] = [] stale_owner_found = False requested_ids = set(device_ids) for device in pool.store.list_devices(): if device.device_id not in requested_ids or device.host_id == host_id: continue owner = hosts.get(device.host_id) if owner is None: continue age_seconds = (now - owner.last_seen_at).total_seconds() if age_seconds <= pool.config.stale_after_seconds: conflicts.append(device.device_id) else: stale_owner_found = True if conflicts: raise HTTPException( status_code=status.HTTP_409_CONFLICT, detail=f"device ownership conflict: {sorted(set(conflicts))}", ) return stale_owner_found