q792602257andClaude Opus 4.6
ec261d57c2
Host Agent now persists step-level execution detail locally (via a real TaskMetadataStore/Timeline wired into TaskRunner) and reports a bounded in-progress snapshot piggybacked on lease renewal. Cloud persists that snapshot per active assignment and exposes it through the existing task list/detail query path; Cloud Console renders it as a live badge. Host Agent's local console gains authenticated, read-only task list and detail/timeline pages (same-origin, server-rendered) with inlined screenshots. Also fixes a pre-existing gap in the shared Timeline: the actual per-step LLM prompt is now recorded instead of the task goal, benefiting both Runtime and Host Agent consoles. When a host uses the cloud planner transport, each decide call's prompt and resulting tool decision are durably logged in a new planner_decision_log table (with bounded retention) and browsable from Cloud Console; direct-transport hosts explicitly surface a "not reported" state. Includes Alembic migrations 0008 (progress columns on scheduled_tasks) and 0009 (planner_decision_log), bounded Host-Agent-local retention, dual-backend repository parity, and Vitest + pytest coverage. Task 6.5 (manual end-to-end device verification) remains. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
483 lines
16 KiB
Python
483 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
from collections.abc import Callable
|
|
from contextlib import asynccontextmanager
|
|
from dataclasses import dataclass
|
|
from datetime import timedelta
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi import FastAPI, status
|
|
from fastapi import Request
|
|
from fastapi.responses import JSONResponse, RedirectResponse
|
|
from fastapi.staticfiles import StaticFiles
|
|
from starlette.exceptions import HTTPException as StarletteHTTPException
|
|
from starlette.types import Scope
|
|
|
|
from cloud.auth import (
|
|
ChainedAuthProvider,
|
|
RepositoryHostAuthProvider,
|
|
UserSessionAuthProvider,
|
|
create_auth_provider,
|
|
)
|
|
from cloud.config import CloudConfig
|
|
from cloud.control_config import (
|
|
CloudConfigurationError,
|
|
CloudControlConfig,
|
|
load_control_config,
|
|
validate_control_config,
|
|
)
|
|
from cloud.database import CloudDatabase
|
|
from cloud.internal_api.api import create_internal_router
|
|
from cloud.llm_providers import LlmProviderService
|
|
from cloud.plugins import PluginRegistry
|
|
from cloud.observability import (
|
|
CORRELATION_HEADER,
|
|
bind_correlation_id,
|
|
new_correlation_id,
|
|
normalize_correlation_id,
|
|
reset_correlation_id,
|
|
)
|
|
from cloud.pool import DevicePool
|
|
from cloud.scheduler import TaskScheduler
|
|
from cloud.schema import require_current_schema
|
|
from cloud.sdk.api import create_cloud_router
|
|
from cloud.sdk.governance_api import create_governance_router
|
|
from cloud.sdk.llm_provider_api import create_llm_provider_router
|
|
from cloud.sdk.user_api import create_user_auth_router
|
|
from cloud.user_auth import USER_CSRF_COOKIE, USER_SESSION_COOKIE, UserAuthService, UserAuthSettings
|
|
from core.models import utc_now
|
|
|
|
|
|
DatabaseFactory = Callable[[CloudControlConfig], CloudDatabase]
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class _RepositoryProxy:
|
|
def __init__(self) -> None:
|
|
self._target: Any | None = None
|
|
|
|
def bind(self, target: Any) -> None:
|
|
self._target = target
|
|
|
|
def unbind(self) -> None:
|
|
self._target = None
|
|
|
|
def __getattr__(self, name: str) -> Any:
|
|
if self._target is None:
|
|
raise RuntimeError("cloud repository is not available outside app lifespan")
|
|
return getattr(self._target, name)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CloudApplicationServices:
|
|
repository: _RepositoryProxy
|
|
pool: DevicePool
|
|
scheduler: TaskScheduler
|
|
plugin_registry: PluginRegistry
|
|
auth_provider: Any
|
|
user_auth_service: UserAuthService
|
|
llm_provider_service: LlmProviderService
|
|
|
|
|
|
class SpaStaticFiles(StaticFiles):
|
|
"""``StaticFiles`` variant that falls back to ``index.html`` for SPA routes.
|
|
|
|
``StaticFiles(html=True)`` only serves ``index.html`` for the mount root and
|
|
for directory roots; an unknown path like ``/console/tasks/abc`` raises a
|
|
plain 404, which breaks deep-link refreshes in a browser-running SPA. This
|
|
subclass intercepts 404s for non-asset paths and re-serves ``index.html``
|
|
so the SPA router can take over.
|
|
"""
|
|
|
|
async def get_response(self, path: str, scope: Scope) -> Any:
|
|
try:
|
|
return await super().get_response(path, scope)
|
|
except StarletteHTTPException as exc:
|
|
if exc.status_code == 404 and path != "index.html":
|
|
return await super().get_response("index.html", scope)
|
|
raise
|
|
|
|
|
|
def create_app(
|
|
*,
|
|
config: CloudControlConfig | None = None,
|
|
database_factory: DatabaseFactory | None = None,
|
|
) -> FastAPI:
|
|
"""Create the independently deployable cloud API application."""
|
|
control_config = config or load_control_config()
|
|
validate_control_config(control_config)
|
|
build_database = database_factory or _default_database_factory
|
|
configured_auth_provider = create_auth_provider(
|
|
allow_insecure_anonymous=control_config.allow_insecure_anonymous,
|
|
)
|
|
repository = _RepositoryProxy()
|
|
user_auth_service = UserAuthService(
|
|
repository,
|
|
settings=UserAuthSettings(
|
|
session_idle_ttl=timedelta(seconds=control_config.user_session_idle_seconds),
|
|
session_absolute_ttl=timedelta(
|
|
seconds=control_config.user_session_absolute_seconds
|
|
),
|
|
login_failure_limit=control_config.login_failure_limit,
|
|
login_failure_window=timedelta(
|
|
seconds=control_config.login_failure_window_seconds
|
|
),
|
|
login_block_duration=timedelta(seconds=control_config.login_block_seconds),
|
|
cookie_secure=control_config.session_cookie_secure,
|
|
),
|
|
)
|
|
llm_provider_service = LlmProviderService(repository)
|
|
auth_provider = ChainedAuthProvider(
|
|
(
|
|
configured_auth_provider,
|
|
RepositoryHostAuthProvider(repository), # type: ignore[arg-type]
|
|
UserSessionAuthProvider(user_auth_service),
|
|
)
|
|
)
|
|
domain_config = CloudConfig(
|
|
lease_duration_seconds=control_config.lease_duration_seconds,
|
|
)
|
|
pool = DevicePool(repository, domain_config) # type: ignore[arg-type]
|
|
scheduler = TaskScheduler(pool, repository, domain_config) # type: ignore[arg-type]
|
|
plugin_registry = PluginRegistry(repository) # type: ignore[arg-type]
|
|
services = CloudApplicationServices(
|
|
repository=repository,
|
|
pool=pool,
|
|
scheduler=scheduler,
|
|
plugin_registry=plugin_registry,
|
|
auth_provider=auth_provider,
|
|
user_auth_service=user_auth_service,
|
|
llm_provider_service=llm_provider_service,
|
|
)
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
database = build_database(control_config)
|
|
repository.bind(database.repository)
|
|
app.state.cloud_config = control_config
|
|
app.state.database = database
|
|
app.state.cloud_services = services
|
|
app.state.startup_complete = False
|
|
app.state.schema_ready = False
|
|
stop_workers = asyncio.Event()
|
|
worker_tasks: list[asyncio.Task[None]] = []
|
|
try:
|
|
repository.health_check()
|
|
if control_config.environment == "production":
|
|
require_current_schema(control_config.database_url)
|
|
app.state.schema_ready = True
|
|
worker_tasks = [
|
|
asyncio.create_task(
|
|
_run_scheduler_loop(
|
|
services,
|
|
stop_workers,
|
|
interval_seconds=control_config.scheduler_interval_seconds,
|
|
),
|
|
name="cloud-scheduler",
|
|
),
|
|
asyncio.create_task(
|
|
_run_lease_reaper_loop(
|
|
services,
|
|
stop_workers,
|
|
interval_seconds=(control_config.lease_reaper_interval_seconds),
|
|
max_attempts=control_config.max_task_attempts,
|
|
),
|
|
name="cloud-lease-reaper",
|
|
),
|
|
asyncio.create_task(
|
|
_run_planner_decision_log_pruner_loop(
|
|
services,
|
|
stop_workers,
|
|
interval_seconds=(
|
|
control_config.planner_decision_log_prune_interval_seconds
|
|
),
|
|
retention_days=(
|
|
control_config.planner_decision_log_retention_days
|
|
),
|
|
),
|
|
name="cloud-planner-decision-log-pruner",
|
|
),
|
|
]
|
|
app.state.worker_tasks = tuple(worker_tasks)
|
|
app.state.startup_complete = True
|
|
yield
|
|
finally:
|
|
app.state.startup_complete = False
|
|
stop_workers.set()
|
|
try:
|
|
if worker_tasks:
|
|
await asyncio.gather(*worker_tasks)
|
|
finally:
|
|
try:
|
|
database.close()
|
|
finally:
|
|
repository.unbind()
|
|
|
|
app = FastAPI(title="Device Cloud API", lifespan=lifespan)
|
|
|
|
if control_config.cors_allowed_origins:
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=list(control_config.cors_allowed_origins),
|
|
allow_credentials=True,
|
|
allow_methods=["*"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
@app.middleware("http")
|
|
async def correlation_logging(request: Request, call_next):
|
|
correlation_id = normalize_correlation_id(
|
|
request.headers.get(CORRELATION_HEADER)
|
|
)
|
|
correlation_token = bind_correlation_id(correlation_id)
|
|
try:
|
|
response = await call_next(request)
|
|
if (
|
|
request.cookies.get(USER_SESSION_COOKIE)
|
|
and not request.headers.get("authorization")
|
|
and response.status_code == status.HTTP_401_UNAUTHORIZED
|
|
):
|
|
_clear_user_auth_cookies(response, control_config)
|
|
logger.info(
|
|
"cloud request completed",
|
|
extra={
|
|
"correlation_id": correlation_id,
|
|
"method": request.method,
|
|
"path": request.url.path,
|
|
"status_code": response.status_code,
|
|
},
|
|
)
|
|
response.headers[CORRELATION_HEADER] = correlation_id
|
|
return response
|
|
finally:
|
|
reset_correlation_id(correlation_token)
|
|
|
|
@app.get("/health/live")
|
|
def health_live() -> dict[str, str]:
|
|
return {"status": "live"}
|
|
|
|
@app.get("/health/ready")
|
|
def health_ready():
|
|
checks = {
|
|
"configuration": bool(getattr(app.state, "startup_complete", False)),
|
|
"database": False,
|
|
"schema": bool(getattr(app.state, "schema_ready", False)),
|
|
"workers": False,
|
|
}
|
|
if checks["configuration"]:
|
|
try:
|
|
services.repository.health_check()
|
|
checks["database"] = True
|
|
except Exception:
|
|
checks["database"] = False
|
|
worker_state = getattr(app.state, "worker_tasks", ())
|
|
checks["workers"] = bool(worker_state) and all(
|
|
not task.done() for task in worker_state
|
|
)
|
|
ready = all(checks.values())
|
|
return JSONResponse(
|
|
status_code=(
|
|
status.HTTP_200_OK if ready else status.HTTP_503_SERVICE_UNAVAILABLE
|
|
),
|
|
content={"status": "ready" if ready else "not_ready", "checks": checks},
|
|
)
|
|
|
|
app.include_router(
|
|
create_cloud_router(
|
|
pool=pool,
|
|
scheduler=scheduler,
|
|
plugin_registry=plugin_registry,
|
|
auth_provider=auth_provider,
|
|
csrf_validator=lambda request, principal: _valid_csrf_request(
|
|
request,
|
|
principal,
|
|
user_auth_service,
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
create_user_auth_router(
|
|
user_auth_service=user_auth_service,
|
|
auth_provider=auth_provider,
|
|
config=control_config,
|
|
)
|
|
)
|
|
app.include_router(
|
|
create_governance_router(
|
|
repository=repository,
|
|
auth_provider=auth_provider,
|
|
)
|
|
)
|
|
app.include_router(
|
|
create_llm_provider_router(
|
|
service=llm_provider_service,
|
|
repository=repository,
|
|
auth_provider=auth_provider,
|
|
csrf_validator=lambda request, principal: _valid_csrf_request(
|
|
request,
|
|
principal,
|
|
user_auth_service,
|
|
),
|
|
)
|
|
)
|
|
app.include_router(
|
|
create_internal_router(
|
|
pool=pool,
|
|
auth_provider=auth_provider,
|
|
lease_duration_seconds=control_config.lease_duration_seconds,
|
|
scheduler=scheduler,
|
|
planner_token_reservation_ceiling=(
|
|
control_config.planner_token_reservation_ceiling
|
|
),
|
|
planner_token_reservation_ttl_seconds=(
|
|
control_config.planner_token_reservation_ttl_seconds
|
|
),
|
|
planner_provider_service=llm_provider_service,
|
|
)
|
|
)
|
|
|
|
if control_config.console_static_dir:
|
|
dist_dir = Path(control_config.console_static_dir)
|
|
if not dist_dir.is_dir():
|
|
raise CloudConfigurationError(
|
|
f"CLOUD_CONSOLE_STATIC_DIR is not a directory: {dist_dir}"
|
|
)
|
|
|
|
@app.get("/", include_in_schema=False)
|
|
async def _redirect_to_console() -> RedirectResponse:
|
|
return RedirectResponse(url="/console/")
|
|
|
|
app.mount(
|
|
"/console",
|
|
SpaStaticFiles(directory=str(dist_dir), html=True),
|
|
name="cloud-console",
|
|
)
|
|
|
|
return app
|
|
|
|
|
|
def _default_database_factory(config: CloudControlConfig) -> CloudDatabase:
|
|
return CloudDatabase(
|
|
config.database_url,
|
|
create_schema=config.environment != "production",
|
|
)
|
|
|
|
|
|
def _valid_csrf_request(
|
|
request: Request,
|
|
principal: Any,
|
|
user_auth_service: UserAuthService,
|
|
) -> bool:
|
|
if principal.session_id is None:
|
|
return True
|
|
return user_auth_service.validate_csrf(
|
|
session_token=request.cookies.get(USER_SESSION_COOKIE),
|
|
csrf_cookie=request.cookies.get(USER_CSRF_COOKIE),
|
|
csrf_header=request.headers.get("x-csrf-token"),
|
|
)
|
|
|
|
|
|
def _clear_user_auth_cookies(response: Any, config: CloudControlConfig) -> None:
|
|
response.delete_cookie(
|
|
USER_SESSION_COOKIE,
|
|
path="/",
|
|
secure=config.session_cookie_secure,
|
|
httponly=True,
|
|
samesite="lax",
|
|
)
|
|
response.delete_cookie(
|
|
USER_CSRF_COOKIE,
|
|
path="/",
|
|
secure=config.session_cookie_secure,
|
|
httponly=False,
|
|
samesite="lax",
|
|
)
|
|
|
|
|
|
async def _run_scheduler_loop(
|
|
services: CloudApplicationServices,
|
|
stop: asyncio.Event,
|
|
*,
|
|
interval_seconds: float,
|
|
) -> None:
|
|
while not stop.is_set():
|
|
correlation_token = bind_correlation_id(new_correlation_id())
|
|
try:
|
|
services.scheduler.assign()
|
|
except Exception:
|
|
logger.exception(
|
|
"cloud lifecycle iteration failed",
|
|
extra={"worker": "scheduler"},
|
|
)
|
|
finally:
|
|
reset_correlation_id(correlation_token)
|
|
if await _wait_for_stop(stop, interval_seconds):
|
|
return
|
|
|
|
|
|
async def _run_lease_reaper_loop(
|
|
services: CloudApplicationServices,
|
|
stop: asyncio.Event,
|
|
*,
|
|
interval_seconds: float,
|
|
max_attempts: int,
|
|
) -> None:
|
|
while not stop.is_set():
|
|
correlation_token = bind_correlation_id(new_correlation_id())
|
|
try:
|
|
services.repository.reap_expired_leases(
|
|
now=utc_now(),
|
|
max_attempts=max_attempts,
|
|
)
|
|
services.repository.cleanup_expired_token_reservations(
|
|
now=utc_now(),
|
|
limit=100,
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"cloud lifecycle iteration failed",
|
|
extra={"worker": "lease_reaper"},
|
|
)
|
|
finally:
|
|
reset_correlation_id(correlation_token)
|
|
if await _wait_for_stop(stop, interval_seconds):
|
|
return
|
|
|
|
|
|
async def _wait_for_stop(stop: asyncio.Event, interval_seconds: float) -> bool:
|
|
try:
|
|
await asyncio.wait_for(stop.wait(), timeout=interval_seconds)
|
|
except TimeoutError:
|
|
return False
|
|
return True
|
|
|
|
|
|
async def _run_planner_decision_log_pruner_loop(
|
|
services: CloudApplicationServices,
|
|
stop: asyncio.Event,
|
|
*,
|
|
interval_seconds: float,
|
|
retention_days: int,
|
|
) -> None:
|
|
while not stop.is_set():
|
|
correlation_token = bind_correlation_id(new_correlation_id())
|
|
try:
|
|
services.repository.prune_planner_decision_log(
|
|
now=utc_now(),
|
|
prune_after_terminal_seconds=retention_days * 86_400,
|
|
)
|
|
except Exception:
|
|
logger.exception(
|
|
"planner decision log prune failed",
|
|
extra={"worker": "planner_decision_log_pruner"},
|
|
)
|
|
finally:
|
|
reset_correlation_id(correlation_token)
|
|
if await _wait_for_stop(stop, interval_seconds):
|
|
return
|