diff --git a/apps/cloud-api/cloud_api/app.py b/apps/cloud-api/cloud_api/app.py index 71f9c51..6b2f74a 100644 --- a/apps/cloud-api/cloud_api/app.py +++ b/apps/cloud-api/cloud_api/app.py @@ -7,7 +7,8 @@ from contextlib import asynccontextmanager from dataclasses import dataclass from typing import Any -from fastapi import FastAPI +from fastapi import FastAPI, status +from fastapi.responses import JSONResponse from cloud.auth import create_auth_provider from cloud.config import CloudConfig @@ -90,12 +91,15 @@ def create_app( app.state.cloud_config = control_config app.state.database = database app.state.cloud_services = services + app.state.startup_complete = False + app.state.schema_ready = False stop_workers = asyncio.Event() worker_tasks: list[asyncio.Task[None]] = [] try: repository.health_check() if control_config.environment == "production": require_current_schema(control_config.database_url) + app.state.schema_ready = True worker_tasks = [ asyncio.create_task( _run_scheduler_loop( @@ -116,8 +120,10 @@ def create_app( ), ] app.state.worker_tasks = tuple(worker_tasks) + app.state.startup_complete = True yield finally: + app.state.startup_complete = False stop_workers.set() try: if worker_tasks: @@ -129,6 +135,37 @@ def create_app( repository.unbind() app = FastAPI(title="Device Cloud API", lifespan=lifespan) + + @app.get("/health/live") + def health_live() -> dict[str, str]: + return {"status": "live"} + + @app.get("/health/ready") + def health_ready(): + checks = { + "configuration": bool(getattr(app.state, "startup_complete", False)), + "database": False, + "schema": bool(getattr(app.state, "schema_ready", False)), + "workers": False, + } + if checks["configuration"]: + try: + services.repository.health_check() + checks["database"] = True + except Exception: + checks["database"] = False + worker_state = getattr(app.state, "worker_tasks", ()) + checks["workers"] = bool(worker_state) and all( + not task.done() for task in worker_state + ) + ready = all(checks.values()) + return JSONResponse( + status_code=( + status.HTTP_200_OK if ready else status.HTTP_503_SERVICE_UNAVAILABLE + ), + content={"status": "ready" if ready else "not_ready", "checks": checks}, + ) + app.include_router( create_cloud_router( pool=pool, diff --git a/apps/cloud-api/tests/test_app.py b/apps/cloud-api/tests/test_app.py index 20de669..479f324 100644 --- a/apps/cloud-api/tests/test_app.py +++ b/apps/cloud-api/tests/test_app.py @@ -182,6 +182,79 @@ def test_lifecycle_workers_log_failures_and_continue(monkeypatch) -> None: } +def test_liveness_stays_up_when_database_readiness_fails() -> None: + database_healthy = True + + class FakeRepository: + def health_check(self) -> None: + if not database_healthy: + raise RuntimeError("database unavailable with secret credentials") + + def list_queued_tasks(self) -> list[object]: + return [] + + def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]: + return [] + + class FakeDatabase: + repository = FakeRepository() + + def close(self) -> None: + return None + + app = create_app( + config=CloudControlConfig(database_url="sqlite:///:memory:"), + database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value] + ) + + with TestClient(app) as client: + assert client.get("/health/live").json() == {"status": "live"} + ready = client.get("/health/ready") + assert ready.status_code == 200 + assert all(ready.json()["checks"].values()) + + database_healthy = False + assert client.get("/health/live").status_code == 200 + not_ready = client.get("/health/ready") + assert not_ready.status_code == 503 + assert not_ready.json()["checks"]["database"] is False + assert "secret credentials" not in not_ready.text + + +def test_readiness_reports_stopped_worker() -> None: + class FakeRepository: + def health_check(self) -> None: + return None + + def list_queued_tasks(self) -> list[object]: + return [] + + def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]: + return [] + + class FakeDatabase: + repository = FakeRepository() + + def close(self) -> None: + return None + + class StoppedWorker: + def done(self) -> bool: + return True + + app = create_app( + config=CloudControlConfig(database_url="sqlite:///:memory:"), + database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value] + ) + + with TestClient(app) as client: + app.state.worker_tasks = (StoppedWorker(),) + response = client.get("/health/ready") + + assert response.status_code == 503 + assert response.json()["checks"]["workers"] is False + + def test_production_app_rejects_missing_credentials() -> None: with pytest.raises(CloudConfigurationError, match="credential"): create_app( diff --git a/openspec/changes/cloud-control-plane-integration/tasks.md b/openspec/changes/cloud-control-plane-integration/tasks.md index e4280db..02158c8 100644 --- a/openspec/changes/cloud-control-plane-integration/tasks.md +++ b/openspec/changes/cloud-control-plane-integration/tasks.md @@ -46,7 +46,7 @@ - [x] 6.1 Compose repository, pool, scheduler, plugin registry, public router, internal router, and auth providers in the cloud app factory. - [x] 6.2 Implement FastAPI lifespan startup/shutdown for configuration validation, database checks, scheduler loop, and lease-reaper loop. - [x] 6.3 Ensure lifecycle iteration failures are logged and retried without terminating later iterations. -- [ ] 6.4 Add `/health/live` and `/health/ready` with separate process, database/schema, and worker-state semantics. +- [x] 6.4 Add `/health/live` and `/health/ready` with separate process, database/schema, and worker-state semantics. - [ ] 6.5 Add structured correlation-aware logging for requests and task lifecycle events with sensitive payload redaction. - [ ] 6.6 Add app-level tests for startup failures, readiness transitions, graceful shutdown, persisted queue recovery, and expired-lease recovery.