feat(cloud-api): run lifecycle workers
This commit is contained in:
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
from contextlib import asynccontextmanager
|
from contextlib import asynccontextmanager
|
||||||
from dataclasses import dataclass
|
from dataclasses import dataclass
|
||||||
@@ -19,7 +20,9 @@ from cloud.internal_api.api import create_internal_router
|
|||||||
from cloud.plugins import PluginRegistry
|
from cloud.plugins import PluginRegistry
|
||||||
from cloud.pool import DevicePool
|
from cloud.pool import DevicePool
|
||||||
from cloud.scheduler import TaskScheduler
|
from cloud.scheduler import TaskScheduler
|
||||||
|
from cloud.schema import require_current_schema
|
||||||
from cloud.sdk.api import create_cloud_router
|
from cloud.sdk.api import create_cloud_router
|
||||||
|
from core.models import utc_now
|
||||||
|
|
||||||
|
|
||||||
DatabaseFactory = Callable[[CloudControlConfig], CloudDatabase]
|
DatabaseFactory = Callable[[CloudControlConfig], CloudDatabase]
|
||||||
@@ -85,13 +88,43 @@ def create_app(
|
|||||||
app.state.cloud_config = control_config
|
app.state.cloud_config = control_config
|
||||||
app.state.database = database
|
app.state.database = database
|
||||||
app.state.cloud_services = services
|
app.state.cloud_services = services
|
||||||
|
stop_workers = asyncio.Event()
|
||||||
|
worker_tasks: list[asyncio.Task[None]] = []
|
||||||
try:
|
try:
|
||||||
|
repository.health_check()
|
||||||
|
if control_config.environment == "production":
|
||||||
|
require_current_schema(control_config.database_url)
|
||||||
|
worker_tasks = [
|
||||||
|
asyncio.create_task(
|
||||||
|
_run_scheduler_loop(
|
||||||
|
services,
|
||||||
|
stop_workers,
|
||||||
|
interval_seconds=control_config.scheduler_interval_seconds,
|
||||||
|
),
|
||||||
|
name="cloud-scheduler",
|
||||||
|
),
|
||||||
|
asyncio.create_task(
|
||||||
|
_run_lease_reaper_loop(
|
||||||
|
services,
|
||||||
|
stop_workers,
|
||||||
|
interval_seconds=(control_config.lease_reaper_interval_seconds),
|
||||||
|
max_attempts=control_config.max_task_attempts,
|
||||||
|
),
|
||||||
|
name="cloud-lease-reaper",
|
||||||
|
),
|
||||||
|
]
|
||||||
|
app.state.worker_tasks = tuple(worker_tasks)
|
||||||
yield
|
yield
|
||||||
finally:
|
finally:
|
||||||
|
stop_workers.set()
|
||||||
try:
|
try:
|
||||||
database.close()
|
if worker_tasks:
|
||||||
|
await asyncio.gather(*worker_tasks)
|
||||||
finally:
|
finally:
|
||||||
repository.unbind()
|
try:
|
||||||
|
database.close()
|
||||||
|
finally:
|
||||||
|
repository.unbind()
|
||||||
|
|
||||||
app = FastAPI(title="Device Cloud API", lifespan=lifespan)
|
app = FastAPI(title="Device Cloud API", lifespan=lifespan)
|
||||||
app.include_router(
|
app.include_router(
|
||||||
@@ -117,3 +150,39 @@ def _default_database_factory(config: CloudControlConfig) -> CloudDatabase:
|
|||||||
config.database_url,
|
config.database_url,
|
||||||
create_schema=config.environment != "production",
|
create_schema=config.environment != "production",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_scheduler_loop(
|
||||||
|
services: CloudApplicationServices,
|
||||||
|
stop: asyncio.Event,
|
||||||
|
*,
|
||||||
|
interval_seconds: float,
|
||||||
|
) -> None:
|
||||||
|
while not stop.is_set():
|
||||||
|
services.scheduler.assign()
|
||||||
|
if await _wait_for_stop(stop, interval_seconds):
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
async def _run_lease_reaper_loop(
|
||||||
|
services: CloudApplicationServices,
|
||||||
|
stop: asyncio.Event,
|
||||||
|
*,
|
||||||
|
interval_seconds: float,
|
||||||
|
max_attempts: int,
|
||||||
|
) -> None:
|
||||||
|
while not stop.is_set():
|
||||||
|
services.repository.reap_expired_leases(
|
||||||
|
now=utc_now(),
|
||||||
|
max_attempts=max_attempts,
|
||||||
|
)
|
||||||
|
if await _wait_for_stop(stop, interval_seconds):
|
||||||
|
return
|
||||||
|
|
||||||
|
|
||||||
|
async def _wait_for_stop(stop: asyncio.Event, interval_seconds: float) -> bool:
|
||||||
|
try:
|
||||||
|
await asyncio.wait_for(stop.wait(), timeout=interval_seconds)
|
||||||
|
except TimeoutError:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
@@ -20,8 +22,20 @@ def test_create_app_returns_independent_cloud_application() -> None:
|
|||||||
def test_cloud_application_owns_database_lifecycle() -> None:
|
def test_cloud_application_owns_database_lifecycle() -> None:
|
||||||
events: list[str] = []
|
events: list[str] = []
|
||||||
|
|
||||||
|
class FakeRepository:
|
||||||
|
def health_check(self) -> None:
|
||||||
|
events.append("healthy")
|
||||||
|
|
||||||
|
def list_queued_tasks(self) -> list[object]:
|
||||||
|
events.append("scheduled")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
||||||
|
events.append("reaped")
|
||||||
|
return []
|
||||||
|
|
||||||
class FakeDatabase:
|
class FakeDatabase:
|
||||||
repository = object()
|
repository = FakeRepository()
|
||||||
|
|
||||||
def close(self) -> None:
|
def close(self) -> None:
|
||||||
events.append("closed")
|
events.append("closed")
|
||||||
@@ -37,9 +51,9 @@ def test_cloud_application_owns_database_lifecycle() -> None:
|
|||||||
assert (
|
assert (
|
||||||
app.state.cloud_services.pool.store is app.state.cloud_services.repository
|
app.state.cloud_services.pool.store is app.state.cloud_services.repository
|
||||||
)
|
)
|
||||||
assert events == []
|
assert "healthy" in events
|
||||||
|
|
||||||
assert events == ["closed"]
|
assert events[-1] == "closed"
|
||||||
|
|
||||||
|
|
||||||
def test_repository_proxy_is_available_only_during_lifespan() -> None:
|
def test_repository_proxy_is_available_only_during_lifespan() -> None:
|
||||||
@@ -47,6 +61,12 @@ def test_repository_proxy_is_available_only_during_lifespan() -> None:
|
|||||||
def health_check(self) -> None:
|
def health_check(self) -> None:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def list_queued_tasks(self) -> list[object]:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
||||||
|
return []
|
||||||
|
|
||||||
class FakeDatabase:
|
class FakeDatabase:
|
||||||
repository = FakeRepository()
|
repository = FakeRepository()
|
||||||
|
|
||||||
@@ -65,6 +85,47 @@ def test_repository_proxy_is_available_only_during_lifespan() -> None:
|
|||||||
app.state.cloud_services.repository.health_check()
|
app.state.cloud_services.repository.health_check()
|
||||||
|
|
||||||
|
|
||||||
|
def test_lifespan_runs_scheduler_and_reaper_until_shutdown() -> None:
|
||||||
|
events: list[str] = []
|
||||||
|
|
||||||
|
class FakeRepository:
|
||||||
|
def health_check(self) -> None:
|
||||||
|
events.append("healthy")
|
||||||
|
|
||||||
|
def list_queued_tasks(self) -> list[object]:
|
||||||
|
events.append("scheduled")
|
||||||
|
return []
|
||||||
|
|
||||||
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
||||||
|
events.append(f"reaped:{max_attempts}")
|
||||||
|
return []
|
||||||
|
|
||||||
|
class FakeDatabase:
|
||||||
|
repository = FakeRepository()
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
events.append("closed")
|
||||||
|
|
||||||
|
app = create_app(
|
||||||
|
config=CloudControlConfig(
|
||||||
|
database_url="sqlite:///:memory:",
|
||||||
|
scheduler_interval_seconds=0.01,
|
||||||
|
lease_reaper_interval_seconds=0.01,
|
||||||
|
max_task_attempts=4,
|
||||||
|
),
|
||||||
|
database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value]
|
||||||
|
)
|
||||||
|
|
||||||
|
with TestClient(app):
|
||||||
|
time.sleep(0.04)
|
||||||
|
assert events.count("scheduled") >= 2
|
||||||
|
assert events.count("reaped:4") >= 2
|
||||||
|
assert all(not task.done() for task in app.state.worker_tasks)
|
||||||
|
|
||||||
|
assert all(task.done() for task in app.state.worker_tasks)
|
||||||
|
assert events[-1] == "closed"
|
||||||
|
|
||||||
|
|
||||||
def test_production_app_rejects_missing_credentials() -> None:
|
def test_production_app_rejects_missing_credentials() -> None:
|
||||||
with pytest.raises(CloudConfigurationError, match="credential"):
|
with pytest.raises(CloudConfigurationError, match="credential"):
|
||||||
create_app(
|
create_app(
|
||||||
|
|||||||
@@ -44,7 +44,7 @@
|
|||||||
## 6. Cloud Control Plane Composition
|
## 6. Cloud Control Plane Composition
|
||||||
|
|
||||||
- [x] 6.1 Compose repository, pool, scheduler, plugin registry, public router, internal router, and auth providers in the cloud app factory.
|
- [x] 6.1 Compose repository, pool, scheduler, plugin registry, public router, internal router, and auth providers in the cloud app factory.
|
||||||
- [ ] 6.2 Implement FastAPI lifespan startup/shutdown for configuration validation, database checks, scheduler loop, and lease-reaper loop.
|
- [x] 6.2 Implement FastAPI lifespan startup/shutdown for configuration validation, database checks, scheduler loop, and lease-reaper loop.
|
||||||
- [ ] 6.3 Ensure lifecycle iteration failures are logged and retried without terminating later iterations.
|
- [ ] 6.3 Ensure lifecycle iteration failures are logged and retried without terminating later iterations.
|
||||||
- [ ] 6.4 Add `/health/live` and `/health/ready` with separate process, database/schema, and worker-state semantics.
|
- [ ] 6.4 Add `/health/live` and `/health/ready` with separate process, database/schema, and worker-state semantics.
|
||||||
- [ ] 6.5 Add structured correlation-aware logging for requests and task lifecycle events with sensitive payload redaction.
|
- [ ] 6.5 Add structured correlation-aware logging for requests and task lifecycle events with sensitive payload redaction.
|
||||||
|
|||||||
Reference in New Issue
Block a user