266 lines
7.8 KiB
Python
266 lines
7.8 KiB
Python
from __future__ import annotations
|
|
|
|
import time
|
|
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
|
|
import cloud_api.app as app_module
|
|
from cloud_api.app import create_app
|
|
from cloud.control_config import CloudConfigurationError, CloudControlConfig
|
|
|
|
|
|
def test_create_app_returns_independent_cloud_application() -> None:
|
|
app = create_app()
|
|
|
|
assert app.title == "Device Cloud API"
|
|
assert callable(create_app)
|
|
paths = set(app.openapi()["paths"])
|
|
assert "/v1/tasks" in paths
|
|
assert "/internal/v1/hosts/{host_id}/heartbeat" in paths
|
|
|
|
|
|
def test_cloud_application_owns_database_lifecycle() -> None:
|
|
events: list[str] = []
|
|
|
|
class FakeRepository:
|
|
def health_check(self) -> None:
|
|
events.append("healthy")
|
|
|
|
def list_queued_tasks(self) -> list[object]:
|
|
events.append("scheduled")
|
|
return []
|
|
|
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
|
events.append("reaped")
|
|
return []
|
|
|
|
class FakeDatabase:
|
|
repository = FakeRepository()
|
|
|
|
def close(self) -> None:
|
|
events.append("closed")
|
|
|
|
fake_database = FakeDatabase()
|
|
app = create_app(
|
|
config=CloudControlConfig(database_url="sqlite:///:memory:"),
|
|
database_factory=lambda _config: fake_database, # type: ignore[arg-type,return-value]
|
|
)
|
|
|
|
with TestClient(app):
|
|
assert app.state.database is fake_database
|
|
assert (
|
|
app.state.cloud_services.pool.store is app.state.cloud_services.repository
|
|
)
|
|
assert "healthy" in events
|
|
|
|
assert events[-1] == "closed"
|
|
|
|
|
|
def test_repository_proxy_is_available_only_during_lifespan() -> None:
|
|
class FakeRepository:
|
|
def health_check(self) -> None:
|
|
return None
|
|
|
|
def list_queued_tasks(self) -> list[object]:
|
|
return []
|
|
|
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
|
return []
|
|
|
|
class FakeDatabase:
|
|
repository = FakeRepository()
|
|
|
|
def close(self) -> None:
|
|
return None
|
|
|
|
app = create_app(
|
|
config=CloudControlConfig(database_url="sqlite:///:memory:"),
|
|
database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value]
|
|
)
|
|
|
|
with TestClient(app):
|
|
app.state.cloud_services.repository.health_check()
|
|
|
|
with pytest.raises(RuntimeError, match="outside app lifespan"):
|
|
app.state.cloud_services.repository.health_check()
|
|
|
|
|
|
def test_lifespan_runs_scheduler_and_reaper_until_shutdown() -> None:
|
|
events: list[str] = []
|
|
|
|
class FakeRepository:
|
|
def health_check(self) -> None:
|
|
events.append("healthy")
|
|
|
|
def list_queued_tasks(self) -> list[object]:
|
|
events.append("scheduled")
|
|
return []
|
|
|
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
|
events.append(f"reaped:{max_attempts}")
|
|
return []
|
|
|
|
class FakeDatabase:
|
|
repository = FakeRepository()
|
|
|
|
def close(self) -> None:
|
|
events.append("closed")
|
|
|
|
app = create_app(
|
|
config=CloudControlConfig(
|
|
database_url="sqlite:///:memory:",
|
|
scheduler_interval_seconds=0.01,
|
|
lease_reaper_interval_seconds=0.01,
|
|
max_task_attempts=4,
|
|
),
|
|
database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value]
|
|
)
|
|
|
|
with TestClient(app):
|
|
time.sleep(0.04)
|
|
assert events.count("scheduled") >= 2
|
|
assert events.count("reaped:4") >= 2
|
|
assert all(not task.done() for task in app.state.worker_tasks)
|
|
|
|
assert all(task.done() for task in app.state.worker_tasks)
|
|
assert events[-1] == "closed"
|
|
|
|
|
|
def test_lifecycle_workers_log_failures_and_continue(monkeypatch) -> None:
|
|
logged_workers: list[str] = []
|
|
|
|
def record_exception(_message: str, *, extra: dict[str, str]) -> None:
|
|
logged_workers.append(extra["worker"])
|
|
|
|
monkeypatch.setattr(app_module.logger, "exception", record_exception)
|
|
scheduler_calls = 0
|
|
reaper_calls = 0
|
|
|
|
class FakeRepository:
|
|
def health_check(self) -> None:
|
|
return None
|
|
|
|
def list_queued_tasks(self) -> list[object]:
|
|
nonlocal scheduler_calls
|
|
scheduler_calls += 1
|
|
if scheduler_calls == 1:
|
|
raise RuntimeError("scheduler transient failure")
|
|
return []
|
|
|
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
|
nonlocal reaper_calls
|
|
reaper_calls += 1
|
|
if reaper_calls == 1:
|
|
raise RuntimeError("reaper transient failure")
|
|
return []
|
|
|
|
class FakeDatabase:
|
|
repository = FakeRepository()
|
|
|
|
def close(self) -> None:
|
|
return None
|
|
|
|
app = create_app(
|
|
config=CloudControlConfig(
|
|
database_url="sqlite:///:memory:",
|
|
scheduler_interval_seconds=0.01,
|
|
lease_reaper_interval_seconds=0.01,
|
|
),
|
|
database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value]
|
|
)
|
|
|
|
with TestClient(app):
|
|
time.sleep(0.04)
|
|
assert scheduler_calls >= 2
|
|
assert reaper_calls >= 2
|
|
assert all(not task.done() for task in app.state.worker_tasks)
|
|
|
|
assert set(logged_workers) >= {
|
|
"scheduler",
|
|
"lease_reaper",
|
|
}
|
|
|
|
|
|
def test_liveness_stays_up_when_database_readiness_fails() -> None:
|
|
database_healthy = True
|
|
|
|
class FakeRepository:
|
|
def health_check(self) -> None:
|
|
if not database_healthy:
|
|
raise RuntimeError("database unavailable with secret credentials")
|
|
|
|
def list_queued_tasks(self) -> list[object]:
|
|
return []
|
|
|
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
|
return []
|
|
|
|
class FakeDatabase:
|
|
repository = FakeRepository()
|
|
|
|
def close(self) -> None:
|
|
return None
|
|
|
|
app = create_app(
|
|
config=CloudControlConfig(database_url="sqlite:///:memory:"),
|
|
database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value]
|
|
)
|
|
|
|
with TestClient(app) as client:
|
|
assert client.get("/health/live").json() == {"status": "live"}
|
|
ready = client.get("/health/ready")
|
|
assert ready.status_code == 200
|
|
assert all(ready.json()["checks"].values())
|
|
|
|
database_healthy = False
|
|
assert client.get("/health/live").status_code == 200
|
|
not_ready = client.get("/health/ready")
|
|
assert not_ready.status_code == 503
|
|
assert not_ready.json()["checks"]["database"] is False
|
|
assert "secret credentials" not in not_ready.text
|
|
|
|
|
|
def test_readiness_reports_stopped_worker() -> None:
|
|
class FakeRepository:
|
|
def health_check(self) -> None:
|
|
return None
|
|
|
|
def list_queued_tasks(self) -> list[object]:
|
|
return []
|
|
|
|
def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
|
|
return []
|
|
|
|
class FakeDatabase:
|
|
repository = FakeRepository()
|
|
|
|
def close(self) -> None:
|
|
return None
|
|
|
|
class StoppedWorker:
|
|
def done(self) -> bool:
|
|
return True
|
|
|
|
app = create_app(
|
|
config=CloudControlConfig(database_url="sqlite:///:memory:"),
|
|
database_factory=lambda _config: FakeDatabase(), # type: ignore[arg-type,return-value]
|
|
)
|
|
|
|
with TestClient(app) as client:
|
|
app.state.worker_tasks = (StoppedWorker(),)
|
|
response = client.get("/health/ready")
|
|
|
|
assert response.status_code == 503
|
|
assert response.json()["checks"]["workers"] is False
|
|
|
|
|
|
def test_production_app_rejects_missing_credentials() -> None:
|
|
with pytest.raises(CloudConfigurationError, match="credential"):
|
|
create_app(
|
|
config=CloudControlConfig(
|
|
environment="production",
|
|
database_url="postgresql://db/cloud",
|
|
)
|
|
)
|