fix(cloud-api): retry lifecycle iterations

apps/cloud-api/tests/test_app.py

@@ -5,6 +5,7 @@ import time
 import pytest
 from fastapi.testclient import TestClient
 
+import cloud_api.app as app_module
 from cloud_api.app import create_app
 from cloud.control_config import CloudConfigurationError, CloudControlConfig
 
@@ -126,6 +127,61 @@ def test_lifespan_runs_scheduler_and_reaper_until_shutdown() -> None:
     assert events[-1] == "closed"
 
 
+def test_lifecycle_workers_log_failures_and_continue(monkeypatch) -> None:
+    logged_workers: list[str] = []
+
+    def record_exception(_message: str, *, extra: dict[str, str]) -> None:
+        logged_workers.append(extra["worker"])
+
+    monkeypatch.setattr(app_module.logger, "exception", record_exception)
+    scheduler_calls = 0
+    reaper_calls = 0
+
+    class FakeRepository:
+        def health_check(self) -> None:
+            return None
+
+        def list_queued_tasks(self) -> list[object]:
+            nonlocal scheduler_calls
+            scheduler_calls += 1
+            if scheduler_calls == 1:
+                raise RuntimeError("scheduler transient failure")
+            return []
+
+        def reap_expired_leases(self, *, now, max_attempts: int) -> list[str]:
+            nonlocal reaper_calls
+            reaper_calls += 1
+            if reaper_calls == 1:
+                raise RuntimeError("reaper transient failure")
+            return []
+
+    class FakeDatabase:
+        repository = FakeRepository()
+
+        def close(self) -> None:
+            return None
+
+    app = create_app(
+        config=CloudControlConfig(
+            database_url="sqlite:///:memory:",
+            scheduler_interval_seconds=0.01,
+            lease_reaper_interval_seconds=0.01,
+        ),
+        database_factory=lambda _config: FakeDatabase(),  # type: ignore[arg-type,return-value]
+    )
+
+    with TestClient(app):
+        time.sleep(0.04)
+        assert scheduler_calls >= 2
+        assert reaper_calls >= 2
+        assert all(not task.done() for task in app.state.worker_tasks)
+
+    assert set(logged_workers) >= {
+        "scheduler",
+        "lease_reaper",
+    }
+
+
 def test_production_app_rejects_missing_credentials() -> None:
     with pytest.raises(CloudConfigurationError, match="credential"):
         create_app(