fix(cloud-api): retry lifecycle iterations

This commit is contained in:
2026-07-12 18:31:19 +08:00
parent 937a4646e1
commit b8d02abf87
3 changed files with 76 additions and 6 deletions
+19 -5
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import logging
from collections.abc import Callable
from contextlib import asynccontextmanager
from dataclasses import dataclass
@@ -26,6 +27,7 @@ from core.models import utc_now
DatabaseFactory = Callable[[CloudControlConfig], CloudDatabase]
logger = logging.getLogger(__name__)
class _RepositoryProxy:
@@ -159,7 +161,13 @@ async def _run_scheduler_loop(
interval_seconds: float,
) -> None:
while not stop.is_set():
services.scheduler.assign()
try:
services.scheduler.assign()
except Exception:
logger.exception(
"cloud lifecycle iteration failed",
extra={"worker": "scheduler"},
)
if await _wait_for_stop(stop, interval_seconds):
return
@@ -172,10 +180,16 @@ async def _run_lease_reaper_loop(
max_attempts: int,
) -> None:
while not stop.is_set():
services.repository.reap_expired_leases(
now=utc_now(),
max_attempts=max_attempts,
)
try:
services.repository.reap_expired_leases(
now=utc_now(),
max_attempts=max_attempts,
)
except Exception:
logger.exception(
"cloud lifecycle iteration failed",
extra={"worker": "lease_reaper"},
)
if await _wait_for_stop(stop, interval_seconds):
return