fix(cloud-api): retry lifecycle iterations
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import Callable
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
@@ -26,6 +27,7 @@ from core.models import utc_now
|
||||
|
||||
|
||||
DatabaseFactory = Callable[[CloudControlConfig], CloudDatabase]
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _RepositoryProxy:
|
||||
@@ -159,7 +161,13 @@ async def _run_scheduler_loop(
|
||||
interval_seconds: float,
|
||||
) -> None:
|
||||
while not stop.is_set():
|
||||
services.scheduler.assign()
|
||||
try:
|
||||
services.scheduler.assign()
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"cloud lifecycle iteration failed",
|
||||
extra={"worker": "scheduler"},
|
||||
)
|
||||
if await _wait_for_stop(stop, interval_seconds):
|
||||
return
|
||||
|
||||
@@ -172,10 +180,16 @@ async def _run_lease_reaper_loop(
|
||||
max_attempts: int,
|
||||
) -> None:
|
||||
while not stop.is_set():
|
||||
services.repository.reap_expired_leases(
|
||||
now=utc_now(),
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
try:
|
||||
services.repository.reap_expired_leases(
|
||||
now=utc_now(),
|
||||
max_attempts=max_attempts,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"cloud lifecycle iteration failed",
|
||||
extra={"worker": "lease_reaper"},
|
||||
)
|
||||
if await _wait_for_stop(stop, interval_seconds):
|
||||
return
|
||||
|
||||
|
||||
Reference in New Issue
Block a user