feat(cloud-scheduler): reap expired leases

2026-07-12 17:32:11 +08:00
parent e1af4403f5
commit c58d9e5316
3 changed files with 242 additions and 1 deletions
@@ -451,6 +451,59 @@ class SQLAlchemyCloudRepository:
            attempt_row.result_json = result_json
            return "recorded"

+    def reap_expired_leases(
+        self,
+        *,
+        now: datetime,
+        max_attempts: int,
+    ) -> list[str]:
+        if max_attempts < 1:
+            raise ValueError("max_attempts must be at least 1")
+
+        with self._sessions.begin() as session:
+            statement = (
+                select(ScheduledTaskRow)
+                .where(
+                    ScheduledTaskRow.status.in_(("assigned", "dispatched")),
+                    ScheduledTaskRow.lease_expires_at.is_not(None),
+                    ScheduledTaskRow.lease_expires_at <= _iso(now),
+                )
+                .order_by(ScheduledTaskRow.lease_expires_at, ScheduledTaskRow.id)
+            )
+            if self.engine.dialect.name == "postgresql":
+                statement = statement.with_for_update(skip_locked=True)
+
+            tasks = session.scalars(statement).all()
+            reaped_task_ids: list[str] = []
+            for task in tasks:
+                attempt_row = session.get(
+                    TaskAttemptRow,
+                    (task.id, task.attempt_count),
+                    with_for_update=self.engine.dialect.name == "postgresql",
+                )
+                if attempt_row is None:
+                    continue
+
+                attempt_row.status = "expired"
+                attempt_row.completed_at = _iso(now)
+                attempt_row.failure_reason = "lease expired"
+                task.updated_at = _iso(now)
+                task.lease_id = None
+                task.lease_expires_at = None
+                task.result_json = None
+                if task.attempt_count < max_attempts:
+                    task.status = "queued"
+                    task.assigned_host_id = None
+                    task.assigned_device_id = None
+                    task.failure_reason = None
+                else:
+                    task.status = "failed"
+                    task.failure_reason = (
+                        f"lease expired after {task.attempt_count} attempts"
+                    )
+                reaped_task_ids.append(task.id)
+            return reaped_task_ids
+
    def list_task_attempts(self, task_id: str) -> list[Any]:
        with self._sessions() as session:
            rows = session.scalars(