"""账号只读查询通道的业务逻辑：状态机、租约、重投、过期与保留期清理 与 `task_queue.py` 是**刻意分开的两套语义**，不是重复代码（见 docs/order-gateway.md §11）： | | 下单任务队列 | 本模块（只读查询） | | --- | --- | --- | | 操作性质 | 不可逆写 | 只读 | | 全局并发度 1 | 是（同账号写操作必须串行） | 否（账号级串行由本地 SiteInteractor 的锁兜底） | | 租约过期 | 置 stale，**绝不自动重投**，等人工 reclaim | 回 queued **自动重投**，重跑无副作用 | | 续租 | 有（下单要几分钟） | 无（超时即重投，比续租简单且安全） | | 心跳 | lease 兼作心跳 | **不刷心跳**（见 `lease` 注释） | 这两张表共用一个 `GatewayDB` 连接，但各自持有自己的锁：查询的长轮询等待不该 把下单任务的 lease/report 一起挂住。SQLite 单连接本身是串行的，两把锁只是各自 保护「检查 + 写入」的原子序列。 """ from __future__ import annotations import asyncio import json import logging import secrets from datetime import datetime, timedelta, timezone from app.gateway.db import GatewayDB, QueryRow from app.gateway.models import ( QueryDetail, QueryError, QueryLeaseData, QueryListData, QueryResultData, SubmitQueryData, ) from app.shared.errors import QueryLeaseInvalidError, QueryNotFoundError from app.shared.task_state import QUERY_TERMINAL_STATUSES, QueryStatus logger = logging.getLogger(__name__) def utcnow() -> datetime: return datetime.now(timezone.utc) def to_iso(dt: datetime) -> str: """统一时间戳格式：ISO8601 UTC，秒精度，Z 结尾（与 task_queue 一致）""" return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") def parse_iso(s: str) -> datetime: if s.endswith("Z"): s = s[:-1] + "+00:00" return datetime.fromisoformat(s) def _generate_query_id() -> str: """服务端生成的 query_id：与 task_id 用不同前缀，日志里一眼能分清两条通道""" return f"q-{to_iso(utcnow())[:10].replace('-', '')}-{secrets.token_hex(4)}" def _row_to_detail(row: QueryRow) -> QueryDetail: error = None if row.error_code is not None or row.error_message: error = QueryError(code=row.error_code, message=row.error_message or "") return QueryDetail( query_id=row.query_id, site=row.site, kind=row.kind, # type: ignore[arg-type] params=row.params, status=QueryStatus(row.status), lease_owner=row.lease_owner, lease_expires_at=row.lease_expires_at, attempts=row.attempts, created_at=row.created_at, updated_at=row.updated_at, completed_at=row.completed_at, result=row.result, error=error, ) class QueryQueue: """账号只读查询单队列 生命周期与 TaskQueue 一致：由 GatewayContainer 持有，共用同一个 GatewayDB。 """ def __init__( self, db: GatewayDB, *, lease_ttl_seconds: int, query_ttl_seconds: int, max_attempts: int, retention_seconds: int, ): self._db = db self._lease_ttl = lease_ttl_seconds self._query_ttl = query_ttl_seconds self._max_attempts = max_attempts self._retention = retention_seconds self._lock = asyncio.Lock() self._cond = asyncio.Condition(self._lock) # ---- 提交 ---- async def submit( self, *, query_id: str | None, site: str, kind: str, params: dict ) -> SubmitQueryData: """上游提交查询单。query_id 缺省时服务端生成；重复提交幂等""" qid = query_id or _generate_query_id() now = to_iso(utcnow()) row = QueryRow( query_id=qid, site=site, kind=kind, params_json=json.dumps(params, ensure_ascii=False), status=QueryStatus.QUEUED.value, lease_owner=None, lease_expires_at=None, attempts=0, result_json=None, error_code=None, error_message=None, created_at=now, updated_at=now, completed_at=None, ) async with self._lock: created = await self._db.insert_query(row) if created: self._cond.notify_all() else: existing = await self._db.get_query(qid) if existing is None: raise RuntimeError(f"查询单 {qid} 既未新建也无法读取，状态异常") row = existing return SubmitQueryData( query_id=row.query_id, status=QueryStatus(row.status), created=created ) # ---- 领取 ---- async def lease( self, *, worker_id: str, wait: int, site: str | None, max_wait: int ) -> QueryLeaseData | None: """本地长轮询领取一张查询单 与下单 lease 的三点不同： 1. **没有全局并发度 1 的闸门**——只读操作允许多张单同时在飞；真正的账号级 串行由本地 SiteInteractor 那把锁保证，网关不必也不该重复实现。 2. **不刷 workers.last_seen_at**。心跳的语义是「下单 worker 还活着」， 如果查询循环活着就刷心跳，下单主循环挂了也看不出来，告警会失真。 3. 领走前先 sweep：把超时的重投回 queued、过期的置 expired。 """ effective_wait = max(0, min(wait, max_wait)) deadline = utcnow() + timedelta(seconds=effective_wait) async with self._lock: await self._sweep_locked() while True: row = await self._db.pick_queued_query(site) if row is not None: return await self._lease_to_locked(row, worker_id) remaining = (deadline - utcnow()).total_seconds() if remaining <= 0: return None try: await asyncio.wait_for(self._cond.wait(), timeout=remaining) except asyncio.TimeoutError: return None async def _lease_to_locked(self, row: QueryRow, worker_id: str) -> QueryLeaseData: """把一张 queued 查询单租给 worker。调用方必须持有 self._lock""" now = utcnow() expires = to_iso(now + timedelta(seconds=self._lease_ttl)) attempt = row.attempts + 1 await self._db.update_query( row.query_id, status=QueryStatus.LEASED.value, lease_owner=worker_id, lease_expires_at=expires, attempts=attempt, updated_at=to_iso(now), ) return QueryLeaseData( query_id=row.query_id, site=row.site, kind=row.kind, # type: ignore[arg-type] params=row.params, lease_expires_at=expires, attempt=attempt, ) # ---- 回结果 ---- async def submit_result( self, query_id: str, *, worker_id: str, success: bool, result: dict | None, error_code: int | None, error_message: str, ) -> QueryResultData: """worker 回报结果 校验 `worker_id == lease_owner` 且查询单未终结，否则 6006——最典型的场景是 执行超时、单子已被重投给下一轮，迟到的结果必须丢弃，不能覆盖新结果。 """ async with self._lock: row = await self._db.get_query(query_id) if row is None: raise QueryNotFoundError(query_id) status = QueryStatus(row.status) if status in QUERY_TERMINAL_STATUSES: raise QueryLeaseInvalidError( f"查询单 {query_id} 已是终态 {status.value}，不再接受结果" ) if row.lease_owner != worker_id: raise QueryLeaseInvalidError( f"worker {worker_id} 不是查询单 {query_id} 的租约持有者" f"（当前持有者 {row.lease_owner}）" ) now = to_iso(utcnow()) if success: await self._db.update_query( query_id, status=QueryStatus.SUCCEEDED.value, result_json=json.dumps(result or {}, ensure_ascii=False), updated_at=now, completed_at=now, clear_lease=True, ) final = QueryStatus.SUCCEEDED else: await self._db.update_query( query_id, status=QueryStatus.FAILED.value, error_code=error_code, error_message=error_message or "worker 未给出失败原因", updated_at=now, completed_at=now, clear_lease=True, ) final = QueryStatus.FAILED logger.warning( "查询单失败：query_id=%s kind=%s code=%s msg=%s", query_id, row.kind, error_code, error_message, ) return QueryResultData(query_id=query_id, status=final) # ---- 清扫：重投 / 过期 / 保留期 ---- async def sweep(self) -> int: """扫一轮：租约超时重投、整体超时置 expired、过保留期的终态单清理 返回本轮发生状态变更的查询单条数（删除的不计入）。 """ async with self._lock: changed = await self._sweep_locked() purged = await self._db.delete_queries_completed_before( to_iso(utcnow() - timedelta(seconds=self._retention)) ) if purged: logger.info("清理了 %s 张过保留期的查询单", purged) return changed async def _sweep_locked(self) -> int: """锁内清扫。调用方必须持有 self._lock""" now = utcnow() changed = 0 rows = await self._db.list_queries_in_statuses( (QueryStatus.QUEUED.value, QueryStatus.LEASED.value) ) for row in rows: # 整体 TTL 优先：单子已经没有意义了，再重投也只是白跑一趟 if parse_iso(row.created_at) + timedelta(seconds=self._query_ttl) < now: await self._db.update_query( row.query_id, status=QueryStatus.EXPIRED.value, error_code=None, error_message=( f"查询单超过 {self._query_ttl} 秒仍未完成" f"（attempts={row.attempts}，多半是本地 worker 不在线）" ), updated_at=to_iso(now), completed_at=to_iso(now), clear_lease=True, ) changed += 1 logger.warning("查询单过期：query_id=%s kind=%s", row.query_id, row.kind) continue if row.status != QueryStatus.LEASED.value: continue if not row.lease_expires_at or parse_iso(row.lease_expires_at) >= now: continue if row.attempts >= self._max_attempts: await self._db.update_query( row.query_id, status=QueryStatus.FAILED.value, error_message=( f"已被领取 {row.attempts} 次仍未回结果，不再重投" "（可能是站点页面卡住或本地 worker 反复崩溃）" ), updated_at=to_iso(now), completed_at=to_iso(now), clear_lease=True, ) logger.warning( "查询单重投次数用尽，置 failed：query_id=%s attempts=%s", row.query_id, row.attempts, ) else: # 只读操作可以安全重投——这正是查询与下单任务最大的语义差别 await self._db.update_query( row.query_id, status=QueryStatus.QUEUED.value, updated_at=to_iso(now), clear_lease=True, ) logger.info( "查询单租约过期，重投回队列：query_id=%s attempts=%s", row.query_id, row.attempts, ) changed += 1 if changed: self._cond.notify_all() return changed # ---- 查询 ---- async def get_detail(self, query_id: str) -> QueryDetail: row = await self._db.get_query(query_id) if row is None: raise QueryNotFoundError(query_id) return _row_to_detail(row) async def list_queries( self, *, status: str | None, kind: str | None, limit: int, offset: int ) -> QueryListData: rows, total = await self._db.list_queries( status=status, kind=kind, limit=limit, offset=offset ) return QueryListData( items=[_row_to_detail(r) for r in rows], total=total, limit=limit, offset=offset ) async def queued_count(self) -> int: """等待领取的查询单数量（/health 用）""" return await self._db.count_queries_by_status(QueryStatus.QUEUED.value)