feat(gateway): 账号只读查询通道——从已登录账号取真实订单

上游要的不只是网关记的任务状态镜像，还有「已登录账号在站点上的真实订单」，但账号只在 NAT 后本地机上，只能经网关队列走。新增独立查询通道（§11）： - gateway 单开 account_queries 表 + QueryStatus 状态机，接口 POST /api/account/queries（幂等）/ lease / {id}/result / {id} - 不复用下单任务队列：查询是只读，租约过期可安全重投（与下单「绝不自动重投」相反），且不该被全局并发度 1 堵死、task_reports 是订单镜像不能污染 - 本地交易服务起第二条常驻循环 query_runner，领到即调 SiteInteractor 真读： order_list 复用已实测的 list_recent_orders（规范化字段 + 站点 orderListData 原文），order_detail 复用 fetch_order_detail（配送阶段 + 页面 __INITIAL_STATE__ 原样透传，结构未经真实样本，不抽字段） - 账号级串行仍由 SiteInteractor 的锁保证；每次执行套超时按失败回报 - 错误码 6005/6006（查询通道，可重试只读区别于 6001-6004）；/health 暴露 queued_query_count；结果体积上限先丢原始 JSON openapi.json 重导，docs/order-gateway.md §11、README、.env.example 补全配置与实测边界。全量测试 404→454 通过。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-16 22:46:01 +08:00
parent b01c9659d1
commit c03158488b
22 changed files with 2575 additions and 43 deletions
@@ -0,0 +1,390 @@
+"""账号只读查询通道测试（docs/order-gateway.md §11）
+
+分两层：
+- HTTP 层：鉴权、幂等、6005/6006、lease → result 全链路、列表与 /health
+- 队列语义层：直接驱动 QueryQueue，测那些靠 HTTP 不好造的时序——租约超时重投、
+  重投次数用尽、整体 TTL 过期、迟到结果被拒、过保留期清理
+
+这里最关键的一条是「只读可重投」：它与下单任务的「绝不自动重投」正好相反，
+两条通道的语义分歧全在这几个用例里，改任何一边前先看它们。
+"""
+from __future__ import annotations
+
+from datetime import timedelta
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+
+from app.gateway.db import GatewayDB, QueryRow
+from app.gateway.query_queue import QueryQueue, to_iso, utcnow
+from app.shared.config import get_settings
+from app.shared.errors import QueryLeaseInvalidError
+from app.shared.task_state import AccountQueryKind, QueryStatus
+
+TOKEN = get_settings().bearer_token
+AUTH = {"Authorization": f"Bearer {TOKEN}"}
+
+QUERIES = "/api/account/queries"
+
+
+@pytest.fixture
+def gateway_client(tmp_path: Path, monkeypatch):
+    """起一个独立 DB 的网关应用（与 test_gateway_api.py 同一套装配方式）"""
+    monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(tmp_path / "gw.db"))
+    get_settings.cache_clear()
+    try:
+        from app.gateway.main import create_app
+
+        app = create_app()
+        with TestClient(app) as client:
+            yield client
+    finally:
+        get_settings.cache_clear()
+
+
+@pytest.fixture
+async def queue(tmp_path: Path):
+    """直接可用的 (QueryQueue, GatewayDB) 对
+
+    队列语义用例要模拟「租约已过期」「结果早就写完了」这类时间流逝，直接改库里的
+    时间戳比 sleep 快得多，所以把 db 一并交出去。
+    """
+    db = GatewayDB(tmp_path / "q.db")
+    await db.start()
+    try:
+        yield (
+            QueryQueue(
+                db,
+                lease_ttl_seconds=60,
+                query_ttl_seconds=600,
+                max_attempts=3,
+                retention_seconds=7 * 24 * 3600,
+            ),
+            db,
+        )
+    finally:
+        await db.close()
+
+
+def _submit(client: TestClient, **overrides) -> dict:
+    payload = {
+        "query_id": "q1",
+        "site": "rakuten",
+        "kind": AccountQueryKind.ORDER_LIST.value,
+        "params": {"max_pages": 1},
+    }
+    payload.update(overrides)
+    return client.post(QUERIES, json=payload, headers=AUTH).json()
+
+
+# ---- HTTP：鉴权与参数校验 ----
+
+
+@pytest.mark.parametrize(
+    "path,method",
+    [
+        (QUERIES, "POST"),
+        (f"{QUERIES}/lease", "GET"),
+        (f"{QUERIES}/q1", "GET"),
+    ],
+)
+def test_query_endpoints_reject_missing_token(gateway_client, path, method):
+    response = gateway_client.request(method, path)
+    assert response.status_code == 401
+    assert response.json()["code"] == 1001
+
+
+def test_unknown_kind_is_rejected_at_submit(gateway_client):
+    """kind 拼错当场 422，而不是等 worker 领走、执行、回报失败才知道"""
+    response = gateway_client.post(
+        QUERIES,
+        json={"site": "rakuten", "kind": "order_lst", "params": {}},
+        headers=AUTH,
+    )
+    assert response.status_code == 422
+    assert response.json()["code"] == 1002
+
+
+# ---- HTTP：提交、幂等、领取、回结果 ----
+
+
+def test_submit_returns_query_id_and_queued(gateway_client):
+    body = _submit(gateway_client)
+    assert body["success"] is True
+    assert body["data"]["query_id"] == "q1"
+    assert body["data"]["status"] == QueryStatus.QUEUED.value
+    assert body["data"]["created"] is True
+
+
+def test_submit_is_idempotent_on_same_query_id(gateway_client):
+    assert _submit(gateway_client)["data"]["created"] is True
+    assert _submit(gateway_client)["data"]["created"] is False
+
+
+def test_submit_generates_query_id_when_absent(gateway_client):
+    body = gateway_client.post(
+        QUERIES,
+        json={"site": "rakuten", "kind": AccountQueryKind.ORDER_DETAIL.value, "params": {}},
+        headers=AUTH,
+    ).json()
+    assert body["data"]["query_id"].startswith("q-")
+
+
+def test_lease_returns_null_when_no_query(gateway_client):
+    response = gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
+    assert response.status_code == 200
+    assert response.json()["data"] is None
+
+
+def test_lease_then_result_completes_the_query(gateway_client):
+    _submit(gateway_client)
+
+    leased = gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH).json()
+    assert leased["data"]["query_id"] == "q1"
+    assert leased["data"]["kind"] == AccountQueryKind.ORDER_LIST.value
+    assert leased["data"]["params"] == {"max_pages": 1}
+    assert leased["data"]["attempt"] == 1
+
+    result = {"kind": "order_list", "orders": [{"order_number": "306087-20260813-0863947697"}]}
+    reported = gateway_client.post(
+        f"{QUERIES}/q1/result",
+        json={"worker_id": "w1", "success": True, "result": result},
+        headers=AUTH,
+    ).json()
+    assert reported["data"]["status"] == QueryStatus.SUCCEEDED.value
+
+    detail = gateway_client.get(f"{QUERIES}/q1", headers=AUTH).json()["data"]
+    assert detail["status"] == QueryStatus.SUCCEEDED.value
+    assert detail["result"] == result
+    assert detail["error"] is None
+    assert detail["lease_owner"] is None
+
+
+def test_failed_result_carries_error_code_and_message(gateway_client):
+    _submit(gateway_client)
+    gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
+    gateway_client.post(
+        f"{QUERIES}/q1/result",
+        json={
+            "worker_id": "w1",
+            "success": False,
+            "error_code": 5001,
+            "error_message": "账号未登录",
+        },
+        headers=AUTH,
+    )
+    detail = gateway_client.get(f"{QUERIES}/q1", headers=AUTH).json()["data"]
+    assert detail["status"] == QueryStatus.FAILED.value
+    assert detail["error"] == {"code": 5001, "message": "账号未登录"}
+
+
+def test_two_queries_can_be_in_flight_at_once(gateway_client):
+    """与下单任务不同：只读查询没有「全局并发度 1」的闸门"""
+    _submit(gateway_client, query_id="q1")
+    _submit(gateway_client, query_id="q2")
+
+    first = gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH).json()
+    second = gateway_client.get(f"{QUERIES}/lease?worker_id=w2&wait=0", headers=AUTH).json()
+
+    assert first["data"]["query_id"] == "q1"
+    assert second["data"]["query_id"] == "q2"
+
+
+def test_order_lease_is_not_disturbed_by_pending_query(gateway_client):
+    """两条通道互不干扰：有查询在飞，下单任务照领"""
+    _submit(gateway_client)
+    gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
+
+    gateway_client.post(
+        "/api/orders",
+        json={"task_id": "t1", "site": "rakuten", "intent": {}},
+        headers=AUTH,
+    )
+    leased = gateway_client.get("/api/orders/lease?worker_id=w1&wait=0", headers=AUTH).json()
+    assert leased["data"]["task_id"] == "t1"
+
+
+# ---- HTTP：错误码 ----
+
+
+def test_get_unknown_query_returns_6005(gateway_client):
+    response = gateway_client.get(f"{QUERIES}/no-such", headers=AUTH)
+    assert response.status_code == 404
+    assert response.json()["code"] == 6005
+
+
+def test_result_from_non_owner_returns_6006(gateway_client):
+    _submit(gateway_client)
+    gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
+
+    response = gateway_client.post(
+        f"{QUERIES}/q1/result",
+        json={"worker_id": "w2", "success": True, "result": {}},
+        headers=AUTH,
+    )
+    assert response.status_code == 409
+    assert response.json()["code"] == 6006
+
+
+def test_result_on_terminal_query_returns_6006(gateway_client):
+    _submit(gateway_client)
+    gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
+    gateway_client.post(
+        f"{QUERIES}/q1/result",
+        json={"worker_id": "w1", "success": True, "result": {"a": 1}},
+        headers=AUTH,
+    )
+
+    response = gateway_client.post(
+        f"{QUERIES}/q1/result",
+        json={"worker_id": "w1", "success": True, "result": {"a": 2}},
+        headers=AUTH,
+    )
+    assert response.status_code == 409
+    assert response.json()["code"] == 6006
+    # 结果没有被第二次回报覆盖
+    detail = gateway_client.get(f"{QUERIES}/q1", headers=AUTH).json()["data"]
+    assert detail["result"] == {"a": 1}
+
+
+# ---- HTTP：列表与健康检查 ----
+
+
+def test_list_queries_filters_by_kind(gateway_client):
+    _submit(gateway_client, query_id="q1", kind=AccountQueryKind.ORDER_LIST.value)
+    _submit(gateway_client, query_id="q2", kind=AccountQueryKind.ORDER_DETAIL.value)
+
+    body = gateway_client.get(
+        f"{QUERIES}?kind={AccountQueryKind.ORDER_DETAIL.value}", headers=AUTH
+    ).json()
+    assert body["data"]["total"] == 1
+    assert body["data"]["items"][0]["query_id"] == "q2"
+
+
+def test_health_reports_queued_query_count(gateway_client):
+    _submit(gateway_client)
+    body = gateway_client.get("/health").json()
+    assert body["data"]["queued_query_count"] == 1
+
+
+# ---- 队列语义：只读可重投（与下单任务的核心分歧）----
+
+
+async def test_expired_lease_goes_back_to_queued(queue):
+    """租约过期→回 queued 自动重投。下单任务在这里是 stale 且绝不重投"""
+    q, db = queue
+    await q.submit(query_id="q1", site="rakuten", kind="order_list", params={})
+    lease = await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
+    assert lease is not None
+
+    await db.update_query("q1", lease_expires_at=to_iso(utcnow() - timedelta(seconds=1)))
+    assert await q.sweep() == 1
+
+    detail = await q.get_detail("q1")
+    assert detail.status == QueryStatus.QUEUED
+    assert detail.lease_owner is None
+
+    again = await q.lease(worker_id="w2", wait=0, site=None, max_wait=60)
+    assert again is not None
+    assert again.attempt == 2
+
+
+async def test_late_result_after_reinvest_is_rejected(queue):
+    """超时重投后，上一轮 worker 迟到的结果必须被拒，不能覆盖新一轮"""
+    q, db = queue
+    await q.submit(query_id="q1", site="rakuten", kind="order_list", params={})
+    await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
+    await db.update_query("q1", lease_expires_at=to_iso(utcnow() - timedelta(seconds=1)))
+    await q.sweep()
+    await q.lease(worker_id="w2", wait=0, site=None, max_wait=60)
+
+    with pytest.raises(QueryLeaseInvalidError):
+        await q.submit_result(
+            "q1", worker_id="w1", success=True, result={"stale": True},
+            error_code=None, error_message="",
+        )
+
+
+async def test_reinvest_stops_at_max_attempts(queue):
+    """重投次数用尽 → failed，不再无限重投同一张卡死的单"""
+    q, db = queue
+    await q.submit(query_id="q1", site="rakuten", kind="order_list", params={})
+    for _ in range(3):
+        await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
+        await db.update_query("q1", lease_expires_at=to_iso(utcnow() - timedelta(seconds=1)))
+        await q.sweep()
+
+    detail = await q.get_detail("q1")
+    assert detail.status == QueryStatus.FAILED
+    assert detail.attempts == 3
+    assert "不再重投" in detail.error.message
+
+
+async def test_query_expires_after_ttl(tmp_path):
+    """整体 TTL 到点 → expired（本地 worker 不在线的典型表现）"""
+    db = GatewayDB(tmp_path / "q.db")
+    await db.start()
+    try:
+        queue = QueryQueue(
+            db, lease_ttl_seconds=60, query_ttl_seconds=60, max_attempts=3,
+            retention_seconds=3600,
+        )
+        # 直接插一张「10 分钟前创建」的单，比 sleep 现实
+        old = to_iso(utcnow() - timedelta(minutes=10))
+        await db.insert_query(
+            QueryRow(
+                query_id="q1", site="rakuten", kind="order_list", params_json="{}",
+                status=QueryStatus.QUEUED.value, lease_owner=None, lease_expires_at=None,
+                attempts=0, result_json=None, error_code=None, error_message=None,
+                created_at=old, updated_at=old, completed_at=None,
+            )
+        )
+
+        assert await queue.sweep() == 1
+        detail = await queue.get_detail("q1")
+        assert detail.status == QueryStatus.EXPIRED
+        assert "worker 不在线" in detail.error.message
+        # 过期的单不会再被领走
+        assert await queue.lease(worker_id="w1", wait=0, site=None, max_wait=60) is None
+    finally:
+        await db.close()
+
+
+async def test_sweep_purges_queries_past_retention(tmp_path):
+    """终态查询单过保留期被清掉——结果里带站点原始 JSON，不清会一直涨"""
+    db = GatewayDB(tmp_path / "q.db")
+    await db.start()
+    try:
+        queue = QueryQueue(
+            db, lease_ttl_seconds=60, query_ttl_seconds=600, max_attempts=3,
+            retention_seconds=1,
+        )
+        await queue.submit(query_id="q1", site="rakuten", kind="order_list", params={})
+        await queue.lease(worker_id="w1", wait=0, site=None, max_wait=60)
+        await queue.submit_result(
+            "q1", worker_id="w1", success=True, result={"a": 1},
+            error_code=None, error_message="",
+        )
+        await db.update_query("q1", completed_at=to_iso(utcnow() - timedelta(seconds=60)))
+
+        await queue.sweep()
+        assert await db.get_query("q1") is None
+    finally:
+        await db.close()
+
+
+async def test_sweep_keeps_unfinished_queries(tmp_path):
+    """保留期清理只动终态单，进行中的一律不碰"""
+    db = GatewayDB(tmp_path / "q.db")
+    await db.start()
+    try:
+        queue = QueryQueue(
+            db, lease_ttl_seconds=60, query_ttl_seconds=600, max_attempts=3,
+            retention_seconds=1,
+        )
+        await queue.submit(query_id="q1", site="rakuten", kind="order_list", params={})
+        await queue.sweep()
+        assert await db.get_query("q1") is not None
+    finally:
+        await db.close()