feat(trading): 浏览器掉线兜底——任务边界自愈重建，中途掉线转 needs_human
SiteInteractor 的 Chromium 是进程级单例，此前启动后即假定永远活着：全仓唯一的 is_connected() 探活在 scraping 侧，交易侧既不探活也不重启。容器里 Chromium 崩溃 是有真实前提的（/dev/shm 不足、OOM kill、seccomp 挡 sandbox，docker-compose.yml 里已有相关注释），一旦发生，进程还活着但之后每一单都会失败，且 /health 恒返回 ok，restart: unless-stopped 永远不会被触发。 更隐蔽的一条：clear_cart / enter_checkout / pay 等处的 new_page()、context.request 写在 try 之外，掉线抛的 TargetClosedError 不是 AppError，会穿过 runner 的 except AppError 落到主循环那个只记日志的兜底里——任务一次都不上报，网关侧要干等 整个 lease_ttl（默认 300s）才被 sweep 置 stale。clear_cart 是 execute() 的 step 0， 浏览器死时最先撞上的正是它。 分三层处理： - 任务边界自愈。_launch() 从 start() 抽出，_context_options() 统一 context 参数 （重建必须与启动完全一致，指纹漂移就是一次风控事件）；_refresh_context_if_stale 改名 _ensure_context_ready()，先探活重建再做原有的 storage_state mtime 检查 （顺序不可换，mtime 重建要用 self._browser）。重建前丢弃 _checkout_pages 里的 残留确认页并记 warning——那些 Page 已随浏览器一起没了。 - 中途掉线不重建，抛 BrowserDeadError（新增，5006）。新增 _new_page() 与 _request() 两个壳收口裸异常；_request() 只在确认浏览器真死了时才改写异常， 站点 5xx 这类正常业务失败原样抛出。submit_order / pay 入口用 _require_live_browser() 直接拒绝：这两步复用 enter_checkout 留存的 Page， 重建救不回服务端订单草稿，而 pay 跑的时候订单已经真的提交了。顺带修掉一个 误诊——浏览器死时 submit_order 原先报「未找到确认按钮」，把「浏览器崩了」 说成「站点改版了」，两者的处置方式完全不同。 - runner 把 BrowserDeadError 转 needs_human 而非 failed，except 分支排在 except AppError 之前（子类，顺序反了就报 failed）。掉线发生在动作中途， 站点侧生效与否无从判断，不能给上游「明确失败」的结论。 /health 暴露 browser 状态，掉线时 degraded + HTTP 503 + code 5006，让 Dockerfile.trading 的 HEALTHCHECK 探到并重启容器。重启不会导致重复下单：网关侧 任务绝不自动重投，租约过期只置 stale 等人工 reclaim（docs/order-gateway.md §5）， 重启只是恢复领新任务的能力。启动窗口期 started=False 不算掉线。 README 错误码表补 5005（此前遗漏）与 5006。 新增 15 个用例覆盖探活三态、is_connected() 自身抛错、边界重建/不重建/丢弃残留页/ 重建失败、两个包装壳的分支、submit/pay 拒绝、runner 转 needs_human、/health 503。 真实 Chromium 崩溃无法在离线测试里制造，用例模拟的是 is_connected() 返回 False 这个唯一可观测信号，覆盖的是代码对该信号的反应而非崩溃本身；容器 HEALTHCHECK 真的触发重启这条链路尚未实跑验证。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
+230
-2
@@ -20,6 +20,7 @@ import pytest
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from app.shared.errors import (
|
||||
BrowserDeadError,
|
||||
CartOperationError,
|
||||
InvalidRequestError,
|
||||
NotLoggedInError,
|
||||
@@ -576,10 +577,24 @@ _ORDER_DETAIL_LANDED_URL = "https://order.my.rakuten.co.jp/purchase-history/?ord
|
||||
_ORDER_LIST_LANDED_URL = "https://order.my.rakuten.co.jp/purchase-history/order-list"
|
||||
|
||||
|
||||
class _FakeBrowser:
|
||||
"""Browser 替身：只实现 _ensure_browser_alive 用到的 is_connected()
|
||||
|
||||
没有它，`_ensure_context_ready` 会判定「浏览器没起来」并尝试真的 launch 一次
|
||||
Chromium。connected 可以翻成 False 来模拟掉线。
|
||||
"""
|
||||
|
||||
def __init__(self, connected: bool = True):
|
||||
self.connected = connected
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
return self.connected
|
||||
|
||||
|
||||
def _build_site(tmp_path: Path, context: _FakeContext, auth: _FakeAuthSession) -> SiteInteractor:
|
||||
"""装配一个不依赖 Playwright 的 SiteInteractor
|
||||
|
||||
auth_state_dir 指向空目录：_refresh_context_if_stale 见不到 storage_state
|
||||
auth_state_dir 指向空目录：_ensure_context_ready 见不到 storage_state
|
||||
文件就直接返回，不会试图重建 context（重建需要真实 browser）。
|
||||
"""
|
||||
from app.shared.config import Settings
|
||||
@@ -588,6 +603,7 @@ def _build_site(tmp_path: Path, context: _FakeContext, auth: _FakeAuthSession) -
|
||||
auth_session=auth, # type: ignore[arg-type]
|
||||
settings=Settings(auth_state_dir=str(tmp_path / "auth"), evidence_dir=str(tmp_path)),
|
||||
)
|
||||
site._browser = _FakeBrowser() # type: ignore[assignment]
|
||||
site._context = context # type: ignore[assignment]
|
||||
return site
|
||||
|
||||
@@ -980,13 +996,16 @@ async def test_check_order_status_still_returns_only_status(tmp_path):
|
||||
|
||||
|
||||
async def test_submit_order_without_checkout_page_raises():
|
||||
"""浏览器活着但没有留存确认页 → OrderOperationError（不是掉线那类错误）"""
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
site._browser = _FakeBrowser() # type: ignore[assignment]
|
||||
with pytest.raises(OrderOperationError):
|
||||
await site.submit_order(_make_task())
|
||||
|
||||
|
||||
async def test_pay_without_checkout_page_raises():
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
site._browser = _FakeBrowser() # type: ignore[assignment]
|
||||
with pytest.raises(OrderOperationError):
|
||||
await site.pay(_make_task(), "ord-1")
|
||||
|
||||
@@ -1314,15 +1333,17 @@ class _FakeLoggedInAuth:
|
||||
def _build_clear_cart_site(*, delete_buttons: int, count_body: str, html: str) -> SiteInteractor:
|
||||
"""构造 clear_cart 可离线跑起来的 SiteInteractor（fake page + fake count API）"""
|
||||
site = SiteInteractor(auth_session=_FakeLoggedInAuth(), settings=None) # type: ignore[arg-type]
|
||||
site._browser = _FakeBrowser() # type: ignore[assignment]
|
||||
site._context = _FakeClearCartContext(
|
||||
page=_FakeClearCartPage(delete_buttons=delete_buttons, html=html),
|
||||
count_body=count_body,
|
||||
) # type: ignore[assignment]
|
||||
|
||||
# settings=None，走不了真实的 mtime 检查；探活那半段由 _FakeBrowser 覆盖
|
||||
async def _noop_refresh() -> None:
|
||||
return None
|
||||
|
||||
site._refresh_context_if_stale = _noop_refresh # type: ignore[assignment]
|
||||
site._ensure_context_ready = _noop_refresh # type: ignore[assignment]
|
||||
return site
|
||||
|
||||
|
||||
@@ -1390,3 +1411,210 @@ async def test_clear_cart_html_capture_failure_keeps_clear_result():
|
||||
|
||||
|
||||
# ---- helper ----
|
||||
|
||||
|
||||
# ---- 浏览器掉线：任务边界就地重建，中途掉线转 BrowserDeadError ----
|
||||
#
|
||||
# 真实的 Chromium 崩溃（容器 /dev/shm 不足、OOM kill、seccomp 挡 sandbox）没法在
|
||||
# 离线测试里制造，这里用 _FakeBrowser.connected 翻成 False 模拟「is_connected()
|
||||
# 返回 False」这个唯一的可观测信号，覆盖的是 SiteInteractor 对该信号的反应。
|
||||
|
||||
|
||||
def _build_dead_browser_site(tmp_path: Path) -> SiteInteractor:
|
||||
"""浏览器已掉线的 SiteInteractor：_browser 在位但 is_connected() 为 False"""
|
||||
from app.shared.config import Settings
|
||||
|
||||
site = SiteInteractor(
|
||||
auth_session=_FakeLoggedInAuth(), # type: ignore[arg-type]
|
||||
settings=Settings(auth_state_dir=str(tmp_path / "auth"), evidence_dir=str(tmp_path)),
|
||||
)
|
||||
site._browser = _FakeBrowser(connected=False) # type: ignore[assignment]
|
||||
return site
|
||||
|
||||
|
||||
def test_browser_alive_reflects_is_connected(tmp_path):
|
||||
"""browser_alive：没起过 / 已掉线 / 正常，三种情况分别为 False/False/True"""
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
assert site.browser_alive is False # 还没 start()
|
||||
|
||||
site._browser = _FakeBrowser(connected=False) # type: ignore[assignment]
|
||||
assert site.browser_alive is False
|
||||
|
||||
site._browser = _FakeBrowser(connected=True) # type: ignore[assignment]
|
||||
assert site.browser_alive is True
|
||||
|
||||
|
||||
def test_browser_alive_survives_is_connected_raising():
|
||||
"""is_connected() 自己抛错（驱动已经没了）也要当掉线处理，不能把异常漏出去"""
|
||||
|
||||
class _ExplodingBrowser:
|
||||
def is_connected(self) -> bool:
|
||||
raise RuntimeError("driver connection closed")
|
||||
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
site._browser = _ExplodingBrowser() # type: ignore[assignment]
|
||||
|
||||
assert site.browser_alive is False
|
||||
|
||||
|
||||
def test_browser_status_distinguishes_not_started_from_dead():
|
||||
"""/health 要能分清「还没启动」和「起过但死了」——只有后者算 degraded"""
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
|
||||
not_started = site.browser_status()
|
||||
assert not_started["started"] is False
|
||||
assert not_started["alive"] is False
|
||||
|
||||
site._browser = _FakeBrowser(connected=False) # type: ignore[assignment]
|
||||
dead = site.browser_status()
|
||||
assert dead["started"] is True
|
||||
assert dead["alive"] is False
|
||||
|
||||
site._browser = _FakeBrowser(connected=True) # type: ignore[assignment]
|
||||
assert site.browser_status() == {
|
||||
"started": True,
|
||||
"alive": True,
|
||||
"pending_checkout_tasks": [],
|
||||
"detail": "连接正常",
|
||||
}
|
||||
|
||||
|
||||
async def test_ensure_browser_alive_relaunches_on_task_boundary(tmp_path):
|
||||
"""任务边界探到掉线 → 重放一遍启动流程，调用方无感知"""
|
||||
site = _build_dead_browser_site(tmp_path)
|
||||
relaunched = []
|
||||
|
||||
async def _fake_launch() -> None:
|
||||
relaunched.append(True)
|
||||
site._browser = _FakeBrowser(connected=True) # type: ignore[assignment]
|
||||
|
||||
site._launch = _fake_launch # type: ignore[assignment]
|
||||
|
||||
await site._ensure_browser_alive()
|
||||
|
||||
assert relaunched == [True]
|
||||
assert site.browser_alive is True
|
||||
|
||||
|
||||
async def test_ensure_browser_alive_is_noop_when_connected(tmp_path):
|
||||
"""浏览器好好的就不能重建——重建一次要几秒，还会丢掉当前 context 的 cookie"""
|
||||
site = _build_dead_browser_site(tmp_path)
|
||||
site._browser = _FakeBrowser(connected=True) # type: ignore[assignment]
|
||||
|
||||
async def _fail_launch() -> None:
|
||||
raise AssertionError("浏览器连接正常时不应重建")
|
||||
|
||||
site._launch = _fail_launch # type: ignore[assignment]
|
||||
|
||||
await site._ensure_browser_alive()
|
||||
|
||||
|
||||
async def test_ensure_browser_alive_drops_orphaned_checkout_pages(tmp_path):
|
||||
"""重建前丢掉残留的确认页：那些 Page 已随浏览器一起没了，留着必然报错"""
|
||||
site = _build_dead_browser_site(tmp_path)
|
||||
site._checkout_pages["t-stuck"] = object() # type: ignore[assignment]
|
||||
|
||||
async def _fake_launch() -> None:
|
||||
site._browser = _FakeBrowser(connected=True) # type: ignore[assignment]
|
||||
|
||||
site._launch = _fake_launch # type: ignore[assignment]
|
||||
|
||||
await site._ensure_browser_alive()
|
||||
|
||||
assert site._checkout_pages == {}
|
||||
|
||||
|
||||
async def test_ensure_browser_alive_raises_browser_dead_when_relaunch_fails(tmp_path):
|
||||
"""重建也起不来 → BrowserDeadError（AppError，runner 接得住），并把半成品收干净"""
|
||||
site = _build_dead_browser_site(tmp_path)
|
||||
|
||||
async def _broken_launch() -> None:
|
||||
raise RuntimeError("chromium 起不来")
|
||||
|
||||
site._launch = _broken_launch # type: ignore[assignment]
|
||||
|
||||
with pytest.raises(BrowserDeadError):
|
||||
await site._ensure_browser_alive()
|
||||
|
||||
assert site._browser is None # 已 teardown，下次调用从头再试
|
||||
|
||||
|
||||
async def test_new_page_wraps_target_closed_into_app_error():
|
||||
"""中途掉线的裸 TargetClosedError 必须被包成 AppError
|
||||
|
||||
这是整条链最关键的一环：不包的话异常会穿过 runner 的 except AppError，
|
||||
落到主循环那个只记日志的兜底里——任务一次都不上报，网关要干等租约过期。
|
||||
"""
|
||||
|
||||
class _DeadContext:
|
||||
async def new_page(self):
|
||||
raise RuntimeError("Target page, context or browser has been closed")
|
||||
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
site._context = _DeadContext() # type: ignore[assignment]
|
||||
|
||||
with pytest.raises(BrowserDeadError) as excinfo:
|
||||
await site._new_page()
|
||||
|
||||
from app.shared.errors import AppError
|
||||
|
||||
assert isinstance(excinfo.value, AppError)
|
||||
assert excinfo.value.err_code == 5006
|
||||
|
||||
|
||||
async def test_request_keeps_original_error_when_browser_is_alive():
|
||||
"""浏览器活着时的请求失败是正常业务失败，不能被误标成掉线"""
|
||||
|
||||
class _FlakyRequest:
|
||||
async def get(self, url: str, **kwargs):
|
||||
raise RuntimeError("站点 502")
|
||||
|
||||
class _Context:
|
||||
request = _FlakyRequest()
|
||||
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
site._browser = _FakeBrowser(connected=True) # type: ignore[assignment]
|
||||
site._context = _Context() # type: ignore[assignment]
|
||||
|
||||
with pytest.raises(RuntimeError, match="站点 502"):
|
||||
await site._request("get", "https://example.test/")
|
||||
|
||||
|
||||
async def test_request_converts_to_browser_dead_when_browser_is_gone():
|
||||
"""同样的请求失败，浏览器确实没了时才改写成 BrowserDeadError"""
|
||||
|
||||
class _FlakyRequest:
|
||||
async def get(self, url: str, **kwargs):
|
||||
raise RuntimeError("Target closed")
|
||||
|
||||
class _Context:
|
||||
request = _FlakyRequest()
|
||||
|
||||
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
|
||||
site._browser = _FakeBrowser(connected=False) # type: ignore[assignment]
|
||||
site._context = _Context() # type: ignore[assignment]
|
||||
|
||||
with pytest.raises(BrowserDeadError):
|
||||
await site._request("get", "https://example.test/")
|
||||
|
||||
|
||||
async def test_submit_order_reports_browser_dead_before_blaming_selectors(tmp_path):
|
||||
"""浏览器死时 submit_order 要报掉线，而不是「找不到确认按钮」
|
||||
|
||||
留存的 Page 是个死对象（不是 None），不先探活的话 selector 轮询会全部超时，
|
||||
最后把「浏览器崩了」误诊成「站点改版了」——这两个结论的处置方式完全不同。
|
||||
"""
|
||||
site = _build_dead_browser_site(tmp_path)
|
||||
site._checkout_pages["t1"] = object() # type: ignore[assignment]
|
||||
|
||||
with pytest.raises(BrowserDeadError):
|
||||
await site.submit_order(_make_task(task_id="t1"))
|
||||
|
||||
|
||||
async def test_pay_reports_browser_dead(tmp_path):
|
||||
"""pay 之前 submit_order 已经真的下过单了，掉线必须当场抛出去交人工"""
|
||||
site = _build_dead_browser_site(tmp_path)
|
||||
site._checkout_pages["t1"] = object() # type: ignore[assignment]
|
||||
|
||||
with pytest.raises(BrowserDeadError):
|
||||
await site.pay(_make_task(task_id="t1"), "ord-1")
|
||||
|
||||
@@ -85,6 +85,7 @@ class StubSiteInteractor:
|
||||
self.clear_calls = 0
|
||||
self.remove_calls: list[str] = []
|
||||
self.fail_with: Exception | None = None
|
||||
self.browser_alive = True
|
||||
|
||||
async def add_to_cart_payload(
|
||||
self,
|
||||
@@ -129,6 +130,15 @@ class StubSiteInteractor:
|
||||
raise self.fail_with
|
||||
return {"removed": True, "item_id": item_id}
|
||||
|
||||
def browser_status(self) -> dict:
|
||||
"""/health 读的浏览器状态；browser_alive 可翻成 False 模拟掉线"""
|
||||
return {
|
||||
"started": True,
|
||||
"alive": self.browser_alive,
|
||||
"pending_checkout_tasks": [],
|
||||
"detail": "连接正常" if self.browser_alive else "浏览器已掉线",
|
||||
}
|
||||
|
||||
async def close(self) -> None:
|
||||
"""lifespan 收尾会调用；桩没有真实浏览器要关"""
|
||||
|
||||
@@ -168,6 +178,26 @@ def test_health_needs_no_token(client):
|
||||
body = response.json()
|
||||
assert body["data"]["status"] == "ok"
|
||||
assert set(body["data"]["auth"]) == {"rakuten"}
|
||||
assert body["data"]["browser"]["alive"] is True
|
||||
|
||||
|
||||
def test_health_returns_503_when_browser_is_dead(client, stub_site):
|
||||
"""浏览器掉线 → 503，让容器 HEALTHCHECK 探到并触发重启
|
||||
|
||||
进程还活着、端口还通，但这个服务的所有站点操作都要靠那一个浏览器，它没了以后
|
||||
每一单都会失败。返回 200 的话 HEALTHCHECK 永远绿灯，缺口就一直挂在那儿。
|
||||
重启是安全的：网关侧任务绝不自动重投（docs/order-gateway.md §5）。
|
||||
"""
|
||||
stub_site.browser_alive = False
|
||||
|
||||
response = client.get("/health")
|
||||
|
||||
assert response.status_code == 503
|
||||
body = response.json()
|
||||
assert body["success"] is False
|
||||
assert body["code"] == 5006 # 与 BrowserDeadError 同码
|
||||
assert body["data"]["status"] == "degraded"
|
||||
assert body["data"]["browser"]["alive"] is False
|
||||
|
||||
|
||||
def test_health_does_not_probe_the_site(client, stub):
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.shared.errors import CheckoutBlockedError, OrderGuardError
|
||||
from app.shared.errors import BrowserDeadError, CheckoutBlockedError, OrderGuardError
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
from app.trading.worker import verify
|
||||
from app.trading.worker.evidence import EvidenceStore
|
||||
@@ -352,6 +352,41 @@ async def test_checkout_blocked_becomes_needs_human(
|
||||
assert "风控" in terminal["detail"]
|
||||
|
||||
|
||||
# ---- 浏览器掉线 → needs_human（不是 failed）----
|
||||
|
||||
|
||||
async def test_browser_dead_becomes_needs_human(runner: WorkerRunner):
|
||||
"""执行途中浏览器掉线 → BrowserDeadError → 转 needs_human
|
||||
|
||||
两件事同时被这个用例守着：
|
||||
|
||||
1. BrowserDeadError 是 AppError 的子类，能被 _execute_with_renewal 接住并**上报**。
|
||||
不是 AppError 的话它会一路飘到主循环的兜底日志里，任务一次都不上报，网关侧
|
||||
要干等整个 lease_ttl 才被 sweep 置 stale。
|
||||
2. 结论是 needs_human 而不是 failed——`except BrowserDeadError` 必须排在
|
||||
`except AppError` 前面，顺序反了这个断言就会挂。浏览器是在动作中途没的，
|
||||
站点侧生效与否无从判断，不能给上游一个「明确失败」的结论。
|
||||
"""
|
||||
|
||||
async def _noop(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _browser_died(task): # noqa: ANN001
|
||||
raise BrowserDeadError("pay：浏览器已掉线，本单是否已提交无法判断，需人工核对订单列表")
|
||||
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _noop # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _browser_died # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
assert "掉线" in terminal["detail"]
|
||||
|
||||
|
||||
# ---- ラクマ 不在交易范围 → needs_human ----
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user