"""worker 主循环测试 不启动真实的后台任务,直接构造 WorkerRunner 与桩 gateway client / local_db / evidence store / site,覆盖 §6 主循环的分支: - 已完成的任务再次被领:直接补报,不重新执行 - lease_count > 1(恢复领取):走核对分支,verify unknown → needs_human - 站点交互未实现(SiteInteractor 默认抛 NotImplementedError):转 needs_human - 金额守卫拦截:转 needs_human,站点侧无提交动作 每一步的证据文件在 report 之前就已落盘(§9 第 11 条)也在这里验证。 """ from __future__ import annotations import asyncio import json from dataclasses import dataclass, field from pathlib import Path from typing import Any import pytest from opentelemetry.sdk.trace import TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import StatusCode from app.shared.errors import BrowserDeadError, CheckoutBlockedError, OrderGuardError from app.shared.task_state import OrderState, TaskStatus from app.trading.worker import runner as runner_module from app.trading.worker import verify from app.trading.worker.evidence import EvidenceStore from app.trading.worker.local_db import LocalDB from app.trading.worker.models import LeaseTask from app.trading.worker.runner import WorkerRunner from app.trading.worker.site_interact import ( CheckoutSummary, OrderStatusSnapshot, PageSnapshot, SiteInteractor, SubmitOutcome, ) # ---- 桩:网关客户端 ---- @dataclass class FakeGateway: """记录所有调用,可控返回值的网关桩""" reports: list[dict[str, Any]] = field(default_factory=list) renews: int = 0 fail_report_with: Exception | None = None # verify_on_site 恢复核对会调 get_task 拿 created_at;测试默认给一个可解析 # 的时间戳,具体核对分支由 monkeypatch 替换 verify.verify_on_site 控制 task_created_at: str = "2026-01-01T00:00:00Z" async def lease( self, worker_id: str, *, wait: int = 30, site: str | None = None ) -> LeaseTask | None: return None # 主循环测试不通过 lease 喂数据,直接调 handle async def get_task(self, task_id: str) -> dict[str, Any]: return {"task_id": task_id, "created_at": self.task_created_at} async def renew(self, task_id: str, worker_id: str) -> dict[str, Any]: self.renews += 1 return {"task_id": task_id, "lease_expires_at": "2099-01-01T00:00:00Z"} async def report( self, task_id: str, worker_id: str, *, state: OrderState, payable_yen: int | None = None, pay_deadline: str | None = None, site_order_id: str | None = None, evidence_ref: str | None = None, detail: str = "", terminal: bool = False, terminal_status: TaskStatus | None = None, ) -> dict[str, Any]: if self.fail_report_with is not None: raise self.fail_report_with record = { "task_id": task_id, "worker_id": worker_id, "state": state, "payable_yen": payable_yen, "pay_deadline": pay_deadline, "site_order_id": site_order_id, "evidence_ref": evidence_ref, "detail": detail, "terminal": terminal, "terminal_status": terminal_status, } self.reports.append(record) return {"task_id": task_id, "status": "running", "recorded": True} def last_terminal_report(self) -> dict[str, Any]: for r in reversed(self.reports): if r["terminal"]: return r raise AssertionError("没有终态 report") # ---- fixtures ---- @pytest.fixture async def local_db(tmp_path: Path) -> LocalDB: db = LocalDB(tmp_path / "trading.db") await db.start() yield db await db.close() @pytest.fixture def evidence(tmp_path: Path) -> EvidenceStore: return EvidenceStore(tmp_path / "evidence") @pytest.fixture def site() -> SiteInteractor: """返回**未启动**的 SiteInteractor(不调 start,不依赖 Playwright) 默认 add_to_cart/verify_cart 已实现(但会抛 NotImplementedError 因为没启动), 具体测试可用 monkeypatch.setattr(site, "method", fn) 替换为桩。 """ # 用最小化的 auth_session / settings 占位;测试不会真的调它们 return SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type] @pytest.fixture def runner(tmp_path, local_db, evidence, site) -> WorkerRunner: """构造一个 worker_id=test、网关 URL 已配置的 runner 使用 fake gateway;settings 取真实 Settings 但 worker_id 注入为 test。 """ class _FakeSettings: worker_id = "test" order_max_total_yen = 30000 # 测试环境不真的等 3 小时:间隔设 0,次数设小,_monitor_order 的循环 # 靠 asyncio.sleep(0) 立刻推进,不拖慢测试 order_monitor_poll_interval_seconds = 0 order_monitor_max_checks = 5 @property def worker_id_effective(self) -> str: return "test" gateway = FakeGateway() r = WorkerRunner( settings=_FakeSettings(), # type: ignore[arg-type] gateway_client=gateway, # type: ignore[arg-type] local_db=local_db, evidence=evidence, site=site, ) # 把 gateway 桩挂到 runner 上,测试通过它断言调用 r._gateway_for_test = gateway # type: ignore[attr-defined] return r def _make_task( *, task_id: str = "t1", site: str = "rakuten", lease_count: int = 1, known_state: str | None = None, intent: dict | None = None, ) -> LeaseTask: return LeaseTask( task_id=task_id, site=site, intent=intent or {}, lease_expires_at="2099-01-01T00:00:00Z", lease_count=lease_count, known_state=known_state, ) def _set_site_clear(site, result: dict) -> None: """把 site.clear_cart 替换成返回指定结果的桩 runner.execute() 现在开单一律先 clear_cart(step 0,见 runner 里注释), 测试若不复用它,site 桩会真的去调未启动的 SiteInteractor 而崩。 """ async def _f() -> dict: return result site.clear_cart = _f # type: ignore[assignment] # ---- 已完成的任务:补报而不重新执行(§9 第 9 条)---- async def test_already_finished_task_is_reported_not_re_executed( runner: WorkerRunner, local_db: LocalDB ): """本地已完成 → 直接补报 final_state,不进入 execute""" await local_db.ensure_started("t1", "rakuten", {"k": "v"}) await local_db.mark_finished("t1", OrderState.PAID.value) await runner.handle(_make_task(task_id="t1")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["state"] == OrderState.PAID assert terminal["terminal"] is True # 没有进入 execute:site 不应被调用,但为保险起见,我们只看 # evidence 目录里没有任何步骤文件 evidence_files = list(local_db._db_path.parent.parent.glob("**/*.html")) # noqa # 上面 glob 会扫到 trading.db 之外的所有 .html;这里只关心 evidence 目录 # (runner 走 execute 才会写证据) # ---- lease_count > 1:核对分支(§9 第 8 条)---- async def test_recovery_task_with_unknown_verdict_reports_needs_human( runner: WorkerRunner, local_db: LocalDB, monkeypatch ): """lease_count > 1 + verify 返回 unknown(intent 缺 item_url)→ 转 needs_human,不执行""" # verify.verify_on_site 已是真实实现(2026-08-13),但 _make_task 默认 intent 为空, # 缺 item_url 时函数第一步就短路返回 unknown,不用 monkeypatch 也能测这条分支 await runner.handle(_make_task(task_id="t1", lease_count=2)) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "unknown" in terminal["detail"] assert "item_url" in terminal["detail"] async def test_recovery_task_with_already_ordered_verdict_reports_succeeded( runner: WorkerRunner, local_db: LocalDB, monkeypatch ): """lease_count > 1 + verify 返回 already_ordered → 补报 succeeded,不重新下单""" async def _already_ordered(task, **kwargs): # noqa: ANN001 return verify.VerifyResult( verdict=verify.VerifyVerdict.ALREADY_ORDERED, site_order_id="ord-1", detail="订单列表命中", ) monkeypatch.setattr(verify, "verify_on_site", _already_ordered) await runner.handle(_make_task(task_id="t1", lease_count=2, known_state="ordered")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["state"] == OrderState.ORDERED assert terminal["site_order_id"] == "ord-1" assert terminal["terminal_status"] == TaskStatus.SUCCEEDED async def test_recovery_task_with_not_ordered_verdict_still_reports_needs_human( runner: WorkerRunner, local_db: LocalDB, monkeypatch ): """lease_count > 1 + verify 确认 not_ordered → 仍转 needs_human,不自动重新下单 刻意的保守选择(见 runner._handle_recovery 里的注释):NOT_ORDERED 分支目前 只有逻辑正确性,没有被真实多单数据验证过,放开自动重新执行前必须显式决定。 """ async def _not_ordered(task, **kwargs): # noqa: ANN001 return verify.VerifyResult( verdict=verify.VerifyVerdict.NOT_ORDERED, detail="已核对任务创建时间之后的全部订单,未找到匹配商品", ) monkeypatch.setattr(verify, "verify_on_site", _not_ordered) await runner.handle(_make_task(task_id="t1", lease_count=2)) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "not_ordered" in terminal["detail"] # needs_human 分支不标记本地完成(与 UNKNOWN 分支一致):只有 ALREADY_ORDERED # 才 mark_finished,避免把「转人工」误当成本地已完成的终态 assert await local_db.has_finished("t1") is False # ---- 站点交互未实现 → needs_human(§10 接口缝)---- async def test_unimplemented_site_interaction_becomes_needs_human( runner: WorkerRunner, local_db: LocalDB ): """站点交互抛 NotImplementedError → runner 转 needs_human add_to_cart / verify_cart / enter_checkout / parse_checkout / submit_order / pay / check_order_status 现在均已实现,这里用桩显式模拟「某一步没实现」, 覆盖 runner._execute_with_renewal 里 NotImplementedError → needs_human 的分支, 与具体哪个方法真的未实现解耦。 """ async def _noop(task): # noqa: ANN001 return None async def _checkout_html(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"fake-checkout-png") async def _unimplemented(html): # noqa: ANN001 raise NotImplementedError("模拟:假装 parse_checkout 还没实现") runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _checkout_html # type: ignore[assignment] runner._site.parse_checkout = _unimplemented # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) await runner.handle(_make_task(task_id="t1")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "未实现" in terminal["detail"] # ---- 结算被站点风控拦截 → needs_human(规格 §10.1)---- async def test_checkout_blocked_becomes_needs_human( runner: WorkerRunner, local_db: LocalDB ): """enter_checkout 抛 CheckoutBlockedError(session upgrade 被拦截)→ 转 needs_human 与普通 AppError(转 FAILED)区分开:这类阻断是站点主动要求人工验证, 不是本服务的失败,需要人工用有头浏览器接管当前登录态完成验证。 """ async def _noop(task): # noqa: ANN001 return None async def _blocked(task): # noqa: ANN001 raise CheckoutBlockedError("session upgrade 提交密码后 30s 内 URL 未变化,判定被站点风控拦截") runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _blocked # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) await runner.handle(_make_task(task_id="t1")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "风控" in terminal["detail"] # ---- 浏览器掉线 → needs_human(不是 failed)---- async def test_browser_dead_becomes_needs_human(runner: WorkerRunner): """执行途中浏览器掉线 → BrowserDeadError → 转 needs_human 两件事同时被这个用例守着: 1. BrowserDeadError 是 AppError 的子类,能被 _execute_with_renewal 接住并**上报**。 不是 AppError 的话它会一路飘到主循环的兜底日志里,任务一次都不上报,网关侧 要干等整个 lease_ttl 才被 sweep 置 stale。 2. 结论是 needs_human 而不是 failed——`except BrowserDeadError` 必须排在 `except AppError` 前面,顺序反了这个断言就会挂。浏览器是在动作中途没的, 站点侧生效与否无从判断,不能给上游一个「明确失败」的结论。 """ async def _noop(task): # noqa: ANN001 return None async def _browser_died(task): # noqa: ANN001 raise BrowserDeadError("pay:浏览器已掉线,本单是否已提交无法判断,需人工核对订单列表") runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _browser_died # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) await runner.handle(_make_task(task_id="t1")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "掉线" in terminal["detail"] # ---- ラクマ 不在交易范围 → needs_human ---- async def test_rakuma_site_reports_needs_human(runner: WorkerRunner): """交易服务只对乐天;其他站点(含 ラクマ)直接转人工""" await runner.handle(_make_task(task_id="t1", site="rakuma")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN # ---- 金额守卫:拦截而不提交(§9 第 10 条)---- async def test_amount_guard_blocks_when_payable_exceeds_limit( runner: WorkerRunner, local_db: LocalDB ): """实际应付超过上限 → OrderGuardError → 转 needs_human,站点侧无提交动作 模拟站点交互:前两步(add_to_cart / verify_cart)正常,enter_checkout 返回 html, parse_checkout 返回超额金额。submit_order 永远不应被调用。 """ call_log: list[str] = [] async def _ok(task): # noqa: ANN001 call_log.append(task.task_id + ":step") async def _checkout_html(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"fake-checkout-png") async def _parse(html: str): return CheckoutSummary(payable_yen=50000) submit_called = False async def _submit(task): # noqa: ANN001 nonlocal submit_called submit_called = True return SubmitOutcome( site_order_id="ord-1", evidence=PageSnapshot(html="complete", screenshot=b"fake-submit-png"), ) runner._site.add_to_cart = _ok # type: ignore[assignment] runner._site.verify_cart = _ok # type: ignore[assignment] runner._site.enter_checkout = _checkout_html # type: ignore[assignment] runner._site.parse_checkout = _parse # type: ignore[assignment] runner._site.submit_order = _submit # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) # intent.max_total_yen 缺省,回落到 settings.order_max_total_yen=30000, # 实际 50000 > 30000 → 拦截 await runner.handle(_make_task(task_id="t1")) assert submit_called is False # 关键:站点侧无提交动作 gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "上限" in terminal["detail"] async def test_amount_guard_honors_intent_max_total_yen( runner: WorkerRunner, local_db: LocalDB ): """intent 显式给 max_total_yen 时优先于全局上限""" async def _noop(task): # noqa: ANN001 return None async def _checkout_html(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"fake-checkout-png") async def _parse(html: str): return CheckoutSummary(payable_yen=8000) submit_called = False async def _submit(task): # noqa: ANN001 nonlocal submit_called submit_called = True raise NotImplementedError("不应到这里") # parse 之后的步骤未实现,到这里即说明守卫过了 runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _checkout_html # type: ignore[assignment] runner._site.parse_checkout = _parse # type: ignore[assignment] runner._site.submit_order = _submit # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) # 全局上限 30000,但 intent 给 5000 → 实际 8000 > 5000 → 拦截 await runner.handle(_make_task(task_id="t1", intent={"max_total_yen": 5000})) assert submit_called is False gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN # ---- 开单前清理购物车(step 0):上一单失败残留不被新单买走 ---- async def test_execute_clears_cart_before_adding( runner: WorkerRunner, local_db: LocalDB ): """开单一律先 clear_cart,且必须先于 add_to_cart 发生 上一单若在下单前失败(加购后、提交前),残留不会自动消失;不清掉就直接 加购会把新商品叠在旧商品上,结算时把上一单的一起买走。这里断言顺序: clear 在 add_to_cart 之前,且 clear 返回 count=0 才继续往下加购。 """ call_order: list[str] = [] async def _clear(): # noqa: ANN001 call_order.append("clear") return {"removed_count": 0, "cart_count": 0} async def _noop(task): # noqa: ANN001 call_order.append("add") return None async def _verify(task): # noqa: ANN001 return None async def _checkout_html(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"fake-checkout-png") async def _parse(html: str): return CheckoutSummary(payable_yen=297) async def _submit(task): # noqa: ANN001 return SubmitOutcome( site_order_id="ord-1", evidence=PageSnapshot(html="complete", screenshot=b"fake-submit-png"), ) async def _pay(task, site_order_id): # noqa: ANN001 return None async def _unchanged(order_id: str): # noqa: ANN001 # 不要让后台监控真的落到未启动的 SiteInteractor 上:found=True 但状态无 # 变化(order_state=None),监控循环会一直轮询到上限退出,不污染断言 return OrderStatusSnapshot(found=True, order_state=None) runner._site.clear_cart = _clear # type: ignore[assignment] runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _verify # type: ignore[assignment] runner._site.enter_checkout = _checkout_html # type: ignore[assignment] runner._site.parse_checkout = _parse # type: ignore[assignment] runner._site.submit_order = _submit # type: ignore[assignment] runner._site.pay = _pay # type: ignore[assignment] runner._site.check_order_status = _unchanged # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) assert call_order == ["clear", "add"], f"clear 必须在 add 之前:{call_order}" async def test_execute_blocks_when_cart_not_empty_after_clear( runner: WorkerRunner, local_db: LocalDB ): """清理后 count 仍非 0 → 视为清理没跑干净,转 needs_human,绝不往下加购 残留可能属于上一单失败留下的商品,带着它加购/结算会把上一单的一起买走。 按闸门语义宁可卡住等人核对,也不赌「残留不会被买走」;关键断言是 add_to_cart/submit_order 都未被调用。 """ any_add_or_submit = {"called": False} async def _clear(): # noqa: ANN001 return {"removed_count": 2, "cart_count": 2} # 清理后仍剩 2 件 async def _add(task): # noqa: ANN001 any_add_or_submit["called"] = True return None runner._site.clear_cart = _clear # type: ignore[assignment] runner._site.add_to_cart = _add # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) assert any_add_or_submit["called"] is False # 关键:没有加购、更没有提交 gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "未清空" in terminal["detail"] async def test_execute_blocks_when_clear_count_unknown( runner: WorkerRunner, local_db: LocalDB ): """清理后 count API 拿不到结果(cart_count=-1)→ 同样按未清空处理 无法确认购物车已空,就不能带着残留往下加购。clear_cart 在末尾 count API 失败时返回 cart_count=-1——此时不能默认它是 0(那等于把「确认已空」当成 「可能非空」),转 needs_human 让现场可查。 """ add_called = False async def _clear(): # noqa: ANN001 return {"removed_count": 0, "cart_count": -1} async def _add(task): # noqa: ANN001 nonlocal add_called add_called = True return None runner._site.clear_cart = _clear # type: ignore[assignment] runner._site.add_to_cart = _add # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) assert add_called is False gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN assert "未清空" in terminal["detail"] async def test_clear_cart_step_writes_evidence_but_no_gateway_report( runner: WorkerRunner, evidence: EvidenceStore ): """step 0 落本地证据:清理后的页面 HTML 与 meta.json 在盘上,但不回报 gateway clear_cart 返回的 html 是清理结束后的 cart 页现场,meta 记 removed_count / cart_count;它仍是本机卫生步骤——evidence_index 可翻查,gateway 收不到 这一步的 evidence_ref。 """ async def _clear(): # noqa: ANN001 return { "removed_count": 1, "cart_count": 0, "html": "cart", "screenshot": b"fake-clear-png", } async def _noop(task): # noqa: ANN001 return None async def _unimplemented(task): # noqa: ANN001 # 流程走到 enter_checkout 为止:step 0 的证据此时已经落盘 raise NotImplementedError("模拟:enter_checkout 未实现,到此为止") runner._site.clear_cart = _clear # type: ignore[assignment] runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _unimplemented # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) step_dir = evidence.step_dir("t1") html_text = (step_dir / "00-cart-clear.html").read_text(encoding="utf-8") assert html_text == "cart" assert (step_dir / "00-cart-clear.png").read_bytes() == b"fake-clear-png" meta = json.loads((step_dir / "00-cart-clear.meta.json").read_text(encoding="utf-8")) assert meta == {"step": "cart-clear", "removed_count": 1, "cart_count": 0} # 本机卫生步骤:gateway 的任何 report 都不携带 step-0 的 evidence_ref gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] assert all(r["evidence_ref"] != "t1/00-cart-clear" for r in gateway.reports) async def test_clear_cart_evidence_written_when_guard_blocks( runner: WorkerRunner, evidence: EvidenceStore ): """闸门拦截(清理后 cart_count 非 0)时 step-0 证据同样已落盘 被拦转 needs_human 时这份现场就是排查依据——证据先于闸门判断落盘。 """ async def _clear(): # noqa: ANN001 return { "removed_count": 2, "cart_count": 2, "html": "leftover", "screenshot": b"fake-leftover-png", } runner._site.clear_cart = _clear # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) step_dir = evidence.step_dir("t1") assert (step_dir / "00-cart-clear.html").exists() assert (step_dir / "00-cart-clear.png").read_bytes() == b"fake-leftover-png" meta = json.loads((step_dir / "00-cart-clear.meta.json").read_text(encoding="utf-8")) assert meta["cart_count"] == 2 async def test_every_order_step_writes_html_png_and_meta( runner: WorkerRunner, evidence: EvidenceStore ): """下单流程每一步(step 0 清车 + step 1-5)都落 html + png + meta 三件套 排查问题时每一步都要有现场可看:页面 HTML、整页截图、步骤 meta 缺一不可。 """ async def _clear(): # noqa: ANN001 return { "removed_count": 0, "cart_count": 0, "html": "cart", "screenshot": b"png-0", } async def _add(task): # noqa: ANN001 return PageSnapshot(html="added", screenshot=b"png-1") async def _verify(task): # noqa: ANN001 return PageSnapshot(html="verified", screenshot=b"png-2") async def _checkout(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"png-3") async def _parse(html: str): return CheckoutSummary(payable_yen=297) async def _submit(task): # noqa: ANN001 return SubmitOutcome( site_order_id="ord-1", evidence=PageSnapshot(html="done", screenshot=b"png-4"), ) async def _pay(task, site_order_id): # noqa: ANN001 return PageSnapshot(html="paid", screenshot=b"png-5") async def _unchanged(order_id: str): # noqa: ANN001 # 监控不污染断言:found=True 但无状态变化,轮询到上限自行退出 return OrderStatusSnapshot(found=True, order_state=None) runner._site.clear_cart = _clear # type: ignore[assignment] runner._site.add_to_cart = _add # type: ignore[assignment] runner._site.verify_cart = _verify # type: ignore[assignment] runner._site.enter_checkout = _checkout # type: ignore[assignment] runner._site.parse_checkout = _parse # type: ignore[assignment] runner._site.submit_order = _submit # type: ignore[assignment] runner._site.pay = _pay # type: ignore[assignment] runner._site.check_order_status = _unchanged # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) step_dir = evidence.step_dir("t1") expected = { "00-cart-clear": b"png-0", "01-cart-add": b"png-1", "02-cart-check": b"png-2", "03-order-confirm": b"png-3", "04-order-submit": b"png-4", "05-payment": b"png-5", } for stem, png in expected.items(): assert (step_dir / f"{stem}.png").read_bytes() == png, f"{stem} 缺整页截图" assert (step_dir / f"{stem}.html").exists(), f"{stem} 缺页面 HTML" assert (step_dir / f"{stem}.meta.json").exists(), f"{stem} 缺 meta" # ---- 证据在 report 之前落盘(§9 第 11 条)---- async def test_evidence_files_exist_before_each_report( runner: WorkerRunner, evidence: EvidenceStore ): """每个 _run_step 调用:动作 → 落证据 → 写本地 → 回报 gateway 用一个 hook 把 report 时机抓出来,断言此时该 step 的 meta.json 已在盘上。 """ seen_evidence_at_report: list[bool] = [] base = evidence.step_dir("t1") real_report = runner._gateway_for_test.report # type: ignore[attr-defined] async def _spy_report(*args, **kwargs): # noqa: ANN001, ANN002, ANN003 # 在 report 触发时检查 evidence_ref 指向的 meta 文件是否已落盘 ev_ref = kwargs.get("evidence_ref") if ev_ref: meta_path = base / (ev_ref.split("/", 1)[-1] + ".meta.json") seen_evidence_at_report.append(meta_path.exists()) return await real_report(*args, **kwargs) runner._gateway_for_test.report = _spy_report # type: ignore[attr-defined] # 让 add_to_cart / verify_cart / enter_checkout 正常,parse_checkout 抛 # NotImplemented(模拟某一步没实现),触发前几步写证据 + report,最终转 needs_human async def _noop(task): # noqa: ANN001 return None async def _checkout_html(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"fake-checkout-png") async def _unimplemented(html): # noqa: ANN001 raise NotImplementedError("模拟:假装 parse_checkout 还没实现") runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _checkout_html # type: ignore[assignment] runner._site.parse_checkout = _unimplemented # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) await runner.handle(_make_task(task_id="t1")) # 至少有一个 step 调了 report,且每次 report 之前证据都在 assert seen_evidence_at_report, "应当至少有一次带 evidence_ref 的 report" assert all(seen_evidence_at_report), "某次 report 之前证据文件未落盘" # ---- 付款后监控(规格 §6「付款后监控」)---- async def test_successful_order_spawns_monitor_task_and_cancel_monitors_cleans_up( runner: WorkerRunner, local_db: LocalDB ): """付款成功后 execute() 应起一个后台监控任务、不阻塞返回;cancel_monitors 能干净收尾 check_order_status 桩故意永不返回(挂在一个手动控制的 Event 上),模拟「监控任务 还在跑」这个可观察状态——用来确认 execute() 没有同步 await 监控循环 (不然 handle() 早该被这个永不返回的调用卡死,测试会超时而不是正常结束)。 """ async def _noop(task): # noqa: ANN001 return None async def _checkout_html(task): # noqa: ANN001 return PageSnapshot(html="checkout", screenshot=b"fake-checkout-png") async def _parse(html: str): return CheckoutSummary(payable_yen=297) async def _submit(task): # noqa: ANN001 return SubmitOutcome( site_order_id="306087-20260813-0863947697", evidence=PageSnapshot(html="complete", screenshot=b"fake-submit-png"), ) async def _pay(task, site_order_id): # noqa: ANN001 return None never_resolves = asyncio.Event() async def _hang_check(order_id: str): # noqa: ANN001 await never_resolves.wait() runner._site.add_to_cart = _noop # type: ignore[assignment] runner._site.verify_cart = _noop # type: ignore[assignment] runner._site.enter_checkout = _checkout_html # type: ignore[assignment] runner._site.parse_checkout = _parse # type: ignore[assignment] runner._site.submit_order = _submit # type: ignore[assignment] runner._site.pay = _pay # type: ignore[assignment] runner._site.check_order_status = _hang_check # type: ignore[assignment] _set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0}) await runner.handle(_make_task(task_id="t1")) gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] terminal = gateway.last_terminal_report() assert terminal["state"] == OrderState.PAID assert terminal["terminal_status"] == TaskStatus.SUCCEEDED assert len(runner._monitor_tasks) == 1 await runner.cancel_monitors() assert len(runner._monitor_tasks) == 0 async def test_monitor_order_reports_state_changes_and_stops_at_delivered( runner: WorkerRunner, evidence: EvidenceStore ): """状态没变化不重复 report;到「配達完了」立刻停止轮询,不再多探测一次""" snapshots = [ OrderStatusSnapshot(found=False), OrderStatusSnapshot(found=True, stage_label="ショップ", order_state=None, html="shop"), OrderStatusSnapshot( found=True, stage_label="出荷", order_state=OrderState.SHIPPED, html="shipped" ), OrderStatusSnapshot( found=True, stage_label="配達完了", order_state=OrderState.DELIVERED, html="delivered" ), ] calls: list[OrderStatusSnapshot] = [] async def _check(order_id: str): # noqa: ANN001 calls.append(snapshots[len(calls)]) return calls[-1] runner._site.check_order_status = _check # type: ignore[assignment] await runner._monitor_order(_make_task(task_id="t1"), "306087-20260813-0863947697") assert len(calls) == 4 # 配達完了那次之后立刻返回,不会再多轮询一次 gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] reported_states = [r["state"] for r in gateway.reports] assert reported_states == [OrderState.SHIPPED, OrderState.DELIVERED] assert all(r["terminal"] is False for r in gateway.reports) # 追加上报,不重新终结任务 base = evidence.step_dir("t1") assert (base / "06-monitor-shipped.html").exists() assert (base / "07-monitor-delivered.html").exists() async def test_monitor_order_stops_after_max_checks_without_finding_order( runner: WorkerRunner, ): """一直查不到订单(found=False):轮询到上限后正常退出,不报错、不 report""" async def _check(order_id: str): # noqa: ANN001 return OrderStatusSnapshot(found=False) runner._site.check_order_status = _check # type: ignore[assignment] await runner._monitor_order(_make_task(task_id="t1"), "ord-1") gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] assert gateway.reports == [] # ---- 链路追踪:一个任务一条 trace ---- @pytest.fixture def spans(monkeypatch) -> InMemorySpanExporter: """把 runner 模块级 tracer 换成写内存的 不动全局 provider:OTel 的全局 provider 只允许设置一次,测试间共享会互相污染。 """ exporter = InMemorySpanExporter() provider = TracerProvider() provider.add_span_processor(SimpleSpanProcessor(exporter)) monkeypatch.setattr(runner_module, "tracer", provider.get_tracer("test")) return exporter def _stub_happy_path(runner: WorkerRunner) -> None: """把整条下单流程替换成桩,让 execute() 一路走到付款完成""" async def _clear(): return {"removed_count": 0, "cart_count": 0, "html": "", "screenshot": b"p"} async def _snapshot(*args, **kwargs): return PageSnapshot(html="", screenshot=b"p") async def _parse(html: str): return CheckoutSummary(payable_yen=297) async def _submit(task): return SubmitOutcome( site_order_id="ord-1", evidence=PageSnapshot(html="", screenshot=b"p") ) async def _unchanged(order_id: str): return OrderStatusSnapshot(found=True, order_state=None) runner._site.clear_cart = _clear # type: ignore[assignment] runner._site.add_to_cart = _snapshot # type: ignore[assignment] runner._site.verify_cart = _snapshot # type: ignore[assignment] runner._site.enter_checkout = _snapshot # type: ignore[assignment] runner._site.parse_checkout = _parse # type: ignore[assignment] runner._site.submit_order = _submit # type: ignore[assignment] runner._site.pay = _snapshot # type: ignore[assignment] runner._site.check_order_status = _unchanged # type: ignore[assignment] async def test_order_task_emits_one_trace_with_a_span_per_step( runner: WorkerRunner, spans: InMemorySpanExporter ): """一笔下单 = 一条 trace,每个步骤一个子 span,全部挂在任务根 span 底下 worker 是后台 asyncio 任务,没有 HTTP 入口。不开这个根 span 的话,下游每次 出站调用都各自成为孤立 trace——观测后台上就只剩「一堆请求记录」,看不出 这是同一笔单、也看不出卡在哪一步。这条用例把链路结构钉住。 """ _stub_happy_path(runner) await runner.handle(_make_task(task_id="t1", intent={"item_url": "https://x/y/", "quantity": 2})) finished = spans.get_finished_spans() root = next(s for s in finished if s.name == "order.task") steps = [s for s in finished if s.name.startswith("order.step.")] # 步骤 span 覆盖整条流水,且顺序与执行顺序一致 assert [s.name for s in steps] == [ "order.step.cart-add", "order.step.cart-check", "order.step.order-confirm", "order.step.order-submit", "order.step.payment", ] # 每个步骤都挂在任务根底下,同一条 trace for step in steps: assert step.parent.span_id == root.context.span_id assert step.context.trace_id == root.context.trace_id assert root.attributes["order.task_id"] == "t1" assert root.attributes["order.route"] == "execute" assert root.attributes["order.item_url"] == "https://x/y/" # 证据路径落在 span 上:看到失败步骤能直接找到现场 assert steps[0].attributes["order.evidence_ref"] assert steps[3].attributes["order.site_order_id"] == "ord-1" async def test_guard_failure_marks_task_span_as_error( runner: WorkerRunner, spans: InMemorySpanExporter ): """金额守卫拦下时任务根 span 要是 ERROR,并带上对外错误码 闸门拦截会被 _execute_with_renewal 吞掉转 needs_human(不抛出 handle), 所以根 span 很容易显示成功——这里确认它没有。 """ _stub_happy_path(runner) async def _pricey(html: str): return CheckoutSummary(payable_yen=999_999) runner._site.parse_checkout = _pricey # type: ignore[assignment] await runner.handle(_make_task(task_id="t1")) root = next(s for s in spans.get_finished_spans() if s.name == "order.task") assert root.status.status_code is StatusCode.ERROR assert root.attributes["error.type"] == "OrderGuardError" # 提交步骤没有发生:拦在确认页之后、提交之前 names = [s.name for s in spans.get_finished_spans()] assert "order.step.order-submit" not in names async def test_idle_lease_does_not_emit_spans(runner: WorkerRunner, spans: InMemorySpanExporter): """空转的长轮询不产生 span——否则每 30 秒一条孤立 trace 会把观测后台刷满""" runner._running = True async def _lease_then_stop(worker_id: str, *, wait: int = 30, site: str | None = None): runner.stop() # 领到空就停,避免死循环 return None runner._gateway.lease = _lease_then_stop # type: ignore[assignment] await runner.run() assert spans.get_finished_spans() == () async def test_monitor_order_swallows_check_errors_and_keeps_polling( runner: WorkerRunner, ): """单次探测异常(如登录态失效)只记日志继续重试,不会让后台任务崩掉""" call_count = 0 async def _check(order_id: str): # noqa: ANN001 nonlocal call_count call_count += 1 if call_count <= 2: raise RuntimeError("模拟登录态失效") return OrderStatusSnapshot( found=True, stage_label="配達完了", order_state=OrderState.DELIVERED, html="ok" ) runner._site.check_order_status = _check # type: ignore[assignment] await runner._monitor_order(_make_task(task_id="t1"), "ord-1") assert call_count == 3 # 前两次异常被吞掉继续重试,第三次成功拿到 DELIVERED 后停止 gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined] assert [r["state"] for r in gateway.reports] == [OrderState.DELIVERED]