实现 site_interact 加购与购物车校验(Playwright 通道)

探针实测发现 httpx 因 TLS/HTTP2 指纹被 Rakuten 拒认(cookie 有效但
站点不识别 session),site_interact 改为全程 Playwright + storage_state。
add_to_cart 与 verify_cart 已实测可用:从商品页 __INITIAL_STATE__ 抽
purchase 块、调用方补 variant_id 与 choice、POST basketDomain、用 cart
count JSONP API 与 SPA 渲染后的 cart 页校验。enter_checkout 及之后仍
NotImplementedError:Rakuten 对 checkout 要求 session upgrade(重输密码),
是自动 checkout 的硬墙,未实测过墙方案前留接口缝。

SiteInteractor 类持有 Playwright BrowserContext,由 WorkerRunner 实例
持有、container/main 接线、lifespan 管理生命周期。测试侧 23 个用例,
全套件 333 passed,架构边界守住。

附 5 个探针脚本(scripts/probe_*.py),记录实测路径与响应结构。

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-07-27 17:59:30 +08:00
co-authored by Claude Opus 4.6
parent 07107094a7
commit 18cf7ae079
13 changed files with 1929 additions and 74 deletions
+289
View File
@@ -0,0 +1,289 @@
"""site_interact 单元测试
重点测模块级纯函数(_parse_initial_state / _extract_purchase_fields /
_extract_error_message)——这三者覆盖了「从商品页 HTML 抽加购表单」的核心逻辑,
是 worker 真实跑起来时最关键的转换。
SiteInteractor 类的 add_to_cart / verify_cart 涉及 Playwright,不在离线测试覆盖
范围;只在 test_worker_runner.py 里用桩站点覆盖 runner 与 site 的契约。
"""
from __future__ import annotations
import json
from pathlib import Path
from typing import Any
import pytest
from app.shared.errors import CartOperationError, InvalidRequestError
from app.trading.worker.models import LeaseTask
from app.trading.worker.site_interact import (
CheckoutSummary,
SiteInteractor,
_extract_error_message,
_extract_purchase_fields,
_parse_initial_state,
)
FIXTURES = Path(__file__).parent / "fixtures"
def _make_task(**kwargs: Any) -> LeaseTask:
defaults = {
"task_id": "t1",
"site": "rakuten",
"intent": {"item_url": "https://item.rakuten.co.jp/shop/x/"},
}
defaults.update(kwargs)
return LeaseTask(**defaults)
def _wrap_state(state: dict[str, Any]) -> str:
"""构造一段模拟的 item page HTML,把 state 嵌进 __INITIAL_STATE__"""
return (
"<html><script>window.__INITIAL_STATE__ = "
+ json.dumps(state, ensure_ascii=False)
+ ";window.__OTHER__ = 1;</script></html>"
)
# ---- _parse_initial_state ----
def test_parse_initial_state_extracts_state_dict():
state = {"item": {"itemId": 123}, "purchase": {}}
parsed = _parse_initial_state(_wrap_state(state))
assert parsed == state
def test_parse_initial_state_returns_none_when_no_marker():
assert _parse_initial_state("<html>no state here</html>") is None
def test_parse_initial_state_returns_none_on_invalid_json():
html = "<script>window.__INITIAL_STATE__ = {invalid};window.__END=1;</script>"
assert _parse_initial_state(html) is None
# ---- _extract_purchase_fields ----
def test_extract_fields_from_multi_inventory_fixture():
"""item_state.json 是多规格商品,应自动选第一个非售罄 variant"""
state = json.loads((FIXTURES / "item_state.json").read_text(encoding="utf-8"))
fields = _extract_purchase_fields(state, intent_override={})
assert fields["purchase_condition"] == "enabled"
assert fields["basket_domain"].startswith("https://")
assert fields["basket_domain"].endswith("/rms/mall/bss/cartadd/set")
# 多规格:form 应包含自动选的 variant_id
assert fields["inventory_flag"] == "2"
assert "variant_id" in fields["form_fields"]
assert fields["form_fields"]["__event"] == "ES01_003_001"
assert fields["form_fields"]["inventory_flag"] == "2"
# quantity_field / variant_field 是约定字段名
assert fields["quantity_field"] == "units"
assert fields["variant_field"] == "variant_id"
def test_extract_fields_single_inventory_uses_item_variant_id():
"""单规格商品应直接用 item.variantId,不需要选 sku.variants"""
state = {
"item": {"itemId": 999, "variantId": "v-1"},
"purchase": {
"sku": {"inventoryType": "single", "variants": []},
"sellType": {
"normalPurchase": {
"basketDomain": "https://example.co.jp/add",
"purchaseCondition": "enabled",
}
},
"information": {},
},
"shop": {"information": {"shopId": 42}},
}
fields = _extract_purchase_fields(state, intent_override={})
assert fields["inventory_flag"] == "1"
assert fields["form_fields"]["variant_id"] == "v-1"
def test_extract_fields_intent_overrides_variant_id():
"""调用方显式给 variant_id 时,跳过自动选"""
state = json.loads((FIXTURES / "item_state.json").read_text(encoding="utf-8"))
fields = _extract_purchase_fields(state, intent_override={"variant_id": "my-choice"})
assert fields["form_fields"]["variant_id"] == "my-choice"
def test_extract_fields_multi_inventory_no_variants_returns_no_variant():
"""多规格但 variants 空:不强行填 variant_id;调用方靠这个判断不可加购"""
state = {
"item": {"itemId": 1},
"purchase": {
"sku": {"inventoryType": "multiple", "variants": []},
"sellType": {"normalPurchase": {"basketDomain": "https://x/add", "purchaseCondition": "enabled"}},
"information": {},
},
"shop": {"information": {"shopId": 1}},
}
fields = _extract_purchase_fields(state, intent_override={})
assert "variant_id" not in fields["form_fields"]
def test_extract_fields_required_options_auto_picks_first_value():
"""有必填选项时,自动用第一个候选值填 choice"""
state = {
"item": {"itemId": 1, "variantId": "v"},
"purchase": {
"sku": {"inventoryType": "single"},
"sellType": {"normalPurchase": {"basketDomain": "https://x/add", "purchaseCondition": "enabled"}},
"information": {
"options": [
{
"id": 1,
"name": "サイズ",
"type": "select",
"isRequired": True,
"values": [
{"id": 10, "name": "S"},
{"id": 11, "name": "M"},
],
}
]
},
},
"shop": {"information": {"shopId": 1}},
}
fields = _extract_purchase_fields(state, intent_override={})
assert fields["has_required_options"] is True
assert fields["form_fields"]["choice"] == "サイズ:S"
def test_extract_fields_intent_choice_override_accepts_list_and_str():
state = {
"item": {"itemId": 1, "variantId": "v"},
"purchase": {
"sku": {"inventoryType": "single"},
"sellType": {"normalPurchase": {"basketDomain": "https://x/add", "purchaseCondition": "enabled"}},
"information": {"options": [{"name": "x", "isRequired": True, "values": [{"name": "a"}]}]},
},
"shop": {"information": {"shopId": 1}},
}
# list 形态
fields = _extract_purchase_fields(state, intent_override={"choice": ["サイズ:M", "色:赤"]})
assert fields["form_fields"]["choice"] == "サイズ:M,色:赤"
# str 形态
fields = _extract_purchase_fields(state, intent_override={"choice": "custom:value"})
assert fields["form_fields"]["choice"] == "custom:value"
def test_extract_fields_basket_domain_unescapes_u002f():
"""JSON encoded 的 / 在 __INITIAL_STATE__ 里可能是 \\u002F,要还原"""
state = {
"item": {"itemId": 1, "variantId": "v"},
"purchase": {
"sku": {"inventoryType": "single"},
"sellType": {
"normalPurchase": {
"basketDomain": "https:\\u002F\\u002Fx.example\\u002Fadd",
"purchaseCondition": "enabled",
}
},
"information": {},
},
"shop": {"information": {"shopId": 1}},
}
fields = _extract_purchase_fields(state, intent_override={})
assert fields["basket_domain"] == "https://x.example/add"
# ---- _extract_error_message ----
def test_extract_error_message_picks_visible_text():
body = """
<html><body>
<div>「商品 X」は商品情報が変更されました。</div>
<div>最新の商品ページから再度かごに追加してください。</div>
<div>ご注文手続き中の通信は、SSLによりすべて暗号化されます</div>
<div>© Rakuten Group, Inc.</div>
</body></html>
"""
msg = _extract_error_message(body)
# 应当包含前两条核心提示;SSL 与版权被过滤
assert "商品情報が変更されました" in msg
assert "SSL" not in msg
assert "Rakuten Group" not in msg
def test_extract_error_message_filters_punctuation_heavy_strings():
body = "<div>未選択の項目からどれか1つ選んでください。</div><div>{json: 'x'}</div>"
msg = _extract_error_message(body)
assert "未選択" in msg
# 含 { 与 ' 的 JSON 片段应被过滤
assert "json" not in msg
def test_extract_error_message_returns_empty_when_no_meaningful_text():
body = "<html></html>"
assert _extract_error_message(body) == ""
# ---- CheckoutSummary 仍是 dataclass ----
def test_checkout_summary_defaults():
s = CheckoutSummary(payable_yen=1000)
assert s.payable_yen == 1000
assert s.site_order_id is None
assert s.pay_deadline is None
# ---- SiteInteractor 构造(不调 start,无 Playwright 依赖)----
def test_site_interactor_construction_does_not_require_playwright():
"""构造 SiteInteractor 不应触发 Playwright import;start() 才会"""
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
assert site._playwright is None
assert site._browser is None
assert site._context is None
assert site._per_task_state == {}
# ---- 未实现的方法仍然抛 NotImplementedError ----
@pytest.mark.parametrize(
"method,args",
[
("enter_checkout", (_make_task(),)),
("submit_order", (_make_task(),)),
("pay", (_make_task(), "ord-1")),
("monitor", (_make_task(), "ord-1")),
],
)
async def test_unimplemented_methods_raise(method: str, args: tuple):
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
fn = getattr(site, method)
with pytest.raises(NotImplementedError):
await fn(*args)
async def test_parse_checkout_unimplemented():
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
with pytest.raises(NotImplementedError):
await site.parse_checkout("<html></html>")
# ---- add_to_cart / verify_cart 在没启动 Playwright 时应失败 ----
async def test_add_to_cart_without_start_raises():
"""没调 start() 就调 add_to_cart,_context 是 None,应抛错"""
site = SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
# require_logged_in 会因为 auth_session=None 抛 AttributeError,比 _context None 更早
# 这里测的是「未启动时不应该静默成功」
with pytest.raises((AttributeError, TypeError)):
await site.add_to_cart(_make_task())
# ---- helper ----
+50 -26
View File
@@ -1,11 +1,11 @@
"""worker 主循环测试
不启动真实的后台任务,直接构造 WorkerRunner 与桩 gateway client / local_db /
evidence store,覆盖 §6 主循环的分支:
evidence store / site,覆盖 §6 主循环的分支:
- 已完成的任务再次被领:直接补报,不重新执行
- lease_count > 1(恢复领取):走核对分支,verify unknown → needs_human
- 站点交互未实现(site_interact 抛 NotImplementedError):转 needs_human
- 站点交互未实现(SiteInteractor 默认抛 NotImplementedError):转 needs_human
- 金额守卫拦截:转 needs_human,站点侧无提交动作
每一步的证据文件在 report 之前就已落盘(§9 第 11 条)也在这里验证。
@@ -20,11 +20,12 @@ import pytest
from app.shared.errors import OrderGuardError
from app.shared.task_state import OrderState, TaskStatus
from app.trading.worker import site_interact, verify
from app.trading.worker import verify
from app.trading.worker.evidence import EvidenceStore
from app.trading.worker.local_db import LocalDB
from app.trading.worker.models import LeaseTask
from app.trading.worker.runner import WorkerRunner
from app.trading.worker.site_interact import CheckoutSummary, SiteInteractor
# ---- 桩:网关客户端 ----
@@ -102,7 +103,18 @@ def evidence(tmp_path: Path) -> EvidenceStore:
@pytest.fixture
def runner(tmp_path, local_db, evidence, monkeypatch) -> WorkerRunner:
def site() -> SiteInteractor:
"""返回**未启动**的 SiteInteractor(不调 start,不依赖 Playwright)
默认 add_to_cart/verify_cart 已实现(但会抛 NotImplementedError 因为没启动),
具体测试可用 monkeypatch.setattr(site, "method", fn) 替换为桩。
"""
# 用最小化的 auth_session / settings 占位;测试不会真的调它们
return SiteInteractor(auth_session=None, settings=None) # type: ignore[arg-type]
@pytest.fixture
def runner(tmp_path, local_db, evidence, site) -> WorkerRunner:
"""构造一个 worker_id=test、网关 URL 已配置的 runner
使用 fake gateway;settings 取真实 Settings 但 worker_id 注入为 test。
@@ -122,6 +134,7 @@ def runner(tmp_path, local_db, evidence, monkeypatch) -> WorkerRunner:
gateway_client=gateway, # type: ignore[arg-type]
local_db=local_db,
evidence=evidence,
site=site,
)
# 把 gateway 桩挂到 runner 上,测试通过它断言调用
r._gateway_for_test = gateway # type: ignore[attr-defined]
@@ -162,7 +175,7 @@ async def test_already_finished_task_is_reported_not_re_executed(
terminal = gateway.last_terminal_report()
assert terminal["state"] == OrderState.PAID
assert terminal["terminal"] is True
# 没有进入 execute:site_interact 不应被调用,但为保险起见,我们只看
# 没有进入 execute:site 不应被调用,但为保险起见,我们只看
# evidence 目录里没有任何步骤文件
evidence_files = list(local_db._db_path.parent.parent.glob("**/*.html")) # noqa
# 上面 glob 会扫到 trading.db 之外的所有 .html;这里只关心 evidence 目录
@@ -218,7 +231,19 @@ async def test_recovery_task_with_already_ordered_verdict_reports_succeeded(
async def test_unimplemented_site_interaction_becomes_needs_human(
runner: WorkerRunner, local_db: LocalDB
):
"""site_interact.add_to_cart 抛 NotImplementedError → runner 转 needs_human"""
"""SiteInteractor 默认 enter_checkout 抛 NotImplementedError → runner 转 needs_human
add_to_cart 与 verify_cart 已实现(但站点未启动时会从 Playwright 调用失败,
测试里先替换为 noop 让流程跑到 enter_checkout)。
"""
# 让已实现的两步 noop,触发未实现的 enter_checkout
async def _noop(task): # noqa: ANN001
return None
runner._site.add_to_cart = _noop # type: ignore[assignment]
runner._site.verify_cart = _noop # type: ignore[assignment]
await runner.handle(_make_task(task_id="t1"))
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
@@ -243,7 +268,7 @@ async def test_rakuma_site_reports_needs_human(runner: WorkerRunner):
async def test_amount_guard_blocks_when_payable_exceeds_limit(
runner: WorkerRunner, local_db: LocalDB, monkeypatch
runner: WorkerRunner, local_db: LocalDB
):
"""实际应付超过上限 → OrderGuardError → 转 needs_human,站点侧无提交动作
@@ -255,14 +280,11 @@ async def test_amount_guard_blocks_when_payable_exceeds_limit(
async def _ok(task): # noqa: ANN001
call_log.append(task.task_id + ":step")
async def _noop(task): # noqa: ANN001
return None
async def _checkout_html(task): # noqa: ANN001
return "<html>checkout</html>"
async def _parse(html: str):
return site_interact.CheckoutSummary(payable_yen=50000)
return CheckoutSummary(payable_yen=50000)
submit_called = False
@@ -271,11 +293,11 @@ async def test_amount_guard_blocks_when_payable_exceeds_limit(
submit_called = True
return "ord-1"
monkeypatch.setattr(site_interact, "add_to_cart", _noop)
monkeypatch.setattr(site_interact, "verify_cart", _noop)
monkeypatch.setattr(site_interact, "enter_checkout", _checkout_html)
monkeypatch.setattr(site_interact, "parse_checkout", _parse)
monkeypatch.setattr(site_interact, "submit_order", _submit)
runner._site.add_to_cart = _ok # type: ignore[assignment]
runner._site.verify_cart = _ok # type: ignore[assignment]
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
runner._site.parse_checkout = _parse # type: ignore[assignment]
runner._site.submit_order = _submit # type: ignore[assignment]
# intent.max_total_yen 缺省,回落到 settings.order_max_total_yen=30000,
# 实际 50000 > 30000 → 拦截
@@ -290,7 +312,7 @@ async def test_amount_guard_blocks_when_payable_exceeds_limit(
async def test_amount_guard_honors_intent_max_total_yen(
runner: WorkerRunner, local_db: LocalDB, monkeypatch
runner: WorkerRunner, local_db: LocalDB
):
"""intent 显式给 max_total_yen 时优先于全局上限"""
@@ -301,7 +323,7 @@ async def test_amount_guard_honors_intent_max_total_yen(
return "<html>checkout</html>"
async def _parse(html: str):
return site_interact.CheckoutSummary(payable_yen=8000)
return CheckoutSummary(payable_yen=8000)
submit_called = False
@@ -310,11 +332,11 @@ async def test_amount_guard_honors_intent_max_total_yen(
submit_called = True
raise NotImplementedError("不应到这里") # parse 之后的步骤未实现,到这里即说明守卫过了
monkeypatch.setattr(site_interact, "add_to_cart", _noop)
monkeypatch.setattr(site_interact, "verify_cart", _noop)
monkeypatch.setattr(site_interact, "enter_checkout", _checkout_html)
monkeypatch.setattr(site_interact, "parse_checkout", _parse)
monkeypatch.setattr(site_interact, "submit_order", _submit)
runner._site.add_to_cart = _noop # type: ignore[assignment]
runner._site.verify_cart = _noop # type: ignore[assignment]
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
runner._site.parse_checkout = _parse # type: ignore[assignment]
runner._site.submit_order = _submit # type: ignore[assignment]
# 全局上限 30000,但 intent 给 5000 → 实际 8000 > 5000 → 拦截
await runner.handle(_make_task(task_id="t1", intent={"max_total_yen": 5000}))
@@ -329,7 +351,7 @@ async def test_amount_guard_honors_intent_max_total_yen(
async def test_evidence_files_exist_before_each_report(
runner: WorkerRunner, evidence: EvidenceStore, monkeypatch
runner: WorkerRunner, evidence: EvidenceStore
):
"""每个 _run_step 调用:动作 → 落证据 → 写本地 → 回报 gateway
@@ -350,11 +372,13 @@ async def test_evidence_files_exist_before_each_report(
runner._gateway_for_test.report = _spy_report # type: ignore[attr-defined]
# 让 add_to_cart 正常,verify_cart 抛 NotImplemented,触发第一个 step 写证据 + report
# 让 add_to_cart / verify_cart 正常,enter_checkout 抛 NotImplemented,
# 触发前两步写证据 + report,到第三步转 needs_human
async def _noop(task): # noqa: ANN001
return None
monkeypatch.setattr(site_interact, "add_to_cart", _noop)
runner._site.add_to_cart = _noop # type: ignore[assignment]
runner._site.verify_cart = _noop # type: ignore[assignment]
await runner.handle(_make_task(task_id="t1"))