q792602257andClaude Opus 5
b577d3ac8d
trading 自动填 choice 时取 values[0]，而必填 select 的 values[0] 恒为 id=0 的
「選択してください」——等于把「请选择」当答案提交。4 份真实样本一致（真值从
id=200 起）。同时 /api/item_detail 完全不返回 options，调用方即使想显式指定
choice 也无从知道合法取值。
- purchase_contract.py：新增 ItemOption / ItemOptionValue 与 parse_options /
auto_choice_for / format_choice。占位判定以结构为主（value_id == 0），日文
文案仅作兜底。放 shared 是因为「接口声明的合法取值」与「下单实际提交的值」
必须同源，否则两边各判一次迟早再次分叉
- item.py / scrape.py：ItemDetailData 增 options、has_required_options、
unfillable_required_options；只解析一次，两个派生结果都取自同一份结果
- site_interact.py：auto_choice_for 取第一个非占位候选；必填项填不出值时
报错点名是哪些选项，让调用方知道该在 intent.choice 里补什么
- auto_choice_for 只自动填必填项：非必填项要不要选是业务决定，不是我们该替
调用方做的选择
- README / docs：补 options[] → intent.choice、variants[] → intent.variant_id
的对照，修掉 order-gateway 示例里已不存在的 "options": {} 字段
真账号验证（scripts/probe_option_choice.py，仅加购不结算不支付）：两个商品
提交 確認した / 了解致しました。均被站点接受，购物车 count=2，跑完清空恢复
原状。探针刻意走生产的 add_to_cart_payload 并从其日志截获实际 payload——
probe_purchase_block_v2.py 自己抄了一遍字段构造，与生产代码同错，正是这个
bug 当初藏住的原因。
未覆盖：这两家店铺本身不校验该选项（旧的占位值当年也被收下），所以只证明新值
走得通、语义上才是真答案，证明不了旧值会被拒；必填自由文本项（
unfillable_required_options）无真实样本，仅离线测试覆盖。
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
221 lines
9.4 KiB
Python
221 lines
9.4 KiB
Python
"""解析器测试：基于真实页面状态样本"""
|
|
import pytest
|
|
|
|
from app.shared.errors import ScrapeParseError
|
|
from app.scraping.parsers.item import parse_item_detail
|
|
from app.scraping.parsers.search import parse_search
|
|
from app.scraping.parsers.state import extract_initial_state
|
|
|
|
|
|
# ---- __INITIAL_STATE__ 抽取 ----
|
|
|
|
def test_extract_initial_state_stops_at_end_of_json_object():
|
|
"""赋值语句后面紧跟其他脚本代码，不能整段吃掉"""
|
|
html = '<script>window.__INITIAL_STATE__ = {"a": {"b": 1}};window.__LOCALE__="ja";</script>'
|
|
assert extract_initial_state(html) == {"a": {"b": 1}}
|
|
|
|
|
|
def test_extract_initial_state_reports_missing_marker():
|
|
with pytest.raises(ScrapeParseError):
|
|
extract_initial_state("<html><body>blocked</body></html>")
|
|
|
|
|
|
def test_extract_initial_state_reports_broken_json():
|
|
with pytest.raises(ScrapeParseError):
|
|
extract_initial_state("window.__INITIAL_STATE__ = {oops")
|
|
|
|
|
|
# ---- 搜索结果 ----
|
|
|
|
def test_search_excludes_ads_by_default_but_still_counts_them(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
assert result.ad_count == 1
|
|
assert all(not item.is_ad for item in result.items)
|
|
assert len(result.items) == 2
|
|
|
|
|
|
def test_search_can_keep_ads_and_resolves_their_real_item_url(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=False)
|
|
ads = [item for item in result.items if item.is_ad]
|
|
assert len(ads) == 1
|
|
# 广告位的 url 是跳转域，必须还原成 originalItemUrl 才能拿到商品编号
|
|
assert ads[0].item_url.startswith("https://item.rakuten.co.jp/")
|
|
assert ads[0].shop.shop_code and ads[0].item_code
|
|
|
|
|
|
def test_search_item_carries_fields_needed_to_call_item_detail(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
item = result.items[0]
|
|
assert item.item_name
|
|
assert item.price > 0
|
|
# shop.shop_code + item_code 就是 /api/item_detail 的入参
|
|
assert item.shop.shop_code and item.item_code
|
|
assert item.image_url.startswith("https://")
|
|
assert item.genre_id.isdigit()
|
|
|
|
|
|
def test_search_pagination_is_capped_by_site_subset(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
assert result.total_count > 0
|
|
# 站点声明命中数很大，但实际只能翻到 subset 条为止
|
|
assert result.reachable_count <= result.total_count
|
|
assert result.reachable_count <= 6750
|
|
|
|
|
|
def test_search_reports_request_url(search_state):
|
|
result = parse_search(search_state, request_url="https://search.rakuten.co.jp/x", page=1, exclude_ads=True)
|
|
assert result.request_url == "https://search.rakuten.co.jp/x"
|
|
|
|
|
|
def test_search_flags_out_of_range_page_and_drops_wrapped_items(search_state):
|
|
"""越过可达窗口时站点会静默回绕到第 1 页，必须识别出来而不是把重复数据交上去"""
|
|
assert search_state["state"]["data"]["effectiveUiQuestion"]["page"] == 1
|
|
result = parse_search(search_state, request_url="https://x", page=10, exclude_ads=True)
|
|
assert result.out_of_range is True
|
|
assert result.items == []
|
|
assert result.has_more is False
|
|
assert result.page == 10 # 回报请求页码，而不是站点回绕后的 1
|
|
|
|
|
|
def test_search_within_range_is_not_flagged_out_of_range(search_state):
|
|
search_state["state"]["data"]["effectiveUiQuestion"]["page"] = 3
|
|
result = parse_search(search_state, request_url="https://x", page=3, exclude_ads=True)
|
|
assert result.out_of_range is False
|
|
assert result.items
|
|
|
|
|
|
def test_search_rejects_state_without_search_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_search({"state": {"data": {}}}, request_url="https://x", page=1, exclude_ads=True)
|
|
|
|
|
|
# ---- 商品详情 ----
|
|
|
|
def test_item_detail_extracts_core_fields(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://item.rakuten.co.jp/fafachai/tx-300/",
|
|
shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.source == "ichiba"
|
|
assert detail.source_url == "https://item.rakuten.co.jp/fafachai/tx-300/"
|
|
assert detail.item_name
|
|
assert detail.item_code
|
|
assert detail.price > 0
|
|
assert detail.images and all(url.startswith("https://") for url in detail.images)
|
|
assert detail.shop.shop_id and detail.shop.shop_code
|
|
assert detail.genre_id.isdigit()
|
|
assert detail.breadcrumbs
|
|
|
|
|
|
def test_item_detail_purchasable_item_is_not_sold_out(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.purchase_condition == "enabled"
|
|
assert detail.is_sold_out is False
|
|
|
|
|
|
def test_item_detail_parses_sku_axis_and_variants(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.sku.inventory_type == "multiple"
|
|
assert detail.sku.axis and detail.sku.axis[0].values
|
|
assert detail.sku.variants
|
|
variant = detail.sku.variants[0]
|
|
assert variant.variant_id and variant.price > 0
|
|
assert len(variant.selector_values) == len(detail.sku.axis)
|
|
|
|
|
|
def test_item_detail_can_omit_variants_but_keeps_their_count(item_state):
|
|
"""SKU 组合可能上百条，关闭后应只省略明细、不丢失规模信息"""
|
|
full = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=True)
|
|
slim = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=False)
|
|
assert slim.sku.variants == []
|
|
assert slim.sku.variant_count == full.sku.variant_count > 0
|
|
assert slim.sku.axis == full.sku.axis
|
|
|
|
|
|
def test_item_detail_falls_back_to_request_shop_code_when_state_lacks_it(item_state):
|
|
item_state["shop"]["information"]["shopUrl"] = ""
|
|
detail = parse_item_detail(item_state, item_url="https://x", shop_code="fallback", include_sku_variants=False)
|
|
assert detail.shop.shop_code == "fallback"
|
|
|
|
|
|
# ---- 店铺自定义选项（下单必填项）----
|
|
|
|
def test_item_detail_exposes_shop_options_for_ordering(item_with_options_state):
|
|
"""下单要用的选项必须能从接口拿到：名字、类型、是否必填、候选取值"""
|
|
detail = parse_item_detail(
|
|
item_with_options_state, item_url="https://x", shop_code="kizamu",
|
|
include_sku_variants=True,
|
|
)
|
|
assert detail.has_required_options is True
|
|
# 真实样本：1 个必填 select（名入れ）+ 1 个非必填 select + 2 个自由文本
|
|
assert [option.name for option in detail.options] == [
|
|
"名入れ",
|
|
" 【お名前】※漢字・かな文字：4文字まで",
|
|
"【定型】",
|
|
"【定型】※〇寿・〇婚式・〇長を選択の場合のみ(1文字まで)",
|
|
]
|
|
required = detail.options[0]
|
|
assert required.is_required is True
|
|
assert required.type == "select"
|
|
assert required.option_id == 100
|
|
|
|
|
|
def test_item_detail_marks_placeholder_option_values_as_unselectable(item_with_options_state):
|
|
"""「選択してください」是占位项，不能作为下单取值——这是 choice 填错的根因"""
|
|
detail = parse_item_detail(
|
|
item_with_options_state, item_url="https://x", shop_code="kizamu",
|
|
include_sku_variants=True,
|
|
)
|
|
required = detail.options[0]
|
|
placeholder, first_real = required.values[0], required.values[1]
|
|
assert placeholder.name == "選択してください"
|
|
assert placeholder.is_placeholder is True
|
|
assert first_real.name == "希望する【次の項目で入力】"
|
|
assert first_real.is_placeholder is False
|
|
# 3 个候选里只有 2 个真正可提交
|
|
assert len(required.values) == 3
|
|
assert required.selectable_value_count == 2
|
|
|
|
|
|
def test_item_detail_free_text_option_has_no_values(item_with_options_state):
|
|
"""type=text 是自由文本，站点不给候选值，无法自动选"""
|
|
detail = parse_item_detail(
|
|
item_with_options_state, item_url="https://x", shop_code="kizamu",
|
|
include_sku_variants=True,
|
|
)
|
|
free_text = detail.options[1]
|
|
assert free_text.type == "text"
|
|
assert free_text.values == []
|
|
assert free_text.selectable_value_count == 0
|
|
|
|
|
|
def test_item_detail_reports_required_options_that_cannot_be_auto_filled(item_with_options_state):
|
|
"""必填自由文本项无法自动选值，必须点名交给调用方，不能静默蒙一个值"""
|
|
options = item_with_options_state["purchase"]["information"]["options"]
|
|
# 把那个自由文本项改成必填：店铺要求填「お名前」时就是这种形态
|
|
options[1]["isRequired"] = True
|
|
detail = parse_item_detail(
|
|
item_with_options_state, item_url="https://x", shop_code="kizamu",
|
|
include_sku_variants=True,
|
|
)
|
|
assert detail.unfillable_required_options == [" 【お名前】※漢字・かな文字：4文字まで"]
|
|
|
|
|
|
def test_item_detail_without_options_reports_empty(item_state):
|
|
"""没有选项的商品（真实样本 purchase.information 只有 unit）不应凭空造出选项"""
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="s", include_sku_variants=False,
|
|
)
|
|
assert detail.options == []
|
|
assert detail.has_required_options is False
|
|
assert detail.unfillable_required_options == []
|
|
|
|
|
|
def test_item_detail_rejects_state_without_item_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_item_detail({}, item_url="https://x", shop_code="s", include_sku_variants=True)
|