148 lines
6.1 KiB
Python
148 lines
6.1 KiB
Python
"""解析器测试：基于真实页面状态样本"""
|
|
import pytest
|
|
|
|
from app.core.errors import ScrapeParseError
|
|
from app.parsers.item import parse_item_detail
|
|
from app.parsers.search import parse_search
|
|
from app.parsers.state import extract_initial_state
|
|
|
|
|
|
# ---- __INITIAL_STATE__ 抽取 ----
|
|
|
|
def test_extract_initial_state_stops_at_end_of_json_object():
|
|
"""赋值语句后面紧跟其他脚本代码，不能整段吃掉"""
|
|
html = '<script>window.__INITIAL_STATE__ = {"a": {"b": 1}};window.__LOCALE__="ja";</script>'
|
|
assert extract_initial_state(html) == {"a": {"b": 1}}
|
|
|
|
|
|
def test_extract_initial_state_reports_missing_marker():
|
|
with pytest.raises(ScrapeParseError):
|
|
extract_initial_state("<html><body>blocked</body></html>")
|
|
|
|
|
|
def test_extract_initial_state_reports_broken_json():
|
|
with pytest.raises(ScrapeParseError):
|
|
extract_initial_state("window.__INITIAL_STATE__ = {oops")
|
|
|
|
|
|
# ---- 搜索结果 ----
|
|
|
|
def test_search_excludes_ads_by_default_but_still_counts_them(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
assert result.ad_count == 1
|
|
assert all(not item.is_ad for item in result.items)
|
|
assert len(result.items) == 2
|
|
|
|
|
|
def test_search_can_keep_ads_and_resolves_their_real_item_url(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=False)
|
|
ads = [item for item in result.items if item.is_ad]
|
|
assert len(ads) == 1
|
|
# 广告位的 url 是跳转域，必须还原成 originalItemUrl 才能拿到商品编号
|
|
assert ads[0].item_url.startswith("https://item.rakuten.co.jp/")
|
|
assert ads[0].shop.shop_code and ads[0].item_code
|
|
|
|
|
|
def test_search_item_carries_fields_needed_to_call_item_detail(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
item = result.items[0]
|
|
assert item.item_name
|
|
assert item.price > 0
|
|
# shop.shop_code + item_code 就是 /api/item_detail 的入参
|
|
assert item.shop.shop_code and item.item_code
|
|
assert item.image_url.startswith("https://")
|
|
assert item.genre_id.isdigit()
|
|
|
|
|
|
def test_search_pagination_is_capped_by_site_subset(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
assert result.total_count > 0
|
|
# 站点声明命中数很大，但实际只能翻到 subset 条为止
|
|
assert result.reachable_count <= result.total_count
|
|
assert result.reachable_count <= 6750
|
|
|
|
|
|
def test_search_reports_request_url(search_state):
|
|
result = parse_search(search_state, request_url="https://search.rakuten.co.jp/x", page=1, exclude_ads=True)
|
|
assert result.request_url == "https://search.rakuten.co.jp/x"
|
|
|
|
|
|
def test_search_flags_out_of_range_page_and_drops_wrapped_items(search_state):
|
|
"""越过可达窗口时站点会静默回绕到第 1 页，必须识别出来而不是把重复数据交上去"""
|
|
assert search_state["state"]["data"]["effectiveUiQuestion"]["page"] == 1
|
|
result = parse_search(search_state, request_url="https://x", page=10, exclude_ads=True)
|
|
assert result.out_of_range is True
|
|
assert result.items == []
|
|
assert result.has_more is False
|
|
assert result.page == 10 # 回报请求页码，而不是站点回绕后的 1
|
|
|
|
|
|
def test_search_within_range_is_not_flagged_out_of_range(search_state):
|
|
search_state["state"]["data"]["effectiveUiQuestion"]["page"] = 3
|
|
result = parse_search(search_state, request_url="https://x", page=3, exclude_ads=True)
|
|
assert result.out_of_range is False
|
|
assert result.items
|
|
|
|
|
|
def test_search_rejects_state_without_search_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_search({"state": {"data": {}}}, request_url="https://x", page=1, exclude_ads=True)
|
|
|
|
|
|
# ---- 商品详情 ----
|
|
|
|
def test_item_detail_extracts_core_fields(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://item.rakuten.co.jp/fafachai/tx-300/",
|
|
shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.source == "ichiba"
|
|
assert detail.source_url == "https://item.rakuten.co.jp/fafachai/tx-300/"
|
|
assert detail.item_name
|
|
assert detail.item_code
|
|
assert detail.price > 0
|
|
assert detail.images and all(url.startswith("https://") for url in detail.images)
|
|
assert detail.shop.shop_id and detail.shop.shop_code
|
|
assert detail.genre_id.isdigit()
|
|
assert detail.breadcrumbs
|
|
|
|
|
|
def test_item_detail_purchasable_item_is_not_sold_out(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.purchase_condition == "enabled"
|
|
assert detail.is_sold_out is False
|
|
|
|
|
|
def test_item_detail_parses_sku_axis_and_variants(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.sku.inventory_type == "multiple"
|
|
assert detail.sku.axis and detail.sku.axis[0].values
|
|
assert detail.sku.variants
|
|
variant = detail.sku.variants[0]
|
|
assert variant.variant_id and variant.price > 0
|
|
assert len(variant.selector_values) == len(detail.sku.axis)
|
|
|
|
|
|
def test_item_detail_can_omit_variants_but_keeps_their_count(item_state):
|
|
"""SKU 组合可能上百条，关闭后应只省略明细、不丢失规模信息"""
|
|
full = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=True)
|
|
slim = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=False)
|
|
assert slim.sku.variants == []
|
|
assert slim.sku.variant_count == full.sku.variant_count > 0
|
|
assert slim.sku.axis == full.sku.axis
|
|
|
|
|
|
def test_item_detail_falls_back_to_request_shop_code_when_state_lacks_it(item_state):
|
|
item_state["shop"]["information"]["shopUrl"] = ""
|
|
detail = parse_item_detail(item_state, item_url="https://x", shop_code="fallback", include_sku_variants=False)
|
|
assert detail.shop.shop_code == "fallback"
|
|
|
|
|
|
def test_item_detail_rejects_state_without_item_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_item_detail({}, item_url="https://x", shop_code="s", include_sku_variants=True)
|