"""解析器测试：基于真实页面状态样本""" import pytest from app.core.errors import ScrapeParseError from app.parsers.item import parse_item_detail from app.parsers.search import parse_search from app.parsers.state import extract_initial_state # ---- __INITIAL_STATE__ 抽取 ---- def test_extract_initial_state_stops_at_end_of_json_object(): """赋值语句后面紧跟其他脚本代码，不能整段吃掉""" html = '' assert extract_initial_state(html) == {"a": {"b": 1}} def test_extract_initial_state_reports_missing_marker(): with pytest.raises(ScrapeParseError): extract_initial_state("blocked") def test_extract_initial_state_reports_broken_json(): with pytest.raises(ScrapeParseError): extract_initial_state("window.__INITIAL_STATE__ = {oops") # ---- 搜索结果 ---- def test_search_excludes_ads_by_default_but_still_counts_them(search_state): result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True) assert result.ad_count == 1 assert all(not item.is_ad for item in result.items) assert len(result.items) == 2 def test_search_can_keep_ads_and_resolves_their_real_item_url(search_state): result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=False) ads = [item for item in result.items if item.is_ad] assert len(ads) == 1 # 广告位的 url 是跳转域，必须还原成 originalItemUrl 才能拿到商品编号 assert ads[0].item_url.startswith("https://item.rakuten.co.jp/") assert ads[0].shop.shop_code and ads[0].item_code def test_search_item_carries_fields_needed_to_call_item_detail(search_state): result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True) item = result.items[0] assert item.item_name assert item.price > 0 # shop.shop_code + item_code 就是 /api/item_detail 的入参 assert item.shop.shop_code and item.item_code assert item.image_url.startswith("https://") assert item.genre_id.isdigit() def test_search_pagination_is_capped_by_site_subset(search_state): result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True) assert result.total_count > 0 # 站点声明命中数很大，但实际只能翻到 subset 条为止 assert result.reachable_count <= result.total_count assert result.reachable_count <= 6750 def test_search_reports_request_url(search_state): result = parse_search(search_state, request_url="https://search.rakuten.co.jp/x", page=1, exclude_ads=True) assert result.request_url == "https://search.rakuten.co.jp/x" def test_search_flags_out_of_range_page_and_drops_wrapped_items(search_state): """越过可达窗口时站点会静默回绕到第 1 页，必须识别出来而不是把重复数据交上去""" assert search_state["state"]["data"]["effectiveUiQuestion"]["page"] == 1 result = parse_search(search_state, request_url="https://x", page=10, exclude_ads=True) assert result.out_of_range is True assert result.items == [] assert result.has_more is False assert result.page == 10 # 回报请求页码，而不是站点回绕后的 1 def test_search_within_range_is_not_flagged_out_of_range(search_state): search_state["state"]["data"]["effectiveUiQuestion"]["page"] = 3 result = parse_search(search_state, request_url="https://x", page=3, exclude_ads=True) assert result.out_of_range is False assert result.items def test_search_rejects_state_without_search_node(): with pytest.raises(ScrapeParseError): parse_search({"state": {"data": {}}}, request_url="https://x", page=1, exclude_ads=True) # ---- 商品详情 ---- def test_item_detail_extracts_core_fields(item_state): detail = parse_item_detail( item_state, item_url="https://item.rakuten.co.jp/fafachai/tx-300/", shop_code="fafachai", include_sku_variants=True, ) assert detail.source == "ichiba" assert detail.source_url == "https://item.rakuten.co.jp/fafachai/tx-300/" assert detail.item_name assert detail.item_code assert detail.price > 0 assert detail.images and all(url.startswith("https://") for url in detail.images) assert detail.shop.shop_id and detail.shop.shop_code assert detail.genre_id.isdigit() assert detail.breadcrumbs def test_item_detail_purchasable_item_is_not_sold_out(item_state): detail = parse_item_detail( item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True, ) assert detail.purchase_condition == "enabled" assert detail.is_sold_out is False def test_item_detail_parses_sku_axis_and_variants(item_state): detail = parse_item_detail( item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True, ) assert detail.sku.inventory_type == "multiple" assert detail.sku.axis and detail.sku.axis[0].values assert detail.sku.variants variant = detail.sku.variants[0] assert variant.variant_id and variant.price > 0 assert len(variant.selector_values) == len(detail.sku.axis) def test_item_detail_can_omit_variants_but_keeps_their_count(item_state): """SKU 组合可能上百条，关闭后应只省略明细、不丢失规模信息""" full = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=True) slim = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=False) assert slim.sku.variants == [] assert slim.sku.variant_count == full.sku.variant_count > 0 assert slim.sku.axis == full.sku.axis def test_item_detail_falls_back_to_request_shop_code_when_state_lacks_it(item_state): item_state["shop"]["information"]["shopUrl"] = "" detail = parse_item_detail(item_state, item_url="https://x", shop_code="fallback", include_sku_variants=False) assert detail.shop.shop_code == "fallback" def test_item_detail_rejects_state_without_item_node(): with pytest.raises(ScrapeParseError): parse_item_detail({}, item_url="https://x", shop_code="s", include_sku_variants=True)