"""子站商品页解析测试：基于真实页面样本 夹具说明：books 的解析依赖 DOM 结构，保留了完整真实页面；brandavenue 与 biccamera 只读页面内联的 JSON，夹具做了裁剪以免把上百 KB 的噪音带进仓库。 """ from pathlib import Path import pytest from app.core.errors import OffIchibaRedirectError, ScrapeParseError from app.parsers.subsites import ( SUBSITE_PARSERS, build_item_page_validator, host_of, parse_subsite_item, ) from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price FIXTURES = Path(__file__).parent / "fixtures" def make_page(filename: str, *, host: str, shop_code: str, item_code: str, variants: bool = True): return SubsitePage( html=(FIXTURES / filename).read_text(encoding="utf-8"), requested_url=f"https://item.rakuten.co.jp/{shop_code}/{item_code}/", final_url=f"https://{host}/item/{item_code}/", shop_code=shop_code, item_code=item_code, include_sku_variants=variants, ) @pytest.fixture def books_page(): return make_page("books_item.html", host="books.rakuten.co.jp", shop_code="book", item_code="17065211") @pytest.fixture def brandavenue_page(): return make_page( "brandavenue_item.html", host="brandavenue.rakuten.co.jp", shop_code="stylife", item_code="md7606" ) @pytest.fixture def biccamera_page(): return make_page( "biccamera_item.html", host="biccamera.rakuten.co.jp", shop_code="biccamera", item_code="4548736147041" ) # ---- 通用工具 ---- @pytest.mark.parametrize( ("raw", "expected"), [("3,091円", 3091), ("1760", 1760), (1760, 1760), ("", 0), (None, 0), ("価格未定", 0)], ) def test_parse_price_handles_site_formats(raw, expected): assert parse_price(raw) == expected @pytest.mark.parametrize( ("status", "expected"), [("在庫あり", False), ("在庫なし", True), ("品切れ", True), ("販売終了", True), ("", False)], ) def test_looks_sold_out_reads_japanese_stock_wording(status, expected): assert looks_sold_out(status) is expected # ---- 分派 ---- def test_registry_covers_the_three_known_subsites(): assert set(SUBSITE_PARSERS) == { "books.rakuten.co.jp", "brandavenue.rakuten.co.jp", "biccamera.rakuten.co.jp", } def test_validator_accepts_ichiba_page_with_state(): validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/") ok = validate("window.__INITIAL_STATE__ = {}", "https://item.rakuten.co.jp/s/c/") assert ok is None def test_validator_rejects_ichiba_page_without_state_as_retryable(): validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/") reason = validate("blocked", "https://item.rakuten.co.jp/s/c/") assert reason and "INITIAL_STATE" in reason def test_validator_raises_for_unregistered_subsite(): """未登记的站点是「不支持」而非「抓取失败」，不能走重试""" validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/") with pytest.raises(OffIchibaRedirectError): validate("whatever", "https://unknown.rakuten.co.jp/x/") def test_validator_delegates_to_subsite_rules(books_page): validate = build_item_page_validator(books_page.requested_url) assert validate(books_page.html, books_page.final_url) is None assert validate("empty", books_page.final_url) is not None def test_parse_subsite_item_rejects_unknown_host(books_page): books_page.final_url = "https://unknown.rakuten.co.jp/x/" with pytest.raises(ScrapeParseError): parse_subsite_item(books_page) def test_host_of_extracts_hostname(): assert host_of("https://books.rakuten.co.jp/rb/1/?a=b") == "books.rakuten.co.jp" # ---- 楽天ブックス ---- def test_books_extracts_core_fields(books_page): d = parse_subsite_item(books_page) assert d.source == "books" assert d.source_url == books_page.final_url assert d.item_url == books_page.requested_url assert d.item_name == "12歳までに知っておきたい語彙力図鑑" assert d.price == 1760 assert d.shop.shop_code == "book" assert d.shop.shop_name == "楽天ブックス" def test_books_maps_review_and_stock(books_page): d = parse_subsite_item(books_page) assert d.review.score == pytest.approx(4.54) assert d.review.count == 218 assert d.purchase_condition == "在庫あり" assert d.is_sold_out is False def test_books_uses_rms_genre_ids_for_breadcrumbs(books_page): """站内分类 ID 对上游没用，要取 data_genres 里的 rmsGenreId（市场侧 genre_id）""" d = parse_subsite_item(books_page) assert [b.name for b in d.breadcrumbs] == ["本", "絵本・児童書・図鑑", "図鑑・ちしき"] assert d.genre_id == "208880" assert d.breadcrumbs[0].url == "https://www.rakuten.co.jp/category/200162/" def test_books_spec_table_becomes_sku_attributes(books_page): d = parse_subsite_item(books_page) spec = {a.title: a.value for a in d.sku.attributes} assert spec["ISBN"] == "9784800590039" assert spec["出版社"] == "日本能率協会マネジメントセンター" assert "齋藤 孝" in spec["著者／編集"] def test_books_description_collects_flat_sibling_sections(books_page): """标题与正文是兄弟节点而非父子，按容器取会漏掉绝大部分内容""" d = parse_subsite_item(books_page) assert len(d.description) > 1000 assert "内容紹介（JPROより）" in d.description assert "目次" in d.description def test_books_images_are_absolute_and_deduplicated(books_page): d = parse_subsite_item(books_page) assert d.images assert all(url.startswith("https://") for url in d.images) assert len(d.images) == len(set(d.images)) def test_books_has_no_sku_variants(books_page): """图书没有规格轴，SKU 只承载规格表""" d = parse_subsite_item(books_page) assert d.sku.inventory_type == "single" assert d.sku.variant_count == 0 assert d.sku.axis == [] # ---- Rakuten Fashion ---- def test_brandavenue_extracts_core_fields(brandavenue_page): d = parse_subsite_item(brandavenue_page) assert d.source == "brandavenue" assert d.item_name == "プリント ロゴ 刺繍 エンブロイダリー 半袖 Tシャツ" assert d.price == 3091 assert d.catch_copy == "andme" # 品牌名 assert d.is_sold_out is False def test_brandavenue_reads_ichiba_ids_from_rms_info(brandavenue_page): """市场侧的商品 ID 与 genre_id 藏在 rms_info 里，不是站内编码""" d = parse_subsite_item(brandavenue_page) assert d.item_id == "14674748" assert d.genre_id == "303656" assert d.shop.shop_id == 279405 assert d.shop.shop_code == "stylife" def test_brandavenue_builds_color_and_size_axes(brandavenue_page): d = parse_subsite_item(brandavenue_page) assert [a.label for a in d.sku.axis] == ["カラー", "サイズ"] assert d.sku.inventory_type == "multiple" assert d.sku.variant_count == len(d.sku.variants) > 1 def test_brandavenue_variant_joins_stock_from_inventory_list(brandavenue_page): """售价来自 product_sku，库存与 variant_id 来自 rms_info.inventory_list，按尺码+颜色对齐""" d = parse_subsite_item(brandavenue_page) by_selector = {tuple(v.selector_values): v for v in d.sku.variants} variant = by_selector[("オフホワイト×レッド", "Sサイズ")] assert variant.price == 2810 assert variant.quantity == 90 assert variant.variant_id def test_brandavenue_variant_without_matching_inventory_stays_purchasable(brandavenue_page): """站点两侧的颜色命名偶有出入导致对不上；此时库存未知，但不能据此判成售罄""" import json import re html = brandavenue_page.html state = json.loads(re.search(r"__INITIAL_STATE__ = (\{.*?\});window", html, re.S).group(1)) state["itemDetail"]["data"]["product"]["rms_info"]["inventory_list"] = [] brandavenue_page.html = re.sub( r"__INITIAL_STATE__ = \{.*?\};window", lambda _: f"__INITIAL_STATE__ = {json.dumps(state, ensure_ascii=False)};window", html, flags=re.S, ) d = parse_subsite_item(brandavenue_page) variant = d.sku.variants[0] assert variant.quantity == 0 assert variant.variant_id == "" assert variant.is_sold_out is False # 以 inventory_exist_flg 为准 def test_brandavenue_can_omit_variants_but_keeps_count(brandavenue_page): brandavenue_page.include_sku_variants = False d = parse_subsite_item(brandavenue_page) assert d.sku.variants == [] assert d.sku.variant_count > 0 assert d.sku.axis def test_brandavenue_takes_images_rendered_in_page(brandavenue_page): """图片按商品编号做了目录分片，分片规则不外露，只能取页面已渲染的地址""" d = parse_subsite_item(brandavenue_page) assert d.images assert all("stylife/cabinet/item" in url for url in d.images) # 主图应排在最前 assert d.images[0].lower().endswith("md7606-13_1.jpg") def test_brandavenue_breadcrumbs_have_names_without_urls(brandavenue_page): """站内分类编码不是市场 genre_id，拼不出可用链接，只给名称""" d = parse_subsite_item(brandavenue_page) assert [b.name for b in d.breadcrumbs] == ["トップス", "カットソー・Tシャツ"] assert all(b.url == "" for b in d.breadcrumbs) # ---- ビックカメラ ---- def test_biccamera_extracts_core_fields(biccamera_page): d = parse_subsite_item(biccamera_page) assert d.source == "biccamera" assert d.item_name.startswith("SONY") assert d.price == 73760 assert d.item_id == "13873289" assert d.item_code == "4548736147041" assert d.shop.shop_id == 269553 def test_biccamera_maps_inventory_and_delivery(biccamera_page): d = parse_subsite_item(biccamera_page) assert d.is_sold_out is False assert d.purchase_condition == "enabled" assert d.sku.quantity == 28 assert d.sku.show_inventory is True assert "出荷予定" in d.shipping.delivery_message assert d.shipping.is_shipping_free is True def test_biccamera_breadcrumbs_use_ichiba_genre_ids(biccamera_page): d = parse_subsite_item(biccamera_page) assert [b.name for b in d.breadcrumbs][0] == "TV・オーディオ・カメラ" assert d.breadcrumbs[0].url == "https://www.rakuten.co.jp/category/211742/" assert d.genre_id == "110110" def test_biccamera_rejects_page_without_nuxt_state(biccamera_page): biccamera_page.html = "" with pytest.raises(ScrapeParseError): parse_subsite_item(biccamera_page)