Init

2026-07-27 10:34:53 +08:00
commit 3527975794
74 changed files with 16405 additions and 0 deletions
@@ -0,0 +1,290 @@
+"""子站商品页解析测试：基于真实页面样本
+
+夹具说明：books 的解析依赖 DOM 结构，保留了完整真实页面；brandavenue 与
+biccamera 只读页面内联的 JSON，夹具做了裁剪以免把上百 KB 的噪音带进仓库。
+"""
+from pathlib import Path
+
+import pytest
+
+from app.core.errors import OffIchibaRedirectError, ScrapeParseError
+from app.parsers.subsites import (
+    SUBSITE_PARSERS,
+    build_item_page_validator,
+    host_of,
+    parse_subsite_item,
+)
+from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
+
+FIXTURES = Path(__file__).parent / "fixtures"
+
+
+def make_page(filename: str, *, host: str, shop_code: str, item_code: str, variants: bool = True):
+    return SubsitePage(
+        html=(FIXTURES / filename).read_text(encoding="utf-8"),
+        requested_url=f"https://item.rakuten.co.jp/{shop_code}/{item_code}/",
+        final_url=f"https://{host}/item/{item_code}/",
+        shop_code=shop_code,
+        item_code=item_code,
+        include_sku_variants=variants,
+    )
+
+
+@pytest.fixture
+def books_page():
+    return make_page("books_item.html", host="books.rakuten.co.jp", shop_code="book", item_code="17065211")
+
+
+@pytest.fixture
+def brandavenue_page():
+    return make_page(
+        "brandavenue_item.html", host="brandavenue.rakuten.co.jp", shop_code="stylife", item_code="md7606"
+    )
+
+
+@pytest.fixture
+def biccamera_page():
+    return make_page(
+        "biccamera_item.html", host="biccamera.rakuten.co.jp", shop_code="biccamera", item_code="4548736147041"
+    )
+
+
+# ---- 通用工具 ----
+
+@pytest.mark.parametrize(
+    ("raw", "expected"),
+    [("3,091円", 3091), ("1760", 1760), (1760, 1760), ("", 0), (None, 0), ("価格未定", 0)],
+)
+def test_parse_price_handles_site_formats(raw, expected):
+    assert parse_price(raw) == expected
+
+
+@pytest.mark.parametrize(
+    ("status", "expected"),
+    [("在庫あり", False), ("在庫なし", True), ("品切れ", True), ("販売終了", True), ("", False)],
+)
+def test_looks_sold_out_reads_japanese_stock_wording(status, expected):
+    assert looks_sold_out(status) is expected
+
+
+# ---- 分派 ----
+
+def test_registry_covers_the_three_known_subsites():
+    assert set(SUBSITE_PARSERS) == {
+        "books.rakuten.co.jp",
+        "brandavenue.rakuten.co.jp",
+        "biccamera.rakuten.co.jp",
+    }
+
+
+def test_validator_accepts_ichiba_page_with_state():
+    validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/")
+    ok = validate("window.__INITIAL_STATE__ = {}", "https://item.rakuten.co.jp/s/c/")
+    assert ok is None
+
+
+def test_validator_rejects_ichiba_page_without_state_as_retryable():
+    validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/")
+    reason = validate("<html>blocked</html>", "https://item.rakuten.co.jp/s/c/")
+    assert reason and "INITIAL_STATE" in reason
+
+
+def test_validator_raises_for_unregistered_subsite():
+    """未登记的站点是「不支持」而非「抓取失败」，不能走重试"""
+    validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/")
+    with pytest.raises(OffIchibaRedirectError):
+        validate("<html>whatever</html>", "https://unknown.rakuten.co.jp/x/")
+
+
+def test_validator_delegates_to_subsite_rules(books_page):
+    validate = build_item_page_validator(books_page.requested_url)
+    assert validate(books_page.html, books_page.final_url) is None
+    assert validate("<html>empty</html>", books_page.final_url) is not None
+
+
+def test_parse_subsite_item_rejects_unknown_host(books_page):
+    books_page.final_url = "https://unknown.rakuten.co.jp/x/"
+    with pytest.raises(ScrapeParseError):
+        parse_subsite_item(books_page)
+
+
+def test_host_of_extracts_hostname():
+    assert host_of("https://books.rakuten.co.jp/rb/1/?a=b") == "books.rakuten.co.jp"
+
+
+# ---- 楽天ブックス ----
+
+def test_books_extracts_core_fields(books_page):
+    d = parse_subsite_item(books_page)
+    assert d.source == "books"
+    assert d.source_url == books_page.final_url
+    assert d.item_url == books_page.requested_url
+    assert d.item_name == "12歳までに知っておきたい語彙力図鑑"
+    assert d.price == 1760
+    assert d.shop.shop_code == "book"
+    assert d.shop.shop_name == "楽天ブックス"
+
+
+def test_books_maps_review_and_stock(books_page):
+    d = parse_subsite_item(books_page)
+    assert d.review.score == pytest.approx(4.54)
+    assert d.review.count == 218
+    assert d.purchase_condition == "在庫あり"
+    assert d.is_sold_out is False
+
+
+def test_books_uses_rms_genre_ids_for_breadcrumbs(books_page):
+    """站内分类 ID 对上游没用，要取 data_genres 里的 rmsGenreId（市场侧 genre_id）"""
+    d = parse_subsite_item(books_page)
+    assert [b.name for b in d.breadcrumbs] == ["本", "絵本・児童書・図鑑", "図鑑・ちしき"]
+    assert d.genre_id == "208880"
+    assert d.breadcrumbs[0].url == "https://www.rakuten.co.jp/category/200162/"
+
+
+def test_books_spec_table_becomes_sku_attributes(books_page):
+    d = parse_subsite_item(books_page)
+    spec = {a.title: a.value for a in d.sku.attributes}
+    assert spec["ISBN"] == "9784800590039"
+    assert spec["出版社"] == "日本能率協会マネジメントセンター"
+    assert "齋藤 孝" in spec["著者／編集"]
+
+
+def test_books_description_collects_flat_sibling_sections(books_page):
+    """标题与正文是兄弟节点而非父子，按容器取会漏掉绝大部分内容"""
+    d = parse_subsite_item(books_page)
+    assert len(d.description) > 1000
+    assert "内容紹介（JPROより）" in d.description
+    assert "目次" in d.description
+
+
+def test_books_images_are_absolute_and_deduplicated(books_page):
+    d = parse_subsite_item(books_page)
+    assert d.images
+    assert all(url.startswith("https://") for url in d.images)
+    assert len(d.images) == len(set(d.images))
+
+
+def test_books_has_no_sku_variants(books_page):
+    """图书没有规格轴，SKU 只承载规格表"""
+    d = parse_subsite_item(books_page)
+    assert d.sku.inventory_type == "single"
+    assert d.sku.variant_count == 0
+    assert d.sku.axis == []
+
+
+# ---- Rakuten Fashion ----
+
+def test_brandavenue_extracts_core_fields(brandavenue_page):
+    d = parse_subsite_item(brandavenue_page)
+    assert d.source == "brandavenue"
+    assert d.item_name == "プリント ロゴ  刺繍 エンブロイダリー 半袖 Tシャツ"
+    assert d.price == 3091
+    assert d.catch_copy == "andme"  # 品牌名
+    assert d.is_sold_out is False
+
+
+def test_brandavenue_reads_ichiba_ids_from_rms_info(brandavenue_page):
+    """市场侧的商品 ID 与 genre_id 藏在 rms_info 里，不是站内编码"""
+    d = parse_subsite_item(brandavenue_page)
+    assert d.item_id == "14674748"
+    assert d.genre_id == "303656"
+    assert d.shop.shop_id == 279405
+    assert d.shop.shop_code == "stylife"
+
+
+def test_brandavenue_builds_color_and_size_axes(brandavenue_page):
+    d = parse_subsite_item(brandavenue_page)
+    assert [a.label for a in d.sku.axis] == ["カラー", "サイズ"]
+    assert d.sku.inventory_type == "multiple"
+    assert d.sku.variant_count == len(d.sku.variants) > 1
+
+
+def test_brandavenue_variant_joins_stock_from_inventory_list(brandavenue_page):
+    """售价来自 product_sku，库存与 variant_id 来自 rms_info.inventory_list，按尺码+颜色对齐"""
+    d = parse_subsite_item(brandavenue_page)
+    by_selector = {tuple(v.selector_values): v for v in d.sku.variants}
+    variant = by_selector[("オフホワイト×レッド", "Sサイズ")]
+    assert variant.price == 2810
+    assert variant.quantity == 90
+    assert variant.variant_id
+
+
+def test_brandavenue_variant_without_matching_inventory_stays_purchasable(brandavenue_page):
+    """站点两侧的颜色命名偶有出入导致对不上；此时库存未知，但不能据此判成售罄"""
+    import json
+    import re
+
+    html = brandavenue_page.html
+    state = json.loads(re.search(r"__INITIAL_STATE__ = (\{.*?\});window", html, re.S).group(1))
+    state["itemDetail"]["data"]["product"]["rms_info"]["inventory_list"] = []
+    brandavenue_page.html = re.sub(
+        r"__INITIAL_STATE__ = \{.*?\};window",
+        lambda _: f"__INITIAL_STATE__ = {json.dumps(state, ensure_ascii=False)};window",
+        html,
+        flags=re.S,
+    )
+
+    d = parse_subsite_item(brandavenue_page)
+    variant = d.sku.variants[0]
+    assert variant.quantity == 0
+    assert variant.variant_id == ""
+    assert variant.is_sold_out is False  # 以 inventory_exist_flg 为准
+
+
+def test_brandavenue_can_omit_variants_but_keeps_count(brandavenue_page):
+    brandavenue_page.include_sku_variants = False
+    d = parse_subsite_item(brandavenue_page)
+    assert d.sku.variants == []
+    assert d.sku.variant_count > 0
+    assert d.sku.axis
+
+
+def test_brandavenue_takes_images_rendered_in_page(brandavenue_page):
+    """图片按商品编号做了目录分片，分片规则不外露，只能取页面已渲染的地址"""
+    d = parse_subsite_item(brandavenue_page)
+    assert d.images
+    assert all("stylife/cabinet/item" in url for url in d.images)
+    # 主图应排在最前
+    assert d.images[0].lower().endswith("md7606-13_1.jpg")
+
+
+def test_brandavenue_breadcrumbs_have_names_without_urls(brandavenue_page):
+    """站内分类编码不是市场 genre_id，拼不出可用链接，只给名称"""
+    d = parse_subsite_item(brandavenue_page)
+    assert [b.name for b in d.breadcrumbs] == ["トップス", "カットソー・Tシャツ"]
+    assert all(b.url == "" for b in d.breadcrumbs)
+
+
+# ---- ビックカメラ ----
+
+def test_biccamera_extracts_core_fields(biccamera_page):
+    d = parse_subsite_item(biccamera_page)
+    assert d.source == "biccamera"
+    assert d.item_name.startswith("SONY")
+    assert d.price == 73760
+    assert d.item_id == "13873289"
+    assert d.item_code == "4548736147041"
+    assert d.shop.shop_id == 269553
+
+
+def test_biccamera_maps_inventory_and_delivery(biccamera_page):
+    d = parse_subsite_item(biccamera_page)
+    assert d.is_sold_out is False
+    assert d.purchase_condition == "enabled"
+    assert d.sku.quantity == 28
+    assert d.sku.show_inventory is True
+    assert "出荷予定" in d.shipping.delivery_message
+    assert d.shipping.is_shipping_free is True
+
+
+def test_biccamera_breadcrumbs_use_ichiba_genre_ids(biccamera_page):
+    d = parse_subsite_item(biccamera_page)
+    assert [b.name for b in d.breadcrumbs][0] == "TV・オーディオ・カメラ"
+    assert d.breadcrumbs[0].url == "https://www.rakuten.co.jp/category/211742/"
+    assert d.genre_id == "110110"
+
+
+def test_biccamera_rejects_page_without_nuxt_state(biccamera_page):
+    biccamera_page.html = "<html><script>window.__NUXT__={\"state\":{}}</script></html>"
+    with pytest.raises(ScrapeParseError):
+        parse_subsite_item(biccamera_page)