Init

app/parsers/subsites/books.py

@@ -0,0 +1,187 @@
+"""楽天ブックス（books.rakuten.co.jp）商品页解析
+
+该站是传统服务端渲染页面，商品数据以 schema.org 微数据标注（Product / Offer /
+AggregateRating），规格与简介在 `.sec-item` 分节里，分类路径则通过页面内联的
+`var data_genres` 给出——其中的 rmsGenreId 就是市场侧的 genre_id。
+
+站点不提供：SKU 组合（图书没有规格轴）、运费明细、店铺评分。
+"""
+from __future__ import annotations
+
+import json
+import re
+
+from selectolax.parser import HTMLParser
+
+from app.core.errors import ScrapeParseError
+from app.models.scrape import (
+    Breadcrumb,
+    ItemDetailData,
+    PurchaseInfo,
+    ReviewSummary,
+    ShippingInfo,
+    ShopSummary,
+    SkuAttribute,
+    SkuInfo,
+)
+from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
+
+HOST = "books.rakuten.co.jp"
+SOURCE = "books"
+SHOP_NAME = "楽天ブックス"
+
+_GENRES_RE = re.compile(r"var\s+data_genres\s*=\s*")
+# 折叠正文里由展开控件（checkbox/label）留下的连续空行
+_BLANK_LINES_RE = re.compile(r"\n{2,}")
+_IMAGE_RE = re.compile(r"//tshop\.r10s\.jp/book/cabinet/[^\"'\s?]+\.(?:jpg|jpeg|png)", re.I)
+_CATEGORY_URL = "https://www.rakuten.co.jp/category/{}/"
+
+
+def validate(html: str) -> str | None:
+    """页面须带 Product 微数据，否则不是正常的商品页"""
+    if 'itemprop="price"' in html and "schema.org/Product" in html:
+        return None
+    return f"books item page markup not found (body {len(html)} bytes)"
+
+
+def _attr(tree: HTMLParser, selector: str, name: str) -> str:
+    node = tree.css_first(selector)
+    return (node.attributes.get(name) or "") if node else ""
+
+
+def _text(tree: HTMLParser, selector: str) -> str:
+    node = tree.css_first(selector)
+    return node.text(strip=True) if node else ""
+
+
+def _parse_spec(tree: HTMLParser) -> list[SkuAttribute]:
+    """商品情報分节：每个 ul 内首个 li.product-title 是字段名，其后是取值"""
+    attributes: list[SkuAttribute] = []
+    for row in tree.css(".sec-item__identifier__list ul"):
+        cells = row.css("li")
+        if len(cells) < 2:
+            continue
+        title = cells[0]
+        if "product-title" not in (title.attributes.get("class") or ""):
+            continue
+        value = " ".join(cell.text(strip=True) for cell in cells[1:] if cell.text(strip=True))
+        if value:
+            attributes.append(SkuAttribute(title=title.text(strip=True), value=value))
+    return attributes
+
+
+def _parse_description(tree: HTMLParser) -> str:
+    """商品説明分节：把「内容紹介」「目次」等若干小节拼起来
+
+    这些小节在 DOM 里是平铺的——标题 h3 与正文 div 互为兄弟节点而非父子，
+    所以要从标题往后找同级的正文块，不能直接按容器取。
+    """
+    parts: list[str] = []
+    for title in tree.css(".sec-item__extra__title"):
+        node = title.next
+        while node is not None:
+            classes = node.attributes.get("class") or "" if node.tag != "-text" else ""
+            if "sec-item__extra__content" in classes:
+                body = _BLANK_LINES_RE.sub("\n", node.text(separator="\n", strip=True)).strip()
+                if body:
+                    parts.append(f"{title.text(strip=True)}\n{body}")
+                break
+            # 只在紧邻的兄弟里找，遇到下一个标题就说明这一节没有正文
+            if node.tag != "-text" and "sec-item__extra__title" in classes:
+                break
+            node = node.next
+    return "\n\n".join(parts)
+
+
+def _parse_genres(html: str) -> tuple[str, list[Breadcrumb]]:
+    """内联的 data_genres 给出分类路径，其中 rmsGenreId 对应市场侧 genre_id"""
+    match = _GENRES_RE.search(html)
+    if match is None:
+        return "", []
+    try:
+        raw, _ = json.JSONDecoder().raw_decode(html, match.end())
+    except ValueError:
+        return "", []
+
+    # 结构是 [[{...}, {...}]]，取第一条路径
+    path = raw[0] if isinstance(raw, list) and raw and isinstance(raw[0], list) else raw
+    crumbs: list[Breadcrumb] = []
+    genre_id = ""
+    for node in path if isinstance(path, list) else []:
+        if not isinstance(node, dict):
+            continue
+        rms_id = str(node.get("rmsGenreId") or "")
+        name = str(node.get("genreName") or "")
+        if not name:
+            continue
+        crumbs.append(Breadcrumb(name=name, url=_CATEGORY_URL.format(rms_id) if rms_id else ""))
+        if rms_id:
+            genre_id = rms_id
+    return genre_id, crumbs
+
+
+def _parse_purchase(tree: HTMLParser) -> PurchaseInfo:
+    """加购信息直接取页面上的购物车表单
+
+    注意表单里的 item_id 与商品 URL 上的编号不是一回事，加购必须用表单里的值。
+    该表单没有数量字段，无法在加购时指定件数。
+    """
+    for form in tree.css("form"):
+        action = form.attributes.get("action") or ""
+        if "/bs/Cart" not in action:
+            continue
+        fields = {
+            name: inp.attributes.get("value") or ""
+            for inp in form.css("input")
+            if (name := inp.attributes.get("name"))
+        }
+        return PurchaseInfo(
+            cart_url=action,
+            cart_method=(form.attributes.get("method") or "POST").upper(),
+            form_fields=fields,
+        )
+    return PurchaseInfo()
+
+
+def parse(page: SubsitePage) -> ItemDetailData:
+    """解析楽天ブックス商品页"""
+    tree = HTMLParser(page.html)
+
+    name = _text(tree, "#productTitle") or _text(tree, '[itemprop="name"]')
+    if not name:
+        raise ScrapeParseError("楽天ブックス页面未找到商品名")
+
+    images = ["https:" + url if url.startswith("//") else url for url in _IMAGE_RE.findall(page.html)]
+    # 同一张图可能带不同裁剪参数重复出现，去重但保留出现顺序
+    images = list(dict.fromkeys(images))
+
+    status = _text(tree, ".status")
+    genre_id, breadcrumbs = _parse_genres(page.html)
+    review_count = _text(tree, '[itemprop="reviewCount"]')
+    purchase = _parse_purchase(tree)
+
+    return ItemDetailData(
+        source=SOURCE,
+        source_url=page.final_url,
+        # 站内商品 ID 与 URL 上的编号不同，以购物车表单里的为准（下单要用它）
+        item_id=purchase.form_fields.get("item_id", "") or page.item_code,
+        item_code=page.item_code,
+        item_name=name,
+        description=_parse_description(tree),
+        item_url=page.requested_url,
+        price=parse_price(_attr(tree, '[itemprop="price"]', "content")),
+        purchase_condition=status,
+        is_sold_out=looks_sold_out(status),
+        images=images,
+        shop=ShopSummary(shop_code=page.shop_code, shop_name=SHOP_NAME),
+        review=ReviewSummary(
+            score=float(_attr(tree, '[itemprop="ratingValue"]', "content") or 0) or 0.0,
+            count=parse_price(review_count),
+        ),
+        genre_id=genre_id,
+        breadcrumbs=breadcrumbs,
+        # 图书统一由楽天ブックス发货，页面只给库存措辞，不给运费明细
+        shipping=ShippingInfo(delivery_message=status),
+        sku=SkuInfo(inventory_type="single", attributes=_parse_spec(tree), delivery_message=status),
+        purchase=purchase,
+    )