拆分抓取与交易服务

把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」，而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询，同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的ラクマ分类接口与登录态基础设施。验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-07-27 15:05:01 +08:00
parent 4250388762
commit 104d7fef6b
80 changed files with 2330 additions and 402 deletions
@@ -0,0 +1,223 @@
+"""ラクマ 店铺页 HTML → 卖家详情与卖家商品列表
+
+C2C 集市里的「商家」就是个人卖家，页面在 fril.jp/shop/{hash}：
+- 店铺页本身：卖家资料 + 该卖家的商品分页列表（含已售出）
+- /review 子页：评价明细与好评/普通/差评分档计数
+
+商品卡片与搜索页共用 `.item-box` 结构，直接复用 search 里的解析。
+"""
+from __future__ import annotations
+
+import json
+import re
+
+from selectolax.parser import HTMLParser
+
+from app.scraping.core import rakuma_site as site
+from app.shared.errors import ScrapeParseError
+from app.scraping.models.scrape import (
+    RakumaRatingBreakdown,
+    RakumaReview,
+    RakumaShopDetailData,
+    RakumaShopItemsData,
+)
+from app.scraping.parsers.rakuma.base import (
+    attr,
+    event_payload,
+    image_url,
+    node_text,
+    parse_float,
+    parse_int,
+    parse_total_count,
+)
+from app.scraping.parsers.rakuma.search import parse_item_cards
+
+# 评价条目标题左侧的图标 class → 评价档位
+_RATING_ICONS = {
+    "icon_review_sun": "good",  # よい
+    "icon_review_cloud": "normal",  # ふつう
+    "icon_review_rain": "bad",  # わるい
+}
+
+# /review 页上三组分档计数的容器 id 前缀
+_ALL_RATINGS_PREFIX = "all"
+_SELLER_RATINGS_PREFIX = "seller"
+
+_LD_JSON_RE = re.compile(
+    r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
+)
+
+
+def _require_shop_page(html: str) -> HTMLParser:
+    tree = HTMLParser(html)
+    if tree.css_first(f".{site.SHOP_PAGE_MARKER}") is None:
+        raise ScrapeParseError("页面不是店铺页（缺少店铺资料区块）")
+    return tree
+
+
+def _parse_rating_breakdown(tree: HTMLParser, prefix: str) -> RakumaRatingBreakdown:
+    """解析一组好评/普通/差评计数
+
+    页面用 `<ul class="nav-pills">` 里的三个链接展示，锚点形如
+    `#all-good` / `#seller-normal`，按锚点前缀区分「全部」与「出品」两组。
+    """
+    counts = {"good": 0, "normal": 0, "bad": 0}
+    for link in tree.css("ul.nav-pills a"):
+        href = attr(link, "href")
+        for rating in counts:
+            if href == f"#{prefix}-{rating}":
+                counts[rating] = parse_int(node_text(link))
+    return RakumaRatingBreakdown(**counts)
+
+
+def _parse_reviews(tree: HTMLParser) -> list[RakumaReview]:
+    """解析评价列表
+
+    站点在「すべての評価」标签页里最多展示最新 100 条，且三个标签页
+    （全部/出品/购入）的条目在 DOM 里重复出现，这里只取第一个激活面板。
+    """
+    panel = tree.css_first("#all-all") or tree.css_first(".tab-pane.active")
+    if panel is None:
+        return []
+
+    reviews: list[RakumaReview] = []
+    for article in panel.css("article.review-item"):
+        title_node = article.css_first(".review-item-title")
+        icon = title_node.css_first("i") if title_node else None
+        classes = attr(icon, "class").split() if icon else []
+        rating = next((_RATING_ICONS[name] for name in classes if name in _RATING_ICONS), "")
+
+        reviews.append(
+            RakumaReview(
+                rating=rating,
+                title=node_text(title_node),
+                comment=node_text(article.css_first(".review-item-text")),
+                reviewer_name=node_text(article.css_first(".review-item-name")),
+                reviewed_at=node_text(article.css_first(".review-item-date")),
+            )
+        )
+    return reviews
+
+
+def _parse_store_rating(html: str) -> tuple[float, int]:
+    """从店铺页的 ld+json Store 节点取评分与评价数
+
+    评价数只有这里给得出来——可见 DOM 上只有星级和分数，没有条数。
+    """
+    for match in _LD_JSON_RE.finditer(html):
+        try:
+            data = json.loads(match.group(1))
+        except ValueError:
+            continue
+        if not isinstance(data, dict) or data.get("@type") != "Store":
+            continue
+        rating = data.get("aggregateRating")
+        if isinstance(rating, dict):
+            return parse_float(rating.get("ratingValue")), parse_int(rating.get("ratingCount"))
+    return 0.0, 0
+
+
+def parse_shop_detail(
+    html: str, *, shop_id: str, shop_url: str, review_html: str | None = None
+) -> RakumaShopDetailData:
+    """把店铺页 HTML 解析为卖家详情
+
+    Args:
+        review_html: /review 子页的 HTML；给出时才填充评价明细与分档计数
+
+    Raises:
+        ScrapeParseError: 页面不是店铺页
+    """
+    tree = _require_shop_page(html)
+
+    total_count, _, _ = parse_total_count(node_text(tree.css_first(".page-count")))
+    badge = tree.css_first(".badge-status")
+    verification_label = node_text(badge)
+
+    # 简介在侧栏「プロフィール」区块；卖家未填写时站点会写一句占位文案
+    introduction = node_text(tree.css_first("[data-test=profile-text-top]"))
+    if "設定されていません" in introduction:
+        introduction = ""
+
+    score, review_count = _parse_store_rating(html)
+    detail = RakumaShopDetailData(
+        shop_id=shop_id,
+        shop_name=node_text(tree.css_first(".profile-area__shop-name")),
+        user_name=node_text(tree.css_first("[data-test=profile_user_name], .profile-area__user-name")),
+        shop_url=shop_url,
+        icon_url=image_url(tree.css_first(".profile-area__user-icon img")),
+        cover_url=_cover_url(tree),
+        introduction=introduction,
+        review_score=score or parse_float(node_text(tree.css_first(".shop_score__score"))),
+        review_count=review_count,
+        is_verified="未完了" not in verification_label and bool(verification_label),
+        verification_label=verification_label,
+        item_count=total_count,
+    )
+
+    # 用户数值 ID：优先取商品卡片埋点里的 seller_user_id（店铺页所有商品都属于
+    # 该卖家），卖家未设头像时头像地址是站点默认图，取不到 ID。
+    detail.user_id = _seller_user_id(tree) or _user_id_from_icon(detail.icon_url)
+
+    if review_html is not None:
+        review_tree = HTMLParser(review_html)
+        detail.rating_breakdown = _parse_rating_breakdown(review_tree, _ALL_RATINGS_PREFIX)
+        detail.seller_rating_breakdown = _parse_rating_breakdown(review_tree, _SELLER_RATINGS_PREFIX)
+        detail.reviews = _parse_reviews(review_tree)
+
+    return detail
+
+
+def _cover_url(tree: HTMLParser) -> str:
+    """封面图挂在 inline style 的 background url() 里"""
+    cover = tree.css_first(".profile-area__shop-cover")
+    style = attr(cover, "style")
+    start = style.find("url(")
+    if start < 0:
+        return ""
+    end = style.find(")", start)
+    return style[start + 4 : end].strip("'\" ") if end > start else ""
+
+
+def _seller_user_id(tree: HTMLParser) -> str:
+    """从店铺页商品卡片的埋点里取卖家数值 ID"""
+    for node in tree.css("[data-gtm-click], [onclick]"):
+        payload = event_payload(node)
+        user_id = payload.get("seller_user_id")
+        if user_id:
+            return str(user_id)
+    return ""
+
+
+def _user_id_from_icon(icon_url: str) -> str:
+    """从头像地址 https://img.fril.jp/user/{id}/s/{id}.jpg 里取用户数值 ID"""
+    marker = "/user/"
+    start = icon_url.find(marker)
+    if start < 0:
+        return ""
+    rest = icon_url[start + len(marker) :]
+    user_id = rest.split("/", 1)[0]
+    return user_id if user_id.isdigit() else ""
+
+
+def parse_shop_items(
+    html: str, *, shop_id: str, request_url: str, page: int
+) -> RakumaShopItemsData:
+    """把店铺页 HTML 解析为该卖家的商品列表
+
+    Raises:
+        ScrapeParseError: 页面不是店铺页
+    """
+    tree = _require_shop_page(html)
+    items = parse_item_cards(tree)
+    total_count, _, end = parse_total_count(node_text(tree.css_first(".page-count")))
+
+    return RakumaShopItemsData(
+        shop_id=shop_id,
+        shop_name=node_text(tree.css_first(".profile-area__shop-name")),
+        page=page,
+        total_count=total_count,
+        has_more=bool(items) and bool(end) and end < total_count,
+        request_url=request_url,
+        items=items,
+    )