224 lines
7.8 KiB
Python
224 lines
7.8 KiB
Python
"""ラクマ 店铺页 HTML → 卖家详情与卖家商品列表
|
|
|
|
C2C 集市里的「商家」就是个人卖家，页面在 fril.jp/shop/{hash}：
|
|
- 店铺页本身：卖家资料 + 该卖家的商品分页列表（含已售出）
|
|
- /review 子页：评价明细与好评/普通/差评分档计数
|
|
|
|
商品卡片与搜索页共用 `.item-box` 结构，直接复用 search 里的解析。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
|
|
from selectolax.parser import HTMLParser
|
|
|
|
from app.core import rakuma_site as site
|
|
from app.core.errors import ScrapeParseError
|
|
from app.models.scrape import (
|
|
RakumaRatingBreakdown,
|
|
RakumaReview,
|
|
RakumaShopDetailData,
|
|
RakumaShopItemsData,
|
|
)
|
|
from app.parsers.rakuma.base import (
|
|
attr,
|
|
event_payload,
|
|
image_url,
|
|
node_text,
|
|
parse_float,
|
|
parse_int,
|
|
parse_total_count,
|
|
)
|
|
from app.parsers.rakuma.search import parse_item_cards
|
|
|
|
# 评价条目标题左侧的图标 class → 评价档位
|
|
_RATING_ICONS = {
|
|
"icon_review_sun": "good", # よい
|
|
"icon_review_cloud": "normal", # ふつう
|
|
"icon_review_rain": "bad", # わるい
|
|
}
|
|
|
|
# /review 页上三组分档计数的容器 id 前缀
|
|
_ALL_RATINGS_PREFIX = "all"
|
|
_SELLER_RATINGS_PREFIX = "seller"
|
|
|
|
_LD_JSON_RE = re.compile(
|
|
r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
|
|
)
|
|
|
|
|
|
def _require_shop_page(html: str) -> HTMLParser:
|
|
tree = HTMLParser(html)
|
|
if tree.css_first(f".{site.SHOP_PAGE_MARKER}") is None:
|
|
raise ScrapeParseError("页面不是店铺页（缺少店铺资料区块）")
|
|
return tree
|
|
|
|
|
|
def _parse_rating_breakdown(tree: HTMLParser, prefix: str) -> RakumaRatingBreakdown:
|
|
"""解析一组好评/普通/差评计数
|
|
|
|
页面用 `<ul class="nav-pills">` 里的三个链接展示，锚点形如
|
|
`#all-good` / `#seller-normal`，按锚点前缀区分「全部」与「出品」两组。
|
|
"""
|
|
counts = {"good": 0, "normal": 0, "bad": 0}
|
|
for link in tree.css("ul.nav-pills a"):
|
|
href = attr(link, "href")
|
|
for rating in counts:
|
|
if href == f"#{prefix}-{rating}":
|
|
counts[rating] = parse_int(node_text(link))
|
|
return RakumaRatingBreakdown(**counts)
|
|
|
|
|
|
def _parse_reviews(tree: HTMLParser) -> list[RakumaReview]:
|
|
"""解析评价列表
|
|
|
|
站点在「すべての評価」标签页里最多展示最新 100 条，且三个标签页
|
|
（全部/出品/购入）的条目在 DOM 里重复出现，这里只取第一个激活面板。
|
|
"""
|
|
panel = tree.css_first("#all-all") or tree.css_first(".tab-pane.active")
|
|
if panel is None:
|
|
return []
|
|
|
|
reviews: list[RakumaReview] = []
|
|
for article in panel.css("article.review-item"):
|
|
title_node = article.css_first(".review-item-title")
|
|
icon = title_node.css_first("i") if title_node else None
|
|
classes = attr(icon, "class").split() if icon else []
|
|
rating = next((_RATING_ICONS[name] for name in classes if name in _RATING_ICONS), "")
|
|
|
|
reviews.append(
|
|
RakumaReview(
|
|
rating=rating,
|
|
title=node_text(title_node),
|
|
comment=node_text(article.css_first(".review-item-text")),
|
|
reviewer_name=node_text(article.css_first(".review-item-name")),
|
|
reviewed_at=node_text(article.css_first(".review-item-date")),
|
|
)
|
|
)
|
|
return reviews
|
|
|
|
|
|
def _parse_store_rating(html: str) -> tuple[float, int]:
|
|
"""从店铺页的 ld+json Store 节点取评分与评价数
|
|
|
|
评价数只有这里给得出来——可见 DOM 上只有星级和分数，没有条数。
|
|
"""
|
|
for match in _LD_JSON_RE.finditer(html):
|
|
try:
|
|
data = json.loads(match.group(1))
|
|
except ValueError:
|
|
continue
|
|
if not isinstance(data, dict) or data.get("@type") != "Store":
|
|
continue
|
|
rating = data.get("aggregateRating")
|
|
if isinstance(rating, dict):
|
|
return parse_float(rating.get("ratingValue")), parse_int(rating.get("ratingCount"))
|
|
return 0.0, 0
|
|
|
|
|
|
def parse_shop_detail(
|
|
html: str, *, shop_id: str, shop_url: str, review_html: str | None = None
|
|
) -> RakumaShopDetailData:
|
|
"""把店铺页 HTML 解析为卖家详情
|
|
|
|
Args:
|
|
review_html: /review 子页的 HTML；给出时才填充评价明细与分档计数
|
|
|
|
Raises:
|
|
ScrapeParseError: 页面不是店铺页
|
|
"""
|
|
tree = _require_shop_page(html)
|
|
|
|
total_count, _, _ = parse_total_count(node_text(tree.css_first(".page-count")))
|
|
badge = tree.css_first(".badge-status")
|
|
verification_label = node_text(badge)
|
|
|
|
# 简介在侧栏「プロフィール」区块；卖家未填写时站点会写一句占位文案
|
|
introduction = node_text(tree.css_first("[data-test=profile-text-top]"))
|
|
if "設定されていません" in introduction:
|
|
introduction = ""
|
|
|
|
score, review_count = _parse_store_rating(html)
|
|
detail = RakumaShopDetailData(
|
|
shop_id=shop_id,
|
|
shop_name=node_text(tree.css_first(".profile-area__shop-name")),
|
|
user_name=node_text(tree.css_first("[data-test=profile_user_name], .profile-area__user-name")),
|
|
shop_url=shop_url,
|
|
icon_url=image_url(tree.css_first(".profile-area__user-icon img")),
|
|
cover_url=_cover_url(tree),
|
|
introduction=introduction,
|
|
review_score=score or parse_float(node_text(tree.css_first(".shop_score__score"))),
|
|
review_count=review_count,
|
|
is_verified="未完了" not in verification_label and bool(verification_label),
|
|
verification_label=verification_label,
|
|
item_count=total_count,
|
|
)
|
|
|
|
# 用户数值 ID：优先取商品卡片埋点里的 seller_user_id（店铺页所有商品都属于
|
|
# 该卖家），卖家未设头像时头像地址是站点默认图，取不到 ID。
|
|
detail.user_id = _seller_user_id(tree) or _user_id_from_icon(detail.icon_url)
|
|
|
|
if review_html is not None:
|
|
review_tree = HTMLParser(review_html)
|
|
detail.rating_breakdown = _parse_rating_breakdown(review_tree, _ALL_RATINGS_PREFIX)
|
|
detail.seller_rating_breakdown = _parse_rating_breakdown(review_tree, _SELLER_RATINGS_PREFIX)
|
|
detail.reviews = _parse_reviews(review_tree)
|
|
|
|
return detail
|
|
|
|
|
|
def _cover_url(tree: HTMLParser) -> str:
|
|
"""封面图挂在 inline style 的 background url() 里"""
|
|
cover = tree.css_first(".profile-area__shop-cover")
|
|
style = attr(cover, "style")
|
|
start = style.find("url(")
|
|
if start < 0:
|
|
return ""
|
|
end = style.find(")", start)
|
|
return style[start + 4 : end].strip("'\" ") if end > start else ""
|
|
|
|
|
|
def _seller_user_id(tree: HTMLParser) -> str:
|
|
"""从店铺页商品卡片的埋点里取卖家数值 ID"""
|
|
for node in tree.css("[data-gtm-click], [onclick]"):
|
|
payload = event_payload(node)
|
|
user_id = payload.get("seller_user_id")
|
|
if user_id:
|
|
return str(user_id)
|
|
return ""
|
|
|
|
|
|
def _user_id_from_icon(icon_url: str) -> str:
|
|
"""从头像地址 https://img.fril.jp/user/{id}/s/{id}.jpg 里取用户数值 ID"""
|
|
marker = "/user/"
|
|
start = icon_url.find(marker)
|
|
if start < 0:
|
|
return ""
|
|
rest = icon_url[start + len(marker) :]
|
|
user_id = rest.split("/", 1)[0]
|
|
return user_id if user_id.isdigit() else ""
|
|
|
|
|
|
def parse_shop_items(
|
|
html: str, *, shop_id: str, request_url: str, page: int
|
|
) -> RakumaShopItemsData:
|
|
"""把店铺页 HTML 解析为该卖家的商品列表
|
|
|
|
Raises:
|
|
ScrapeParseError: 页面不是店铺页
|
|
"""
|
|
tree = _require_shop_page(html)
|
|
items = parse_item_cards(tree)
|
|
total_count, _, end = parse_total_count(node_text(tree.css_first(".page-count")))
|
|
|
|
return RakumaShopItemsData(
|
|
shop_id=shop_id,
|
|
shop_name=node_text(tree.css_first(".profile-area__shop-name")),
|
|
page=page,
|
|
total_count=total_count,
|
|
has_more=bool(items) and bool(end) and end < total_count,
|
|
request_url=request_url,
|
|
items=items,
|
|
)
|