拆分抓取与交易服务
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」, 而是抓取无状态、幂等、可多开实例,而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询, 同一账号会被并发操作。 - app/shared:配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping:站点常量、会话、解析器与 10 个抓取接口,:31107,可多开 - app/trading:登录态查询/重载与健康检查,:31108,只能单实例 - 依赖方向锁为 scraping→shared、trading→shared,两侧互不 import; tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有:与抓取 UA 值相同但变更理由不同,抓取 UA 为绕 反爬可随时调整,登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in,判据只写一遍 - 同一镜像两个启动命令,交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证:239 个离线用例全绿;两个入口真实启动,/health 与鉴权正常。 未验证:真实探测登录态(当前开发机无外网,对站点的连接全部超时)。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,9 @@
|
||||
"""ラクマ(fril.jp)页面解析器
|
||||
|
||||
站点是服务端渲染的 HTML,没有内联状态 JSON,因此各模块都走 DOM 解析:
|
||||
- base — 埋点属性与文本取值的公共工具
|
||||
- search — 搜索页(商品卡片解析同时被店铺页复用)
|
||||
- item — 商品详情页
|
||||
- shop — 店铺页与评价页
|
||||
- category — 分类一览页(唯一的例外:Next.js 页面,数据在 RSC flight payload 里)
|
||||
"""
|
||||
@@ -0,0 +1,144 @@
|
||||
"""ラクマ(fril.jp)页面解析的公共工具
|
||||
|
||||
站点是服务端渲染的 HTML,没有 `window.__INITIAL_STATE__` 之类的内联状态,
|
||||
因此全部走 DOM 解析。好在页面上挂了成套的埋点属性(`data-rat-*` 与
|
||||
`onclick` 里的 dataLayer JSON),它们比可见文案稳定得多,也带有可见 DOM
|
||||
上没有的字段(商品数值 ID、卖家 ID、分类 ID、品牌 ID),所以优先取这些。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from selectolax.parser import Node
|
||||
|
||||
# 「約1,190,000件中 1 - 40件」里的总数与区间
|
||||
_COUNT_RE = re.compile(r"([\d,]+)\s*件中\s*([\d,]+)\s*[-−–]\s*([\d,]+)\s*件")
|
||||
_DIGITS_RE = re.compile(r"-?\d+")
|
||||
# 页面级埋点属性 data-rat-cp-{key}="{value}"
|
||||
_RAT_PARAM_RE = re.compile(r'data-rat-cp-([\w]+)="([^"]*)"')
|
||||
|
||||
|
||||
def parse_int(text: str | int | float | None) -> int:
|
||||
"""从 "¥6,299" / "6399" / 6399 这类值里取出整数金额或计数"""
|
||||
if isinstance(text, bool) or text is None:
|
||||
return 0
|
||||
if isinstance(text, (int, float)):
|
||||
return int(text)
|
||||
digits = _DIGITS_RE.findall(text.replace(",", ""))
|
||||
return int(digits[0]) if digits else 0
|
||||
|
||||
|
||||
def parse_float(text: str | int | float | None) -> float:
|
||||
"""从 "5.0" 这类文本里取出评分"""
|
||||
if isinstance(text, bool) or text is None:
|
||||
return 0.0
|
||||
if isinstance(text, (int, float)):
|
||||
return float(text)
|
||||
match = re.search(r"\d+(?:\.\d+)?", text.replace(",", ""))
|
||||
return float(match.group()) if match else 0.0
|
||||
|
||||
|
||||
def node_text(node: Node | None) -> str:
|
||||
"""取节点的可见文本,压掉多余空白;节点不存在时返回空串"""
|
||||
if node is None:
|
||||
return ""
|
||||
return re.sub(r"\s+", " ", node.text(strip=True)).strip()
|
||||
|
||||
|
||||
def attr(node: Node | None, name: str) -> str:
|
||||
"""取节点属性,缺失时返回空串"""
|
||||
if node is None:
|
||||
return ""
|
||||
return (node.attributes.get(name) or "").strip()
|
||||
|
||||
|
||||
def image_url(node: Node | None) -> str:
|
||||
"""取图片地址:站点用 lazy load,真实地址在 data-original 上,src 是占位图"""
|
||||
if node is None:
|
||||
return ""
|
||||
return attr(node, "data-original") or attr(node, "src")
|
||||
|
||||
|
||||
def parse_total_count(text: str) -> tuple[int, int, int]:
|
||||
"""解析「N件中 X - Y件」,返回 (总数, 起, 止);解析不出时全为 0
|
||||
|
||||
注意搜索页这里的总数是四舍五入后的展示值(約1,190,000件),
|
||||
精确值要从埋点属性 data-rat-cp-totalresults 取;店铺页则是精确值。
|
||||
"""
|
||||
match = _COUNT_RE.search(text.replace("\xa0", " "))
|
||||
if match is None:
|
||||
return 0, 0, 0
|
||||
return (
|
||||
parse_int(match.group(1)),
|
||||
parse_int(match.group(2)),
|
||||
parse_int(match.group(3)),
|
||||
)
|
||||
|
||||
|
||||
def event_payload(node: Node | None) -> dict[str, Any]:
|
||||
"""从埋点里取出商品参数字典
|
||||
|
||||
商品链接的 onclick / data-gtm-click 上挂着一段 dataLayer JSON,形如:
|
||||
{"event":"fireEvent","eventData":{"event_parameter":{
|
||||
"item_id":"844649627","seller_user_id":"12073120",
|
||||
"category_id":"788","brand_id":"5296","price":6299, ...}}}
|
||||
这里面有可见 DOM 上没有的数值 ID,是搜索结果里最可靠的数据来源。
|
||||
"""
|
||||
if node is None:
|
||||
return {}
|
||||
|
||||
for source in (node.attributes.get("data-gtm-click"), node.attributes.get("onclick")):
|
||||
if not source:
|
||||
continue
|
||||
for raw in _iter_json_objects(html.unescape(source)):
|
||||
parameter = (
|
||||
raw.get("eventData", {}).get("event_parameter")
|
||||
if isinstance(raw.get("eventData"), dict)
|
||||
else None
|
||||
)
|
||||
if isinstance(parameter, dict) and "item_id" in parameter:
|
||||
return parameter
|
||||
return {}
|
||||
|
||||
|
||||
def find_item_payload(tree: Any) -> dict[str, Any]:
|
||||
"""在整页里找出第一段带 item_id 的埋点参数
|
||||
|
||||
详情页的这段 JSON 挂在哪个元素上并不固定(在售商品挂在品牌链接上,
|
||||
已售商品的页面结构不同),因此按属性扫描而不是写死选择器。
|
||||
"""
|
||||
for node in tree.css("[data-gtm-click], [onclick]"):
|
||||
payload = event_payload(node)
|
||||
if payload:
|
||||
return payload
|
||||
return {}
|
||||
|
||||
|
||||
def rat_params(html_text: str) -> dict[str, str]:
|
||||
"""取出页面级埋点属性 `data-rat-cp-*`
|
||||
|
||||
详情页把成色、运费负担、发货地等信息也写在这组属性里。已售出商品的
|
||||
页面会换成另一套布局、规格表消失,但这组属性仍在,可用作兜底。
|
||||
"""
|
||||
return {
|
||||
match.group(1): html.unescape(match.group(2))
|
||||
for match in _RAT_PARAM_RE.finditer(html_text)
|
||||
}
|
||||
|
||||
|
||||
def _iter_json_objects(text: str):
|
||||
"""从一段掺杂着 JS 代码的文本里增量解析出所有顶层 JSON 对象"""
|
||||
decoder = json.JSONDecoder()
|
||||
index = text.find("{")
|
||||
while index >= 0:
|
||||
try:
|
||||
value, end = decoder.raw_decode(text, index)
|
||||
except ValueError:
|
||||
index = text.find("{", index + 1)
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
yield value
|
||||
index = text.find("{", max(end, index + 1))
|
||||
@@ -0,0 +1,155 @@
|
||||
"""ラクマ 分类一览页 → RakumaCategoryData
|
||||
|
||||
站点其余页面都是 Rails 服务端渲染的老模板,只有 `/category` 换成了 Next.js
|
||||
App Router:整棵分类树写在 RSC flight payload 里,一段段挂在
|
||||
`self.__next_f.push([1, "<字符串>"])` 上。把这些字符串按顺序拼回去,就能拿到
|
||||
`"categoryList":[{"id":...,"parentId":...,"name":...,"hasChild":...}, ...]`。
|
||||
|
||||
这份 categoryList 是**全量扁平树**(实测 1686 条:14 个顶层 + 169 个二级 +
|
||||
1503 个三级),且与 URL 上的 `?category_id=` 无关——传任意分类或完全不传,
|
||||
内容都一样。所以一次请求即可满足任意层级的查询,不需要像乐天 genre 那样
|
||||
逐层下钻。
|
||||
|
||||
站点只给 id / parentId / name / hasChild 四个字段,没有商品数:商品数只在
|
||||
`/category/{id}` 列表页的埋点属性上,要逐个分类多打一次请求,这里不做。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from app.shared.errors import ItemNotFoundError, ScrapeParseError
|
||||
from app.scraping.models.scrape import RakumaCategoryData, RakumaCategoryNode
|
||||
from app.scraping.utils.rakuma_urls import build_category_url
|
||||
|
||||
# flight payload 的分片:self.__next_f.push([1,"...JSON 字符串字面量..."])
|
||||
_FLIGHT_CHUNK_RE = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)')
|
||||
# 站点用 parentId=0 表示顶层分类,0 本身不是一个真实分类
|
||||
ROOT_PARENT_ID = 0
|
||||
|
||||
|
||||
def _flight_payload(html: str) -> str:
|
||||
"""把 RSC flight payload 的所有分片按顺序拼成一整段文本
|
||||
|
||||
每个分片是一个 JS 字符串字面量,转义规则与 JSON 一致,因此直接用
|
||||
json.loads 解码;单个分片解不开时跳过它而不是放弃整页——分类数据可能
|
||||
落在其他分片上。
|
||||
"""
|
||||
parts: list[str] = []
|
||||
for match in _FLIGHT_CHUNK_RE.finditer(html):
|
||||
try:
|
||||
parts.append(json.loads(match.group(1)))
|
||||
except ValueError:
|
||||
continue
|
||||
return "".join(parts)
|
||||
|
||||
|
||||
def _raw_categories(html: str) -> list[dict[str, Any]]:
|
||||
"""从页面里取出扁平分类列表
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面里没有 categoryList,或它不是非空数组
|
||||
"""
|
||||
payload = _flight_payload(html)
|
||||
marker = payload.find('"categoryList":')
|
||||
if marker < 0:
|
||||
raise ScrapeParseError(
|
||||
"分类页中缺少 categoryList 数据;站点可能改版或返回了非预期页面"
|
||||
)
|
||||
|
||||
start = payload.find("[", marker)
|
||||
if start < 0:
|
||||
raise ScrapeParseError("分类页的 categoryList 不是数组")
|
||||
|
||||
try:
|
||||
raw, _ = json.JSONDecoder().raw_decode(payload[start:])
|
||||
except ValueError as exc:
|
||||
raise ScrapeParseError(f"分类页的 categoryList 解析失败:{exc}") from exc
|
||||
|
||||
items = [item for item in raw if isinstance(item, dict) and item.get("id") is not None]
|
||||
if not items:
|
||||
raise ScrapeParseError("分类页的 categoryList 为空")
|
||||
return items
|
||||
|
||||
|
||||
def _to_node(raw: dict[str, Any]) -> RakumaCategoryNode:
|
||||
category_id = str(raw.get("id"))
|
||||
return RakumaCategoryNode(
|
||||
category_id=category_id,
|
||||
name=str(raw.get("name") or ""),
|
||||
parent_id=str(raw.get("parentId") if raw.get("parentId") is not None else ROOT_PARENT_ID),
|
||||
is_leaf=not bool(raw.get("hasChild")),
|
||||
url=build_category_url(category_id),
|
||||
)
|
||||
|
||||
|
||||
def _build_subtree(
|
||||
node: RakumaCategoryNode,
|
||||
children_of: dict[str, list[RakumaCategoryNode]],
|
||||
) -> RakumaCategoryNode:
|
||||
"""递归把下级分类挂到 children 上
|
||||
|
||||
分类树只有三层,递归深度可控;节点在 categoryList 里 id 唯一,
|
||||
不会出现自环。
|
||||
"""
|
||||
return node.model_copy(
|
||||
update={
|
||||
"children": [
|
||||
_build_subtree(child, children_of)
|
||||
for child in children_of.get(node.category_id, [])
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def parse_categories(
|
||||
html: str, *, category_id: str | None, include_descendants: bool
|
||||
) -> RakumaCategoryData:
|
||||
"""解析分类树
|
||||
|
||||
Args:
|
||||
category_id: 目标分类;None 表示取顶层分类列表
|
||||
include_descendants: children 里带上完整子树而非只有直接子级
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面里没有分类树
|
||||
ItemNotFoundError: 目标分类不存在于站点分类树中
|
||||
"""
|
||||
raw_items = _raw_categories(html)
|
||||
nodes = {str(raw["id"]): _to_node(raw) for raw in raw_items}
|
||||
|
||||
children_of: dict[str, list[RakumaCategoryNode]] = {}
|
||||
for node in nodes.values():
|
||||
children_of.setdefault(node.parent_id, []).append(node)
|
||||
|
||||
def resolve(node: RakumaCategoryNode) -> RakumaCategoryNode:
|
||||
return _build_subtree(node, children_of) if include_descendants else node
|
||||
|
||||
root_children = children_of.get(str(ROOT_PARENT_ID), [])
|
||||
if category_id is None:
|
||||
return RakumaCategoryData(
|
||||
total_count=len(nodes),
|
||||
children=[resolve(node) for node in root_children],
|
||||
)
|
||||
|
||||
target = nodes.get(category_id.strip())
|
||||
if target is None:
|
||||
raise ItemNotFoundError(f"分类树中未找到分类 {category_id}")
|
||||
|
||||
ancestors: list[RakumaCategoryNode] = []
|
||||
parent = nodes.get(target.parent_id)
|
||||
while parent is not None:
|
||||
ancestors.insert(0, parent)
|
||||
parent = nodes.get(parent.parent_id)
|
||||
|
||||
return RakumaCategoryData(
|
||||
category_id=target.category_id,
|
||||
name=target.name,
|
||||
full_name=" / ".join([node.name for node in ancestors] + [target.name]),
|
||||
is_leaf=target.is_leaf,
|
||||
url=target.url,
|
||||
total_count=len(nodes),
|
||||
ancestors=ancestors,
|
||||
children=[resolve(node) for node in children_of.get(target.category_id, [])],
|
||||
)
|
||||
@@ -0,0 +1,211 @@
|
||||
"""ラクマ 商品详情页 HTML → RakumaItemDetailData
|
||||
|
||||
页面数据分三处,各取所长:
|
||||
- `<script type="application/ld+json">` 的 Product 微数据:名称、价格、描述、图片
|
||||
- `.item__details` 规格表:成色、尺码、配送方式与地区(每行 th 上的
|
||||
`item-status-{key}` class 是稳定键,比日文标签文案可靠)
|
||||
- 埋点属性 `data-rat-cp-*` 与 dataLayer JSON:数值 ID、分类 ID、品牌 ID
|
||||
|
||||
售罄判定用页面上的 SOLD OUT 标记,而不是 ld+json 的 availability——
|
||||
实测已售出商品的 ld+json 仍写 InStock,不可信。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.errors import ScrapeParseError
|
||||
from app.scraping.models.scrape import Breadcrumb, RakumaItemDetailData, RakumaSeller
|
||||
from app.scraping.parsers.rakuma.base import (
|
||||
attr,
|
||||
find_item_payload,
|
||||
image_url,
|
||||
node_text,
|
||||
parse_float,
|
||||
parse_int,
|
||||
rat_params,
|
||||
)
|
||||
from app.scraping.utils.rakuma_urls import split_shop_url
|
||||
|
||||
_LD_JSON_RE = re.compile(
|
||||
r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
|
||||
)
|
||||
# 商品主图:站点用 slider 展示,主图挂在 .sp-image 上(推荐位的图不在其中)
|
||||
_MAIN_IMAGE_SELECTOR = ".sp-slide img.sp-image, .item-photos img, .slider img.sp-image"
|
||||
|
||||
# 规格表里 th 图标 class 上的稳定键 → 模型字段
|
||||
_SPEC_KEYS = {
|
||||
"item-status-status": "condition",
|
||||
"item-status-size": "size",
|
||||
"item-status-carriage": "shipping_payer",
|
||||
"item-status-delivery_method": "shipping_method",
|
||||
"item-status-delivery_date": "shipping_date_estimate",
|
||||
"item-status-delivery_area": "shipping_from",
|
||||
}
|
||||
|
||||
# 站点上表示「没有填写该项」的占位文案
|
||||
_SPEC_EMPTY_VALUES = ("なし", "未定", "指定なし", "-", "―")
|
||||
|
||||
# 规格表缺失时的兜底:页面级埋点属性 data-rat-cp-{key} → 模型字段。
|
||||
# 注意这组值的措辞与规格表不完全一致(如运费负担规格表写「送料込」,
|
||||
# 埋点写「出品者」),原样透出,不做归一。
|
||||
_RAT_SPEC_KEYS = {
|
||||
"condition": "item_condition",
|
||||
"shipping_payer": "shipping_cost_payer",
|
||||
"shipping_date_estimate": "shipping_date_estimate",
|
||||
"shipping_from": "shipping_from",
|
||||
}
|
||||
|
||||
|
||||
def _parse_ld_product(html: str) -> dict[str, Any]:
|
||||
"""取出 ld+json 里的 Product 节点"""
|
||||
for match in _LD_JSON_RE.finditer(html):
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(data, dict) and data.get("@type") == "Product":
|
||||
return data
|
||||
return {}
|
||||
|
||||
|
||||
def _parse_specs(tree: HTMLParser, rat: dict[str, str]) -> dict[str, str]:
|
||||
"""解析商品情報规格表
|
||||
|
||||
每行的 th 里有个 `<i class="icon-status ... item-status-{key}">`,
|
||||
这个 key 比日文标签稳定,用它做映射。
|
||||
|
||||
已售出商品的页面会换成另一套布局、规格表整体消失,此时退回页面级埋点
|
||||
属性——它给的项少一些(没有配送方法与尺码),但成色、运费负担与发货地
|
||||
仍在,好过整片留空。
|
||||
"""
|
||||
specs: dict[str, str] = {}
|
||||
for row in tree.css("table.item__details tr"):
|
||||
icon = row.css_first("th i")
|
||||
value_node = row.css_first("td")
|
||||
if icon is None or value_node is None:
|
||||
continue
|
||||
classes = attr(icon, "class").split()
|
||||
field = next((_SPEC_KEYS[name] for name in classes if name in _SPEC_KEYS), None)
|
||||
if field is None:
|
||||
continue
|
||||
value = node_text(value_node)
|
||||
specs[field] = "" if value in _SPEC_EMPTY_VALUES else value
|
||||
|
||||
for field, key in _RAT_SPEC_KEYS.items():
|
||||
if not specs.get(field) and rat.get(key):
|
||||
specs[field] = rat[key]
|
||||
return specs
|
||||
|
||||
|
||||
def _parse_breadcrumbs(tree: HTMLParser) -> tuple[list[Breadcrumb], str]:
|
||||
"""解析分类面包屑,并返回最具体的一级分类 ID
|
||||
|
||||
取规格表里的分类行而非页头面包屑:页头那条会把品牌也混进来,
|
||||
规格表里的是纯分类链。
|
||||
"""
|
||||
crumbs: list[Breadcrumb] = []
|
||||
category_id = ""
|
||||
for row in tree.css("table.item__details tr"):
|
||||
icon = row.css_first("th i")
|
||||
if icon is None or "item-status-category" not in attr(icon, "class"):
|
||||
continue
|
||||
for link in row.css("td a"):
|
||||
url = attr(link, "href")
|
||||
crumbs.append(Breadcrumb(name=node_text(link), url=url))
|
||||
segments = [segment for segment in url.split("/") if segment]
|
||||
if segments:
|
||||
category_id = segments[-1]
|
||||
break
|
||||
return crumbs, category_id
|
||||
|
||||
|
||||
def _parse_seller(tree: HTMLParser, payload: dict[str, Any]) -> RakumaSeller:
|
||||
"""解析出品者信息块"""
|
||||
link = tree.css_first("a.shop_link, a[href*='/shop/']")
|
||||
shop_url = attr(link, "href")
|
||||
shop_id = ""
|
||||
if shop_url:
|
||||
try:
|
||||
shop_id = split_shop_url(shop_url)
|
||||
except Exception:
|
||||
shop_id = ""
|
||||
|
||||
seller_user_id = payload.get("seller_user_id")
|
||||
return RakumaSeller(
|
||||
shop_id=shop_id,
|
||||
user_id=str(seller_user_id) if seller_user_id is not None else "",
|
||||
shop_name=node_text(tree.css_first(".header-shopinfo__shop-name")),
|
||||
user_name=node_text(tree.css_first(".header-shopinfo__user-name")),
|
||||
shop_url=shop_url,
|
||||
icon_url=image_url(tree.css_first(".header-shopinfo__user-icon img")),
|
||||
seller_type=str(payload.get("seller_user_type") or ""),
|
||||
review_score=parse_float(node_text(tree.css_first(".shop_score__score"))),
|
||||
# 商品页只给评分不给评价数,需要评价数请调 /api/rakuma/shop_detail
|
||||
is_verified=tree.css_first(".header-shopinfo__verified-badge-item") is not None,
|
||||
)
|
||||
|
||||
|
||||
def parse_item_detail(html: str, *, item_id: str, item_url: str) -> RakumaItemDetailData:
|
||||
"""把商品详情页 HTML 解析为商品详情
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是商品详情页
|
||||
"""
|
||||
tree = HTMLParser(html)
|
||||
info = tree.css_first(f".{site.ITEM_PAGE_MARKER}")
|
||||
if info is None:
|
||||
raise ScrapeParseError("页面不是商品详情页(缺少商品信息区块)")
|
||||
|
||||
product = _parse_ld_product(html)
|
||||
# 这段埋点挂在哪个元素上因页面状态而异,按属性全页扫描
|
||||
payload = find_item_payload(tree)
|
||||
rat = rat_params(html)
|
||||
specs = _parse_specs(tree, rat)
|
||||
breadcrumbs, category_id = _parse_breadcrumbs(tree)
|
||||
|
||||
images = [
|
||||
url
|
||||
for url in dict.fromkeys(image_url(node) for node in tree.css(_MAIN_IMAGE_SELECTOR))
|
||||
if url and "img.fril.jp" in url
|
||||
]
|
||||
if not images and isinstance(product.get("image"), str):
|
||||
images = [product["image"]]
|
||||
|
||||
# ld+json 的 availability 对已售商品仍写 InStock,只能按页面标记判断
|
||||
page_text = info.text()
|
||||
is_sold_out = any(marker in page_text for marker in site.SOLD_OUT_MARKERS)
|
||||
|
||||
brand = product.get("brand") if isinstance(product.get("brand"), dict) else {}
|
||||
offers = product.get("offers") if isinstance(product.get("offers"), dict) else {}
|
||||
|
||||
return RakumaItemDetailData(
|
||||
item_id=item_id,
|
||||
item_number=str(payload.get("item_id") or ""),
|
||||
item_name=str(product.get("name") or "") or node_text(tree.css_first("h1.item__name")),
|
||||
description=str(product.get("description") or "")
|
||||
or node_text(tree.css_first(".item__description__line-limited")),
|
||||
item_url=item_url,
|
||||
price=parse_int(offers.get("price")) or parse_int(node_text(tree.css_first(".item__price"))),
|
||||
is_sold_out=is_sold_out,
|
||||
images=images,
|
||||
condition=specs.get("condition", ""),
|
||||
size=specs.get("size", ""),
|
||||
brand_id=str(payload.get("brand_id") or "") or rat.get("brand_id", ""),
|
||||
brand_name=str(brand.get("name") or "") or str(payload.get("brand_name") or ""),
|
||||
category_id=category_id or str(payload.get("category_id") or ""),
|
||||
breadcrumbs=breadcrumbs,
|
||||
shipping_payer=specs.get("shipping_payer", ""),
|
||||
shipping_method=specs.get("shipping_method", ""),
|
||||
shipping_date_estimate=specs.get("shipping_date_estimate", ""),
|
||||
shipping_from=specs.get("shipping_from", ""),
|
||||
is_anonymous_shipping=tree.css_first(".item__icon.anonymous") is not None,
|
||||
like_count=parse_int(node_text(tree.css_first(".like_button_set"))),
|
||||
comment_count=parse_int(node_text(tree.css_first(".go-to-comment-button"))),
|
||||
posted_at=node_text(tree.css_first(".time_ago")),
|
||||
seller=_parse_seller(tree, payload),
|
||||
)
|
||||
@@ -0,0 +1,135 @@
|
||||
"""ラクマ 搜索页 / 店铺商品列表 HTML → 商品列表
|
||||
|
||||
搜索页与店铺页的商品卡片是同一套 `.item-box` 结构(只有链接的 class 前缀
|
||||
不同:搜索页 link_search_image、店铺页 link_shop_image),因此共用一个卡片
|
||||
解析函数。
|
||||
|
||||
页面上的可见总数是四舍五入的展示值(約1,190,000件),精确值在埋点属性
|
||||
`data-rat-cp-totalresults` 上,优先取后者。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.errors import ScrapeParseError
|
||||
from app.scraping.models.scrape import RakumaSearchItem, RakumaSearchResultData
|
||||
from app.scraping.parsers.rakuma.base import (
|
||||
attr,
|
||||
event_payload,
|
||||
image_url,
|
||||
node_text,
|
||||
parse_int,
|
||||
parse_total_count,
|
||||
)
|
||||
from app.scraping.utils.rakuma_urls import item_id_from_url
|
||||
|
||||
# 埋点属性里的精确命中总数
|
||||
_TOTAL_RESULTS_RE = re.compile(r'data-rat-cp-totalresults="(\d+)"')
|
||||
|
||||
|
||||
def _as_str(value: object) -> str:
|
||||
"""埋点 JSON 里的值可能是数字、字符串或 null,统一收敛为字符串"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return ""
|
||||
if isinstance(value, (int, float)):
|
||||
return str(int(value))
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def parse_item_card(card: Node) -> RakumaSearchItem:
|
||||
"""解析一张商品卡片
|
||||
|
||||
优先从埋点 JSON 取结构化字段(数值 ID、分类、品牌、价格),
|
||||
可见 DOM 只用于取图片与售罄标记。
|
||||
"""
|
||||
link = (
|
||||
card.css_first("a.link_search_image")
|
||||
or card.css_first("a.link_shop_image")
|
||||
or card.css_first("a[href*='item.fril.jp']")
|
||||
)
|
||||
payload = event_payload(link)
|
||||
|
||||
item_url = attr(link, "href")
|
||||
category_names = [
|
||||
name
|
||||
for name in (
|
||||
_as_str(payload.get("first_category")),
|
||||
_as_str(payload.get("second_category")),
|
||||
_as_str(payload.get("third_category")),
|
||||
)
|
||||
if name
|
||||
]
|
||||
|
||||
# 价格优先取埋点里的数值,回退到卡片上的展示价
|
||||
price = parse_int(payload.get("price")) or parse_int(
|
||||
node_text(card.css_first(".item-box__item-price"))
|
||||
)
|
||||
|
||||
return RakumaSearchItem(
|
||||
item_id=item_id_from_url(item_url),
|
||||
item_number=_as_str(payload.get("item_id")),
|
||||
item_name=_as_str(payload.get("item_name"))
|
||||
or node_text(card.css_first(".item-box__item-name, .item-box__item-name__limited-three-lines")),
|
||||
item_url=item_url,
|
||||
price=price,
|
||||
image_url=image_url(card.css_first("img")),
|
||||
is_sold_out=card.css_first(".item-box__soldout_ribbon") is not None,
|
||||
brand_id=_as_str(payload.get("brand_id")),
|
||||
brand_name=_as_str(payload.get("brand_name"))
|
||||
or node_text(card.css_first(".item-box__item-sub-name")),
|
||||
category_id=_as_str(payload.get("category_id")),
|
||||
category_names=category_names,
|
||||
seller_user_id=_as_str(payload.get("seller_user_id")),
|
||||
seller_type=_as_str(payload.get("seller_user_type")),
|
||||
)
|
||||
|
||||
|
||||
def parse_item_cards(tree: HTMLParser) -> list[RakumaSearchItem]:
|
||||
"""解析页面上的全部商品卡片
|
||||
|
||||
只取有真实商品链接的卡片:页面上还有一批用于占位的骨架卡片
|
||||
(懒加载的推荐位),它们没有 item.fril.jp 链接。
|
||||
"""
|
||||
items: list[RakumaSearchItem] = []
|
||||
for card in tree.css(".item-box"):
|
||||
link = card.css_first("a[href*='item.fril.jp']")
|
||||
if link is None:
|
||||
continue
|
||||
items.append(parse_item_card(card))
|
||||
return items
|
||||
|
||||
|
||||
def parse_search(html: str, *, request_url: str, page: int, keyword: str) -> RakumaSearchResultData:
|
||||
"""把搜索页 HTML 解析为搜索结果
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是搜索结果页(站点对无法识别的参数值会静默返回首页)
|
||||
"""
|
||||
tree = HTMLParser(html)
|
||||
count_node = tree.css_first(f".{site.SEARCH_PAGE_MARKER}")
|
||||
if count_node is None:
|
||||
raise ScrapeParseError(
|
||||
"页面不是搜索结果页(缺少命中数区块);"
|
||||
"站点对无法识别的筛选取值会静默返回首页,请检查筛选参数"
|
||||
)
|
||||
|
||||
items = parse_item_cards(tree)
|
||||
|
||||
# 展示值是四舍五入过的(約1,190,000件),埋点里才是精确命中数
|
||||
display_total, start, end = parse_total_count(node_text(count_node))
|
||||
match = _TOTAL_RESULTS_RE.search(html)
|
||||
total_count = int(match.group(1)) if match else display_total
|
||||
|
||||
return RakumaSearchResultData(
|
||||
keyword=keyword,
|
||||
page=page,
|
||||
page_size=len(items),
|
||||
total_count=total_count,
|
||||
# 站点 page>100 直接 404,超出可达窗口时没有下一页
|
||||
has_more=bool(items) and page < site.MAX_PAGE and (end or start + len(items) - 1) < total_count,
|
||||
request_url=request_url,
|
||||
items=items,
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""ラクマ 店铺页 HTML → 卖家详情与卖家商品列表
|
||||
|
||||
C2C 集市里的「商家」就是个人卖家,页面在 fril.jp/shop/{hash}:
|
||||
- 店铺页本身:卖家资料 + 该卖家的商品分页列表(含已售出)
|
||||
- /review 子页:评价明细与好评/普通/差评分档计数
|
||||
|
||||
商品卡片与搜索页共用 `.item-box` 结构,直接复用 search 里的解析。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.errors import ScrapeParseError
|
||||
from app.scraping.models.scrape import (
|
||||
RakumaRatingBreakdown,
|
||||
RakumaReview,
|
||||
RakumaShopDetailData,
|
||||
RakumaShopItemsData,
|
||||
)
|
||||
from app.scraping.parsers.rakuma.base import (
|
||||
attr,
|
||||
event_payload,
|
||||
image_url,
|
||||
node_text,
|
||||
parse_float,
|
||||
parse_int,
|
||||
parse_total_count,
|
||||
)
|
||||
from app.scraping.parsers.rakuma.search import parse_item_cards
|
||||
|
||||
# 评价条目标题左侧的图标 class → 评价档位
|
||||
_RATING_ICONS = {
|
||||
"icon_review_sun": "good", # よい
|
||||
"icon_review_cloud": "normal", # ふつう
|
||||
"icon_review_rain": "bad", # わるい
|
||||
}
|
||||
|
||||
# /review 页上三组分档计数的容器 id 前缀
|
||||
_ALL_RATINGS_PREFIX = "all"
|
||||
_SELLER_RATINGS_PREFIX = "seller"
|
||||
|
||||
_LD_JSON_RE = re.compile(
|
||||
r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
|
||||
)
|
||||
|
||||
|
||||
def _require_shop_page(html: str) -> HTMLParser:
|
||||
tree = HTMLParser(html)
|
||||
if tree.css_first(f".{site.SHOP_PAGE_MARKER}") is None:
|
||||
raise ScrapeParseError("页面不是店铺页(缺少店铺资料区块)")
|
||||
return tree
|
||||
|
||||
|
||||
def _parse_rating_breakdown(tree: HTMLParser, prefix: str) -> RakumaRatingBreakdown:
|
||||
"""解析一组好评/普通/差评计数
|
||||
|
||||
页面用 `<ul class="nav-pills">` 里的三个链接展示,锚点形如
|
||||
`#all-good` / `#seller-normal`,按锚点前缀区分「全部」与「出品」两组。
|
||||
"""
|
||||
counts = {"good": 0, "normal": 0, "bad": 0}
|
||||
for link in tree.css("ul.nav-pills a"):
|
||||
href = attr(link, "href")
|
||||
for rating in counts:
|
||||
if href == f"#{prefix}-{rating}":
|
||||
counts[rating] = parse_int(node_text(link))
|
||||
return RakumaRatingBreakdown(**counts)
|
||||
|
||||
|
||||
def _parse_reviews(tree: HTMLParser) -> list[RakumaReview]:
|
||||
"""解析评价列表
|
||||
|
||||
站点在「すべての評価」标签页里最多展示最新 100 条,且三个标签页
|
||||
(全部/出品/购入)的条目在 DOM 里重复出现,这里只取第一个激活面板。
|
||||
"""
|
||||
panel = tree.css_first("#all-all") or tree.css_first(".tab-pane.active")
|
||||
if panel is None:
|
||||
return []
|
||||
|
||||
reviews: list[RakumaReview] = []
|
||||
for article in panel.css("article.review-item"):
|
||||
title_node = article.css_first(".review-item-title")
|
||||
icon = title_node.css_first("i") if title_node else None
|
||||
classes = attr(icon, "class").split() if icon else []
|
||||
rating = next((_RATING_ICONS[name] for name in classes if name in _RATING_ICONS), "")
|
||||
|
||||
reviews.append(
|
||||
RakumaReview(
|
||||
rating=rating,
|
||||
title=node_text(title_node),
|
||||
comment=node_text(article.css_first(".review-item-text")),
|
||||
reviewer_name=node_text(article.css_first(".review-item-name")),
|
||||
reviewed_at=node_text(article.css_first(".review-item-date")),
|
||||
)
|
||||
)
|
||||
return reviews
|
||||
|
||||
|
||||
def _parse_store_rating(html: str) -> tuple[float, int]:
|
||||
"""从店铺页的 ld+json Store 节点取评分与评价数
|
||||
|
||||
评价数只有这里给得出来——可见 DOM 上只有星级和分数,没有条数。
|
||||
"""
|
||||
for match in _LD_JSON_RE.finditer(html):
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(data, dict) or data.get("@type") != "Store":
|
||||
continue
|
||||
rating = data.get("aggregateRating")
|
||||
if isinstance(rating, dict):
|
||||
return parse_float(rating.get("ratingValue")), parse_int(rating.get("ratingCount"))
|
||||
return 0.0, 0
|
||||
|
||||
|
||||
def parse_shop_detail(
|
||||
html: str, *, shop_id: str, shop_url: str, review_html: str | None = None
|
||||
) -> RakumaShopDetailData:
|
||||
"""把店铺页 HTML 解析为卖家详情
|
||||
|
||||
Args:
|
||||
review_html: /review 子页的 HTML;给出时才填充评价明细与分档计数
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是店铺页
|
||||
"""
|
||||
tree = _require_shop_page(html)
|
||||
|
||||
total_count, _, _ = parse_total_count(node_text(tree.css_first(".page-count")))
|
||||
badge = tree.css_first(".badge-status")
|
||||
verification_label = node_text(badge)
|
||||
|
||||
# 简介在侧栏「プロフィール」区块;卖家未填写时站点会写一句占位文案
|
||||
introduction = node_text(tree.css_first("[data-test=profile-text-top]"))
|
||||
if "設定されていません" in introduction:
|
||||
introduction = ""
|
||||
|
||||
score, review_count = _parse_store_rating(html)
|
||||
detail = RakumaShopDetailData(
|
||||
shop_id=shop_id,
|
||||
shop_name=node_text(tree.css_first(".profile-area__shop-name")),
|
||||
user_name=node_text(tree.css_first("[data-test=profile_user_name], .profile-area__user-name")),
|
||||
shop_url=shop_url,
|
||||
icon_url=image_url(tree.css_first(".profile-area__user-icon img")),
|
||||
cover_url=_cover_url(tree),
|
||||
introduction=introduction,
|
||||
review_score=score or parse_float(node_text(tree.css_first(".shop_score__score"))),
|
||||
review_count=review_count,
|
||||
is_verified="未完了" not in verification_label and bool(verification_label),
|
||||
verification_label=verification_label,
|
||||
item_count=total_count,
|
||||
)
|
||||
|
||||
# 用户数值 ID:优先取商品卡片埋点里的 seller_user_id(店铺页所有商品都属于
|
||||
# 该卖家),卖家未设头像时头像地址是站点默认图,取不到 ID。
|
||||
detail.user_id = _seller_user_id(tree) or _user_id_from_icon(detail.icon_url)
|
||||
|
||||
if review_html is not None:
|
||||
review_tree = HTMLParser(review_html)
|
||||
detail.rating_breakdown = _parse_rating_breakdown(review_tree, _ALL_RATINGS_PREFIX)
|
||||
detail.seller_rating_breakdown = _parse_rating_breakdown(review_tree, _SELLER_RATINGS_PREFIX)
|
||||
detail.reviews = _parse_reviews(review_tree)
|
||||
|
||||
return detail
|
||||
|
||||
|
||||
def _cover_url(tree: HTMLParser) -> str:
|
||||
"""封面图挂在 inline style 的 background url() 里"""
|
||||
cover = tree.css_first(".profile-area__shop-cover")
|
||||
style = attr(cover, "style")
|
||||
start = style.find("url(")
|
||||
if start < 0:
|
||||
return ""
|
||||
end = style.find(")", start)
|
||||
return style[start + 4 : end].strip("'\" ") if end > start else ""
|
||||
|
||||
|
||||
def _seller_user_id(tree: HTMLParser) -> str:
|
||||
"""从店铺页商品卡片的埋点里取卖家数值 ID"""
|
||||
for node in tree.css("[data-gtm-click], [onclick]"):
|
||||
payload = event_payload(node)
|
||||
user_id = payload.get("seller_user_id")
|
||||
if user_id:
|
||||
return str(user_id)
|
||||
return ""
|
||||
|
||||
|
||||
def _user_id_from_icon(icon_url: str) -> str:
|
||||
"""从头像地址 https://img.fril.jp/user/{id}/s/{id}.jpg 里取用户数值 ID"""
|
||||
marker = "/user/"
|
||||
start = icon_url.find(marker)
|
||||
if start < 0:
|
||||
return ""
|
||||
rest = icon_url[start + len(marker) :]
|
||||
user_id = rest.split("/", 1)[0]
|
||||
return user_id if user_id.isdigit() else ""
|
||||
|
||||
|
||||
def parse_shop_items(
|
||||
html: str, *, shop_id: str, request_url: str, page: int
|
||||
) -> RakumaShopItemsData:
|
||||
"""把店铺页 HTML 解析为该卖家的商品列表
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是店铺页
|
||||
"""
|
||||
tree = _require_shop_page(html)
|
||||
items = parse_item_cards(tree)
|
||||
total_count, _, end = parse_total_count(node_text(tree.css_first(".page-count")))
|
||||
|
||||
return RakumaShopItemsData(
|
||||
shop_id=shop_id,
|
||||
shop_name=node_text(tree.css_first(".profile-area__shop-name")),
|
||||
page=page,
|
||||
total_count=total_count,
|
||||
has_more=bool(items) and bool(end) and end < total_count,
|
||||
request_url=request_url,
|
||||
items=items,
|
||||
)
|
||||
Reference in New Issue
Block a user