拆分抓取与交易服务
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,44 @@
|
||||
"""站点 JSON 取值的类型收敛工具
|
||||
|
||||
`__INITIAL_STATE__` 里同一字段在不同商品/店铺上可能是 null、数字或字符串
|
||||
（例如运费既可能是 0 也可能是 null，评分既可能是 4.5 也可能是 "4.5"），
|
||||
统一在这里做容错转换，避免解析器里到处写 isinstance 判断。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def as_dict(value: Any) -> dict[str, Any]:
|
||||
"""非 dict（含 None）一律收敛为空 dict"""
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
"""非 list（含 None）一律收敛为空 list"""
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
|
||||
def as_str(value: Any) -> str:
|
||||
"""非字符串一律收敛为空串"""
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def as_int(value: Any, default: int = 0) -> int:
|
||||
"""尽力转成 int；布尔值不视为数字，避免 True 被当成 1"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return default
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def as_float(value: Any, default: float = 0.0) -> float:
|
||||
"""尽力转成 float；布尔值不视为数字"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
@@ -0,0 +1,167 @@
|
||||
"""ラクマ（fril.jp）页面 URL 的构建与解析
|
||||
|
||||
搜索页 URL 形如：
|
||||
https://fril.jp/s?query=switch&category_id=788&statuses=5,4&min=1000&transaction=selling
|
||||
|
||||
参数名与取值取自站点前端 bundle 中 SearchPanel 组件的 `_url()` 方法，
|
||||
不是猜测。注意站点对无法识别的参数值不会报错，而是**静默返回首页**
|
||||
（如 statuses=99 返回 75KB 的首页 HTML），因此所有枚举值都必须来自码表。
|
||||
|
||||
商品页 URL 形如：https://item.fril.jp/{商品hash}
|
||||
店铺页 URL 形如：https://fril.jp/shop/{店铺hash}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote, urlencode, urlsplit, urlunsplit, parse_qsl
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.errors import InvalidRequestError
|
||||
from app.scraping.models.scrape import RakumaSearchRequest
|
||||
|
||||
|
||||
def build_search_url(payload: RakumaSearchRequest) -> str:
|
||||
"""根据搜索请求参数构建 ラクマ 搜索页 URL"""
|
||||
params: list[tuple[str, str]] = []
|
||||
|
||||
keyword = payload.keyword.strip()
|
||||
if keyword:
|
||||
params.append(("query", keyword))
|
||||
if payload.exclude_keyword:
|
||||
params.append(("excluded_query", payload.exclude_keyword))
|
||||
if payload.category_id:
|
||||
params.append(("category_id", str(payload.category_id).strip()))
|
||||
# 站点前端把这两项做成互斥分支：勾选「排除无品牌」时不下发 brand_id
|
||||
if payload.except_for_no_brand:
|
||||
params.append(("except_for_no_brand", "true"))
|
||||
elif payload.brand_id:
|
||||
params.append(("brand_id", str(payload.brand_id).strip()))
|
||||
|
||||
if payload.min_price is not None:
|
||||
params.append(("min", str(payload.min_price)))
|
||||
if payload.max_price is not None:
|
||||
params.append(("max", str(payload.max_price)))
|
||||
|
||||
if payload.authenticity_types:
|
||||
codes = [site.AUTHENTICITY_CODES[value.value] for value in payload.authenticity_types]
|
||||
params.append(("authenticity_types", ",".join(codes)))
|
||||
if payload.conditions:
|
||||
codes = [site.CONDITION_CODES[value.value] for value in payload.conditions]
|
||||
params.append(("statuses", ",".join(codes)))
|
||||
if payload.free_shipping:
|
||||
params.append(("carriage", site.CARRIAGE_INCLUDED))
|
||||
if payload.transaction is not None:
|
||||
params.append(("transaction", site.TRANSACTION_CODES[payload.transaction.value]))
|
||||
if payload.anonymous_shipping:
|
||||
params.append(("anonymous_shipping", "true"))
|
||||
|
||||
sort_code, order_code = site.SORT_CODES[payload.sort.value]
|
||||
params.append(("sort", sort_code))
|
||||
params.append(("order", order_code))
|
||||
|
||||
if payload.page > 1:
|
||||
params.append(("page", str(payload.page)))
|
||||
|
||||
return f"{site.SEARCH_BASE_URL}?{urlencode(params)}"
|
||||
|
||||
|
||||
def normalize_search_url(raw_url: str, page: int) -> str:
|
||||
"""校验透传的搜索 URL，并在需要时覆盖其中的页码
|
||||
|
||||
page 大于 1 时以入参为准覆盖 URL 中的 `page`，这样上游可以复用同一条 URL 翻页。
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.SEARCH_HOST:
|
||||
raise InvalidRequestError(f"search_url 必须是 {site.SEARCH_HOST} 下的搜索页地址")
|
||||
|
||||
if page <= 1:
|
||||
return raw_url
|
||||
|
||||
query = [
|
||||
(key, value)
|
||||
for key, value in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
if key != "page"
|
||||
]
|
||||
query.append(("page", str(page)))
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query), parsed.fragment))
|
||||
|
||||
|
||||
def build_category_url(category_id: str) -> str:
|
||||
"""由分类 ID 拼出分类商品列表页 URL（`/category/{id}`）
|
||||
|
||||
这是对外返回的分类页地址；抓取分类树用的是 `/category` 一览页
|
||||
（见 rakuma_site.CATEGORY_LIST_URL），两者不是同一个页面。
|
||||
"""
|
||||
category = quote(str(category_id).strip().strip("/"), safe="")
|
||||
return f"{site.CATEGORY_BASE_URL}{category}" if category else ""
|
||||
|
||||
|
||||
def build_item_url(item_id: str) -> str:
|
||||
"""由商品 hash 拼出商品详情页 URL"""
|
||||
item = quote(item_id.strip().strip("/"), safe="")
|
||||
if not item:
|
||||
raise InvalidRequestError("item_id 不能为空")
|
||||
return f"{site.ITEM_BASE_URL}{item}"
|
||||
|
||||
|
||||
def split_item_url(raw_url: str) -> str:
|
||||
"""从商品页 URL 中解析出商品 hash
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 item.fril.jp 下的商品页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
raise InvalidRequestError(f"item_url 必须是 {site.ITEM_HOST} 下的商品页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if not segments:
|
||||
raise InvalidRequestError(f"无法从 item_url 中解析商品编号：{raw_url}")
|
||||
return segments[0]
|
||||
|
||||
|
||||
def build_shop_url(shop_id: str, *, page: int = 1, review: bool = False) -> str:
|
||||
"""由店铺 hash 拼出店铺页 URL
|
||||
|
||||
Args:
|
||||
page: 商品列表页码，1 时不带参数
|
||||
review: 取评价页（/review）而非商品列表页
|
||||
"""
|
||||
shop = quote(shop_id.strip().strip("/"), safe="")
|
||||
if not shop:
|
||||
raise InvalidRequestError("shop_id 不能为空")
|
||||
|
||||
url = f"{site.SHOP_BASE_URL}{shop}"
|
||||
if review:
|
||||
return f"{url}/review"
|
||||
return f"{url}?page={page}" if page > 1 else url
|
||||
|
||||
|
||||
def split_shop_url(raw_url: str) -> str:
|
||||
"""从店铺页 URL 中解析出店铺 hash
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 fril.jp/shop/ 下的店铺页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.SEARCH_HOST:
|
||||
raise InvalidRequestError(f"shop_url 必须是 {site.SEARCH_HOST} 下的店铺页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if len(segments) < 2 or segments[0] != "shop":
|
||||
raise InvalidRequestError(f"无法从 shop_url 中解析店铺编号：{raw_url}")
|
||||
return segments[1]
|
||||
|
||||
|
||||
def item_id_from_url(url: str) -> str:
|
||||
"""尽力从任意商品链接中解析商品 hash，失败时返回空串
|
||||
|
||||
用于列表页解析——单条链接解析不出来不应中断整页解析。
|
||||
"""
|
||||
try:
|
||||
parsed = urlsplit(url)
|
||||
except ValueError:
|
||||
return ""
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
return ""
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
return segments[0] if segments else ""
|
||||
@@ -0,0 +1,164 @@
|
||||
"""乐天页面 URL 的构建与解析
|
||||
|
||||
搜索页 URL 形如：
|
||||
https://search.rakuten.co.jp/search/mall/{关键词}/{分类ID}/?p=2&s=2&min=1000&f=2&f=101
|
||||
其中关键词段在只按分类检索时用 `-` 占位；`f` 可重复出现，每个值代表一项筛选。
|
||||
|
||||
商品页 URL 形如：
|
||||
https://item.rakuten.co.jp/{店铺代码}/{商品编号}/
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote, urlencode, urlparse, urlsplit, urlunsplit, parse_qsl
|
||||
|
||||
from app.scraping.core import site
|
||||
from app.shared.errors import InvalidRequestError
|
||||
from app.scraping.models.scrape import SearchRequest
|
||||
|
||||
# 只按分类检索（无关键词）时，关键词段的占位符
|
||||
_KEYWORD_PLACEHOLDER = "-"
|
||||
|
||||
# www.rakuten.co.jp 下已被站点自身占用、不可能是店铺代码的一级路径
|
||||
_RESERVED_WWW_PATHS = frozenset({"category", "event", "search", "rms", "info", "help"})
|
||||
|
||||
|
||||
def build_search_url(payload: SearchRequest) -> str:
|
||||
"""根据搜索请求参数构建乐天搜索页 URL"""
|
||||
keyword = payload.keyword.strip()
|
||||
path = quote(keyword, safe="") if keyword else _KEYWORD_PLACEHOLDER
|
||||
url = f"{site.SEARCH_BASE_URL}{path}/"
|
||||
if payload.genre_id:
|
||||
url = f"{url}{quote(str(payload.genre_id), safe='')}/"
|
||||
|
||||
params: list[tuple[str, str]] = []
|
||||
if payload.page > 1:
|
||||
params.append(("p", str(payload.page)))
|
||||
|
||||
sort_code = site.SORT_CODES.get(payload.sort.value)
|
||||
if sort_code:
|
||||
params.append(("s", sort_code))
|
||||
|
||||
if payload.min_price is not None:
|
||||
params.append(("min", str(payload.min_price)))
|
||||
if payload.max_price is not None:
|
||||
params.append(("max", str(payload.max_price)))
|
||||
if payload.shop_id is not None:
|
||||
params.append(("sid", str(payload.shop_id)))
|
||||
if payload.exclude_keyword:
|
||||
params.append(("nitem", payload.exclude_keyword))
|
||||
if payload.title_only:
|
||||
params.append(("sf", "1"))
|
||||
if payload.or_query:
|
||||
params.append(("st", "O"))
|
||||
if payload.min_review_score is not None:
|
||||
params.append(("review", str(payload.min_review_score)))
|
||||
if payload.tags:
|
||||
params.append(("tg", ",".join(str(tag) for tag in payload.tags)))
|
||||
|
||||
# 成色与各布尔筛选共用可重复的 `f` 参数
|
||||
if payload.condition is not None:
|
||||
params.append(("f", site.CONDITION_CODES[payload.condition.value]))
|
||||
for field_name, code in site.BOOL_FILTER_CODES.items():
|
||||
if getattr(payload, field_name):
|
||||
params.append(("f", code))
|
||||
|
||||
if not params:
|
||||
return url
|
||||
return f"{url}?{urlencode(params)}"
|
||||
|
||||
|
||||
def normalize_search_url(raw_url: str, page: int) -> str:
|
||||
"""校验透传的搜索 URL，并在需要时覆盖其中的页码
|
||||
|
||||
page 大于 1 时以入参为准覆盖 URL 中的 `p`，这样上游可以复用同一条 URL 翻页。
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.SEARCH_HOST:
|
||||
raise InvalidRequestError(f"search_url 必须是 {site.SEARCH_HOST} 下的搜索页地址")
|
||||
|
||||
if page <= 1:
|
||||
return raw_url
|
||||
|
||||
query = [(key, value) for key, value in parse_qsl(parsed.query, keep_blank_values=True) if key != "p"]
|
||||
query.append(("p", str(page)))
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query), parsed.fragment))
|
||||
|
||||
|
||||
def build_genre_url(genre_id: str | None) -> str:
|
||||
"""构建取分类树用的页面地址
|
||||
|
||||
指定分类时用分类页——它比搜索页多给 genreInfo（完整分类名与描述）。
|
||||
不指定时退回搜索页：分类分面与查询内容无关，用哨兵关键词能拿到干净的顶层列表。
|
||||
"""
|
||||
if genre_id:
|
||||
return f"{site.CATEGORY_BASE_URL}{quote(str(genre_id).strip().strip('/'), safe='')}/"
|
||||
return f"{site.SEARCH_BASE_URL}{site.GENRE_FACET_PROBE_KEYWORD}/"
|
||||
|
||||
|
||||
def build_item_url(shop_code: str, item_code: str) -> str:
|
||||
"""由店铺代码与商品编号拼出商品详情页 URL"""
|
||||
shop = quote(shop_code.strip().strip("/"), safe="")
|
||||
item = quote(item_code.strip().strip("/"), safe="")
|
||||
if not shop or not item:
|
||||
raise InvalidRequestError("shop_code 与 item_code 均不能为空")
|
||||
return f"{site.ITEM_BASE_URL}{shop}/{item}/"
|
||||
|
||||
|
||||
def split_item_url(raw_url: str) -> tuple[str, str]:
|
||||
"""从商品页 URL 中解析出 (店铺代码, 商品编号)
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 item.rakuten.co.jp 下的商品页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
raise InvalidRequestError(f"item_url 必须是 {site.ITEM_HOST} 下的商品页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if len(segments) < 2:
|
||||
raise InvalidRequestError(f"无法从 item_url 中解析店铺代码与商品编号：{raw_url}")
|
||||
return segments[0], segments[1]
|
||||
|
||||
|
||||
def build_shop_url(shop_code: str) -> str:
|
||||
"""由店铺代码拼出店铺首页 URL"""
|
||||
shop = quote(shop_code.strip().strip("/"), safe="")
|
||||
if not shop:
|
||||
raise InvalidRequestError("shop_code 不能为空")
|
||||
return f"{site.WWW_BASE_URL}{shop}/"
|
||||
|
||||
|
||||
def split_shop_url(raw_url: str) -> str:
|
||||
"""从店铺页 URL 中解析出店铺代码
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 www.rakuten.co.jp 下的店铺页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.WWW_HOST:
|
||||
raise InvalidRequestError(f"shop_url 必须是 {site.WWW_HOST} 下的店铺页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if not segments:
|
||||
raise InvalidRequestError(f"无法从 shop_url 中解析店铺代码：{raw_url}")
|
||||
# 排除分类页等非店铺路径，避免把 /category/101205/ 当成店铺代码 category
|
||||
if segments[0] in _RESERVED_WWW_PATHS:
|
||||
raise InvalidRequestError(f"shop_url 不是店铺首页地址：{raw_url}")
|
||||
return segments[0]
|
||||
|
||||
|
||||
def item_url_parts(url: str) -> tuple[str, str]:
|
||||
"""尽力从任意商品链接中解析 (店铺代码, 商品编号)，失败时返回空串
|
||||
|
||||
用于搜索结果——广告位链接可能指向跳转域名，解析不出来不应中断整页解析。
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError:
|
||||
return "", ""
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
return "", ""
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if len(segments) < 2:
|
||||
return "", ""
|
||||
return segments[0], segments[1]
|
||||
Reference in New Issue
Block a user