拆分抓取与交易服务
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,155 @@
|
||||
"""ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据
|
||||
|
||||
五个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）：
|
||||
搜索、分类、商品详情、卖家详情、卖家商品列表。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.config import Settings
|
||||
from app.scraping.models.scrape import (
|
||||
RakumaCategoryData,
|
||||
RakumaCategoryRequest,
|
||||
RakumaItemDetailData,
|
||||
RakumaItemDetailRequest,
|
||||
RakumaSearchRequest,
|
||||
RakumaSearchResultData,
|
||||
RakumaShopDetailData,
|
||||
RakumaShopDetailRequest,
|
||||
RakumaShopItemsData,
|
||||
RakumaShopItemsRequest,
|
||||
)
|
||||
from app.scraping.parsers.rakuma.category import parse_categories
|
||||
from app.scraping.parsers.rakuma.item import parse_item_detail
|
||||
from app.scraping.parsers.rakuma.search import parse_search
|
||||
from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
||||
from app.scraping.services.rakuma_session import RakumaSession
|
||||
from app.scraping.utils.rakuma_urls import (
|
||||
build_item_url,
|
||||
build_search_url,
|
||||
build_shop_url,
|
||||
normalize_search_url,
|
||||
split_item_url,
|
||||
split_shop_url,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RakumaClient:
|
||||
"""ラクマ（fril.jp）抓取客户端"""
|
||||
|
||||
def __init__(self, settings: Settings, session: RakumaSession):
|
||||
self._settings = settings
|
||||
self._session = session
|
||||
|
||||
async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData:
|
||||
"""抓取搜索结果列表"""
|
||||
if payload.search_url is not None:
|
||||
url = normalize_search_url(str(payload.search_url), payload.page)
|
||||
else:
|
||||
url = build_search_url(payload)
|
||||
|
||||
logger.info("抓取 ラクマ 搜索页：url=%s", url)
|
||||
html = await self._session.fetch_html(url)
|
||||
result = parse_search(
|
||||
html,
|
||||
request_url=url,
|
||||
page=payload.page,
|
||||
keyword=payload.keyword.strip(),
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 搜索完成：url=%s items=%s total=%s",
|
||||
url, len(result.items), result.total_count,
|
||||
)
|
||||
return result
|
||||
|
||||
async def categories(self, payload: RakumaCategoryRequest) -> RakumaCategoryData:
|
||||
"""抓取分类树
|
||||
|
||||
分类一览页一次就返回整棵树（与 URL 上的 category_id 无关），因此不论
|
||||
查哪一层都只打一次请求，页面地址也固定不带参数。
|
||||
"""
|
||||
url = site.CATEGORY_LIST_URL
|
||||
|
||||
logger.info("抓取 ラクマ 分类页：category_id=%s", payload.category_id)
|
||||
html = await self._session.fetch_html(url)
|
||||
data = parse_categories(
|
||||
html,
|
||||
category_id=payload.category_id,
|
||||
include_descendants=payload.include_descendants,
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 分类完成：category_id=%s name=%s children=%s tree=%s",
|
||||
data.category_id, data.name, len(data.children), data.total_count,
|
||||
)
|
||||
return data
|
||||
|
||||
async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData:
|
||||
"""抓取商品详情"""
|
||||
if payload.item_url is not None:
|
||||
item_id = split_item_url(str(payload.item_url))
|
||||
else:
|
||||
item_id = (payload.item_id or "").strip()
|
||||
url = build_item_url(item_id)
|
||||
|
||||
logger.info("抓取 ラクマ 商品详情：url=%s", url)
|
||||
html = await self._session.fetch_html(url)
|
||||
detail = parse_item_detail(html, item_id=item_id, item_url=url)
|
||||
logger.info(
|
||||
"ラクマ 详情完成：url=%s name=%s price=%s sold_out=%s",
|
||||
url, detail.item_name[:40], detail.price, detail.is_sold_out,
|
||||
)
|
||||
return detail
|
||||
|
||||
async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData:
|
||||
"""抓取卖家详情
|
||||
|
||||
评价明细在单独的 /review 页上，仅在 include_reviews=true 时并发多取一次。
|
||||
"""
|
||||
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
||||
url = build_shop_url(shop_id)
|
||||
|
||||
logger.info("抓取 ラクマ 卖家详情：url=%s reviews=%s", url, payload.include_reviews)
|
||||
if payload.include_reviews:
|
||||
html, review_html = await asyncio.gather(
|
||||
self._session.fetch_html(url),
|
||||
self._session.fetch_html(build_shop_url(shop_id, review=True)),
|
||||
)
|
||||
else:
|
||||
html, review_html = await self._session.fetch_html(url), None
|
||||
|
||||
detail = parse_shop_detail(
|
||||
html, shop_id=shop_id, shop_url=url, review_html=review_html
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 卖家详情完成：shop_id=%s name=%s items=%s reviews=%s",
|
||||
shop_id, detail.shop_name, detail.item_count, detail.review_count,
|
||||
)
|
||||
return detail
|
||||
|
||||
async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData:
|
||||
"""抓取卖家的商品列表"""
|
||||
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
||||
url = build_shop_url(shop_id, page=payload.page)
|
||||
|
||||
logger.info("抓取 ラクマ 卖家商品：url=%s", url)
|
||||
html = await self._session.fetch_html(url)
|
||||
result = parse_shop_items(
|
||||
html, shop_id=shop_id, request_url=url, page=payload.page
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 卖家商品完成：shop_id=%s items=%s total=%s",
|
||||
shop_id, len(result.items), result.total_count,
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str:
|
||||
"""从 shop_url 或 shop_id 里取出店铺 hash（模型校验已保证两者不同时为空）"""
|
||||
if shop_url is not None:
|
||||
return split_shop_url(str(shop_url))
|
||||
return (shop_id or "").strip()
|
||||
Reference in New Issue
Block a user