131 lines
4.8 KiB
Python
131 lines
4.8 KiB
Python
"""ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据
|
|
|
|
四个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）：
|
|
搜索、商品详情、卖家详情、卖家商品列表。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
from app.core.config import Settings
|
|
from app.models.scrape import (
|
|
RakumaItemDetailData,
|
|
RakumaItemDetailRequest,
|
|
RakumaSearchRequest,
|
|
RakumaSearchResultData,
|
|
RakumaShopDetailData,
|
|
RakumaShopDetailRequest,
|
|
RakumaShopItemsData,
|
|
RakumaShopItemsRequest,
|
|
)
|
|
from app.parsers.rakuma.item import parse_item_detail
|
|
from app.parsers.rakuma.search import parse_search
|
|
from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
|
from app.services.rakuma_session import RakumaSession
|
|
from app.utils.rakuma_urls import (
|
|
build_item_url,
|
|
build_search_url,
|
|
build_shop_url,
|
|
normalize_search_url,
|
|
split_item_url,
|
|
split_shop_url,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class RakumaClient:
|
|
"""ラクマ（fril.jp）抓取客户端"""
|
|
|
|
def __init__(self, settings: Settings, session: RakumaSession):
|
|
self._settings = settings
|
|
self._session = session
|
|
|
|
async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData:
|
|
"""抓取搜索结果列表"""
|
|
if payload.search_url is not None:
|
|
url = normalize_search_url(str(payload.search_url), payload.page)
|
|
else:
|
|
url = build_search_url(payload)
|
|
|
|
logger.info("抓取 ラクマ 搜索页：url=%s", url)
|
|
html = await self._session.fetch_html(url)
|
|
result = parse_search(
|
|
html,
|
|
request_url=url,
|
|
page=payload.page,
|
|
keyword=payload.keyword.strip(),
|
|
)
|
|
logger.info(
|
|
"ラクマ 搜索完成：url=%s items=%s total=%s",
|
|
url, len(result.items), result.total_count,
|
|
)
|
|
return result
|
|
|
|
async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData:
|
|
"""抓取商品详情"""
|
|
if payload.item_url is not None:
|
|
item_id = split_item_url(str(payload.item_url))
|
|
else:
|
|
item_id = (payload.item_id or "").strip()
|
|
url = build_item_url(item_id)
|
|
|
|
logger.info("抓取 ラクマ 商品详情：url=%s", url)
|
|
html = await self._session.fetch_html(url)
|
|
detail = parse_item_detail(html, item_id=item_id, item_url=url)
|
|
logger.info(
|
|
"ラクマ 详情完成：url=%s name=%s price=%s sold_out=%s",
|
|
url, detail.item_name[:40], detail.price, detail.is_sold_out,
|
|
)
|
|
return detail
|
|
|
|
async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData:
|
|
"""抓取卖家详情
|
|
|
|
评价明细在单独的 /review 页上，仅在 include_reviews=true 时并发多取一次。
|
|
"""
|
|
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
|
url = build_shop_url(shop_id)
|
|
|
|
logger.info("抓取 ラクマ 卖家详情：url=%s reviews=%s", url, payload.include_reviews)
|
|
if payload.include_reviews:
|
|
html, review_html = await asyncio.gather(
|
|
self._session.fetch_html(url),
|
|
self._session.fetch_html(build_shop_url(shop_id, review=True)),
|
|
)
|
|
else:
|
|
html, review_html = await self._session.fetch_html(url), None
|
|
|
|
detail = parse_shop_detail(
|
|
html, shop_id=shop_id, shop_url=url, review_html=review_html
|
|
)
|
|
logger.info(
|
|
"ラクマ 卖家详情完成：shop_id=%s name=%s items=%s reviews=%s",
|
|
shop_id, detail.shop_name, detail.item_count, detail.review_count,
|
|
)
|
|
return detail
|
|
|
|
async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData:
|
|
"""抓取卖家的商品列表"""
|
|
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
|
url = build_shop_url(shop_id, page=payload.page)
|
|
|
|
logger.info("抓取 ラクマ 卖家商品：url=%s", url)
|
|
html = await self._session.fetch_html(url)
|
|
result = parse_shop_items(
|
|
html, shop_id=shop_id, request_url=url, page=payload.page
|
|
)
|
|
logger.info(
|
|
"ラクマ 卖家商品完成：shop_id=%s items=%s total=%s",
|
|
shop_id, len(result.items), result.total_count,
|
|
)
|
|
return result
|
|
|
|
@staticmethod
|
|
def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str:
|
|
"""从 shop_url 或 shop_id 里取出店铺 hash（模型校验已保证两者不同时为空）"""
|
|
if shop_url is not None:
|
|
return split_shop_url(str(shop_url))
|
|
return (shop_id or "").strip()
|