rakuten-api/app/scraping/services/rakuma_client.py

"""ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据

五个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）：
搜索、分类、商品详情、卖家详情、卖家商品列表。
"""
from __future__ import annotations

import asyncio
import logging

from app.scraping.core import rakuma_site as site
from app.shared.config import Settings
from app.scraping.models.scrape import (
    RakumaCategoryData,
    RakumaCategoryRequest,
    RakumaItemDetailData,
    RakumaItemDetailRequest,
    RakumaSearchRequest,
    RakumaSearchResultData,
    RakumaShopDetailData,
    RakumaShopDetailRequest,
    RakumaShopItemsData,
    RakumaShopItemsRequest,
)
from app.scraping.parsers.rakuma.category import parse_categories
from app.scraping.parsers.rakuma.item import parse_item_detail
from app.scraping.parsers.rakuma.search import parse_search
from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
from app.scraping.services.rakuma_session import RakumaSession
from app.scraping.utils.rakuma_urls import (
    build_item_url,
    build_search_url,
    build_shop_url,
    normalize_search_url,
    split_item_url,
    split_shop_url,
)

logger = logging.getLogger(__name__)


class RakumaClient:
    """ラクマ（fril.jp）抓取客户端"""

    def __init__(self, settings: Settings, session: RakumaSession):
        self._settings = settings
        self._session = session

    async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData:
        """抓取搜索结果列表"""
        if payload.search_url is not None:
            url = normalize_search_url(str(payload.search_url), payload.page)
        else:
            url = build_search_url(payload)

        logger.info("抓取 ラクマ 搜索页：url=%s", url)
        html = await self._session.fetch_html(url)
        result = parse_search(
            html,
            request_url=url,
            page=payload.page,
            keyword=payload.keyword.strip(),
        )
        logger.info(
            "ラクマ 搜索完成：url=%s items=%s total=%s",
            url, len(result.items), result.total_count,
        )
        return result

    async def categories(self, payload: RakumaCategoryRequest) -> RakumaCategoryData:
        """抓取分类树

        分类一览页一次就返回整棵树（与 URL 上的 category_id 无关），因此不论
        查哪一层都只打一次请求，页面地址也固定不带参数。
        """
        url = site.CATEGORY_LIST_URL

        logger.info("抓取 ラクマ 分类页：category_id=%s", payload.category_id)
        html = await self._session.fetch_html(url)
        data = parse_categories(
            html,
            category_id=payload.category_id,
            include_descendants=payload.include_descendants,
        )
        logger.info(
            "ラクマ 分类完成：category_id=%s name=%s children=%s tree=%s",
            data.category_id, data.name, len(data.children), data.total_count,
        )
        return data

    async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData:
        """抓取商品详情"""
        if payload.item_url is not None:
            item_id = split_item_url(str(payload.item_url))
        else:
            item_id = (payload.item_id or "").strip()
        url = build_item_url(item_id)

        logger.info("抓取 ラクマ 商品详情：url=%s", url)
        html = await self._session.fetch_html(url)
        detail = parse_item_detail(html, item_id=item_id, item_url=url)
        logger.info(
            "ラクマ 详情完成：url=%s name=%s price=%s sold_out=%s",
            url, detail.item_name[:40], detail.price, detail.is_sold_out,
        )
        return detail

    async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData:
        """抓取卖家详情

        评价明细在单独的 /review 页上，仅在 include_reviews=true 时并发多取一次。
        """
        shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
        url = build_shop_url(shop_id)

        logger.info("抓取 ラクマ 卖家详情：url=%s reviews=%s", url, payload.include_reviews)
        if payload.include_reviews:
            html, review_html = await asyncio.gather(
                self._session.fetch_html(url),
                self._session.fetch_html(build_shop_url(shop_id, review=True)),
            )
        else:
            html, review_html = await self._session.fetch_html(url), None

        detail = parse_shop_detail(
            html, shop_id=shop_id, shop_url=url, review_html=review_html
        )
        logger.info(
            "ラクマ 卖家详情完成：shop_id=%s name=%s items=%s reviews=%s",
            shop_id, detail.shop_name, detail.item_count, detail.review_count,
        )
        return detail

    async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData:
        """抓取卖家的商品列表"""
        shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
        url = build_shop_url(shop_id, page=payload.page)

        logger.info("抓取 ラクマ 卖家商品：url=%s", url)
        html = await self._session.fetch_html(url)
        result = parse_shop_items(
            html, shop_id=shop_id, request_url=url, page=payload.page
        )
        logger.info(
            "ラクマ 卖家商品完成：shop_id=%s items=%s total=%s",
            shop_id, len(result.items), result.total_count,
        )
        return result

    @staticmethod
    def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str:
        """从 shop_url 或 shop_id 里取出店铺 hash（模型校验已保证两者不同时为空）"""
        if shop_url is not None:
            return split_shop_url(str(shop_url))
        return (shop_id or "").strip()