Init

2026-07-27 10:34:53 +08:00
commit 3527975794
74 changed files with 16405 additions and 0 deletions
@@ -0,0 +1,130 @@
+"""ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据
+
+四个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）：
+搜索、商品详情、卖家详情、卖家商品列表。
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+
+from app.core.config import Settings
+from app.models.scrape import (
+    RakumaItemDetailData,
+    RakumaItemDetailRequest,
+    RakumaSearchRequest,
+    RakumaSearchResultData,
+    RakumaShopDetailData,
+    RakumaShopDetailRequest,
+    RakumaShopItemsData,
+    RakumaShopItemsRequest,
+)
+from app.parsers.rakuma.item import parse_item_detail
+from app.parsers.rakuma.search import parse_search
+from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
+from app.services.rakuma_session import RakumaSession
+from app.utils.rakuma_urls import (
+    build_item_url,
+    build_search_url,
+    build_shop_url,
+    normalize_search_url,
+    split_item_url,
+    split_shop_url,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class RakumaClient:
+    """ラクマ（fril.jp）抓取客户端"""
+
+    def __init__(self, settings: Settings, session: RakumaSession):
+        self._settings = settings
+        self._session = session
+
+    async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData:
+        """抓取搜索结果列表"""
+        if payload.search_url is not None:
+            url = normalize_search_url(str(payload.search_url), payload.page)
+        else:
+            url = build_search_url(payload)
+
+        logger.info("抓取 ラクマ 搜索页：url=%s", url)
+        html = await self._session.fetch_html(url)
+        result = parse_search(
+            html,
+            request_url=url,
+            page=payload.page,
+            keyword=payload.keyword.strip(),
+        )
+        logger.info(
+            "ラクマ 搜索完成：url=%s items=%s total=%s",
+            url, len(result.items), result.total_count,
+        )
+        return result
+
+    async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData:
+        """抓取商品详情"""
+        if payload.item_url is not None:
+            item_id = split_item_url(str(payload.item_url))
+        else:
+            item_id = (payload.item_id or "").strip()
+        url = build_item_url(item_id)
+
+        logger.info("抓取 ラクマ 商品详情：url=%s", url)
+        html = await self._session.fetch_html(url)
+        detail = parse_item_detail(html, item_id=item_id, item_url=url)
+        logger.info(
+            "ラクマ 详情完成：url=%s name=%s price=%s sold_out=%s",
+            url, detail.item_name[:40], detail.price, detail.is_sold_out,
+        )
+        return detail
+
+    async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData:
+        """抓取卖家详情
+
+        评价明细在单独的 /review 页上，仅在 include_reviews=true 时并发多取一次。
+        """
+        shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
+        url = build_shop_url(shop_id)
+
+        logger.info("抓取 ラクマ 卖家详情：url=%s reviews=%s", url, payload.include_reviews)
+        if payload.include_reviews:
+            html, review_html = await asyncio.gather(
+                self._session.fetch_html(url),
+                self._session.fetch_html(build_shop_url(shop_id, review=True)),
+            )
+        else:
+            html, review_html = await self._session.fetch_html(url), None
+
+        detail = parse_shop_detail(
+            html, shop_id=shop_id, shop_url=url, review_html=review_html
+        )
+        logger.info(
+            "ラクマ 卖家详情完成：shop_id=%s name=%s items=%s reviews=%s",
+            shop_id, detail.shop_name, detail.item_count, detail.review_count,
+        )
+        return detail
+
+    async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData:
+        """抓取卖家的商品列表"""
+        shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
+        url = build_shop_url(shop_id, page=payload.page)
+
+        logger.info("抓取 ラクマ 卖家商品：url=%s", url)
+        html = await self._session.fetch_html(url)
+        result = parse_shop_items(
+            html, shop_id=shop_id, request_url=url, page=payload.page
+        )
+        logger.info(
+            "ラクマ 卖家商品完成：shop_id=%s items=%s total=%s",
+            shop_id, len(result.items), result.total_count,
+        )
+        return result
+
+    @staticmethod
+    def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str:
+        """从 shop_url 或 shop_id 里取出店铺 hash（模型校验已保证两者不同时为空）"""
+        if shop_url is not None:
+            return split_shop_url(str(shop_url))
+        return (shop_id or "").strip()