"""ラクマ 抓取客户端:把请求参数翻译成站点 URL,抓取后解析为结构化数据 五个接口都走同一条 HTTP 通道(ラクマ 无 Akamai 限速,不需要分指纹通道): 搜索、分类、商品详情、卖家详情、卖家商品列表。 """ from __future__ import annotations import asyncio import logging from opentelemetry import trace from app.scraping.core import rakuma_site as site from app.shared.config import Settings from app.scraping.models.scrape import ( RakumaCategoryData, RakumaCategoryRequest, RakumaItemDetailData, RakumaItemDetailRequest, RakumaSearchRequest, RakumaSearchResultData, RakumaShopDetailData, RakumaShopDetailRequest, RakumaShopItemsData, RakumaShopItemsRequest, ) from app.scraping.parsers.rakuma.category import parse_categories from app.scraping.parsers.rakuma.item import parse_item_detail from app.scraping.parsers.rakuma.search import parse_search from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items from app.scraping.services.rakuma_session import RakumaSession from app.scraping.utils.rakuma_urls import ( build_item_url, build_search_url, build_shop_url, normalize_search_url, split_item_url, split_shop_url, ) from app.shared.telemetry import snapshot logger = logging.getLogger(__name__) tracer = trace.get_tracer(__name__) class RakumaClient: """ラクマ(fril.jp)抓取客户端""" def __init__(self, settings: Settings, session: RakumaSession): self._settings = settings self._session = session async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData: """抓取搜索结果列表""" if payload.search_url is not None: url = normalize_search_url(str(payload.search_url), payload.page) else: url = build_search_url(payload) logger.info("抓取 ラクマ 搜索页:url=%s", url) html: str | None = None with tracer.start_as_current_span("parse.rakuma.search") as span: span.set_attribute("parse.site", "rakuma") span.set_attribute("parse.op", "search") try: html = await self._session.fetch_html(url) result = parse_search( html, request_url=url, page=payload.page, keyword=payload.keyword.strip(), ) span.set_attribute("parse.items", len(result.items)) span.set_attribute("parse.total", result.total_count) logger.info( "ラクマ 搜索完成:url=%s items=%s total=%s", url, len(result.items), result.total_count, ) return result except Exception: span.record_exception() span.set_attribute("parse.fail_reason", "parse_error") snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes) raise async def categories(self, payload: RakumaCategoryRequest) -> RakumaCategoryData: """抓取分类树 分类一览页一次就返回整棵树(与 URL 上的 category_id 无关),因此不论 查哪一层都只打一次请求,页面地址也固定不带参数。 """ url = site.CATEGORY_LIST_URL logger.info("抓取 ラクマ 分类页:category_id=%s", payload.category_id) html: str | None = None with tracer.start_as_current_span("parse.rakuma.categories") as span: span.set_attribute("parse.site", "rakuma") span.set_attribute("parse.op", "categories") try: html = await self._session.fetch_html(url) data = parse_categories( html, category_id=payload.category_id, include_descendants=payload.include_descendants, ) span.set_attribute("parse.children", len(data.children)) span.set_attribute("parse.tree", data.total_count) logger.info( "ラクマ 分类完成:category_id=%s name=%s children=%s tree=%s", data.category_id, data.name, len(data.children), data.total_count, ) return data except Exception: span.record_exception() span.set_attribute("parse.fail_reason", "parse_error") snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes) raise async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData: """抓取商品详情""" if payload.item_url is not None: item_id = split_item_url(str(payload.item_url)) else: item_id = (payload.item_id or "").strip() url = build_item_url(item_id) logger.info("抓取 ラクマ 商品详情:url=%s", url) html: str | None = None with tracer.start_as_current_span("parse.rakuma.item_detail") as span: span.set_attribute("parse.site", "rakuma") span.set_attribute("parse.op", "item_detail") try: html = await self._session.fetch_html(url) detail = parse_item_detail(html, item_id=item_id, item_url=url) span.set_attribute("parse.price", detail.price) span.set_attribute("parse.sold_out", detail.is_sold_out) logger.info( "ラクマ 详情完成:url=%s name=%s price=%s sold_out=%s", url, detail.item_name[:40], detail.price, detail.is_sold_out, ) return detail except Exception: span.record_exception() span.set_attribute("parse.fail_reason", "parse_error") snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes) raise async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData: """抓取卖家详情 评价明细在单独的 /review 页上,仅在 include_reviews=true 时并发多取一次。 """ shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id) url = build_shop_url(shop_id) logger.info("抓取 ラクマ 卖家详情:url=%s reviews=%s", url, payload.include_reviews) html: str | None = None with tracer.start_as_current_span("parse.rakuma.shop_detail") as span: span.set_attribute("parse.site", "rakuma") span.set_attribute("parse.op", "shop_detail") try: if payload.include_reviews: html, review_html = await asyncio.gather( self._session.fetch_html(url), self._session.fetch_html(build_shop_url(shop_id, review=True)), ) else: html, review_html = await self._session.fetch_html(url), None detail = parse_shop_detail( html, shop_id=shop_id, shop_url=url, review_html=review_html ) span.set_attribute("parse.items", detail.item_count) span.set_attribute("parse.reviews", detail.review_count) logger.info( "ラクマ 卖家详情完成:shop_id=%s name=%s items=%s reviews=%s", shop_id, detail.shop_name, detail.item_count, detail.review_count, ) return detail except Exception: span.record_exception() span.set_attribute("parse.fail_reason", "parse_error") snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes) raise async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData: """抓取卖家的商品列表""" shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id) url = build_shop_url(shop_id, page=payload.page) logger.info("抓取 ラクマ 卖家商品:url=%s", url) html: str | None = None with tracer.start_as_current_span("parse.rakuma.shop_items") as span: span.set_attribute("parse.site", "rakuma") span.set_attribute("parse.op", "shop_items") try: html = await self._session.fetch_html(url) result = parse_shop_items( html, shop_id=shop_id, request_url=url, page=payload.page ) span.set_attribute("parse.items", len(result.items)) span.set_attribute("parse.total", result.total_count) logger.info( "ラクマ 卖家商品完成:shop_id=%s items=%s total=%s", shop_id, len(result.items), result.total_count, ) return result except Exception: span.record_exception() span.set_attribute("parse.fail_reason", "parse_error") snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes) raise @staticmethod def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str: """从 shop_url 或 shop_id 里取出店铺 hash(模型校验已保证两者不同时为空)""" if shop_url is not None: return split_shop_url(str(shop_url)) return (shop_id or "").strip()