q792602257andClaude Opus 4.6
2b7db521c4
抓取-解析链路原本只有日志，出现"抓回内容但解析不出预期字段"时定位慢。 接入 OpenTelemetry traces（FastAPI/httpx 自动 + 手写 fetch/parse span）， 解析失败时把页面 HTML 作为 span event 上报，便于事后复现。 Dockerfile 默认开启，镜像一启动即导出到自建 OTLP endpoint。 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
220 lines
9.5 KiB
Python
220 lines
9.5 KiB
Python
"""ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据
|
|
|
|
五个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）：
|
|
搜索、分类、商品详情、卖家详情、卖家商品列表。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
|
|
from opentelemetry import trace
|
|
|
|
from app.scraping.core import rakuma_site as site
|
|
from app.shared.config import Settings
|
|
from app.scraping.models.scrape import (
|
|
RakumaCategoryData,
|
|
RakumaCategoryRequest,
|
|
RakumaItemDetailData,
|
|
RakumaItemDetailRequest,
|
|
RakumaSearchRequest,
|
|
RakumaSearchResultData,
|
|
RakumaShopDetailData,
|
|
RakumaShopDetailRequest,
|
|
RakumaShopItemsData,
|
|
RakumaShopItemsRequest,
|
|
)
|
|
from app.scraping.parsers.rakuma.category import parse_categories
|
|
from app.scraping.parsers.rakuma.item import parse_item_detail
|
|
from app.scraping.parsers.rakuma.search import parse_search
|
|
from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
|
from app.scraping.services.rakuma_session import RakumaSession
|
|
from app.scraping.utils.rakuma_urls import (
|
|
build_item_url,
|
|
build_search_url,
|
|
build_shop_url,
|
|
normalize_search_url,
|
|
split_item_url,
|
|
split_shop_url,
|
|
)
|
|
from app.shared.telemetry import snapshot
|
|
|
|
logger = logging.getLogger(__name__)
|
|
tracer = trace.get_tracer(__name__)
|
|
|
|
|
|
class RakumaClient:
|
|
"""ラクマ（fril.jp）抓取客户端"""
|
|
|
|
def __init__(self, settings: Settings, session: RakumaSession):
|
|
self._settings = settings
|
|
self._session = session
|
|
|
|
async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData:
|
|
"""抓取搜索结果列表"""
|
|
if payload.search_url is not None:
|
|
url = normalize_search_url(str(payload.search_url), payload.page)
|
|
else:
|
|
url = build_search_url(payload)
|
|
|
|
logger.info("抓取 ラクマ 搜索页：url=%s", url)
|
|
html: str | None = None
|
|
with tracer.start_as_current_span("parse.rakuma.search") as span:
|
|
span.set_attribute("parse.site", "rakuma")
|
|
span.set_attribute("parse.op", "search")
|
|
try:
|
|
html = await self._session.fetch_html(url)
|
|
result = parse_search(
|
|
html,
|
|
request_url=url,
|
|
page=payload.page,
|
|
keyword=payload.keyword.strip(),
|
|
)
|
|
span.set_attribute("parse.items", len(result.items))
|
|
span.set_attribute("parse.total", result.total_count)
|
|
logger.info(
|
|
"ラクマ 搜索完成：url=%s items=%s total=%s",
|
|
url, len(result.items), result.total_count,
|
|
)
|
|
return result
|
|
except Exception:
|
|
span.record_exception()
|
|
span.set_attribute("parse.fail_reason", "parse_error")
|
|
snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes)
|
|
raise
|
|
|
|
async def categories(self, payload: RakumaCategoryRequest) -> RakumaCategoryData:
|
|
"""抓取分类树
|
|
|
|
分类一览页一次就返回整棵树（与 URL 上的 category_id 无关），因此不论
|
|
查哪一层都只打一次请求，页面地址也固定不带参数。
|
|
"""
|
|
url = site.CATEGORY_LIST_URL
|
|
|
|
logger.info("抓取 ラクマ 分类页：category_id=%s", payload.category_id)
|
|
html: str | None = None
|
|
with tracer.start_as_current_span("parse.rakuma.categories") as span:
|
|
span.set_attribute("parse.site", "rakuma")
|
|
span.set_attribute("parse.op", "categories")
|
|
try:
|
|
html = await self._session.fetch_html(url)
|
|
data = parse_categories(
|
|
html,
|
|
category_id=payload.category_id,
|
|
include_descendants=payload.include_descendants,
|
|
)
|
|
span.set_attribute("parse.children", len(data.children))
|
|
span.set_attribute("parse.tree", data.total_count)
|
|
logger.info(
|
|
"ラクマ 分类完成：category_id=%s name=%s children=%s tree=%s",
|
|
data.category_id, data.name, len(data.children), data.total_count,
|
|
)
|
|
return data
|
|
except Exception:
|
|
span.record_exception()
|
|
span.set_attribute("parse.fail_reason", "parse_error")
|
|
snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes)
|
|
raise
|
|
|
|
async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData:
|
|
"""抓取商品详情"""
|
|
if payload.item_url is not None:
|
|
item_id = split_item_url(str(payload.item_url))
|
|
else:
|
|
item_id = (payload.item_id or "").strip()
|
|
url = build_item_url(item_id)
|
|
|
|
logger.info("抓取 ラクマ 商品详情：url=%s", url)
|
|
html: str | None = None
|
|
with tracer.start_as_current_span("parse.rakuma.item_detail") as span:
|
|
span.set_attribute("parse.site", "rakuma")
|
|
span.set_attribute("parse.op", "item_detail")
|
|
try:
|
|
html = await self._session.fetch_html(url)
|
|
detail = parse_item_detail(html, item_id=item_id, item_url=url)
|
|
span.set_attribute("parse.price", detail.price)
|
|
span.set_attribute("parse.sold_out", detail.is_sold_out)
|
|
logger.info(
|
|
"ラクマ 详情完成：url=%s name=%s price=%s sold_out=%s",
|
|
url, detail.item_name[:40], detail.price, detail.is_sold_out,
|
|
)
|
|
return detail
|
|
except Exception:
|
|
span.record_exception()
|
|
span.set_attribute("parse.fail_reason", "parse_error")
|
|
snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes)
|
|
raise
|
|
|
|
async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData:
|
|
"""抓取卖家详情
|
|
|
|
评价明细在单独的 /review 页上，仅在 include_reviews=true 时并发多取一次。
|
|
"""
|
|
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
|
url = build_shop_url(shop_id)
|
|
|
|
logger.info("抓取 ラクマ 卖家详情：url=%s reviews=%s", url, payload.include_reviews)
|
|
html: str | None = None
|
|
with tracer.start_as_current_span("parse.rakuma.shop_detail") as span:
|
|
span.set_attribute("parse.site", "rakuma")
|
|
span.set_attribute("parse.op", "shop_detail")
|
|
try:
|
|
if payload.include_reviews:
|
|
html, review_html = await asyncio.gather(
|
|
self._session.fetch_html(url),
|
|
self._session.fetch_html(build_shop_url(shop_id, review=True)),
|
|
)
|
|
else:
|
|
html, review_html = await self._session.fetch_html(url), None
|
|
|
|
detail = parse_shop_detail(
|
|
html, shop_id=shop_id, shop_url=url, review_html=review_html
|
|
)
|
|
span.set_attribute("parse.items", detail.item_count)
|
|
span.set_attribute("parse.reviews", detail.review_count)
|
|
logger.info(
|
|
"ラクマ 卖家详情完成：shop_id=%s name=%s items=%s reviews=%s",
|
|
shop_id, detail.shop_name, detail.item_count, detail.review_count,
|
|
)
|
|
return detail
|
|
except Exception:
|
|
span.record_exception()
|
|
span.set_attribute("parse.fail_reason", "parse_error")
|
|
snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes)
|
|
raise
|
|
|
|
async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData:
|
|
"""抓取卖家的商品列表"""
|
|
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
|
url = build_shop_url(shop_id, page=payload.page)
|
|
|
|
logger.info("抓取 ラクマ 卖家商品：url=%s", url)
|
|
html: str | None = None
|
|
with tracer.start_as_current_span("parse.rakuma.shop_items") as span:
|
|
span.set_attribute("parse.site", "rakuma")
|
|
span.set_attribute("parse.op", "shop_items")
|
|
try:
|
|
html = await self._session.fetch_html(url)
|
|
result = parse_shop_items(
|
|
html, shop_id=shop_id, request_url=url, page=payload.page
|
|
)
|
|
span.set_attribute("parse.items", len(result.items))
|
|
span.set_attribute("parse.total", result.total_count)
|
|
logger.info(
|
|
"ラクマ 卖家商品完成：shop_id=%s items=%s total=%s",
|
|
shop_id, len(result.items), result.total_count,
|
|
)
|
|
return result
|
|
except Exception:
|
|
span.record_exception()
|
|
span.set_attribute("parse.fail_reason", "parse_error")
|
|
snapshot(span, "parse.failed_html", html, self._settings.otel_snapshot_max_bytes)
|
|
raise
|
|
|
|
@staticmethod
|
|
def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str:
|
|
"""从 shop_url 或 shop_id 里取出店铺 hash（模型校验已保证两者不同时为空）"""
|
|
if shop_url is not None:
|
|
return split_shop_url(str(shop_url))
|
|
return (shop_id or "").strip()
|