rakuten-api/app/scraping/services/rakuma_session.py

"""ラクマ（fril.jp）站点会话：单通道 HTTP 抓取

与乐天市场那条链路（app/services/site_session.py）分开维护，因为两站的
抓取前提完全不同：

- 乐天前置 Akamai Bot Manager，无 cookie 时每个响应被拖到 ~11s，必须先访问
  首页预热再复用 cookie；且搜索页与详情页要用不同 UA，需要两条指纹通道。
- ラクマ 实测没有这类限速：冷请求（无 cookie、无预热）即 0.5-1.1s，与预热后
  持平；PC UA 在搜索页、详情页、店铺页上都能拿到完整模板。

因此这里只有一条通道、不做预热，也不接浏览器兜底——没有需要兜底的拦截行为。
若将来站点加了防护，再按乐天那套补预热与升级链路。
"""
from __future__ import annotations

import asyncio
import logging
from typing import Any

import httpx

from app.scraping.core import rakuma_site as site
from app.shared.config import Settings
from app.shared.errors import (
    ItemNotFoundError,
    ResourceBusyError,
    UpstreamBlockedError,
    UpstreamRequestError,
)

logger = logging.getLogger(__name__)


class RakumaSession:
    """ラクマ 站点抓取会话，管理并发限流与失败重试"""

    def __init__(self, settings: Settings):
        self._settings = settings
        self._semaphore = asyncio.Semaphore(settings.max_site_concurrency)
        self._client: httpx.AsyncClient | None = None

    # ---- 生命周期 ----

    async def start(self) -> None:
        """创建 HTTP 客户端"""
        if self._client is not None:
            return
        self._client = httpx.AsyncClient(
            headers=site.default_headers(),
            timeout=self._settings.request_timeout_seconds,
            follow_redirects=True,
            proxy=self._settings.httpx_proxy,
            http2=True,
        )
        logger.info(
            "ラクマ 会话已就绪：concurrency=%s proxy=%s",
            self._settings.max_site_concurrency,
            bool(self._settings.proxy_server),
        )

    async def close(self) -> None:
        """关闭 HTTP 客户端"""
        if self._client is None:
            return
        try:
            await self._client.aclose()
        except Exception:
            logger.debug("关闭 ラクマ HTTP 客户端失败", exc_info=True)
        self._client = None

    # ---- 状态 ----

    def status(self) -> dict[str, Any]:
        """会话状态，供健康检查展示"""
        return {"ready": self._client is not None}

    # ---- 抓取 ----

    async def fetch_html(self, url: str) -> str:
        """抓取页面 HTML

        Raises:
            ItemNotFoundError: 目标页面 404（商品已下架或 ID 不存在）
            UpstreamRequestError: 网络异常或上游 5xx
            UpstreamBlockedError: 反复取不到正常页面
            ResourceBusyError: 等待并发槽位超时
        """
        client = self._client
        if client is None:
            raise UpstreamRequestError("ラクマ 会话尚未初始化")

        max_attempts = max(1, self._settings.http_max_attempts)
        last_error = "unknown error"

        try:
            await asyncio.wait_for(
                self._semaphore.acquire(),
                timeout=self._settings.request_timeout_seconds,
            )
        except TimeoutError as exc:
            raise ResourceBusyError() from exc

        try:
            for attempt in range(1, max_attempts + 1):
                try:
                    response = await client.get(url)
                except httpx.HTTPError as exc:
                    last_error = f"{type(exc).__name__}: {exc}"
                    logger.warning(
                        "ラクマ 抓取请求异常：url=%s attempt=%s/%s err=%s",
                        url, attempt, max_attempts, last_error,
                    )
                    continue

                if response.status_code == 404:
                    raise ItemNotFoundError(f"Page not found: {url}")

                if response.status_code < 400:
                    return response.text

                last_error = (
                    f"upstream status {response.status_code}"
                    if response.status_code >= 500
                    else f"status {response.status_code}"
                )
                logger.warning(
                    "ラクマ 抓取结果异常：url=%s attempt=%s/%s %s",
                    url, attempt, max_attempts, last_error,
                )

            if last_error.startswith("upstream status") or ":" in last_error:
                raise UpstreamRequestError(f"Upstream request failed: {last_error}")
            raise UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
        finally:
            self._semaphore.release()