"""浏览器兜底:纯 HTTP 被 Akamai 拦截时,用 Playwright 取回一套可用 cookie 为什么只是「兜底」而不是主路径:乐天的反爬是 Akamai Bot Manager,正常携带 完整浏览器请求头 + 复用其下发的 cookie 就能通过,不像骏河屋的 Cloudflare Turnstile 必须真浏览器交互。因此这里的浏览器是冷路径——按需惰性启动, 长时间不用会被回收,未安装 playwright 时整体降级为「不兜底」而非报错。 导航结果里的 HTML 会一并返回:既然浏览器已经把页面取到了,就没必要让调用方 再发一次 HTTP 请求。 """ from __future__ import annotations import asyncio import logging from dataclasses import dataclass, field from typing import Any from app.shared.config import Settings from app.scraping.core import site logger = logging.getLogger(__name__) @dataclass(slots=True) class BrowserVisit: """一次浏览器导航的产出:cookie 快照 + 页面 HTML""" cookies: list[dict[str, Any]] = field(default_factory=list) html: str = "" class BrowserFallback: """按需启动的 Playwright 实例,用于取回通过反爬校验的 cookie 仅在 HTTP 抓取被判定为拦截后才会被调用;调用之间浏览器保持存活, 由 close() 在应用关闭时释放。 """ def __init__(self, settings: Settings): self._settings = settings self._playwright: Any | None = None self._browser: Any | None = None self._lock = asyncio.Lock() self._unavailable_reason: str | None = None @property def enabled(self) -> bool: """配置是否允许使用浏览器兜底""" return self._settings.browser_fallback_enabled @property def unavailable_reason(self) -> str | None: """浏览器不可用的原因(未启用 / 未安装 / 启动失败)""" if not self.enabled: return "browser fallback is disabled" return self._unavailable_reason @property def ready(self) -> bool: """浏览器当前是否已启动且连接正常""" browser = self._browser if browser is None: return False is_connected = getattr(browser, "is_connected", None) if callable(is_connected): try: return bool(is_connected()) except Exception: return False return True async def _ensure_browser(self) -> Any | None: """惰性启动浏览器;不可用时返回 None 并记录原因,不抛异常。""" if not self.enabled: return None if self.ready: return self._browser async with self._lock: if self.ready: return self._browser # 上一轮已经启动失败过,直接沿用结论,避免每次请求都吃一次启动超时 if self._unavailable_reason is not None and self._playwright is None: return None try: from playwright.async_api import async_playwright except ImportError as exc: self._unavailable_reason = ( f"playwright is not installed: {exc}; " 'install it with pip install ".[browser]" && python -m playwright install chromium' ) logger.warning("浏览器兜底不可用:%s", self._unavailable_reason) return None await self._close_locked() try: self._playwright = await async_playwright().start() self._browser = await self._playwright.chromium.launch( headless=self._settings.browser_headless_effective, channel=self._settings.browser_channel or None, proxy=self._settings.playwright_proxy, timeout=self._settings.browser_launch_timeout_seconds * 1000, args=["--no-first-run", "--disable-blink-features=AutomationControlled"], ) self._unavailable_reason = None logger.info( "浏览器兜底已启动:headless=%s channel=%s proxy=%s", self._settings.browser_headless_effective, self._settings.browser_channel, bool(self._settings.proxy_server), ) except Exception as exc: self._unavailable_reason = str(exc) logger.exception("浏览器兜底启动失败:%s", self._unavailable_reason) await self._close_locked() return None return self._browser async def visit(self, url: str, *, mobile: bool) -> BrowserVisit | None: """用浏览器访问 url,返回 cookie 与页面 HTML;不可用时返回 None。 UA 与请求头必须和后续 HTTP 客户端使用的那一套保持一致——Akamai 会把 cookie 与指纹绑定,用 PC 指纹拿到的 cookie 拿去发手机 UA 请求可能失效。 """ browser = await self._ensure_browser() if browser is None: return None headers = site.default_headers(mobile=mobile) context = None try: context = await browser.new_context( user_agent=headers["User-Agent"], locale="ja-JP", timezone_id="Asia/Tokyo", viewport={"width": 390, "height": 844} if mobile else {"width": 1440, "height": 900}, is_mobile=mobile, has_touch=mobile, extra_http_headers={"Accept-Language": site.ACCEPT_LANGUAGE}, ) page = await context.new_page() await page.goto( url, wait_until="domcontentloaded", timeout=self._settings.browser_nav_timeout_seconds * 1000, ) html = await page.content() cookies = await context.cookies() logger.info("浏览器兜底导航完成:url=%s cookies=%s html_len=%s", url, len(cookies), len(html)) return BrowserVisit(cookies=cookies, html=html) except Exception as exc: logger.warning("浏览器兜底导航失败:url=%s err=%s", url, exc) # 导航失败常常意味着浏览器已掉线,标记后由下次调用重建 if not self.ready: self._unavailable_reason = str(exc) return None finally: if context is not None: try: await context.close() except Exception: logger.debug("关闭兜底浏览器上下文失败", exc_info=True) async def _close_locked(self) -> None: """释放浏览器与 Playwright 运行时(调用方需已持锁或处于关闭流程)""" if self._browser is not None: try: await self._browser.close() except Exception: logger.debug("关闭兜底浏览器失败", exc_info=True) self._browser = None if self._playwright is not None: try: await self._playwright.stop() except Exception: logger.debug("停止兜底 Playwright 运行时失败", exc_info=True) self._playwright = None async def close(self) -> None: """应用关闭时释放浏览器资源""" async with self._lock: await self._close_locked()