拆分抓取与交易服务
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,185 @@
|
||||
"""浏览器兜底：纯 HTTP 被 Akamai 拦截时，用 Playwright 取回一套可用 cookie
|
||||
|
||||
为什么只是「兜底」而不是主路径：乐天的反爬是 Akamai Bot Manager，正常携带
|
||||
完整浏览器请求头 + 复用其下发的 cookie 就能通过，不像骏河屋的 Cloudflare
|
||||
Turnstile 必须真浏览器交互。因此这里的浏览器是冷路径——按需惰性启动，
|
||||
长时间不用会被回收，未安装 playwright 时整体降级为「不兜底」而非报错。
|
||||
|
||||
导航结果里的 HTML 会一并返回：既然浏览器已经把页面取到了，就没必要让调用方
|
||||
再发一次 HTTP 请求。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.scraping.core import site
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BrowserVisit:
|
||||
"""一次浏览器导航的产出：cookie 快照 + 页面 HTML"""
|
||||
|
||||
cookies: list[dict[str, Any]] = field(default_factory=list)
|
||||
html: str = ""
|
||||
|
||||
|
||||
class BrowserFallback:
|
||||
"""按需启动的 Playwright 实例，用于取回通过反爬校验的 cookie
|
||||
|
||||
仅在 HTTP 抓取被判定为拦截后才会被调用；调用之间浏览器保持存活，
|
||||
由 close() 在应用关闭时释放。
|
||||
"""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self._settings = settings
|
||||
self._playwright: Any | None = None
|
||||
self._browser: Any | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._unavailable_reason: str | None = None
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""配置是否允许使用浏览器兜底"""
|
||||
return self._settings.browser_fallback_enabled
|
||||
|
||||
@property
|
||||
def unavailable_reason(self) -> str | None:
|
||||
"""浏览器不可用的原因（未启用 / 未安装 / 启动失败）"""
|
||||
if not self.enabled:
|
||||
return "browser fallback is disabled"
|
||||
return self._unavailable_reason
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
"""浏览器当前是否已启动且连接正常"""
|
||||
browser = self._browser
|
||||
if browser is None:
|
||||
return False
|
||||
is_connected = getattr(browser, "is_connected", None)
|
||||
if callable(is_connected):
|
||||
try:
|
||||
return bool(is_connected())
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _ensure_browser(self) -> Any | None:
|
||||
"""惰性启动浏览器；不可用时返回 None 并记录原因，不抛异常。"""
|
||||
if not self.enabled:
|
||||
return None
|
||||
if self.ready:
|
||||
return self._browser
|
||||
|
||||
async with self._lock:
|
||||
if self.ready:
|
||||
return self._browser
|
||||
|
||||
# 上一轮已经启动失败过，直接沿用结论，避免每次请求都吃一次启动超时
|
||||
if self._unavailable_reason is not None and self._playwright is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except ImportError as exc:
|
||||
self._unavailable_reason = (
|
||||
f"playwright is not installed: {exc}; "
|
||||
'install it with pip install ".[browser]" && python -m playwright install chromium'
|
||||
)
|
||||
logger.warning("浏览器兜底不可用：%s", self._unavailable_reason)
|
||||
return None
|
||||
|
||||
await self._close_locked()
|
||||
try:
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self._settings.browser_headless_effective,
|
||||
channel=self._settings.browser_channel or None,
|
||||
proxy=self._settings.playwright_proxy,
|
||||
timeout=self._settings.browser_launch_timeout_seconds * 1000,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
self._unavailable_reason = None
|
||||
logger.info(
|
||||
"浏览器兜底已启动：headless=%s channel=%s proxy=%s",
|
||||
self._settings.browser_headless_effective,
|
||||
self._settings.browser_channel,
|
||||
bool(self._settings.proxy_server),
|
||||
)
|
||||
except Exception as exc:
|
||||
self._unavailable_reason = str(exc)
|
||||
logger.exception("浏览器兜底启动失败：%s", self._unavailable_reason)
|
||||
await self._close_locked()
|
||||
return None
|
||||
|
||||
return self._browser
|
||||
|
||||
async def visit(self, url: str, *, mobile: bool) -> BrowserVisit | None:
|
||||
"""用浏览器访问 url，返回 cookie 与页面 HTML；不可用时返回 None。
|
||||
|
||||
UA 与请求头必须和后续 HTTP 客户端使用的那一套保持一致——Akamai 会把
|
||||
cookie 与指纹绑定，用 PC 指纹拿到的 cookie 拿去发手机 UA 请求可能失效。
|
||||
"""
|
||||
browser = await self._ensure_browser()
|
||||
if browser is None:
|
||||
return None
|
||||
|
||||
headers = site.default_headers(mobile=mobile)
|
||||
context = None
|
||||
try:
|
||||
context = await browser.new_context(
|
||||
user_agent=headers["User-Agent"],
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844} if mobile else {"width": 1440, "height": 900},
|
||||
is_mobile=mobile,
|
||||
has_touch=mobile,
|
||||
extra_http_headers={"Accept-Language": site.ACCEPT_LANGUAGE},
|
||||
)
|
||||
page = await context.new_page()
|
||||
await page.goto(
|
||||
url,
|
||||
wait_until="domcontentloaded",
|
||||
timeout=self._settings.browser_nav_timeout_seconds * 1000,
|
||||
)
|
||||
html = await page.content()
|
||||
cookies = await context.cookies()
|
||||
logger.info("浏览器兜底导航完成：url=%s cookies=%s html_len=%s", url, len(cookies), len(html))
|
||||
return BrowserVisit(cookies=cookies, html=html)
|
||||
except Exception as exc:
|
||||
logger.warning("浏览器兜底导航失败：url=%s err=%s", url, exc)
|
||||
# 导航失败常常意味着浏览器已掉线，标记后由下次调用重建
|
||||
if not self.ready:
|
||||
self._unavailable_reason = str(exc)
|
||||
return None
|
||||
finally:
|
||||
if context is not None:
|
||||
try:
|
||||
await context.close()
|
||||
except Exception:
|
||||
logger.debug("关闭兜底浏览器上下文失败", exc_info=True)
|
||||
|
||||
async def _close_locked(self) -> None:
|
||||
"""释放浏览器与 Playwright 运行时（调用方需已持锁或处于关闭流程）"""
|
||||
if self._browser is not None:
|
||||
try:
|
||||
await self._browser.close()
|
||||
except Exception:
|
||||
logger.debug("关闭兜底浏览器失败", exc_info=True)
|
||||
self._browser = None
|
||||
if self._playwright is not None:
|
||||
try:
|
||||
await self._playwright.stop()
|
||||
except Exception:
|
||||
logger.debug("停止兜底 Playwright 运行时失败", exc_info=True)
|
||||
self._playwright = None
|
||||
|
||||
async def close(self) -> None:
|
||||
"""应用关闭时释放浏览器资源"""
|
||||
async with self._lock:
|
||||
await self._close_locked()
|
||||
Reference in New Issue
Block a user