import asyncio from collections import defaultdict from pathlib import Path from typing import Dict from playwright.async_api import async_playwright, Playwright, BrowserContext from loguru import logger from surugaya_common.browser_profile import DEFAULT_USER_AGENT, LOCALE, TIMEZONE_ID, stealth_launch_args from src.config.settings import get_settings from src.config.accounts import AccountConfig, load_accounts_config # 注入到所有页面的 JS 脚本,用于绕过 Cloudflare、Akamai 等自动化检测 STEALTH_JS = """ // 1. 隐藏 navigator.webdriver Object.defineProperty(navigator, 'webdriver', { get: () => undefined }); // 2. 覆盖语言列表,优先日语(与目标地区一致) Object.defineProperty(navigator, 'languages', { get: () => ['ja-JP', 'ja', 'en-US', 'en'] }); // 3. 伪造 WebGL 渲染器信息,伪装为标准硬件 const getParameter = WebGLRenderingContext.prototype.getParameter; WebGLRenderingContext.prototype.getParameter = function(parameter) { // UNMASKED_VENDOR_WEBGL (37445) if (parameter === 37445) { return 'Intel Inc.'; } // UNMASKED_RENDERER_WEBGL (37446) if (parameter === 37446) { return 'Intel(R) UHD Graphics'; } return getParameter.apply(this, arguments); }; // 4. 伪造设备规格 Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 }); Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 }); // 5. 模拟完整插件列表(无头浏览器常见特征为空列表) const mockPlugins = () => { const makePlugin = (name, filename, description) => { const p = Object.create(Plugin.prototype); Object.defineProperties(p, { name: { get: () => name }, filename: { get: () => filename }, description: { get: () => description }, length: { get: () => 0 } }); return p; }; const plugins = [ makePlugin("PDF Viewer", "internal-pdf-viewer", "Portable Document Format"), makePlugin("Chrome PDF Viewer", "mhjfbgoacfdegokjbbpboihdaibdccjg", "Chrome PDF Viewer"), makePlugin("Chromium PDF Viewer", "internal-pdf-viewer", "Chromium PDF Viewer") ]; const pList = Object.create(PluginArray.prototype); Object.defineProperties(pList, { length: { get: () => plugins.length }, item: { value: (index) => plugins[index] }, namedItem: { value: (name) => plugins.find(p => p.name === name) } }); for (let i = 0; i < plugins.length; i++) { Object.defineProperty(pList, i, { get: () => plugins[i] }); } return pList; }; Object.defineProperty(navigator, 'plugins', { get: () => mockPlugins() }); // 6. 模拟 window.chrome(无头浏览器通常没有,Cloudflare 会检测) window.chrome = { app: { isInstalled: false, InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' }, RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' } }, runtime: { OnInstalledReason: { CHROME_UPDATE: 'chrome_update', INSTALL: 'install', SHARED_MODULE_UPDATE: 'shared_module_update', UPDATE: 'update' }, OnRestartRequiredReason: { APP_UPDATE: 'app_update', OS_UPDATE: 'os_update', PERIODIC: 'periodic' }, PlatformArch: { ARM: 'arm', ARM64: 'arm64', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, PlatformNaclArch: { ARM: 'arm', MIPS: 'mips', MIPS64: 'mips64', X86_32: 'x86-32', X86_64: 'x86-64' }, PlatformOs: { ANDROID: 'android', CROS: 'cros', LINUX: 'linux', MAC: 'mac', OPENBSD: 'openbsd', WIN: 'win' }, RequestUpdateCheckStatus: { NO_UPDATE: 'no_update', THROTTLED: 'throttled', UPDATE_AVAILABLE: 'update_available' } } }; // 7. 模拟 Permissions 查询行为一致性 const originalQuery = window.navigator.permissions.query; window.navigator.permissions.query = (parameters) => ( parameters.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : originalQuery(parameters) ); """ class BrowserManager: """ 进程级浏览器池:为每个账号维护一个常驻的持久化浏览器上下文。 - 启动时可通过 prewarm() 为所有已配置账号预先拉起浏览器,任务到达时即开即用。 - 上下文跨任务复用;任务只负责开关自己的标签页(Page),不关闭浏览器。 - 上下文崩溃或被手动关闭时自动从池中剔除,下次获取时重新启动。 - 按账号隔离数据环境(user_data_dir),并注入反爬脚本以规避 Cloudflare、Akamai 等检测。 """ def __init__(self) -> None: self.settings = get_settings() self._playwright: Playwright | None = None self._contexts: Dict[str, BrowserContext] = {} # 按账号维护启动锁,避免同一账号的 user_data_dir 被并发启动两次 self._launch_locks: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock) async def prewarm(self) -> None: """ 为所有已配置账号预启动常驻浏览器上下文。 单个账号启动失败不阻断其他账号,失败账号会在任务执行时自动重试启动。 """ config = load_accounts_config() accounts = [ account for site_accounts in (config.sites.surugaya, config.sites.mercari, config.sites.yahoo) for account in site_accounts ] if not accounts: logger.info("未配置任何站点账号,跳过浏览器预热。") return logger.info(f"正在预热浏览器,共 {len(accounts)} 个账号...") for account in accounts: try: await self.get_context(account) except Exception as e: logger.error( f"账号 '{account.account_id}' 浏览器预热失败(任务执行时将自动重试启动): {e}" ) logger.success(f"浏览器预热完成,当前常驻上下文 {len(self._contexts)} 个。") async def get_context(self, account: AccountConfig) -> BrowserContext: """ 获取指定账号的常驻浏览器上下文:已存在且存活则直接复用,否则启动新实例。 上下文由本管理器统一持有,调用方不要关闭它,只需管理自己打开的 Page。 Args: account: 包含 account_id、user_data_dir 及可选 proxy 的账号配置。 Returns: BrowserContext: 该账号的常驻浏览器上下文。 """ async with self._launch_locks[account.account_id]: cached = self._contexts.get(account.account_id) if cached is not None: logger.debug(f"账号 '{account.account_id}' 复用常驻浏览器上下文。") return cached return await self._launch_context(account) async def close_all(self) -> None: """ 进程退出时关闭所有常驻上下文并停止 Playwright 驱动。 """ for account_id, context in list(self._contexts.items()): try: await context.close() logger.info(f"账号 '{account_id}' 的常驻浏览器上下文已关闭。") except Exception as e: logger.error(f"关闭账号 '{account_id}' 的浏览器上下文时出错: {e}") self._contexts.clear() if self._playwright: try: await self._playwright.stop() except Exception as e: logger.error(f"停止 Playwright 驱动时出错: {e}") self._playwright = None async def _get_playwright(self) -> Playwright: """惰性启动并复用进程内唯一的 Playwright 驱动。""" if self._playwright is None: self._playwright = await async_playwright().start() return self._playwright def _on_context_close(self, account_id: str) -> None: """上下文关闭(崩溃或被手动关掉)时从池中剔除,下次任务自动重启。""" if self._contexts.pop(account_id, None) is not None: logger.warning( f"账号 '{account_id}' 的浏览器上下文已断开,已从常驻池移除,下次任务将自动重启浏览器。" ) async def _launch_context(self, account: AccountConfig) -> BrowserContext: """ 为指定账号启动持久化浏览器上下文并纳入常驻池。 """ # 1. 解析并确保本地持久化目录存在 data_dir = Path(account.user_data_dir) try: data_dir.mkdir(parents=True, exist_ok=True) # logger.debug(f"持久化用户数据目录: {data_dir.absolute()}") except Exception as e: logger.error(f"创建账号 '{account.account_id}' 的持久化目录失败: {e}") raise e # 2. 配置隐身 Chromium 启动参数(共享反检测参数 + 本进程追加项) chrome_args = stealth_launch_args() + [ "--start-maximized", # 启动时全屏(最大化) ] executable_path_raw = self.settings.browser_executable_path executable_path = None if executable_path_raw: candidate_path = Path(executable_path_raw).expanduser() if candidate_path.is_file(): executable_path = str(candidate_path) logger.info(f"使用自定义浏览器可执行文件: {executable_path}") else: logger.warning( f"配置的浏览器可执行文件无效,将使用 Playwright 默认浏览器: {executable_path_raw}" ) headless_mode = self.settings.browser_headless # 使用与 API 服务一致的 Windows Chrome User-Agent(共享指纹配置) user_agent = DEFAULT_USER_AGENT logger.info(f"正在为账号 {account.account_id} 启动 Playwright | 无头模式: {headless_mode}") # 3. 按账号配置可选代理(服务器 IP 段绕过 Cloudflare 封锁时常用) proxy_config = None if account.proxy: proxy_config = {"server": account.proxy} logger.info(f"账号 '{account.account_id}' 已启用代理: {account.proxy}") playwright_driver = await self._get_playwright() try: # 4. 使用隔离目录创建持久化上下文 launch_options = { "user_data_dir": str(data_dir.absolute()), "headless": headless_mode, "args": chrome_args, "user_agent": user_agent, "no_viewport": True, # 配合 --start-maximized 使得网页视口也铺满全屏 "locale": LOCALE, # 区域语言一致(ja-JP) "timezone_id": TIMEZONE_ID, # 日本本地时区(Asia/Tokyo) "accept_downloads": True, # 允许自动下载(如需要) "proxy": proxy_config # 住宅代理等 } if executable_path: launch_options["executable_path"] = executable_path context = await playwright_driver.chromium.launch_persistent_context(**launch_options) # 5. 注入高级反指纹隐身脚本 await context.add_init_script(STEALTH_JS) # 6. 纳入常驻池,并监听关闭事件以便自愈重启 context.on("close", lambda: self._on_context_close(account.account_id)) self._contexts[account.account_id] = context logger.success(f"账号 '{account.account_id}' 的常驻 Playwright 上下文启动成功。") return context except Exception as e: logger.error(f"账号 '{account.account_id}' 启动持久化上下文失败: {e}") # Playwright 驱动为进程共享,启动失败时仅向上抛出,不停止驱动 raise e _browser_manager: BrowserManager | None = None def get_browser_manager() -> BrowserManager: """ 获取进程级 BrowserManager 单例,保证所有任务共享同一个浏览器池。 """ global _browser_manager if _browser_manager is None: _browser_manager = BrowserManager() return _browser_manager