You've already forked pruchase_jhw
319 lines
13 KiB
Python
319 lines
13 KiB
Python
import asyncio
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
from typing import Dict
|
|
from playwright.async_api import async_playwright, Playwright, BrowserContext
|
|
from loguru import logger
|
|
from surugaya_common.browser_profile import DEFAULT_USER_AGENT, LOCALE, TIMEZONE_ID, stealth_launch_args
|
|
from src.config.settings import get_settings
|
|
from src.config.accounts import AccountConfig, load_accounts_config
|
|
|
|
# 注入到所有页面的 JS 脚本，用于绕过 Cloudflare、Akamai 等自动化检测
|
|
STEALTH_JS = """
|
|
// 1. 隐藏 navigator.webdriver
|
|
Object.defineProperty(navigator, 'webdriver', {
|
|
get: () => undefined
|
|
});
|
|
|
|
// 2. 覆盖语言列表，优先日语（与目标地区一致）
|
|
Object.defineProperty(navigator, 'languages', {
|
|
get: () => ['ja-JP', 'ja', 'en-US', 'en']
|
|
});
|
|
|
|
// 3. 伪造 WebGL 渲染器信息，伪装为标准硬件
|
|
const getParameter = WebGLRenderingContext.prototype.getParameter;
|
|
WebGLRenderingContext.prototype.getParameter = function(parameter) {
|
|
// UNMASKED_VENDOR_WEBGL (37445)
|
|
if (parameter === 37445) {
|
|
return 'Intel Inc.';
|
|
}
|
|
// UNMASKED_RENDERER_WEBGL (37446)
|
|
if (parameter === 37446) {
|
|
return 'Intel(R) UHD Graphics';
|
|
}
|
|
return getParameter.apply(this, arguments);
|
|
};
|
|
|
|
// 4. 伪造设备规格
|
|
Object.defineProperty(navigator, 'hardwareConcurrency', { get: () => 8 });
|
|
Object.defineProperty(navigator, 'deviceMemory', { get: () => 8 });
|
|
|
|
// 5. 模拟完整插件列表（无头浏览器常见特征为空列表）
|
|
const mockPlugins = () => {
|
|
const makePlugin = (name, filename, description) => {
|
|
const p = Object.create(Plugin.prototype);
|
|
Object.defineProperties(p, {
|
|
name: { get: () => name },
|
|
filename: { get: () => filename },
|
|
description: { get: () => description },
|
|
length: { get: () => 0 }
|
|
});
|
|
return p;
|
|
};
|
|
const plugins = [
|
|
makePlugin("PDF Viewer", "internal-pdf-viewer", "Portable Document Format"),
|
|
makePlugin("Chrome PDF Viewer", "mhjfbgoacfdegokjbbpboihdaibdccjg", "Chrome PDF Viewer"),
|
|
makePlugin("Chromium PDF Viewer", "internal-pdf-viewer", "Chromium PDF Viewer")
|
|
];
|
|
const pList = Object.create(PluginArray.prototype);
|
|
Object.defineProperties(pList, {
|
|
length: { get: () => plugins.length },
|
|
item: { value: (index) => plugins[index] },
|
|
namedItem: { value: (name) => plugins.find(p => p.name === name) }
|
|
});
|
|
for (let i = 0; i < plugins.length; i++) {
|
|
Object.defineProperty(pList, i, { get: () => plugins[i] });
|
|
}
|
|
return pList;
|
|
};
|
|
Object.defineProperty(navigator, 'plugins', {
|
|
get: () => mockPlugins()
|
|
});
|
|
|
|
// 6. 模拟 window.chrome（无头浏览器通常没有，Cloudflare 会检测）
|
|
window.chrome = {
|
|
app: {
|
|
isInstalled: false,
|
|
InstallState: {
|
|
DISABLED: 'disabled',
|
|
INSTALLED: 'installed',
|
|
NOT_INSTALLED: 'not_installed'
|
|
},
|
|
RunningState: {
|
|
CANNOT_RUN: 'cannot_run',
|
|
READY_TO_RUN: 'ready_to_run',
|
|
RUNNING: 'running'
|
|
}
|
|
},
|
|
runtime: {
|
|
OnInstalledReason: {
|
|
CHROME_UPDATE: 'chrome_update',
|
|
INSTALL: 'install',
|
|
SHARED_MODULE_UPDATE: 'shared_module_update',
|
|
UPDATE: 'update'
|
|
},
|
|
OnRestartRequiredReason: {
|
|
APP_UPDATE: 'app_update',
|
|
OS_UPDATE: 'os_update',
|
|
PERIODIC: 'periodic'
|
|
},
|
|
PlatformArch: {
|
|
ARM: 'arm',
|
|
ARM64: 'arm64',
|
|
MIPS: 'mips',
|
|
MIPS64: 'mips64',
|
|
X86_32: 'x86-32',
|
|
X86_64: 'x86-64'
|
|
},
|
|
PlatformNaclArch: {
|
|
ARM: 'arm',
|
|
MIPS: 'mips',
|
|
MIPS64: 'mips64',
|
|
X86_32: 'x86-32',
|
|
X86_64: 'x86-64'
|
|
},
|
|
PlatformOs: {
|
|
ANDROID: 'android',
|
|
CROS: 'cros',
|
|
LINUX: 'linux',
|
|
MAC: 'mac',
|
|
OPENBSD: 'openbsd',
|
|
WIN: 'win'
|
|
},
|
|
RequestUpdateCheckStatus: {
|
|
NO_UPDATE: 'no_update',
|
|
THROTTLED: 'throttled',
|
|
UPDATE_AVAILABLE: 'update_available'
|
|
}
|
|
}
|
|
};
|
|
|
|
// 7. 模拟 Permissions 查询行为一致性
|
|
const originalQuery = window.navigator.permissions.query;
|
|
window.navigator.permissions.query = (parameters) => (
|
|
parameters.name === 'notifications' ?
|
|
Promise.resolve({ state: Notification.permission }) :
|
|
originalQuery(parameters)
|
|
);
|
|
"""
|
|
|
|
class BrowserManager:
|
|
"""
|
|
进程级浏览器池：为每个账号维护一个常驻的持久化浏览器上下文。
|
|
- 启动时可通过 prewarm() 为所有已配置账号预先拉起浏览器，任务到达时即开即用。
|
|
- 上下文跨任务复用；任务只负责开关自己的标签页（Page），不关闭浏览器。
|
|
- 上下文崩溃或被手动关闭时自动从池中剔除，下次获取时重新启动。
|
|
- 按账号隔离数据环境（user_data_dir），并注入反爬脚本以规避 Cloudflare、Akamai 等检测。
|
|
"""
|
|
def __init__(self) -> None:
|
|
self.settings = get_settings()
|
|
self._playwright: Playwright | None = None
|
|
self._contexts: Dict[str, BrowserContext] = {}
|
|
# 按账号维护启动锁，避免同一账号的 user_data_dir 被并发启动两次
|
|
self._launch_locks: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
|
|
|
async def prewarm(self) -> None:
|
|
"""
|
|
为所有已配置账号预启动常驻浏览器上下文。
|
|
单个账号启动失败不阻断其他账号，失败账号会在任务执行时自动重试启动。
|
|
"""
|
|
config = load_accounts_config()
|
|
accounts = [
|
|
account
|
|
for site_accounts in (config.sites.surugaya, config.sites.mercari, config.sites.yahoo)
|
|
for account in site_accounts
|
|
]
|
|
if not accounts:
|
|
logger.info("未配置任何站点账号，跳过浏览器预热。")
|
|
return
|
|
|
|
logger.info(f"正在预热浏览器，共 {len(accounts)} 个账号...")
|
|
for account in accounts:
|
|
try:
|
|
await self.get_context(account)
|
|
except Exception as e:
|
|
logger.error(
|
|
f"账号 '{account.account_id}' 浏览器预热失败（任务执行时将自动重试启动）: {e}"
|
|
)
|
|
logger.success(f"浏览器预热完成，当前常驻上下文 {len(self._contexts)} 个。")
|
|
|
|
async def get_context(self, account: AccountConfig) -> BrowserContext:
|
|
"""
|
|
获取指定账号的常驻浏览器上下文：已存在且存活则直接复用，否则启动新实例。
|
|
上下文由本管理器统一持有，调用方不要关闭它，只需管理自己打开的 Page。
|
|
|
|
Args:
|
|
account: 包含 account_id、user_data_dir 及可选 proxy 的账号配置。
|
|
|
|
Returns:
|
|
BrowserContext: 该账号的常驻浏览器上下文。
|
|
"""
|
|
async with self._launch_locks[account.account_id]:
|
|
cached = self._contexts.get(account.account_id)
|
|
if cached is not None:
|
|
logger.debug(f"账号 '{account.account_id}' 复用常驻浏览器上下文。")
|
|
return cached
|
|
return await self._launch_context(account)
|
|
|
|
async def close_all(self) -> None:
|
|
"""
|
|
进程退出时关闭所有常驻上下文并停止 Playwright 驱动。
|
|
"""
|
|
for account_id, context in list(self._contexts.items()):
|
|
try:
|
|
await context.close()
|
|
logger.info(f"账号 '{account_id}' 的常驻浏览器上下文已关闭。")
|
|
except Exception as e:
|
|
logger.error(f"关闭账号 '{account_id}' 的浏览器上下文时出错: {e}")
|
|
self._contexts.clear()
|
|
|
|
if self._playwright:
|
|
try:
|
|
await self._playwright.stop()
|
|
except Exception as e:
|
|
logger.error(f"停止 Playwright 驱动时出错: {e}")
|
|
self._playwright = None
|
|
|
|
async def _get_playwright(self) -> Playwright:
|
|
"""惰性启动并复用进程内唯一的 Playwright 驱动。"""
|
|
if self._playwright is None:
|
|
self._playwright = await async_playwright().start()
|
|
return self._playwright
|
|
|
|
def _on_context_close(self, account_id: str) -> None:
|
|
"""上下文关闭（崩溃或被手动关掉）时从池中剔除，下次任务自动重启。"""
|
|
if self._contexts.pop(account_id, None) is not None:
|
|
logger.warning(
|
|
f"账号 '{account_id}' 的浏览器上下文已断开，已从常驻池移除，下次任务将自动重启浏览器。"
|
|
)
|
|
|
|
async def _launch_context(self, account: AccountConfig) -> BrowserContext:
|
|
"""
|
|
为指定账号启动持久化浏览器上下文并纳入常驻池。
|
|
"""
|
|
# 1. 解析并确保本地持久化目录存在
|
|
data_dir = Path(account.user_data_dir)
|
|
try:
|
|
data_dir.mkdir(parents=True, exist_ok=True)
|
|
# logger.debug(f"持久化用户数据目录: {data_dir.absolute()}")
|
|
except Exception as e:
|
|
logger.error(f"创建账号 '{account.account_id}' 的持久化目录失败: {e}")
|
|
raise e
|
|
|
|
# 2. 配置隐身 Chromium 启动参数（共享反检测参数 + 本进程追加项）
|
|
chrome_args = stealth_launch_args() + [
|
|
"--start-maximized", # 启动时全屏（最大化）
|
|
]
|
|
|
|
executable_path_raw = self.settings.browser_executable_path
|
|
executable_path = None
|
|
if executable_path_raw:
|
|
candidate_path = Path(executable_path_raw).expanduser()
|
|
if candidate_path.is_file():
|
|
executable_path = str(candidate_path)
|
|
logger.info(f"使用自定义浏览器可执行文件: {executable_path}")
|
|
else:
|
|
logger.warning(
|
|
f"配置的浏览器可执行文件无效，将使用 Playwright 默认浏览器: {executable_path_raw}"
|
|
)
|
|
headless_mode = self.settings.browser_headless
|
|
|
|
# 使用与 API 服务一致的 Windows Chrome User-Agent（共享指纹配置）
|
|
user_agent = DEFAULT_USER_AGENT
|
|
|
|
logger.info(f"正在为账号 {account.account_id} 启动 Playwright | 无头模式: {headless_mode}")
|
|
|
|
# 3. 按账号配置可选代理（服务器 IP 段绕过 Cloudflare 封锁时常用）
|
|
proxy_config = None
|
|
if account.proxy:
|
|
proxy_config = {"server": account.proxy}
|
|
logger.info(f"账号 '{account.account_id}' 已启用代理: {account.proxy}")
|
|
|
|
playwright_driver = await self._get_playwright()
|
|
|
|
try:
|
|
# 4. 使用隔离目录创建持久化上下文
|
|
launch_options = {
|
|
"user_data_dir": str(data_dir.absolute()),
|
|
"headless": headless_mode,
|
|
"args": chrome_args,
|
|
"user_agent": user_agent,
|
|
"no_viewport": True, # 配合 --start-maximized 使得网页视口也铺满全屏
|
|
"locale": LOCALE, # 区域语言一致（ja-JP）
|
|
"timezone_id": TIMEZONE_ID, # 日本本地时区（Asia/Tokyo）
|
|
"accept_downloads": True, # 允许自动下载（如需要）
|
|
"proxy": proxy_config # 住宅代理等
|
|
}
|
|
if executable_path:
|
|
launch_options["executable_path"] = executable_path
|
|
|
|
context = await playwright_driver.chromium.launch_persistent_context(**launch_options)
|
|
|
|
# 5. 注入高级反指纹隐身脚本
|
|
await context.add_init_script(STEALTH_JS)
|
|
|
|
# 6. 纳入常驻池，并监听关闭事件以便自愈重启
|
|
context.on("close", lambda: self._on_context_close(account.account_id))
|
|
self._contexts[account.account_id] = context
|
|
logger.success(f"账号 '{account.account_id}' 的常驻 Playwright 上下文启动成功。")
|
|
|
|
return context
|
|
|
|
except Exception as e:
|
|
logger.error(f"账号 '{account.account_id}' 启动持久化上下文失败: {e}")
|
|
# Playwright 驱动为进程共享，启动失败时仅向上抛出，不停止驱动
|
|
raise e
|
|
|
|
|
|
_browser_manager: BrowserManager | None = None
|
|
|
|
|
|
def get_browser_manager() -> BrowserManager:
|
|
"""
|
|
获取进程级 BrowserManager 单例，保证所有任务共享同一个浏览器池。
|
|
"""
|
|
global _browser_manager
|
|
if _browser_manager is None:
|
|
_browser_manager = BrowserManager()
|
|
return _browser_manager
|