This commit is contained in:
2026-06-11 13:02:20 +08:00
parent ab9842a140
commit 781ebdd348
8 changed files with 140 additions and 45 deletions
+112 -21
View File
@@ -1,10 +1,11 @@
import os
import asyncio
from collections import defaultdict
from pathlib import Path
from typing import Tuple
from typing import Dict
from playwright.async_api import async_playwright, Playwright, BrowserContext
from loguru import logger
from src.config.settings import get_settings
from src.config.accounts import AccountConfig
from src.config.accounts import AccountConfig, load_accounts_config
# 注入到所有页面的 JS 脚本,用于绕过 Cloudflare、Akamai 等自动化检测
STEALTH_JS = """
@@ -137,23 +138,97 @@ window.navigator.permissions.query = (parameters) => (
class BrowserManager:
"""
浏览器管理器:负责初始化 Playwright 上下文。
按账号隔离数据环境(user_data_dir),并注入反爬脚本以规避 Cloudflare、Akamai 等检测
进程级浏览器池:为每个账号维护一个常驻的持久化浏览器上下文。
- 启动时可通过 prewarm() 为所有已配置账号预先拉起浏览器,任务到达时即开即用
- 上下文跨任务复用;任务只负责开关自己的标签页(Page),不关闭浏览器。
- 上下文崩溃或被手动关闭时自动从池中剔除,下次获取时重新启动。
- 按账号隔离数据环境(user_data_dir),并注入反爬脚本以规避 Cloudflare、Akamai 等检测。
"""
def __init__(self) -> None:
self.settings = get_settings()
self._playwright: Playwright | None = None
self._contexts: Dict[str, BrowserContext] = {}
# 按账号维护启动锁,避免同一账号的 user_data_dir 被并发启动两次
self._launch_locks: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
async def get_context(self, account: AccountConfig) -> Tuple[Playwright, BrowserContext]:
async def prewarm(self) -> None:
"""
指定账号启动持久化浏览器上下文。
返回 (Playwright, BrowserContext) 元组
调用方必须在 try-finally 中确保关闭 context 与 playwright 实例。
所有已配置账号启动常驻浏览器上下文。
单个账号启动失败不阻断其他账号,失败账号会在任务执行时自动重试启动
"""
config = load_accounts_config()
accounts = [
account
for site_accounts in (config.sites.surugaya, config.sites.mercari, config.sites.yahoo)
for account in site_accounts
]
if not accounts:
logger.info("未配置任何站点账号,跳过浏览器预热。")
return
logger.info(f"正在预热浏览器,共 {len(accounts)} 个账号...")
for account in accounts:
try:
await self.get_context(account)
except Exception as e:
logger.error(
f"账号 '{account.account_id}' 浏览器预热失败(任务执行时将自动重试启动): {e}"
)
logger.success(f"浏览器预热完成,当前常驻上下文 {len(self._contexts)} 个。")
async def get_context(self, account: AccountConfig) -> BrowserContext:
"""
获取指定账号的常驻浏览器上下文:已存在且存活则直接复用,否则启动新实例。
上下文由本管理器统一持有,调用方不要关闭它,只需管理自己打开的 Page。
Args:
account: 包含 account_id、user_data_dir 及可选 proxy 的账号配置。
Returns:
Tuple[Playwright, BrowserContext]: 已启动的驱动与隔离浏览器上下文实例
BrowserContext: 该账号的常驻浏览器上下文。
"""
async with self._launch_locks[account.account_id]:
cached = self._contexts.get(account.account_id)
if cached is not None:
logger.debug(f"账号 '{account.account_id}' 复用常驻浏览器上下文。")
return cached
return await self._launch_context(account)
async def close_all(self) -> None:
"""
进程退出时关闭所有常驻上下文并停止 Playwright 驱动。
"""
for account_id, context in list(self._contexts.items()):
try:
await context.close()
logger.info(f"账号 '{account_id}' 的常驻浏览器上下文已关闭。")
except Exception as e:
logger.error(f"关闭账号 '{account_id}' 的浏览器上下文时出错: {e}")
self._contexts.clear()
if self._playwright:
try:
await self._playwright.stop()
except Exception as e:
logger.error(f"停止 Playwright 驱动时出错: {e}")
self._playwright = None
async def _get_playwright(self) -> Playwright:
"""惰性启动并复用进程内唯一的 Playwright 驱动。"""
if self._playwright is None:
self._playwright = await async_playwright().start()
return self._playwright
def _on_context_close(self, account_id: str) -> None:
"""上下文关闭(崩溃或被手动关掉)时从池中剔除,下次任务自动重启。"""
if self._contexts.pop(account_id, None) is not None:
logger.warning(
f"账号 '{account_id}' 的浏览器上下文已断开,已从常驻池移除,下次任务将自动重启浏览器。"
)
async def _launch_context(self, account: AccountConfig) -> BrowserContext:
"""
为指定账号启动持久化浏览器上下文并纳入常驻池。
"""
# 1. 解析并确保本地持久化目录存在
data_dir = Path(account.user_data_dir)
@@ -204,8 +279,8 @@ class BrowserManager:
proxy_config = {"server": account.proxy}
logger.info(f"账号 '{account.account_id}' 已启用代理: {account.proxy}")
playwright_driver = await async_playwright().start()
playwright_driver = await self._get_playwright()
try:
# 4. 使用隔离目录创建持久化上下文
launch_options = {
@@ -223,15 +298,31 @@ class BrowserManager:
launch_options["executable_path"] = executable_path
context = await playwright_driver.chromium.launch_persistent_context(**launch_options)
# 5. 注入高级反指纹隐身脚本
await context.add_init_script(STEALTH_JS)
logger.success(f"账号 '{account.account_id}' 的 Playwright 上下文启动成功。")
return playwright_driver, context
# 6. 纳入常驻池,并监听关闭事件以便自愈重启
context.on("close", lambda: self._on_context_close(account.account_id))
self._contexts[account.account_id] = context
logger.success(f"账号 '{account.account_id}' 的常驻 Playwright 上下文启动成功。")
return context
except Exception as e:
logger.error(f"账号 '{account.account_id}' 启动持久化上下文失败: {e}")
# 启动失败时清理驱动进程
await playwright_driver.stop()
# Playwright 驱动为进程共享,启动失败时仅向上抛出,不停止驱动
raise e
_browser_manager: BrowserManager | None = None
def get_browser_manager() -> BrowserManager:
"""
获取进程级 BrowserManager 单例,保证所有任务共享同一个浏览器池。
"""
global _browser_manager
if _browser_manager is None:
_browser_manager = BrowserManager()
return _browser_manager
+8 -3
View File
@@ -3,6 +3,7 @@ import signal
import sys
from loguru import logger
from src.browser.manager import get_browser_manager
from src.config.settings import get_settings
from src.config.accounts import load_accounts_config
from src.notifications.service import NotificationService
@@ -37,12 +38,15 @@ async def main() -> None:
logger.critical(f"由于配置错误,守护进程启动失败: {e}")
sys.exit(1)
# 3. 实例化核心任务分发器与 Stream 监听器
# 3. 预热浏览器:为所有已配置账号提前启动常驻浏览器上下文,任务到达时即开即用
await get_browser_manager().prewarm()
# 4. 实例化核心任务分发器与 Stream 监听器
dispatcher = TaskDispatcher()
stream_listener = RedisStreamListener(dispatcher)
delay_listener = RedisZSetDelayListener(dispatcher)
# 4. 优雅停机信号处理 (跨平台支持)
# 5. 优雅停机信号处理 (跨平台支持)
loop = asyncio.get_running_loop()
def shutdown_handler(sig_name: str) -> None:
@@ -62,7 +66,7 @@ async def main() -> None:
else:
logger.debug("运行于 Windows 平台。跳过系统信号注册,将直接捕获 KeyboardInterrupt。")
# 5. 启动消息监听消费主循环
# 6. 启动消息监听消费主循环
try:
stream_task = asyncio.create_task(stream_listener.listen())
delay_task = asyncio.create_task(delay_listener.listen())
@@ -76,6 +80,7 @@ async def main() -> None:
await asyncio.gather(
stream_listener.stop(),
delay_listener.stop(),
get_browser_manager().close_all(),
get_redis_manager().close(),
return_exceptions=True,
)
+12 -15
View File
@@ -6,12 +6,12 @@ from pathlib import Path
from typing import Dict, Any
import redis.asyncio as aioredis
from playwright.async_api import Playwright, BrowserContext, Page
from playwright.async_api import BrowserContext, Page
from loguru import logger
from src.config.settings import get_settings
from src.config.accounts import get_account_config, AccountConfig
from src.browser.manager import BrowserManager
from src.browser.manager import get_browser_manager
from src.notifications.service import NotificationService
from src.utils.video_recorder import TaskVideoRecorder
@@ -30,7 +30,7 @@ class BasePurchaseTask(ABC):
def __init__(self, message_data: Dict[str, Any]) -> None:
self.message = message_data
self.settings = get_settings()
self.browser_manager = BrowserManager()
self.browser_manager = get_browser_manager()
self.notification_service = NotificationService(self.settings)
# 从消息中提取的任务参数
@@ -39,8 +39,7 @@ class BasePurchaseTask(ABC):
self.account_id: str = message_data.get("purchase_account", "")
self.site_id: str = message_data.get("site_id", "unknown")
# 需在结束时清理的 Playwright 对象
self.playwright_driver: Playwright | None = None
# 浏览器上下文由进程级 BrowserManager 常驻持有,任务只负责自己的 Page
self.context: BrowserContext | None = None
self.page: Page | None = None
self.video_recorder: TaskVideoRecorder | None = None
@@ -87,8 +86,8 @@ class BasePurchaseTask(ABC):
f"{self.video_output_path.name} | 路径: {self.video_output_path.absolute()}"
)
# 通过 BrowserManager 启动浏览器
self.playwright_driver, self.context = await self.browser_manager.get_context(account_config)
# 从进程级浏览器池获取该账号的常驻上下文(已预热则即取即用)
self.context = await self.browser_manager.get_context(account_config)
# 打开新标签页
self.page = await self.context.new_page()
@@ -120,16 +119,14 @@ class BasePurchaseTask(ABC):
# 正常结束或未被 except 拦截的其他情况(如主动 Cancel),确保停止录屏
await self._stop_video_record_safely()
# 无论成败都回收 Playwright 后台进程
logger.debug(f"[任务 {self.task_id}] 正在清理浏览器上下文并停止驱动...")
# 浏览器上下文常驻复用,任务只关闭自己打开的标签页
logger.debug(f"[任务 {self.task_id}] 正在关闭任务标签页(浏览器保持常驻)...")
try:
if self.context:
await self.context.close()
if self.playwright_driver:
await self.playwright_driver.stop()
logger.debug(f"[任务 {self.task_id}] 浏览器资源已安全释放。")
if self.page and not self.page.is_closed():
await self.page.close()
logger.debug(f"[任务 {self.task_id}] 任务标签页已关闭。")
except Exception as close_err:
logger.error(f"[任务 {self.task_id}] 释放浏览器资源时出错: {close_err}")
logger.error(f"[任务 {self.task_id}] 关闭任务标签页时出错: {close_err}")
# 上报任务结果
if self.is_report:
+2 -2
View File
@@ -14,11 +14,11 @@ from loguru import logger
from playwright.async_api import Page
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
from notifications import NotificationEvent
from src.notifications import NotificationEvent
from src.tasks.base import BasePurchaseTask
from src.config.accounts import get_account_config
from src.utils.page_utils import format_japanese_price
from task_queue.redis_manager import get_redis_client
from src.task_queue.redis_manager import get_redis_client
from surugaya_common.models import SurugayaOrderDetail
from surugaya_common.parsers.surugaya.order_detail import SurugayaOrderDetailParser
from surugaya_common.browser_utils import human_scroll
+1 -1
View File
@@ -6,7 +6,7 @@ from loguru import logger
from surugaya_common.parsers.surugaya import SurugayaOrderDetailParser
from src.tasks.base import BasePurchaseTask
from task_queue.redis_manager import get_redis_client
from src.task_queue.redis_manager import get_redis_client
class SurugayaTradeMonitorTask(BasePurchaseTask):