diff --git a/.gitignore b/.gitignore index e65f103..6250ecd 100644 --- a/.gitignore +++ b/.gitignore @@ -40,3 +40,5 @@ src/archives/ .DS_Store Thumbs.db /tmp/ + +chrome-profile/ \ No newline at end of file diff --git a/accounts.yaml b/accounts.yaml index e0f7a89..11281bd 100644 --- a/accounts.yaml +++ b/accounts.yaml @@ -17,7 +17,7 @@ sites: - account_id: "trdian023@tokugen.jp" username: "trdian023@tokugen.jp" password: "trdian023.." - user_data_dir: "F:\\user\\chrome\\chrome-profile\\787235076" + user_data_dir: "C:\\Users\\yan1h\\Code\\jp_pruchase_zdh_5\\chrome-profile\\787235076" proxy: "" # Optional residential proxy (e.g. "http://127.0.0.1:10808") # Future sites can be appended here diff --git a/src/browser/manager.py b/src/browser/manager.py index 4602ad7..0d2942c 100644 --- a/src/browser/manager.py +++ b/src/browser/manager.py @@ -1,10 +1,11 @@ -import os +import asyncio +from collections import defaultdict from pathlib import Path -from typing import Tuple +from typing import Dict from playwright.async_api import async_playwright, Playwright, BrowserContext from loguru import logger from src.config.settings import get_settings -from src.config.accounts import AccountConfig +from src.config.accounts import AccountConfig, load_accounts_config # 注入到所有页面的 JS 脚本，用于绕过 Cloudflare、Akamai 等自动化检测 STEALTH_JS = """ @@ -137,23 +138,97 @@ window.navigator.permissions.query = (parameters) => ( class BrowserManager: """ - 浏览器管理器：负责初始化 Playwright 上下文。 - 按账号隔离数据环境（user_data_dir），并注入反爬脚本以规避 Cloudflare、Akamai 等检测。 + 进程级浏览器池：为每个账号维护一个常驻的持久化浏览器上下文。 + - 启动时可通过 prewarm() 为所有已配置账号预先拉起浏览器，任务到达时即开即用。 + - 上下文跨任务复用；任务只负责开关自己的标签页（Page），不关闭浏览器。 + - 上下文崩溃或被手动关闭时自动从池中剔除，下次获取时重新启动。 + - 按账号隔离数据环境（user_data_dir），并注入反爬脚本以规避 Cloudflare、Akamai 等检测。 """ def __init__(self) -> None: self.settings = get_settings() + self._playwright: Playwright | None = None + self._contexts: Dict[str, BrowserContext] = {} + # 按账号维护启动锁，避免同一账号的 user_data_dir 被并发启动两次 + self._launch_locks: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock) - async def get_context(self, account: AccountConfig) -> Tuple[Playwright, BrowserContext]: + async def prewarm(self) -> None: """ - 为指定账号启动持久化浏览器上下文。 - 返回 (Playwright, BrowserContext) 元组。 - 调用方必须在 try-finally 中确保关闭 context 与 playwright 实例。 - + 为所有已配置账号预启动常驻浏览器上下文。 + 单个账号启动失败不阻断其他账号，失败账号会在任务执行时自动重试启动。 + """ + config = load_accounts_config() + accounts = [ + account + for site_accounts in (config.sites.surugaya, config.sites.mercari, config.sites.yahoo) + for account in site_accounts + ] + if not accounts: + logger.info("未配置任何站点账号，跳过浏览器预热。") + return + + logger.info(f"正在预热浏览器，共 {len(accounts)} 个账号...") + for account in accounts: + try: + await self.get_context(account) + except Exception as e: + logger.error( + f"账号 '{account.account_id}' 浏览器预热失败（任务执行时将自动重试启动）: {e}" + ) + logger.success(f"浏览器预热完成，当前常驻上下文 {len(self._contexts)} 个。") + + async def get_context(self, account: AccountConfig) -> BrowserContext: + """ + 获取指定账号的常驻浏览器上下文：已存在且存活则直接复用，否则启动新实例。 + 上下文由本管理器统一持有，调用方不要关闭它，只需管理自己打开的 Page。 + Args: account: 包含 account_id、user_data_dir 及可选 proxy 的账号配置。 - + Returns: - Tuple[Playwright, BrowserContext]: 已启动的驱动与隔离浏览器上下文实例。 + BrowserContext: 该账号的常驻浏览器上下文。 + """ + async with self._launch_locks[account.account_id]: + cached = self._contexts.get(account.account_id) + if cached is not None: + logger.debug(f"账号 '{account.account_id}' 复用常驻浏览器上下文。") + return cached + return await self._launch_context(account) + + async def close_all(self) -> None: + """ + 进程退出时关闭所有常驻上下文并停止 Playwright 驱动。 + """ + for account_id, context in list(self._contexts.items()): + try: + await context.close() + logger.info(f"账号 '{account_id}' 的常驻浏览器上下文已关闭。") + except Exception as e: + logger.error(f"关闭账号 '{account_id}' 的浏览器上下文时出错: {e}") + self._contexts.clear() + + if self._playwright: + try: + await self._playwright.stop() + except Exception as e: + logger.error(f"停止 Playwright 驱动时出错: {e}") + self._playwright = None + + async def _get_playwright(self) -> Playwright: + """惰性启动并复用进程内唯一的 Playwright 驱动。""" + if self._playwright is None: + self._playwright = await async_playwright().start() + return self._playwright + + def _on_context_close(self, account_id: str) -> None: + """上下文关闭（崩溃或被手动关掉）时从池中剔除，下次任务自动重启。""" + if self._contexts.pop(account_id, None) is not None: + logger.warning( + f"账号 '{account_id}' 的浏览器上下文已断开，已从常驻池移除，下次任务将自动重启浏览器。" + ) + + async def _launch_context(self, account: AccountConfig) -> BrowserContext: + """ + 为指定账号启动持久化浏览器上下文并纳入常驻池。 """ # 1. 解析并确保本地持久化目录存在 data_dir = Path(account.user_data_dir) @@ -204,8 +279,8 @@ class BrowserManager: proxy_config = {"server": account.proxy} logger.info(f"账号 '{account.account_id}' 已启用代理: {account.proxy}") - playwright_driver = await async_playwright().start() - + playwright_driver = await self._get_playwright() + try: # 4. 使用隔离目录创建持久化上下文 launch_options = { @@ -223,15 +298,31 @@ class BrowserManager: launch_options["executable_path"] = executable_path context = await playwright_driver.chromium.launch_persistent_context(**launch_options) - + # 5. 注入高级反指纹隐身脚本 await context.add_init_script(STEALTH_JS) - logger.success(f"账号 '{account.account_id}' 的 Playwright 上下文启动成功。") - - return playwright_driver, context - + + # 6. 纳入常驻池，并监听关闭事件以便自愈重启 + context.on("close", lambda: self._on_context_close(account.account_id)) + self._contexts[account.account_id] = context + logger.success(f"账号 '{account.account_id}' 的常驻 Playwright 上下文启动成功。") + + return context + except Exception as e: logger.error(f"账号 '{account.account_id}' 启动持久化上下文失败: {e}") - # 启动失败时清理驱动进程 - await playwright_driver.stop() + # Playwright 驱动为进程共享，启动失败时仅向上抛出，不停止驱动 raise e + + +_browser_manager: BrowserManager | None = None + + +def get_browser_manager() -> BrowserManager: + """ + 获取进程级 BrowserManager 单例，保证所有任务共享同一个浏览器池。 + """ + global _browser_manager + if _browser_manager is None: + _browser_manager = BrowserManager() + return _browser_manager diff --git a/src/main.py b/src/main.py index 319db1f..b468286 100644 --- a/src/main.py +++ b/src/main.py @@ -3,6 +3,7 @@ import signal import sys from loguru import logger +from src.browser.manager import get_browser_manager from src.config.settings import get_settings from src.config.accounts import load_accounts_config from src.notifications.service import NotificationService @@ -37,12 +38,15 @@ async def main() -> None: logger.critical(f"由于配置错误，守护进程启动失败: {e}") sys.exit(1) - # 3. 实例化核心任务分发器与 Stream 监听器 + # 3. 预热浏览器：为所有已配置账号提前启动常驻浏览器上下文，任务到达时即开即用 + await get_browser_manager().prewarm() + + # 4. 实例化核心任务分发器与 Stream 监听器 dispatcher = TaskDispatcher() stream_listener = RedisStreamListener(dispatcher) delay_listener = RedisZSetDelayListener(dispatcher) - # 4. 优雅停机信号处理 (跨平台支持) + # 5. 优雅停机信号处理 (跨平台支持) loop = asyncio.get_running_loop() def shutdown_handler(sig_name: str) -> None: @@ -62,7 +66,7 @@ async def main() -> None: else: logger.debug("运行于 Windows 平台。跳过系统信号注册，将直接捕获 KeyboardInterrupt。") - # 5. 启动消息监听消费主循环 + # 6. 启动消息监听消费主循环 try: stream_task = asyncio.create_task(stream_listener.listen()) delay_task = asyncio.create_task(delay_listener.listen()) @@ -76,6 +80,7 @@ async def main() -> None: await asyncio.gather( stream_listener.stop(), delay_listener.stop(), + get_browser_manager().close_all(), get_redis_manager().close(), return_exceptions=True, ) diff --git a/src/tasks/base.py b/src/tasks/base.py index a304628..6ee921b 100644 --- a/src/tasks/base.py +++ b/src/tasks/base.py @@ -6,12 +6,12 @@ from pathlib import Path from typing import Dict, Any import redis.asyncio as aioredis -from playwright.async_api import Playwright, BrowserContext, Page +from playwright.async_api import BrowserContext, Page from loguru import logger from src.config.settings import get_settings from src.config.accounts import get_account_config, AccountConfig -from src.browser.manager import BrowserManager +from src.browser.manager import get_browser_manager from src.notifications.service import NotificationService from src.utils.video_recorder import TaskVideoRecorder @@ -30,7 +30,7 @@ class BasePurchaseTask(ABC): def __init__(self, message_data: Dict[str, Any]) -> None: self.message = message_data self.settings = get_settings() - self.browser_manager = BrowserManager() + self.browser_manager = get_browser_manager() self.notification_service = NotificationService(self.settings) # 从消息中提取的任务参数 @@ -39,8 +39,7 @@ class BasePurchaseTask(ABC): self.account_id: str = message_data.get("purchase_account", "") self.site_id: str = message_data.get("site_id", "unknown") - # 需在结束时清理的 Playwright 对象 - self.playwright_driver: Playwright | None = None + # 浏览器上下文由进程级 BrowserManager 常驻持有，任务只负责自己的 Page self.context: BrowserContext | None = None self.page: Page | None = None self.video_recorder: TaskVideoRecorder | None = None @@ -87,8 +86,8 @@ class BasePurchaseTask(ABC): f"{self.video_output_path.name} | 路径: {self.video_output_path.absolute()}" ) - # 通过 BrowserManager 启动浏览器 - self.playwright_driver, self.context = await self.browser_manager.get_context(account_config) + # 从进程级浏览器池获取该账号的常驻上下文（已预热则即取即用） + self.context = await self.browser_manager.get_context(account_config) # 打开新标签页 self.page = await self.context.new_page() @@ -120,16 +119,14 @@ class BasePurchaseTask(ABC): # 正常结束或未被 except 拦截的其他情况（如主动 Cancel），确保停止录屏 await self._stop_video_record_safely() - # 无论成败都回收 Playwright 后台进程 - logger.debug(f"[任务 {self.task_id}] 正在清理浏览器上下文并停止驱动...") + # 浏览器上下文常驻复用，任务只关闭自己打开的标签页 + logger.debug(f"[任务 {self.task_id}] 正在关闭任务标签页（浏览器保持常驻）...") try: - if self.context: - await self.context.close() - if self.playwright_driver: - await self.playwright_driver.stop() - logger.debug(f"[任务 {self.task_id}] 浏览器资源已安全释放。") + if self.page and not self.page.is_closed(): + await self.page.close() + logger.debug(f"[任务 {self.task_id}] 任务标签页已关闭。") except Exception as close_err: - logger.error(f"[任务 {self.task_id}] 释放浏览器资源时出错: {close_err}") + logger.error(f"[任务 {self.task_id}] 关闭任务标签页时出错: {close_err}") # 上报任务结果 if self.is_report: diff --git a/src/tasks/surugaya.py b/src/tasks/surugaya.py index 9c97fd0..7ebabb0 100644 --- a/src/tasks/surugaya.py +++ b/src/tasks/surugaya.py @@ -14,11 +14,11 @@ from loguru import logger from playwright.async_api import Page from playwright.async_api import TimeoutError as PlaywrightTimeoutError -from notifications import NotificationEvent +from src.notifications import NotificationEvent from src.tasks.base import BasePurchaseTask from src.config.accounts import get_account_config from src.utils.page_utils import format_japanese_price -from task_queue.redis_manager import get_redis_client +from src.task_queue.redis_manager import get_redis_client from surugaya_common.models import SurugayaOrderDetail from surugaya_common.parsers.surugaya.order_detail import SurugayaOrderDetailParser from surugaya_common.browser_utils import human_scroll diff --git a/src/tasks/surugaya_trade_monitor.py b/src/tasks/surugaya_trade_monitor.py index b6a770a..c7c3080 100644 --- a/src/tasks/surugaya_trade_monitor.py +++ b/src/tasks/surugaya_trade_monitor.py @@ -6,7 +6,7 @@ from loguru import logger from surugaya_common.parsers.surugaya import SurugayaOrderDetailParser from src.tasks.base import BasePurchaseTask -from task_queue.redis_manager import get_redis_client +from src.task_queue.redis_manager import get_redis_client class SurugayaTradeMonitorTask(BasePurchaseTask): diff --git a/uv.lock b/uv.lock index 35052f2..6d4c6d8 100644 --- a/uv.lock +++ b/uv.lock @@ -189,7 +189,7 @@ requires-dist = [ { name = "pyscreeze", specifier = ">=1.0.1" }, { name = "pyyaml", specifier = ">=6.0.1" }, { name = "redis", specifier = "==8.0.0" }, - { name = "surugaya-common", specifier = ">=0.2.0", index = "https://git.jerryyan.top/api/packages/jp/pypi/simple" }, + { name = "surugaya-common", specifier = ">=0.2.0", index = "https://git.jerryyan.net/api/packages/jp/pypi/simple" }, ] [[package]] @@ -688,7 +688,7 @@ wheels = [ [[package]] name = "surugaya-common" version = "0.2.0" -source = { registry = "https://git.jerryyan.top/api/packages/jp/pypi/simple" } +source = { registry = "https://git.jerryyan.net/api/packages/jp/pypi/simple" } dependencies = [ { name = "pydantic" }, { name = "redis" },