You've already forked pruchase_jhw
修改
This commit is contained in:
@@ -40,3 +40,5 @@ src/archives/
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
/tmp/
|
||||
|
||||
chrome-profile/
|
||||
+1
-1
@@ -17,7 +17,7 @@ sites:
|
||||
- account_id: "trdian023@tokugen.jp"
|
||||
username: "trdian023@tokugen.jp"
|
||||
password: "trdian023.."
|
||||
user_data_dir: "F:\\user\\chrome\\chrome-profile\\787235076"
|
||||
user_data_dir: "C:\\Users\\yan1h\\Code\\jp_pruchase_zdh_5\\chrome-profile\\787235076"
|
||||
proxy: "" # Optional residential proxy (e.g. "http://127.0.0.1:10808")
|
||||
|
||||
# Future sites can be appended here
|
||||
|
||||
+106
-15
@@ -1,10 +1,11 @@
|
||||
import os
|
||||
import asyncio
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
from typing import Dict
|
||||
from playwright.async_api import async_playwright, Playwright, BrowserContext
|
||||
from loguru import logger
|
||||
from src.config.settings import get_settings
|
||||
from src.config.accounts import AccountConfig
|
||||
from src.config.accounts import AccountConfig, load_accounts_config
|
||||
|
||||
# 注入到所有页面的 JS 脚本,用于绕过 Cloudflare、Akamai 等自动化检测
|
||||
STEALTH_JS = """
|
||||
@@ -137,23 +138,97 @@ window.navigator.permissions.query = (parameters) => (
|
||||
|
||||
class BrowserManager:
|
||||
"""
|
||||
浏览器管理器:负责初始化 Playwright 上下文。
|
||||
按账号隔离数据环境(user_data_dir),并注入反爬脚本以规避 Cloudflare、Akamai 等检测。
|
||||
进程级浏览器池:为每个账号维护一个常驻的持久化浏览器上下文。
|
||||
- 启动时可通过 prewarm() 为所有已配置账号预先拉起浏览器,任务到达时即开即用。
|
||||
- 上下文跨任务复用;任务只负责开关自己的标签页(Page),不关闭浏览器。
|
||||
- 上下文崩溃或被手动关闭时自动从池中剔除,下次获取时重新启动。
|
||||
- 按账号隔离数据环境(user_data_dir),并注入反爬脚本以规避 Cloudflare、Akamai 等检测。
|
||||
"""
|
||||
def __init__(self) -> None:
|
||||
self.settings = get_settings()
|
||||
self._playwright: Playwright | None = None
|
||||
self._contexts: Dict[str, BrowserContext] = {}
|
||||
# 按账号维护启动锁,避免同一账号的 user_data_dir 被并发启动两次
|
||||
self._launch_locks: Dict[str, asyncio.Lock] = defaultdict(asyncio.Lock)
|
||||
|
||||
async def get_context(self, account: AccountConfig) -> Tuple[Playwright, BrowserContext]:
|
||||
async def prewarm(self) -> None:
|
||||
"""
|
||||
为指定账号启动持久化浏览器上下文。
|
||||
返回 (Playwright, BrowserContext) 元组。
|
||||
调用方必须在 try-finally 中确保关闭 context 与 playwright 实例。
|
||||
为所有已配置账号预启动常驻浏览器上下文。
|
||||
单个账号启动失败不阻断其他账号,失败账号会在任务执行时自动重试启动。
|
||||
"""
|
||||
config = load_accounts_config()
|
||||
accounts = [
|
||||
account
|
||||
for site_accounts in (config.sites.surugaya, config.sites.mercari, config.sites.yahoo)
|
||||
for account in site_accounts
|
||||
]
|
||||
if not accounts:
|
||||
logger.info("未配置任何站点账号,跳过浏览器预热。")
|
||||
return
|
||||
|
||||
logger.info(f"正在预热浏览器,共 {len(accounts)} 个账号...")
|
||||
for account in accounts:
|
||||
try:
|
||||
await self.get_context(account)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
f"账号 '{account.account_id}' 浏览器预热失败(任务执行时将自动重试启动): {e}"
|
||||
)
|
||||
logger.success(f"浏览器预热完成,当前常驻上下文 {len(self._contexts)} 个。")
|
||||
|
||||
async def get_context(self, account: AccountConfig) -> BrowserContext:
|
||||
"""
|
||||
获取指定账号的常驻浏览器上下文:已存在且存活则直接复用,否则启动新实例。
|
||||
上下文由本管理器统一持有,调用方不要关闭它,只需管理自己打开的 Page。
|
||||
|
||||
Args:
|
||||
account: 包含 account_id、user_data_dir 及可选 proxy 的账号配置。
|
||||
|
||||
Returns:
|
||||
Tuple[Playwright, BrowserContext]: 已启动的驱动与隔离浏览器上下文实例。
|
||||
BrowserContext: 该账号的常驻浏览器上下文。
|
||||
"""
|
||||
async with self._launch_locks[account.account_id]:
|
||||
cached = self._contexts.get(account.account_id)
|
||||
if cached is not None:
|
||||
logger.debug(f"账号 '{account.account_id}' 复用常驻浏览器上下文。")
|
||||
return cached
|
||||
return await self._launch_context(account)
|
||||
|
||||
async def close_all(self) -> None:
|
||||
"""
|
||||
进程退出时关闭所有常驻上下文并停止 Playwright 驱动。
|
||||
"""
|
||||
for account_id, context in list(self._contexts.items()):
|
||||
try:
|
||||
await context.close()
|
||||
logger.info(f"账号 '{account_id}' 的常驻浏览器上下文已关闭。")
|
||||
except Exception as e:
|
||||
logger.error(f"关闭账号 '{account_id}' 的浏览器上下文时出错: {e}")
|
||||
self._contexts.clear()
|
||||
|
||||
if self._playwright:
|
||||
try:
|
||||
await self._playwright.stop()
|
||||
except Exception as e:
|
||||
logger.error(f"停止 Playwright 驱动时出错: {e}")
|
||||
self._playwright = None
|
||||
|
||||
async def _get_playwright(self) -> Playwright:
|
||||
"""惰性启动并复用进程内唯一的 Playwright 驱动。"""
|
||||
if self._playwright is None:
|
||||
self._playwright = await async_playwright().start()
|
||||
return self._playwright
|
||||
|
||||
def _on_context_close(self, account_id: str) -> None:
|
||||
"""上下文关闭(崩溃或被手动关掉)时从池中剔除,下次任务自动重启。"""
|
||||
if self._contexts.pop(account_id, None) is not None:
|
||||
logger.warning(
|
||||
f"账号 '{account_id}' 的浏览器上下文已断开,已从常驻池移除,下次任务将自动重启浏览器。"
|
||||
)
|
||||
|
||||
async def _launch_context(self, account: AccountConfig) -> BrowserContext:
|
||||
"""
|
||||
为指定账号启动持久化浏览器上下文并纳入常驻池。
|
||||
"""
|
||||
# 1. 解析并确保本地持久化目录存在
|
||||
data_dir = Path(account.user_data_dir)
|
||||
@@ -204,7 +279,7 @@ class BrowserManager:
|
||||
proxy_config = {"server": account.proxy}
|
||||
logger.info(f"账号 '{account.account_id}' 已启用代理: {account.proxy}")
|
||||
|
||||
playwright_driver = await async_playwright().start()
|
||||
playwright_driver = await self._get_playwright()
|
||||
|
||||
try:
|
||||
# 4. 使用隔离目录创建持久化上下文
|
||||
@@ -226,12 +301,28 @@ class BrowserManager:
|
||||
|
||||
# 5. 注入高级反指纹隐身脚本
|
||||
await context.add_init_script(STEALTH_JS)
|
||||
logger.success(f"账号 '{account.account_id}' 的 Playwright 上下文启动成功。")
|
||||
|
||||
return playwright_driver, context
|
||||
# 6. 纳入常驻池,并监听关闭事件以便自愈重启
|
||||
context.on("close", lambda: self._on_context_close(account.account_id))
|
||||
self._contexts[account.account_id] = context
|
||||
logger.success(f"账号 '{account.account_id}' 的常驻 Playwright 上下文启动成功。")
|
||||
|
||||
return context
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"账号 '{account.account_id}' 启动持久化上下文失败: {e}")
|
||||
# 启动失败时清理驱动进程
|
||||
await playwright_driver.stop()
|
||||
# Playwright 驱动为进程共享,启动失败时仅向上抛出,不停止驱动
|
||||
raise e
|
||||
|
||||
|
||||
_browser_manager: BrowserManager | None = None
|
||||
|
||||
|
||||
def get_browser_manager() -> BrowserManager:
|
||||
"""
|
||||
获取进程级 BrowserManager 单例,保证所有任务共享同一个浏览器池。
|
||||
"""
|
||||
global _browser_manager
|
||||
if _browser_manager is None:
|
||||
_browser_manager = BrowserManager()
|
||||
return _browser_manager
|
||||
|
||||
+8
-3
@@ -3,6 +3,7 @@ import signal
|
||||
import sys
|
||||
from loguru import logger
|
||||
|
||||
from src.browser.manager import get_browser_manager
|
||||
from src.config.settings import get_settings
|
||||
from src.config.accounts import load_accounts_config
|
||||
from src.notifications.service import NotificationService
|
||||
@@ -37,12 +38,15 @@ async def main() -> None:
|
||||
logger.critical(f"由于配置错误,守护进程启动失败: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
# 3. 实例化核心任务分发器与 Stream 监听器
|
||||
# 3. 预热浏览器:为所有已配置账号提前启动常驻浏览器上下文,任务到达时即开即用
|
||||
await get_browser_manager().prewarm()
|
||||
|
||||
# 4. 实例化核心任务分发器与 Stream 监听器
|
||||
dispatcher = TaskDispatcher()
|
||||
stream_listener = RedisStreamListener(dispatcher)
|
||||
delay_listener = RedisZSetDelayListener(dispatcher)
|
||||
|
||||
# 4. 优雅停机信号处理 (跨平台支持)
|
||||
# 5. 优雅停机信号处理 (跨平台支持)
|
||||
loop = asyncio.get_running_loop()
|
||||
|
||||
def shutdown_handler(sig_name: str) -> None:
|
||||
@@ -62,7 +66,7 @@ async def main() -> None:
|
||||
else:
|
||||
logger.debug("运行于 Windows 平台。跳过系统信号注册,将直接捕获 KeyboardInterrupt。")
|
||||
|
||||
# 5. 启动消息监听消费主循环
|
||||
# 6. 启动消息监听消费主循环
|
||||
try:
|
||||
stream_task = asyncio.create_task(stream_listener.listen())
|
||||
delay_task = asyncio.create_task(delay_listener.listen())
|
||||
@@ -76,6 +80,7 @@ async def main() -> None:
|
||||
await asyncio.gather(
|
||||
stream_listener.stop(),
|
||||
delay_listener.stop(),
|
||||
get_browser_manager().close_all(),
|
||||
get_redis_manager().close(),
|
||||
return_exceptions=True,
|
||||
)
|
||||
|
||||
+12
-15
@@ -6,12 +6,12 @@ from pathlib import Path
|
||||
from typing import Dict, Any
|
||||
|
||||
import redis.asyncio as aioredis
|
||||
from playwright.async_api import Playwright, BrowserContext, Page
|
||||
from playwright.async_api import BrowserContext, Page
|
||||
from loguru import logger
|
||||
|
||||
from src.config.settings import get_settings
|
||||
from src.config.accounts import get_account_config, AccountConfig
|
||||
from src.browser.manager import BrowserManager
|
||||
from src.browser.manager import get_browser_manager
|
||||
from src.notifications.service import NotificationService
|
||||
from src.utils.video_recorder import TaskVideoRecorder
|
||||
|
||||
@@ -30,7 +30,7 @@ class BasePurchaseTask(ABC):
|
||||
def __init__(self, message_data: Dict[str, Any]) -> None:
|
||||
self.message = message_data
|
||||
self.settings = get_settings()
|
||||
self.browser_manager = BrowserManager()
|
||||
self.browser_manager = get_browser_manager()
|
||||
self.notification_service = NotificationService(self.settings)
|
||||
|
||||
# 从消息中提取的任务参数
|
||||
@@ -39,8 +39,7 @@ class BasePurchaseTask(ABC):
|
||||
self.account_id: str = message_data.get("purchase_account", "")
|
||||
self.site_id: str = message_data.get("site_id", "unknown")
|
||||
|
||||
# 需在结束时清理的 Playwright 对象
|
||||
self.playwright_driver: Playwright | None = None
|
||||
# 浏览器上下文由进程级 BrowserManager 常驻持有,任务只负责自己的 Page
|
||||
self.context: BrowserContext | None = None
|
||||
self.page: Page | None = None
|
||||
self.video_recorder: TaskVideoRecorder | None = None
|
||||
@@ -87,8 +86,8 @@ class BasePurchaseTask(ABC):
|
||||
f"{self.video_output_path.name} | 路径: {self.video_output_path.absolute()}"
|
||||
)
|
||||
|
||||
# 通过 BrowserManager 启动浏览器
|
||||
self.playwright_driver, self.context = await self.browser_manager.get_context(account_config)
|
||||
# 从进程级浏览器池获取该账号的常驻上下文(已预热则即取即用)
|
||||
self.context = await self.browser_manager.get_context(account_config)
|
||||
|
||||
# 打开新标签页
|
||||
self.page = await self.context.new_page()
|
||||
@@ -120,16 +119,14 @@ class BasePurchaseTask(ABC):
|
||||
# 正常结束或未被 except 拦截的其他情况(如主动 Cancel),确保停止录屏
|
||||
await self._stop_video_record_safely()
|
||||
|
||||
# 无论成败都回收 Playwright 后台进程
|
||||
logger.debug(f"[任务 {self.task_id}] 正在清理浏览器上下文并停止驱动...")
|
||||
# 浏览器上下文常驻复用,任务只关闭自己打开的标签页
|
||||
logger.debug(f"[任务 {self.task_id}] 正在关闭任务标签页(浏览器保持常驻)...")
|
||||
try:
|
||||
if self.context:
|
||||
await self.context.close()
|
||||
if self.playwright_driver:
|
||||
await self.playwright_driver.stop()
|
||||
logger.debug(f"[任务 {self.task_id}] 浏览器资源已安全释放。")
|
||||
if self.page and not self.page.is_closed():
|
||||
await self.page.close()
|
||||
logger.debug(f"[任务 {self.task_id}] 任务标签页已关闭。")
|
||||
except Exception as close_err:
|
||||
logger.error(f"[任务 {self.task_id}] 释放浏览器资源时出错: {close_err}")
|
||||
logger.error(f"[任务 {self.task_id}] 关闭任务标签页时出错: {close_err}")
|
||||
|
||||
# 上报任务结果
|
||||
if self.is_report:
|
||||
|
||||
@@ -14,11 +14,11 @@ from loguru import logger
|
||||
from playwright.async_api import Page
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from notifications import NotificationEvent
|
||||
from src.notifications import NotificationEvent
|
||||
from src.tasks.base import BasePurchaseTask
|
||||
from src.config.accounts import get_account_config
|
||||
from src.utils.page_utils import format_japanese_price
|
||||
from task_queue.redis_manager import get_redis_client
|
||||
from src.task_queue.redis_manager import get_redis_client
|
||||
from surugaya_common.models import SurugayaOrderDetail
|
||||
from surugaya_common.parsers.surugaya.order_detail import SurugayaOrderDetailParser
|
||||
from surugaya_common.browser_utils import human_scroll
|
||||
|
||||
@@ -6,7 +6,7 @@ from loguru import logger
|
||||
|
||||
from surugaya_common.parsers.surugaya import SurugayaOrderDetailParser
|
||||
from src.tasks.base import BasePurchaseTask
|
||||
from task_queue.redis_manager import get_redis_client
|
||||
from src.task_queue.redis_manager import get_redis_client
|
||||
|
||||
|
||||
class SurugayaTradeMonitorTask(BasePurchaseTask):
|
||||
|
||||
@@ -189,7 +189,7 @@ requires-dist = [
|
||||
{ name = "pyscreeze", specifier = ">=1.0.1" },
|
||||
{ name = "pyyaml", specifier = ">=6.0.1" },
|
||||
{ name = "redis", specifier = "==8.0.0" },
|
||||
{ name = "surugaya-common", specifier = ">=0.2.0", index = "https://git.jerryyan.top/api/packages/jp/pypi/simple" },
|
||||
{ name = "surugaya-common", specifier = ">=0.2.0", index = "https://git.jerryyan.net/api/packages/jp/pypi/simple" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -688,7 +688,7 @@ wheels = [
|
||||
[[package]]
|
||||
name = "surugaya-common"
|
||||
version = "0.2.0"
|
||||
source = { registry = "https://git.jerryyan.top/api/packages/jp/pypi/simple" }
|
||||
source = { registry = "https://git.jerryyan.net/api/packages/jp/pypi/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "redis" },
|
||||
|
||||
Reference in New Issue
Block a user