Init
This commit is contained in:
@@ -0,0 +1,299 @@
|
||||
"""浏览器池：管理 Playwright 浏览器实例的生命周期和并发访问
|
||||
|
||||
核心能力：
|
||||
- 启动/关闭 Chromium 浏览器实例（进程级单例）
|
||||
- 通过信号量控制并发页面数量，避免 API 并发把浏览器资源打爆
|
||||
- 支持本地调试模式下保留浏览器窗口（出错或成功时），方便人工排查
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import BrowserUnavailableError, ResourceBusyError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BrowserPool:
|
||||
"""浏览器实例池，提供并发安全的页面会话获取"""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self._settings = settings
|
||||
self._playwright: Any | None = None
|
||||
self._browser: Any | None = None
|
||||
self._semaphore = asyncio.Semaphore(settings.max_site_concurrency)
|
||||
self._restart_lock = asyncio.Lock()
|
||||
self._started = False
|
||||
self._startup_error: str | None = None
|
||||
self._retained_contexts: list[Any] = []
|
||||
self._retained_pages: list[Any] = []
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""浏览器功能是否启用"""
|
||||
return self._settings.browser_enabled
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
"""浏览器是否已就绪（未启用或已启动成功均视为就绪）"""
|
||||
return not self.enabled or self._started
|
||||
|
||||
@property
|
||||
def startup_error(self) -> str | None:
|
||||
"""浏览器启动失败时的错误信息"""
|
||||
return self._startup_error
|
||||
|
||||
async def start(self) -> None:
|
||||
"""启动 Playwright 浏览器实例
|
||||
|
||||
仅在 browser_enabled=True 且尚未启动时执行。
|
||||
启动失败会记录错误信息到 startup_error，但不会抛异常（服务仍可启动）。
|
||||
"""
|
||||
if not self.enabled or self._started:
|
||||
return
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
headless = self._settings.browser_headless_effective
|
||||
logger.info(
|
||||
"启动浏览器：enabled=%s headless=%s keep_open_on_error=%s channel=%s proxy=%s env=%s",
|
||||
self._settings.browser_enabled,
|
||||
headless,
|
||||
self._settings.browser_keep_open_on_error_effective,
|
||||
self._settings.browser_channel,
|
||||
bool(self._settings.proxy_server),
|
||||
self._settings.app_env,
|
||||
)
|
||||
self._playwright = await async_playwright().start()
|
||||
chromium = self._playwright.chromium
|
||||
launch_args = [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-web-security",
|
||||
"--no-first-run",
|
||||
"--disable-infobars",
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-gpu",
|
||||
"--use-mock-keychain",
|
||||
"--password-store=basic",
|
||||
]
|
||||
# 本地可视化调试时强制窗口化，避免浏览器启动后不可见
|
||||
if not headless:
|
||||
launch_args.extend(["--start-maximized", "--window-position=120,120"])
|
||||
self._browser = await chromium.launch(
|
||||
headless=headless,
|
||||
channel=self._settings.browser_channel,
|
||||
proxy=self._settings.proxy,
|
||||
args=launch_args,
|
||||
)
|
||||
self._started = True
|
||||
self._startup_error = None
|
||||
logger.info("浏览器启动完成")
|
||||
except Exception as exc:
|
||||
self._playwright = None
|
||||
self._browser = None
|
||||
self._started = False
|
||||
self._startup_error = str(exc)
|
||||
logger.exception("浏览器启动失败：%s", self._startup_error)
|
||||
|
||||
def _is_browser_connected(self) -> bool:
|
||||
"""检查当前浏览器对象是否仍与 Playwright driver 保持连接。"""
|
||||
browser = self._browser
|
||||
if not self._started or browser is None:
|
||||
return False
|
||||
|
||||
is_connected = getattr(browser, "is_connected", None)
|
||||
if callable(is_connected):
|
||||
try:
|
||||
return bool(is_connected())
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭浏览器与 Playwright 运行时，通常发生在服务退出时"""
|
||||
self._retained_pages.clear()
|
||||
for context in self._retained_contexts:
|
||||
try:
|
||||
await context.close()
|
||||
except Exception:
|
||||
logger.exception("关闭保留的调试上下文失败")
|
||||
self._retained_contexts.clear()
|
||||
if self._browser is not None:
|
||||
try:
|
||||
await self._browser.close()
|
||||
except Exception:
|
||||
logger.exception("关闭浏览器实例失败")
|
||||
if self._playwright is not None:
|
||||
try:
|
||||
await self._playwright.stop()
|
||||
except Exception:
|
||||
logger.exception("停止 Playwright 运行时失败")
|
||||
self._browser = None
|
||||
self._playwright = None
|
||||
self._started = False
|
||||
|
||||
async def _restart_browser(self, reason: str) -> None:
|
||||
"""在浏览器掉线后串行重建 Playwright 运行时。"""
|
||||
async with self._restart_lock:
|
||||
if self._is_browser_connected():
|
||||
return
|
||||
|
||||
logger.warning("检测到浏览器连接不可用，准备重建浏览器实例：reason=%s", reason)
|
||||
await self.close()
|
||||
await self.start()
|
||||
if not self._is_browser_connected():
|
||||
raise BrowserUnavailableError(self._startup_error or f"Browser restart failed: {reason}")
|
||||
|
||||
async def _ensure_browser_ready(self) -> None:
|
||||
"""确保浏览器池处于可用状态；若连接已断开则自动重建。"""
|
||||
if not self.enabled:
|
||||
raise BrowserUnavailableError("Browser support is disabled")
|
||||
|
||||
if self._is_browser_connected():
|
||||
return
|
||||
|
||||
await self._restart_browser("browser is disconnected before acquiring page session")
|
||||
|
||||
@staticmethod
|
||||
def _should_retry_context_creation(exc: Exception) -> bool:
|
||||
"""识别 Playwright driver 断链类错误，允许执行一次重建后重试。"""
|
||||
message = str(exc).lower()
|
||||
retry_markers = (
|
||||
"connection closed while reading from the driver",
|
||||
"browser has been closed",
|
||||
"target page, context or browser has been closed",
|
||||
"connection closed",
|
||||
)
|
||||
return any(marker in message for marker in retry_markers)
|
||||
|
||||
async def _retain_context(self, context: Any, page: Any | None, reason: str) -> None:
|
||||
"""保留浏览器上下文用于本地调试排查
|
||||
|
||||
当本地调试模式开启时，出错或成功后不关闭浏览器窗口，
|
||||
方便人工查看页面状态。保留数量超过池大小时自动关闭最早的。
|
||||
"""
|
||||
if page is not None:
|
||||
try:
|
||||
await page.bring_to_front()
|
||||
logger.debug("%s：已将页面置顶：url=%s", reason, page.url)
|
||||
except Exception:
|
||||
logger.exception("%s：页面置顶失败", reason)
|
||||
self._retained_pages.append(page)
|
||||
self._retained_contexts.append(context)
|
||||
|
||||
# 超出保留上限时关闭最早的上下文
|
||||
limit = max(1, int(self._settings.browser_pool_size))
|
||||
while len(self._retained_contexts) > limit:
|
||||
old = self._retained_contexts.pop(0)
|
||||
try:
|
||||
await old.close()
|
||||
except Exception:
|
||||
logger.exception("关闭旧的保留调试上下文失败")
|
||||
|
||||
@asynccontextmanager
|
||||
async def page_session(
|
||||
self,
|
||||
storage_state: dict[str, Any] | None = None,
|
||||
*,
|
||||
keep_open_on_success: bool | None = None,
|
||||
) -> AsyncIterator[tuple[Any, Any]]:
|
||||
"""获取一个浏览器页面会话（上下文管理器）
|
||||
|
||||
内部通过信号量做并发控制，超时未获取到槽位会抛出 ResourceBusyError。
|
||||
页面创建时会设置日语环境、User-Agent 等反检测参数。
|
||||
|
||||
Args:
|
||||
storage_state: 浏览器存储状态（含 Cookie），用于会话复用
|
||||
keep_open_on_success: 请求成功后是否保留浏览器窗口（用于本地调试）
|
||||
|
||||
Yields:
|
||||
(context, page) 元组，context 为浏览器上下文，page 为页面实例
|
||||
|
||||
Raises:
|
||||
BrowserUnavailableError: 浏览器未启用或未启动
|
||||
ResourceBusyError: 等待可用槽位超时
|
||||
"""
|
||||
await self._ensure_browser_ready()
|
||||
|
||||
should_keep_open_on_success = (
|
||||
self._settings.browser_keep_open_effective
|
||||
if keep_open_on_success is None
|
||||
else keep_open_on_success
|
||||
)
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.page_acquire_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise ResourceBusyError() from exc
|
||||
|
||||
context = None
|
||||
page = None
|
||||
try:
|
||||
for attempt in range(2):
|
||||
try:
|
||||
context = await self._browser.new_context(
|
||||
locale=self._settings.browser_locale,
|
||||
timezone_id=self._settings.browser_timezone_id,
|
||||
user_agent=self._settings.browser_user_agent,
|
||||
viewport=None if not self._settings.browser_headless_effective else {"width": 1440, "height": 900},
|
||||
storage_state=storage_state,
|
||||
)
|
||||
break
|
||||
except Exception as exc:
|
||||
if attempt == 0 and self._should_retry_context_creation(exc):
|
||||
logger.warning("创建浏览器上下文失败，尝试重建浏览器后重试一次：err=%s", exc)
|
||||
await self._restart_browser(f"new_context failed: {exc}")
|
||||
continue
|
||||
raise
|
||||
|
||||
if self._settings.browser_stealth_enabled:
|
||||
try:
|
||||
from playwright_stealth import Stealth
|
||||
|
||||
await Stealth().apply_stealth_async(context)
|
||||
# logger.debug("已为浏览器上下文应用 Stealth 模式")
|
||||
except Exception as e:
|
||||
logger.warning("应用 playwright-stealth 失败，隐身模式未生效, err=%s", e)
|
||||
|
||||
page = await context.new_page()
|
||||
await page.set_extra_http_headers(
|
||||
{
|
||||
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Cache-Control": "no-cache",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
)
|
||||
yield context, page
|
||||
except Exception:
|
||||
if self._settings.browser_keep_open_on_error_effective and context is not None:
|
||||
await self._retain_context(context, page, reason="抓取失败，已保留页面用于排查")
|
||||
logger.warning(
|
||||
"本地调试模式已保留浏览器页面用于排查，窗口不会自动关闭；请手动查看页面。当前保留上下文数=%s",
|
||||
len(self._retained_contexts),
|
||||
)
|
||||
page = None
|
||||
context = None
|
||||
raise
|
||||
finally:
|
||||
# 本地调试模式下保留窗口
|
||||
if context is not None and should_keep_open_on_success:
|
||||
await self._retain_context(context, page, reason="本地调试：已保留页面（请求成功）")
|
||||
page = None
|
||||
context = None
|
||||
if page is not None:
|
||||
await page.close()
|
||||
if context is not None:
|
||||
await context.close()
|
||||
self._semaphore.release()
|
||||
Reference in New Issue
Block a user