diff --git a/README.md b/README.md index bac59e6..68dd203 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,9 @@ curl -H "Authorization: Bearer " ^ - `SURUGAYA_APP_PORT`(默认 `31106`) - 鉴权: - `SURUGAYA_BEARER_TOKEN` +- 浏览器回收(防止 Chromium 长时间运行内存累积导致 driver 掉线): + - `SURUGAYA_BROWSER_RECYCLE_AFTER_CONTEXTS`(默认 `200`,累计创建上下文达到该值后在空闲时主动重建浏览器,`0` 关闭) + - `SURUGAYA_BROWSER_RECYCLE_AFTER_SECONDS`(默认 `3600`,浏览器运行超过该秒数后在空闲时主动重建,`0` 关闭) - 代理(日本 IP): - `SURUGAYA_PROXY_SERVER`(例:`http://127.0.0.1:7890`) - `SURUGAYA_PROXY_USERNAME` diff --git a/app/core/config.py b/app/core/config.py index b1f44d6..50734b1 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -53,6 +53,8 @@ class Settings(BaseSettings): browser_channel: str | None = "chrome" browser_pool_size: int = 3 max_site_concurrency: int = 5 # 最大并发页面数 + browser_recycle_after_contexts: int = 200 # 浏览器累计创建上下文达到该值后主动回收,0 关闭 + browser_recycle_after_seconds: float = 3600.0 # 浏览器运行超过该秒数后主动回收,0 关闭 page_acquire_timeout_seconds: float = 60.0 # 获取页面槽位超时 request_timeout_seconds: float = 120.0 # 页面请求超时 cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时 diff --git a/app/services/browser_pool.py b/app/services/browser_pool.py index 2d6ecc1..7d4d1b6 100644 --- a/app/services/browser_pool.py +++ b/app/services/browser_pool.py @@ -32,6 +32,10 @@ class BrowserPool: self._startup_error: str | None = None self._retained_contexts: list[Any] = [] self._retained_pages: list[Any] = [] + # 主动回收相关计数:累计创建的上下文数、本次浏览器启动时刻、当前在用上下文数 + self._contexts_since_start = 0 + self._browser_started_at: float | None = None + self._active_contexts = 0 @property def enabled(self) -> bool: @@ -95,6 +99,8 @@ class BrowserPool: ) self._started = True self._startup_error = None + self._contexts_since_start = 0 + self._browser_started_at = asyncio.get_running_loop().time() logger.info("浏览器启动完成") except Exception as exc: self._playwright = None @@ -163,6 +169,51 @@ class BrowserPool: await self._restart_browser("browser is disconnected before acquiring page session") + def _recycle_due(self) -> bool: + """判断浏览器是否达到主动回收阈值(累计上下文数或运行时长任一超限)。""" + max_contexts = self._settings.browser_recycle_after_contexts + if max_contexts > 0 and self._contexts_since_start >= max_contexts: + return True + + max_age = self._settings.browser_recycle_after_seconds + if max_age > 0 and self._browser_started_at is not None: + elapsed = asyncio.get_running_loop().time() - self._browser_started_at + if elapsed >= max_age: + return True + + return False + + async def _maybe_recycle(self) -> None: + """达到回收阈值且当前无在用上下文时,主动重建浏览器以释放累积内存。 + + 采用「空闲时回收」策略:若仍有其他会话在使用上下文,则跳过本次, + 交由后续空闲时机处理,避免关闭浏览器时打断进行中的请求。 + """ + if not self._recycle_due(): + return + + async with self._restart_lock: + # 持锁后二次确认,避免与并发回收/重建重复执行 + if not self._recycle_due(): + return + if self._active_contexts > 0: + logger.debug( + "浏览器达到回收阈值但仍有 %s 个上下文在用,延迟回收", + self._active_contexts, + ) + return + + logger.info( + "浏览器达到回收阈值,主动重建:contexts=%s elapsed=%.0fs", + self._contexts_since_start, + 0.0 if self._browser_started_at is None + else asyncio.get_running_loop().time() - self._browser_started_at, + ) + await self.close() + await self.start() + if not self._is_browser_connected(): + raise BrowserUnavailableError(self._startup_error or "Browser recycle failed") + @staticmethod def _should_retry_context_creation(exc: Exception) -> bool: """识别 Playwright driver 断链类错误,允许执行一次重建后重试。""" @@ -240,7 +291,10 @@ class BrowserPool: context = None page = None + context_counted = False try: + # 创建上下文前检查是否需要主动回收(仅在空闲时执行,不打断进行中的会话) + await self._maybe_recycle() for attempt in range(2): try: context = await self._browser.new_context( @@ -258,6 +312,11 @@ class BrowserPool: continue raise + # 上下文创建成功,计入回收阈值统计并标记为在用 + self._contexts_since_start += 1 + self._active_contexts += 1 + context_counted = True + if self._settings.browser_stealth_enabled: try: from playwright_stealth import Stealth @@ -296,4 +355,6 @@ class BrowserPool: await page.close() if context is not None: await context.close() + if context_counted: + self._active_contexts -= 1 self._semaphore.release() diff --git a/tests/test_browser_recycle.py b/tests/test_browser_recycle.py new file mode 100644 index 0000000..ab7bc92 --- /dev/null +++ b/tests/test_browser_recycle.py @@ -0,0 +1,116 @@ +"""浏览器池主动回收逻辑的单元测试(不启动真实 Chromium)。""" +from __future__ import annotations + +import asyncio + +import pytest + +from app.core.config import Settings +from app.services.browser_pool import BrowserPool + + +class _FakeBrowser: + """最小化的浏览器替身,仅实现连接状态查询。""" + + def __init__(self, connected: bool = True): + self._connected = connected + + def is_connected(self) -> bool: + return self._connected + + +def _make_pool(**overrides) -> BrowserPool: + settings = Settings(**overrides) + pool = BrowserPool(settings) + pool._browser = _FakeBrowser() + pool._started = True + return pool + + +@pytest.mark.asyncio +async def test_recycle_due_by_context_count(): + pool = _make_pool(browser_recycle_after_contexts=10, browser_recycle_after_seconds=0) + pool._browser_started_at = asyncio.get_running_loop().time() + + pool._contexts_since_start = 9 + assert pool._recycle_due() is False + + pool._contexts_since_start = 10 + assert pool._recycle_due() is True + + +@pytest.mark.asyncio +async def test_recycle_due_by_age(): + pool = _make_pool(browser_recycle_after_contexts=0, browser_recycle_after_seconds=100) + now = asyncio.get_running_loop().time() + + pool._browser_started_at = now - 99 + assert pool._recycle_due() is False + + pool._browser_started_at = now - 101 + assert pool._recycle_due() is True + + +@pytest.mark.asyncio +async def test_recycle_disabled_when_thresholds_zero(): + pool = _make_pool(browser_recycle_after_contexts=0, browser_recycle_after_seconds=0) + pool._browser_started_at = asyncio.get_running_loop().time() - 99999 + pool._contexts_since_start = 99999 + assert pool._recycle_due() is False + + +@pytest.mark.asyncio +async def test_maybe_recycle_rebuilds_when_idle(monkeypatch): + pool = _make_pool(browser_recycle_after_contexts=10, browser_recycle_after_seconds=0) + pool._browser_started_at = asyncio.get_running_loop().time() + pool._contexts_since_start = 10 + pool._active_contexts = 0 + + calls: list[str] = [] + + async def fake_close(): + calls.append("close") + + async def fake_start(): + calls.append("start") + pool._browser = _FakeBrowser() + pool._started = True + pool._contexts_since_start = 0 # start() 正常会重置计数 + + monkeypatch.setattr(pool, "close", fake_close) + monkeypatch.setattr(pool, "start", fake_start) + + await pool._maybe_recycle() + + assert calls == ["close", "start"] + + +@pytest.mark.asyncio +async def test_maybe_recycle_skips_when_busy(monkeypatch): + pool = _make_pool(browser_recycle_after_contexts=10, browser_recycle_after_seconds=0) + pool._browser_started_at = asyncio.get_running_loop().time() + pool._contexts_since_start = 10 + pool._active_contexts = 1 # 仍有会话在用,不应回收 + + calls: list[str] = [] + monkeypatch.setattr(pool, "close", lambda: calls.append("close")) + monkeypatch.setattr(pool, "start", lambda: calls.append("start")) + + await pool._maybe_recycle() + + assert calls == [] + + +@pytest.mark.asyncio +async def test_maybe_recycle_noop_when_not_due(monkeypatch): + pool = _make_pool(browser_recycle_after_contexts=10, browser_recycle_after_seconds=0) + pool._browser_started_at = asyncio.get_running_loop().time() + pool._contexts_since_start = 3 + + calls: list[str] = [] + monkeypatch.setattr(pool, "close", lambda: calls.append("close")) + monkeypatch.setattr(pool, "start", lambda: calls.append("start")) + + await pool._maybe_recycle() + + assert calls == []