优化浏览器context复用

2026-06-09 16:04:44 +08:00
parent bc0f7161c0
commit 3b8bdf09f7
4 changed files with 399 additions and 93 deletions
@@ -250,6 +250,107 @@ class BrowserPool:
            except Exception:
                logger.exception("关闭旧的保留调试上下文失败")

+    async def _build_context_obj(self, storage_state: dict[str, Any] | None) -> Any:
+        """创建浏览器上下文（含断链重试与 stealth），仅负责创建本身。
+
+        计入回收阈值统计；不负责并发信号量与关闭，由调用方管理生命周期。
+        """
+        context = None
+        for attempt in range(2):
+            try:
+                context = await self._browser.new_context(
+                    locale=self._settings.browser_locale,
+                    timezone_id=self._settings.browser_timezone_id,
+                    user_agent=self._settings.browser_user_agent,
+                    viewport=None if not self._settings.browser_headless_effective else {"width": 1440, "height": 900},
+                    storage_state=storage_state,
+                )
+                break
+            except Exception as exc:
+                if attempt == 0 and self._should_retry_context_creation(exc):
+                    logger.warning("创建浏览器上下文失败，尝试重建浏览器后重试一次：err=%s", exc)
+                    await self._restart_browser(f"new_context failed: {exc}")
+                    continue
+                raise
+
+        # 上下文创建成功，计入回收阈值统计
+        self._contexts_since_start += 1
+
+        if self._settings.browser_stealth_enabled:
+            try:
+                from playwright_stealth import Stealth
+
+                await Stealth().apply_stealth_async(context)
+            except Exception as e:
+                logger.warning("应用 playwright-stealth 失败，隐身模式未生效, err=%s", e)
+
+        return context
+
+    @staticmethod
+    async def _configure_page(context: Any) -> Any:
+        """在上下文中创建页面并设置反检测请求头。"""
+        page = await context.new_page()
+        await page.set_extra_http_headers(
+            {
+                "Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
+                "Cache-Control": "no-cache",
+                "Pragma": "no-cache",
+            }
+        )
+        return page
+
+    async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any:
+        """创建一个由调用方负责生命周期的长存上下文。
+
+        用于 cf_clearance 保活复用：通过挑战后保留该上下文，后续请求在其内
+        开新页面访问目标页（同一上下文内 cf_clearance 有效）。调用方需在不再
+        使用时调用 close_context 释放。
+        """
+        await self._ensure_browser_ready()
+        return await self._build_context_obj(storage_state)
+
+    @asynccontextmanager
+    async def page_in_context(self, context: Any) -> AsyncIterator[Any]:
+        """在既有（长存）上下文中开一个页面，复用并发信号量。
+
+        结束时仅关闭页面，不关闭上下文（上下文生命周期由调用方维护）。
+
+        Raises:
+            ResourceBusyError: 等待可用槽位超时
+        """
+        await self._ensure_browser_ready()
+        try:
+            await asyncio.wait_for(
+                self._semaphore.acquire(),
+                timeout=self._settings.page_acquire_timeout_seconds,
+            )
+        except TimeoutError as exc:
+            raise ResourceBusyError() from exc
+
+        page = None
+        counted = False
+        try:
+            self._active_contexts += 1
+            counted = True
+            page = await self._configure_page(context)
+            yield page
+        finally:
+            if page is not None:
+                try:
+                    await page.close()
+                except Exception:
+                    logger.debug("关闭复用上下文页面失败", exc_info=True)
+            if counted:
+                self._active_contexts -= 1
+            self._semaphore.release()
+
+    async def close_context(self, context: Any) -> None:
+        """关闭一个长存上下文（忽略已关闭等异常）。"""
+        try:
+            await context.close()
+        except Exception:
+            logger.debug("关闭长存上下文失败", exc_info=True)
+
    @asynccontextmanager
    async def page_session(
        self,
@@ -295,45 +396,12 @@ class BrowserPool:
        try:
            # 创建上下文前检查是否需要主动回收（仅在空闲时执行，不打断进行中的会话）
            await self._maybe_recycle()
-            for attempt in range(2):
-                try:
-                    context = await self._browser.new_context(
-                        locale=self._settings.browser_locale,
-                        timezone_id=self._settings.browser_timezone_id,
-                        user_agent=self._settings.browser_user_agent,
-                        viewport=None if not self._settings.browser_headless_effective else {"width": 1440, "height": 900},
-                        storage_state=storage_state,
-                    )
-                    break
-                except Exception as exc:
-                    if attempt == 0 and self._should_retry_context_creation(exc):
-                        logger.warning("创建浏览器上下文失败，尝试重建浏览器后重试一次：err=%s", exc)
-                        await self._restart_browser(f"new_context failed: {exc}")
-                        continue
-                    raise
-
-            # 上下文创建成功，计入回收阈值统计并标记为在用
-            self._contexts_since_start += 1
+            context = await self._build_context_obj(storage_state)
+            # 上下文创建成功，标记为在用（回收阈值计数已在 _build_context_obj 内完成）
            self._active_contexts += 1
            context_counted = True

-            if self._settings.browser_stealth_enabled:
-                try:
-                    from playwright_stealth import Stealth
-
-                    await Stealth().apply_stealth_async(context)
-                    # logger.debug("已为浏览器上下文应用 Stealth 模式")
-                except Exception as e:
-                    logger.warning("应用 playwright-stealth 失败，隐身模式未生效, err=%s", e)
-
-            page = await context.new_page()
-            await page.set_extra_http_headers(
-                {
-                    "Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
-                    "Cache-Control": "no-cache",
-                    "Pragma": "no-cache",
-                }
-            )
+            page = await self._configure_page(context)
            yield context, page
        except Exception:
            if self._settings.browser_keep_open_on_error_effective and context is not None: