优化浏览器context复用

app/services/cloudflare_session.py

@@ -120,28 +120,42 @@ class CloudflareSessionManager:
         session_name = self._session_name_for_url(target_url)
 
         # 1. 优先复用已有保活上下文
-        warm = self._warm_contexts.get(session_name)
-        if warm is not None and not self._warm_expired(warm):
-            html = await self._fetch_via_warm(warm, target_url)
-            if html is not None:
+        reused = await self._reuse_warm(session_name, target_url)
+        if reused is not None:
+            html, warm = reused
             return html, self._warm_to_session(warm)
-            await self._retire_warm(session_name, warm)
 
         # 2. 在构建锁内创建/复用保活上下文，避免并发重建风暴
         lock = self._build_locks.setdefault(session_name, asyncio.Lock())
         async with lock:
             # 双重检查：可能已被其他协程在等待锁期间建好
-            warm = self._warm_contexts.get(session_name)
-            if warm is not None and not self._warm_expired(warm):
-                html = await self._fetch_via_warm(warm, target_url)
-                if html is not None:
+            reused = await self._reuse_warm(session_name, target_url)
+            if reused is not None:
+                html, warm = reused
                 return html, self._warm_to_session(warm)
-                await self._retire_warm(session_name, warm)
 
             logger.info("创建保活上下文：session_name=%s", session_name)
             warm, html = await self._build_warm_context(session_name, target_url)
             return html, self._warm_to_session(warm)
 
+    async def _reuse_warm(self, session_name: str, target_url: str) -> tuple[str, WarmContext] | None:
+        """尝试复用已有保活上下文。
+
+        过期或复用失败时退役（关闭）该上下文并返回 None；
+        成功时返回 (html, warm)。
+        """
+        warm = self._warm_contexts.get(session_name)
+        if warm is None:
+            return None
+        if self._warm_expired(warm):
+            await self._retire_warm(session_name, warm)
+            return None
+        html = await self._fetch_via_warm(warm, target_url)
+        if html is None:
+            await self._retire_warm(session_name, warm)
+            return None
+        return html, warm
+
     def _warm_expired(self, warm: WarmContext) -> bool:
         """保活上下文是否已退役或超过最大存活时长（沿用 session_ttl）。"""
         if warm.retired:
@@ -158,10 +172,13 @@ class CloudflareSessionManager:
         warm.in_use += 1
         try:
             async with self._browser_pool.page_in_context(warm.context) as page:
+                # 携带同站 Referer，使复用请求看起来像站内跳转而非「凭空直达」，
+                # 规避 CF 对「持有 clearance 却无浏览上下文」请求的 403 判定。
                 response = await page.goto(
                     target_url,
                     wait_until="domcontentloaded",
                     timeout=int(self._settings.request_timeout_seconds * 1000),
+                    referer=self._settings.base_url,
                 )
                 status = response.status if response is not None else None
                 if status != 200:
@@ -190,6 +207,11 @@ class CloudflareSessionManager:
 
     async def _build_warm_context(self, session_name: str, target_url: str) -> tuple[WarmContext, str]:
         """创建一个通过 Cloudflare 挑战的保活上下文，并返回本次访问的页面 HTML。"""
+        # 退役并关闭可能残留的旧上下文（如已过期未复用的），避免覆盖注册表导致泄漏
+        stale = self._warm_contexts.get(session_name)
+        if stale is not None:
+            await self._retire_warm(session_name, stale)
+
         context = await self._browser_pool.create_persistent_context()
         try:
             async with self._browser_pool.page_in_context(context) as page:

tests/test_warm_context.py

@@ -111,6 +111,23 @@ async def test_warm_context_rebuilds_and_closes_on_failure():
     assert sess2.session_id != sess1.session_id   # 换了新会话
 
 
+@pytest.mark.asyncio
+async def test_expired_warm_context_is_closed_on_rebuild():
+    pool = _FakeBrowserPool()
+    # 存活时长设为 0：任何已存在的保活上下文都立即判定过期
+    settings = Settings(session_ttl_seconds=0, cloudflare_reuse_wait_timeout_seconds=8.0)
+    mgr = CloudflareSessionManager(settings, pool, SessionStore(settings))
+
+    await mgr.fetch_html(_DETAIL_URL.format(1))
+    assert pool.create_calls == 1
+
+    # 第二次请求：旧上下文过期 -> 退役关闭 -> 重建，旧上下文不应泄漏
+    await mgr.fetch_html(_DETAIL_URL.format(2))
+    assert pool.create_calls == 2
+    assert pool.contexts[0].closed is True
+    assert len(mgr._warm_contexts) == 1  # 注册表中只保留新的那个
+
+
 @pytest.mark.asyncio
 async def test_close_releases_all_warm_contexts():
     pool = _FakeBrowserPool()