diff --git a/app/services/cloudflare_session.py b/app/services/cloudflare_session.py index 41d11bf..a43bf38 100644 --- a/app/services/cloudflare_session.py +++ b/app/services/cloudflare_session.py @@ -120,28 +120,42 @@ class CloudflareSessionManager: session_name = self._session_name_for_url(target_url) # 1. 优先复用已有保活上下文 - warm = self._warm_contexts.get(session_name) - if warm is not None and not self._warm_expired(warm): - html = await self._fetch_via_warm(warm, target_url) - if html is not None: - return html, self._warm_to_session(warm) - await self._retire_warm(session_name, warm) + reused = await self._reuse_warm(session_name, target_url) + if reused is not None: + html, warm = reused + return html, self._warm_to_session(warm) # 2. 在构建锁内创建/复用保活上下文,避免并发重建风暴 lock = self._build_locks.setdefault(session_name, asyncio.Lock()) async with lock: # 双重检查:可能已被其他协程在等待锁期间建好 - warm = self._warm_contexts.get(session_name) - if warm is not None and not self._warm_expired(warm): - html = await self._fetch_via_warm(warm, target_url) - if html is not None: - return html, self._warm_to_session(warm) - await self._retire_warm(session_name, warm) + reused = await self._reuse_warm(session_name, target_url) + if reused is not None: + html, warm = reused + return html, self._warm_to_session(warm) logger.info("创建保活上下文:session_name=%s", session_name) warm, html = await self._build_warm_context(session_name, target_url) return html, self._warm_to_session(warm) + async def _reuse_warm(self, session_name: str, target_url: str) -> tuple[str, WarmContext] | None: + """尝试复用已有保活上下文。 + + 过期或复用失败时退役(关闭)该上下文并返回 None; + 成功时返回 (html, warm)。 + """ + warm = self._warm_contexts.get(session_name) + if warm is None: + return None + if self._warm_expired(warm): + await self._retire_warm(session_name, warm) + return None + html = await self._fetch_via_warm(warm, target_url) + if html is None: + await self._retire_warm(session_name, warm) + return None + return html, warm + def _warm_expired(self, warm: WarmContext) -> bool: """保活上下文是否已退役或超过最大存活时长(沿用 session_ttl)。""" if warm.retired: @@ -158,10 +172,13 @@ class CloudflareSessionManager: warm.in_use += 1 try: async with self._browser_pool.page_in_context(warm.context) as page: + # 携带同站 Referer,使复用请求看起来像站内跳转而非「凭空直达」, + # 规避 CF 对「持有 clearance 却无浏览上下文」请求的 403 判定。 response = await page.goto( target_url, wait_until="domcontentloaded", timeout=int(self._settings.request_timeout_seconds * 1000), + referer=self._settings.base_url, ) status = response.status if response is not None else None if status != 200: @@ -190,6 +207,11 @@ class CloudflareSessionManager: async def _build_warm_context(self, session_name: str, target_url: str) -> tuple[WarmContext, str]: """创建一个通过 Cloudflare 挑战的保活上下文,并返回本次访问的页面 HTML。""" + # 退役并关闭可能残留的旧上下文(如已过期未复用的),避免覆盖注册表导致泄漏 + stale = self._warm_contexts.get(session_name) + if stale is not None: + await self._retire_warm(session_name, stale) + context = await self._browser_pool.create_persistent_context() try: async with self._browser_pool.page_in_context(context) as page: diff --git a/tests/test_warm_context.py b/tests/test_warm_context.py index 4fc487e..65bcfdc 100644 --- a/tests/test_warm_context.py +++ b/tests/test_warm_context.py @@ -111,6 +111,23 @@ async def test_warm_context_rebuilds_and_closes_on_failure(): assert sess2.session_id != sess1.session_id # 换了新会话 +@pytest.mark.asyncio +async def test_expired_warm_context_is_closed_on_rebuild(): + pool = _FakeBrowserPool() + # 存活时长设为 0:任何已存在的保活上下文都立即判定过期 + settings = Settings(session_ttl_seconds=0, cloudflare_reuse_wait_timeout_seconds=8.0) + mgr = CloudflareSessionManager(settings, pool, SessionStore(settings)) + + await mgr.fetch_html(_DETAIL_URL.format(1)) + assert pool.create_calls == 1 + + # 第二次请求:旧上下文过期 -> 退役关闭 -> 重建,旧上下文不应泄漏 + await mgr.fetch_html(_DETAIL_URL.format(2)) + assert pool.create_calls == 2 + assert pool.contexts[0].closed is True + assert len(mgr._warm_contexts) == 1 # 注册表中只保留新的那个 + + @pytest.mark.asyncio async def test_close_releases_all_warm_contexts(): pool = _FakeBrowserPool()