diff --git a/app/scraping/services/site_session.py b/app/scraping/services/site_session.py index 30ed7de..21d643e 100644 --- a/app/scraping/services/site_session.py +++ b/app/scraping/services/site_session.py @@ -266,7 +266,7 @@ class SiteSession: return reason.startswith("upstream status") or reason.startswith("httpx") or "Error:" in reason async def _ensure_warm(self, profile: _Profile) -> None: - """确保通道持有新鲜的 Akamai cookie；过期或缺失时访问首页预热""" + """确保通道有一次新鲜的首页预热；过期时重新访问首页""" if self._is_warm(profile): return @@ -275,7 +275,10 @@ class SiteSession: return try: response = await profile.client.get(self._settings.home_url) - profile.warmed_at = time.monotonic() + # Akamai 不保证每次都下发 cookie；首页探测成功本身就是可复用的 + # 预热结果，cookie 只用于观测和失败升级时的回灌。 + if response.status_code < 400: + profile.warmed_at = time.monotonic() logger.info( "会话预热完成：profile=%s status=%s cookies=%s", profile.name, @@ -285,14 +288,14 @@ class SiteSession: except httpx.HTTPError as exc: # 预热失败不阻断本次抓取：直连目标页仍可能成功，只是慢 logger.warning("会话预热失败：profile=%s err=%s", profile.name, exc) - profile.warmed_at = time.monotonic() + profile.warmed_at = 0.0 def _is_warm(self, profile: _Profile) -> bool: if not profile.warmed_at: return False if time.monotonic() - profile.warmed_at > self._settings.session_ttl_seconds: return False - return bool(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES)) + return True async def _invalidate(self, profile: _Profile) -> None: """清空通道 cookie 并强制下次重新预热""" diff --git a/tests/test_site_session.py b/tests/test_site_session.py index cb85c59..d650fdc 100644 --- a/tests/test_site_session.py +++ b/tests/test_site_session.py @@ -89,6 +89,25 @@ async def test_warmup_happens_before_first_fetch_and_is_reused(): assert seen.count(TARGET) == 2 +async def test_successful_warmup_is_reused_when_upstream_sets_no_cookie(): + """首页可能返回 200 但不下发 Akamai cookie，仍不得重复预热。""" + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(str(request.url)) + return httpx.Response(200, text=GOOD_PAGE) + + session = await build_session(handler) + try: + await session.fetch_html(TARGET, mobile=False) + await session.fetch_html(TARGET, mobile=False) + finally: + await session.close() + + assert seen.count("https://www.rakuten.co.jp/") == 1 + assert seen.count(TARGET) == 2 + + async def test_missing_state_marker_is_treated_as_blocked(): def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(200, text="no state here")