probe search latency

2026-08-16 23:31:59 +08:00
parent 3b53b8f28b
commit 6b77f3bcc3
1 changed files with 73 additions and 0 deletions
@@ -0,0 +1,73 @@
+"""搜索页抓取延迟探针：复现线上 ~5s 的慢请求，定位耗时阶段
+
+测量：
+1. 首次抓取的 预热耗时 / 目标页耗时
+2. 同会话连续抓取的稳态耗时（预热复用是否生效）
+3. 对比「无预热直连」的耗时（验证 Akamai 限速行为是否变化）
+"""
+from __future__ import annotations
+
+import asyncio
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from app.shared.config import get_settings  # noqa: E402
+from app.scraping.services.browser_fallback import BrowserFallback  # noqa: E402
+from app.scraping.services.site_session import SiteSession  # noqa: E402
+from app.scraping.core import site  # noqa: E402
+
+URL = "https://search.rakuten.co.jp/search/mall/kitty/?p=2"
+
+
+async def main() -> int:
+    settings = get_settings()
+    print(f"proxy={bool(settings.proxy_server)} timeout={settings.request_timeout_seconds} "
+          f"ttl={settings.session_ttl_seconds} attempts={settings.http_max_attempts}")
+
+    session = SiteSession(settings, BrowserFallback(settings))
+    await session.start()
+    profile = session._profiles["pc"]
+
+    # 1) 首次抓取（含预热）
+    t0 = time.monotonic()
+    await session._ensure_warm(profile)
+    t_warm = time.monotonic() - t0
+    print(f"[1] 预热: {t_warm:.2f}s cookies={sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES))}")
+
+    t0 = time.monotonic()
+    page = await session.fetch_html(URL, mobile=False)
+    t_first = time.monotonic() - t0
+    print(f"[1] 首次抓取: {t_first:.2f}s html={len(page)}")
+
+    # 2) 稳态：同会话连抓 3 次
+    for i in range(2, 5):
+        t0 = time.monotonic()
+        page = await session.fetch_html(URL, mobile=False)
+        print(f"[{i}] 稳态抓取: {time.monotonic() - t0:.2f}s html={len(page)}")
+
+    await session.close()
+
+    # 3) 对照：全新客户端、无预热、直接打目标页（Akamai 限速基线）
+    import httpx
+    from app.shared.proxy import httpx_client_options
+
+    async with httpx.AsyncClient(
+        headers=site.default_headers(mobile=False),
+        timeout=settings.request_timeout_seconds,
+        follow_redirects=True,
+        http2=True,
+        **httpx_client_options(settings),
+    ) as cold:
+        t0 = time.monotonic()
+        resp = await cold.get(URL)
+        print(f"[对照] 无预热直连: {time.monotonic() - t0:.2f}s status={resp.status_code} "
+              f"html={len(resp.text)} cookies={sorted(c for c in resp.cookies.keys())}")
+
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(asyncio.run(main()))