74 lines
2.5 KiB
Python
74 lines
2.5 KiB
Python
"""搜索页抓取延迟探针：复现线上 ~5s 的慢请求，定位耗时阶段
|
|
|
|
测量：
|
|
1. 首次抓取的 预热耗时 / 目标页耗时
|
|
2. 同会话连续抓取的稳态耗时（预热复用是否生效）
|
|
3. 对比「无预热直连」的耗时（验证 Akamai 限速行为是否变化）
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
|
|
|
from app.shared.config import get_settings # noqa: E402
|
|
from app.scraping.services.browser_fallback import BrowserFallback # noqa: E402
|
|
from app.scraping.services.site_session import SiteSession # noqa: E402
|
|
from app.scraping.core import site # noqa: E402
|
|
|
|
URL = "https://search.rakuten.co.jp/search/mall/kitty/?p=2"
|
|
|
|
|
|
async def main() -> int:
|
|
settings = get_settings()
|
|
print(f"proxy={bool(settings.proxy_server)} timeout={settings.request_timeout_seconds} "
|
|
f"ttl={settings.session_ttl_seconds} attempts={settings.http_max_attempts}")
|
|
|
|
session = SiteSession(settings, BrowserFallback(settings))
|
|
await session.start()
|
|
profile = session._profiles["pc"]
|
|
|
|
# 1) 首次抓取（含预热）
|
|
t0 = time.monotonic()
|
|
await session._ensure_warm(profile)
|
|
t_warm = time.monotonic() - t0
|
|
print(f"[1] 预热: {t_warm:.2f}s cookies={sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES))}")
|
|
|
|
t0 = time.monotonic()
|
|
page = await session.fetch_html(URL, mobile=False)
|
|
t_first = time.monotonic() - t0
|
|
print(f"[1] 首次抓取: {t_first:.2f}s html={len(page)}")
|
|
|
|
# 2) 稳态：同会话连抓 3 次
|
|
for i in range(2, 5):
|
|
t0 = time.monotonic()
|
|
page = await session.fetch_html(URL, mobile=False)
|
|
print(f"[{i}] 稳态抓取: {time.monotonic() - t0:.2f}s html={len(page)}")
|
|
|
|
await session.close()
|
|
|
|
# 3) 对照：全新客户端、无预热、直接打目标页（Akamai 限速基线）
|
|
import httpx
|
|
from app.shared.proxy import httpx_client_options
|
|
|
|
async with httpx.AsyncClient(
|
|
headers=site.default_headers(mobile=False),
|
|
timeout=settings.request_timeout_seconds,
|
|
follow_redirects=True,
|
|
http2=True,
|
|
**httpx_client_options(settings),
|
|
) as cold:
|
|
t0 = time.monotonic()
|
|
resp = await cold.get(URL)
|
|
print(f"[对照] 无预热直连: {time.monotonic() - t0:.2f}s status={resp.status_code} "
|
|
f"html={len(resp.text)} cookies={sorted(c for c in resp.cookies.keys())}")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(asyncio.run(main()))
|