311 lines
12 KiB
Python
311 lines
12 KiB
Python
"""站点会话：带 Akamai cookie 复用的 HTTP 抓取通道，附带浏览器兜底
|
|
|
|
乐天前置 Akamai Bot Manager，行为特征（实测）：
|
|
- 请求头不完整、无 cookie 时不封禁，而是把每个响应拖到 ~11s（与响应体大小无关）
|
|
- 补齐浏览器导航请求头并复用 Akamai 下发的 cookie 后，稳定在 ~0.6-0.9s
|
|
|
|
因此这里按「指纹画像」维护两条独立通道：搜索页用 PC 画像，商品详情页用手机
|
|
画像（详情页只有手机 UA 才返回带 __INITIAL_STATE__ 的统一模板）。两条通道
|
|
各自持有独立 cookie 罐，避免把 PC 指纹拿到的 cookie 混用到手机请求上。
|
|
|
|
抓取失败时的升级路径：重新预热 → 浏览器兜底取 cookie → 放弃。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from app.core import site
|
|
from app.core.config import Settings
|
|
from app.core.errors import (
|
|
ItemNotFoundError,
|
|
ResourceBusyError,
|
|
UpstreamBlockedError,
|
|
UpstreamRequestError,
|
|
)
|
|
from app.parsers.state import PageValidator, require_state_marker
|
|
from app.services.browser_fallback import BrowserFallback
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class FetchedPage:
|
|
"""一次成功抓取的产物"""
|
|
|
|
html: str
|
|
url: str # 最终落地 URL；发生跳转时与请求地址不同
|
|
|
|
# Akamai 拦截页/挑战页的特征串
|
|
_BLOCK_MARKERS = (
|
|
"access denied",
|
|
"pardon our interruption",
|
|
"reference #",
|
|
"errors.edgesuite.net",
|
|
"/_sec/cp_challenge/",
|
|
)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _Profile:
|
|
"""一条指纹通道：独立的 httpx 客户端、cookie 罐与预热状态"""
|
|
|
|
name: str
|
|
mobile: bool
|
|
client: httpx.AsyncClient
|
|
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
|
warmed_at: float = 0.0
|
|
|
|
@property
|
|
def cookie_names(self) -> set[str]:
|
|
return {cookie.name for cookie in self.client.cookies.jar}
|
|
|
|
|
|
class SiteSession:
|
|
"""乐天站点抓取会话，管理 cookie 预热、并发限流与失败升级"""
|
|
|
|
def __init__(self, settings: Settings, browser_fallback: BrowserFallback):
|
|
self._settings = settings
|
|
self._browser = browser_fallback
|
|
self._semaphore = asyncio.Semaphore(settings.max_site_concurrency)
|
|
self._profiles: dict[str, _Profile] = {}
|
|
|
|
# ---- 生命周期 ----
|
|
|
|
async def start(self) -> None:
|
|
"""创建两条指纹通道的 HTTP 客户端"""
|
|
for name, mobile in (("pc", False), ("sp", True)):
|
|
if name in self._profiles:
|
|
continue
|
|
self._profiles[name] = _Profile(
|
|
name=name,
|
|
mobile=mobile,
|
|
client=httpx.AsyncClient(
|
|
headers=site.default_headers(mobile=mobile),
|
|
timeout=self._settings.request_timeout_seconds,
|
|
follow_redirects=True,
|
|
proxy=self._settings.httpx_proxy,
|
|
http2=True,
|
|
),
|
|
)
|
|
logger.info(
|
|
"站点会话已就绪：profiles=%s concurrency=%s proxy=%s",
|
|
list(self._profiles),
|
|
self._settings.max_site_concurrency,
|
|
bool(self._settings.proxy_server),
|
|
)
|
|
|
|
async def close(self) -> None:
|
|
"""关闭所有 HTTP 客户端"""
|
|
for profile in self._profiles.values():
|
|
try:
|
|
await profile.client.aclose()
|
|
except Exception:
|
|
logger.debug("关闭 HTTP 客户端失败：profile=%s", profile.name, exc_info=True)
|
|
self._profiles.clear()
|
|
|
|
# ---- 状态 ----
|
|
|
|
def profile_status(self) -> dict[str, dict[str, Any]]:
|
|
"""各通道的预热状态，供健康检查展示"""
|
|
now = time.monotonic()
|
|
return {
|
|
name: {
|
|
"warmed": profile.warmed_at > 0,
|
|
"age_seconds": round(now - profile.warmed_at, 1) if profile.warmed_at else None,
|
|
"cookies": sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES)),
|
|
}
|
|
for name, profile in self._profiles.items()
|
|
}
|
|
|
|
# ---- 抓取 ----
|
|
|
|
async def fetch_html(self, url: str, *, mobile: bool) -> str:
|
|
"""抓取要求含 __INITIAL_STATE__ 的页面，只取 HTML"""
|
|
page = await self.fetch(url, mobile=mobile)
|
|
return page.html
|
|
|
|
async def fetch(
|
|
self,
|
|
url: str,
|
|
*,
|
|
mobile: bool,
|
|
validator: PageValidator | None = None,
|
|
) -> FetchedPage:
|
|
"""抓取页面，返回 HTML 与最终落地地址
|
|
|
|
失败时按 重新预热 → 浏览器兜底 的顺序逐级升级重试。
|
|
|
|
Args:
|
|
validator: 页面校验器，默认要求页面含 __INITIAL_STATE__。跨站抓取时
|
|
由调用方注入按落地域名分派的校验逻辑。
|
|
|
|
Raises:
|
|
ItemNotFoundError: 目标页面 404
|
|
UpstreamBlockedError: 反复被反爬阻断
|
|
UpstreamRequestError: 网络异常或上游 5xx
|
|
ResourceBusyError: 等待并发槽位超时
|
|
AppError: 校验器判定为不可重试的失败（如落到不支持的站点）
|
|
"""
|
|
profile = self._profiles["sp" if mobile else "pc"]
|
|
max_attempts = max(1, self._settings.http_max_attempts)
|
|
validate = validator or require_state_marker
|
|
last_error: str = "unknown error"
|
|
|
|
try:
|
|
await asyncio.wait_for(
|
|
self._semaphore.acquire(),
|
|
timeout=self._settings.request_timeout_seconds,
|
|
)
|
|
except TimeoutError as exc:
|
|
raise ResourceBusyError() from exc
|
|
|
|
try:
|
|
for attempt in range(1, max_attempts + 1):
|
|
await self._ensure_warm(profile)
|
|
try:
|
|
response = await profile.client.get(url)
|
|
except httpx.HTTPError as exc:
|
|
last_error = f"{type(exc).__name__}: {exc}"
|
|
logger.warning(
|
|
"抓取请求异常：url=%s profile=%s attempt=%s/%s err=%s",
|
|
url, profile.name, attempt, max_attempts, last_error,
|
|
)
|
|
continue
|
|
|
|
if response.status_code == 404:
|
|
raise ItemNotFoundError(f"Page not found: {url}")
|
|
|
|
text = response.text
|
|
final_url = str(response.url)
|
|
if response.status_code < 400:
|
|
# 校验器可能直接抛 AppError 表示「重试也没用」，此处不拦截
|
|
reason = validate(text, final_url)
|
|
if reason is None:
|
|
return FetchedPage(html=text, url=final_url)
|
|
last_error = self._refine_failure(reason, text)
|
|
else:
|
|
last_error = self._describe_error_status(response.status_code)
|
|
logger.warning(
|
|
"抓取结果异常：url=%s profile=%s attempt=%s/%s %s",
|
|
url, profile.name, attempt, max_attempts, last_error,
|
|
)
|
|
|
|
if attempt >= max_attempts:
|
|
break
|
|
|
|
# 第一次失败先便宜地换一套 cookie；仍失败才动用浏览器
|
|
if attempt == 1:
|
|
await self._invalidate(profile)
|
|
else:
|
|
page = await self._escalate_to_browser(profile, url, validate)
|
|
if page is not None:
|
|
return page
|
|
|
|
if self._is_server_error(last_error):
|
|
raise UpstreamRequestError(f"Upstream request failed: {last_error}")
|
|
raise UpstreamBlockedError(f"Blocked while fetching {url}: {last_error}")
|
|
finally:
|
|
self._semaphore.release()
|
|
|
|
# ---- 内部 ----
|
|
|
|
@staticmethod
|
|
def _describe_error_status(status_code: int) -> str:
|
|
return (
|
|
f"upstream status {status_code}"
|
|
if status_code >= 500
|
|
else f"status {status_code}"
|
|
)
|
|
|
|
@staticmethod
|
|
def _refine_failure(reason: str, text: str) -> str:
|
|
"""页面内容校验失败时，优先报出更具体的反爬挑战页特征"""
|
|
lowered = text[:4000].lower()
|
|
matched = next((marker for marker in _BLOCK_MARKERS if marker in lowered), None)
|
|
return f"challenge page detected ({matched})" if matched else reason
|
|
|
|
@staticmethod
|
|
def _is_server_error(reason: str) -> bool:
|
|
return reason.startswith("upstream status") or reason.startswith("httpx") or "Error:" in reason
|
|
|
|
async def _ensure_warm(self, profile: _Profile) -> None:
|
|
"""确保通道持有新鲜的 Akamai cookie；过期或缺失时访问首页预热"""
|
|
if self._is_warm(profile):
|
|
return
|
|
|
|
async with profile.lock:
|
|
if self._is_warm(profile):
|
|
return
|
|
try:
|
|
response = await profile.client.get(self._settings.home_url)
|
|
profile.warmed_at = time.monotonic()
|
|
logger.info(
|
|
"会话预热完成：profile=%s status=%s cookies=%s",
|
|
profile.name,
|
|
response.status_code,
|
|
sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES)),
|
|
)
|
|
except httpx.HTTPError as exc:
|
|
# 预热失败不阻断本次抓取：直连目标页仍可能成功，只是慢
|
|
logger.warning("会话预热失败：profile=%s err=%s", profile.name, exc)
|
|
profile.warmed_at = time.monotonic()
|
|
|
|
def _is_warm(self, profile: _Profile) -> bool:
|
|
if not profile.warmed_at:
|
|
return False
|
|
if time.monotonic() - profile.warmed_at > self._settings.session_ttl_seconds:
|
|
return False
|
|
return bool(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES))
|
|
|
|
async def _invalidate(self, profile: _Profile) -> None:
|
|
"""清空通道 cookie 并强制下次重新预热"""
|
|
async with profile.lock:
|
|
profile.client.cookies.clear()
|
|
profile.warmed_at = 0.0
|
|
logger.info("已清空会话 cookie，将重新预热：profile=%s", profile.name)
|
|
|
|
async def _escalate_to_browser(
|
|
self, profile: _Profile, url: str, validate: PageValidator
|
|
) -> FetchedPage | None:
|
|
"""用浏览器访问目标页，把 cookie 回灌给 HTTP 客户端
|
|
|
|
浏览器已经拿到合格页面时直接返回，省掉一次重复请求。
|
|
"""
|
|
visit = await self._browser.visit(url, mobile=profile.mobile)
|
|
if visit is None:
|
|
logger.warning(
|
|
"浏览器兜底不可用，放弃升级：profile=%s reason=%s",
|
|
profile.name,
|
|
self._browser.unavailable_reason,
|
|
)
|
|
return None
|
|
|
|
async with profile.lock:
|
|
for cookie in visit.cookies:
|
|
name = cookie.get("name")
|
|
value = cookie.get("value")
|
|
if not name or value is None:
|
|
continue
|
|
profile.client.cookies.set(
|
|
name,
|
|
value,
|
|
domain=cookie.get("domain") or "",
|
|
path=cookie.get("path") or "/",
|
|
)
|
|
profile.warmed_at = time.monotonic()
|
|
|
|
# 浏览器不回报最终 URL，这里以请求地址为准；跨站跳转场景下 HTTP 通道已先行
|
|
# 报错，走不到这一步。
|
|
if validate(visit.html, url) is None:
|
|
logger.info("浏览器兜底直接取回页面：url=%s profile=%s", url, profile.name)
|
|
return FetchedPage(html=visit.html, url=url)
|
|
|
|
logger.warning("浏览器兜底页面仍未通过校验：url=%s profile=%s", url, profile.name)
|
|
return None
|