接入 OTel 链路追踪 + 镜像默认启用
抓取-解析链路原本只有日志，出现"抓回内容但解析不出预期字段"时定位慢。 接入 OpenTelemetry traces（FastAPI/httpx 自动 + 手写 fetch/parse span）， 解析失败时把页面 HTML 作为 span event 上报，便于事后复现。 Dockerfile 默认开启，镜像一启动即导出到自建 OTLP endpoint。 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.6
@@ -19,6 +19,7 @@ from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from opentelemetry import trace
|
||||
|
||||
from app.scraping.core import site
|
||||
from app.shared.config import Settings
|
||||
@@ -30,6 +31,7 @@ from app.shared.errors import (
|
||||
)
|
||||
from app.scraping.parsers.state import PageValidator, require_state_marker
|
||||
from app.scraping.services.browser_fallback import BrowserFallback
|
||||
from app.shared.telemetry import snapshot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -152,66 +154,94 @@ class SiteSession:
|
||||
ResourceBusyError: 等待并发槽位超时
|
||||
AppError: 校验器判定为不可重试的失败（如落到不支持的站点）
|
||||
"""
|
||||
tracer = trace.get_tracer(__name__)
|
||||
profile = self._profiles["sp" if mobile else "pc"]
|
||||
max_attempts = max(1, self._settings.http_max_attempts)
|
||||
validate = validator or require_state_marker
|
||||
last_error: str = "unknown error"
|
||||
# 失败时供 trace snapshot 上报：保留最后一次拿到的响应体（含挑战页 / 5xx 响应）。
|
||||
last_text: str | None = None
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise ResourceBusyError() from exc
|
||||
with tracer.start_as_current_span("scrape.fetch") as span:
|
||||
span.set_attribute("scrape.url", url)
|
||||
span.set_attribute("scrape.profile", profile.name)
|
||||
|
||||
try:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
await self._ensure_warm(profile)
|
||||
try:
|
||||
response = await profile.client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
err = ResourceBusyError()
|
||||
span.record_exception(err)
|
||||
span.set_attribute("scrape.fail_reason", "ResourceBusyError")
|
||||
raise err from exc
|
||||
|
||||
try:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
span.set_attribute("scrape.attempts", attempt)
|
||||
await self._ensure_warm(profile)
|
||||
try:
|
||||
response = await profile.client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
logger.warning(
|
||||
"抓取请求异常：url=%s profile=%s attempt=%s/%s err=%s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
err = ItemNotFoundError(f"Page not found: {url}")
|
||||
span.record_exception(err)
|
||||
span.set_attribute("scrape.fail_reason", "ItemNotFoundError")
|
||||
raise err
|
||||
|
||||
text = response.text
|
||||
final_url = str(response.url)
|
||||
span.set_attribute("scrape.last_status_code", response.status_code)
|
||||
span.set_attribute("scrape.final_url", final_url)
|
||||
span.set_attribute("scrape.html_bytes", len(text))
|
||||
|
||||
if response.status_code < 400:
|
||||
# 校验器可能直接抛 AppError 表示「重试也没用」，此处不拦截
|
||||
reason = validate(text, final_url)
|
||||
if reason is None:
|
||||
return FetchedPage(html=text, url=final_url)
|
||||
last_error = self._refine_failure(reason, text)
|
||||
if last_error.startswith("challenge page detected"):
|
||||
span.set_attribute("scrape.challenge_detected", True)
|
||||
else:
|
||||
last_error = self._describe_error_status(response.status_code)
|
||||
last_text = text
|
||||
logger.warning(
|
||||
"抓取请求异常：url=%s profile=%s attempt=%s/%s err=%s",
|
||||
"抓取结果异常：url=%s profile=%s attempt=%s/%s %s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
raise ItemNotFoundError(f"Page not found: {url}")
|
||||
if attempt >= max_attempts:
|
||||
break
|
||||
|
||||
text = response.text
|
||||
final_url = str(response.url)
|
||||
if response.status_code < 400:
|
||||
# 校验器可能直接抛 AppError 表示「重试也没用」，此处不拦截
|
||||
reason = validate(text, final_url)
|
||||
if reason is None:
|
||||
return FetchedPage(html=text, url=final_url)
|
||||
last_error = self._refine_failure(reason, text)
|
||||
# 第一次失败先便宜地换一套 cookie；仍失败才动用浏览器
|
||||
if attempt == 1:
|
||||
await self._invalidate(profile)
|
||||
else:
|
||||
page = await self._escalate_to_browser(profile, url, validate)
|
||||
if page is not None:
|
||||
span.set_attribute("scrape.fell_back_to_browser", True)
|
||||
span.set_attribute("scrape.final_url", page.url)
|
||||
return page
|
||||
|
||||
if self._is_server_error(last_error):
|
||||
err = UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
else:
|
||||
last_error = self._describe_error_status(response.status_code)
|
||||
logger.warning(
|
||||
"抓取结果异常：url=%s profile=%s attempt=%s/%s %s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
)
|
||||
|
||||
if attempt >= max_attempts:
|
||||
break
|
||||
|
||||
# 第一次失败先便宜地换一套 cookie；仍失败才动用浏览器
|
||||
if attempt == 1:
|
||||
await self._invalidate(profile)
|
||||
else:
|
||||
page = await self._escalate_to_browser(profile, url, validate)
|
||||
if page is not None:
|
||||
return page
|
||||
|
||||
if self._is_server_error(last_error):
|
||||
raise UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
raise UpstreamBlockedError(f"Blocked while fetching {url}: {last_error}")
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
err = UpstreamBlockedError(f"Blocked while fetching {url}: {last_error}")
|
||||
span.record_exception(err)
|
||||
span.set_attribute("scrape.fail_reason", type(err).__name__)
|
||||
snapshot(span, "scrape.failed_html", last_text, self._settings.otel_snapshot_max_bytes)
|
||||
raise err
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
|
||||
# ---- 内部 ----
|
||||
|
||||
|
||||
Reference in New Issue
Block a user