接入 OTel 链路追踪 + 镜像默认启用
抓取-解析链路原本只有日志,出现"抓回内容但解析不出预期字段"时定位慢。 接入 OpenTelemetry traces(FastAPI/httpx 自动 + 手写 fetch/parse span), 解析失败时把页面 HTML 作为 span event 上报,便于事后复现。 Dockerfile 默认开启,镜像一启动即导出到自建 OTLP endpoint。 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
@@ -18,6 +18,7 @@ import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from opentelemetry import trace
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.config import Settings
|
||||
@@ -27,6 +28,7 @@ from app.shared.errors import (
|
||||
UpstreamBlockedError,
|
||||
UpstreamRequestError,
|
||||
)
|
||||
from app.shared.telemetry import snapshot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -89,47 +91,71 @@ class RakumaSession:
|
||||
if client is None:
|
||||
raise UpstreamRequestError("ラクマ 会话尚未初始化")
|
||||
|
||||
tracer = trace.get_tracer(__name__)
|
||||
max_attempts = max(1, self._settings.http_max_attempts)
|
||||
last_error = "unknown error"
|
||||
last_text: str | None = None
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise ResourceBusyError() from exc
|
||||
with tracer.start_as_current_span("scrape.fetch") as span:
|
||||
span.set_attribute("scrape.url", url)
|
||||
span.set_attribute("scrape.profile", "rakuma")
|
||||
|
||||
try:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
err = ResourceBusyError()
|
||||
span.record_exception(err)
|
||||
span.set_attribute("scrape.fail_reason", "ResourceBusyError")
|
||||
raise err from exc
|
||||
|
||||
try:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
span.set_attribute("scrape.attempts", attempt)
|
||||
try:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
logger.warning(
|
||||
"ラクマ 抓取请求异常:url=%s attempt=%s/%s err=%s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
err = ItemNotFoundError(f"Page not found: {url}")
|
||||
span.record_exception(err)
|
||||
span.set_attribute("scrape.fail_reason", "ItemNotFoundError")
|
||||
raise err
|
||||
|
||||
if response.status_code < 400:
|
||||
span.set_attribute("scrape.last_status_code", response.status_code)
|
||||
span.set_attribute("scrape.final_url", str(response.url))
|
||||
span.set_attribute("scrape.html_bytes", len(response.text))
|
||||
return response.text
|
||||
|
||||
last_error = (
|
||||
f"upstream status {response.status_code}"
|
||||
if response.status_code >= 500
|
||||
else f"status {response.status_code}"
|
||||
)
|
||||
last_text = response.text
|
||||
span.set_attribute("scrape.last_status_code", response.status_code)
|
||||
span.set_attribute("scrape.html_bytes", len(response.text))
|
||||
logger.warning(
|
||||
"ラクマ 抓取请求异常:url=%s attempt=%s/%s err=%s",
|
||||
"ラクマ 抓取结果异常:url=%s attempt=%s/%s %s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
raise ItemNotFoundError(f"Page not found: {url}")
|
||||
|
||||
if response.status_code < 400:
|
||||
return response.text
|
||||
|
||||
last_error = (
|
||||
f"upstream status {response.status_code}"
|
||||
if response.status_code >= 500
|
||||
else f"status {response.status_code}"
|
||||
)
|
||||
logger.warning(
|
||||
"ラクマ 抓取结果异常:url=%s attempt=%s/%s %s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
)
|
||||
|
||||
if last_error.startswith("upstream status") or ":" in last_error:
|
||||
raise UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
raise UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
if last_error.startswith("upstream status") or ":" in last_error:
|
||||
err = UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
else:
|
||||
err = UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
|
||||
span.record_exception(err)
|
||||
span.set_attribute("scrape.fail_reason", type(err).__name__)
|
||||
snapshot(span, "scrape.failed_html", last_text, self._settings.otel_snapshot_max_bytes)
|
||||
raise err
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
|
||||
Reference in New Issue
Block a user