接入 OTel 链路追踪 + 镜像默认启用

抓取-解析链路原本只有日志,出现"抓回内容但解析不出预期字段"时定位慢。
接入 OpenTelemetry traces(FastAPI/httpx 自动 + 手写 fetch/parse span),
解析失败时把页面 HTML 作为 span event 上报,便于事后复现。
Dockerfile 默认开启,镜像一启动即导出到自建 OTLP endpoint。

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
2026-07-27 15:43:19 +08:00
co-authored by Claude Opus 4.6
parent 5376ced511
commit 2b7db521c4
13 changed files with 1021 additions and 203 deletions
+62 -36
View File
@@ -18,6 +18,7 @@ import logging
from typing import Any
import httpx
from opentelemetry import trace
from app.scraping.core import rakuma_site as site
from app.shared.config import Settings
@@ -27,6 +28,7 @@ from app.shared.errors import (
UpstreamBlockedError,
UpstreamRequestError,
)
from app.shared.telemetry import snapshot
logger = logging.getLogger(__name__)
@@ -89,47 +91,71 @@ class RakumaSession:
if client is None:
raise UpstreamRequestError("ラクマ 会话尚未初始化")
tracer = trace.get_tracer(__name__)
max_attempts = max(1, self._settings.http_max_attempts)
last_error = "unknown error"
last_text: str | None = None
try:
await asyncio.wait_for(
self._semaphore.acquire(),
timeout=self._settings.request_timeout_seconds,
)
except TimeoutError as exc:
raise ResourceBusyError() from exc
with tracer.start_as_current_span("scrape.fetch") as span:
span.set_attribute("scrape.url", url)
span.set_attribute("scrape.profile", "rakuma")
try:
for attempt in range(1, max_attempts + 1):
try:
response = await client.get(url)
except httpx.HTTPError as exc:
last_error = f"{type(exc).__name__}: {exc}"
try:
await asyncio.wait_for(
self._semaphore.acquire(),
timeout=self._settings.request_timeout_seconds,
)
except TimeoutError as exc:
err = ResourceBusyError()
span.record_exception(err)
span.set_attribute("scrape.fail_reason", "ResourceBusyError")
raise err from exc
try:
for attempt in range(1, max_attempts + 1):
span.set_attribute("scrape.attempts", attempt)
try:
response = await client.get(url)
except httpx.HTTPError as exc:
last_error = f"{type(exc).__name__}: {exc}"
logger.warning(
"ラクマ 抓取请求异常:url=%s attempt=%s/%s err=%s",
url, attempt, max_attempts, last_error,
)
continue
if response.status_code == 404:
err = ItemNotFoundError(f"Page not found: {url}")
span.record_exception(err)
span.set_attribute("scrape.fail_reason", "ItemNotFoundError")
raise err
if response.status_code < 400:
span.set_attribute("scrape.last_status_code", response.status_code)
span.set_attribute("scrape.final_url", str(response.url))
span.set_attribute("scrape.html_bytes", len(response.text))
return response.text
last_error = (
f"upstream status {response.status_code}"
if response.status_code >= 500
else f"status {response.status_code}"
)
last_text = response.text
span.set_attribute("scrape.last_status_code", response.status_code)
span.set_attribute("scrape.html_bytes", len(response.text))
logger.warning(
"ラクマ 抓取请求异常:url=%s attempt=%s/%s err=%s",
"ラクマ 抓取结果异常:url=%s attempt=%s/%s %s",
url, attempt, max_attempts, last_error,
)
continue
if response.status_code == 404:
raise ItemNotFoundError(f"Page not found: {url}")
if response.status_code < 400:
return response.text
last_error = (
f"upstream status {response.status_code}"
if response.status_code >= 500
else f"status {response.status_code}"
)
logger.warning(
"ラクマ 抓取结果异常:url=%s attempt=%s/%s %s",
url, attempt, max_attempts, last_error,
)
if last_error.startswith("upstream status") or ":" in last_error:
raise UpstreamRequestError(f"Upstream request failed: {last_error}")
raise UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
finally:
self._semaphore.release()
if last_error.startswith("upstream status") or ":" in last_error:
err = UpstreamRequestError(f"Upstream request failed: {last_error}")
else:
err = UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
span.record_exception(err)
span.set_attribute("scrape.fail_reason", type(err).__name__)
snapshot(span, "scrape.failed_html", last_text, self._settings.otel_snapshot_max_bytes)
raise err
finally:
self._semaphore.release()