feat(observability): 抓取会话逐次尝试与升级路径记成 event
重试链路的关键信息是「升级路径」，而属性表达不了过程：同名属性后写覆盖先写， 三次尝试跑完只剩最后一次的状态码，前两次为什么失败、升到哪一级全被盖掉。 - site_session / rakuma_session：每次尝试各记一条 scrape.attempt event，带 attempt / outcome / status_code / 截断后的错误串；outcome 区分 http_error、 bad_status、challenge、validate_failed - site_session 的每次升级各记一条 scrape.escalate：rewarm_on_home、浏览器兜底 recovered / failed。兜底失败也要记——不然链路里只剩「最终失败」，看不出浏览器 这一级试过没有，而「没装 playwright」和「装了也被挡」是两个查法（reason 取 BrowserFallback.unavailable_reason，为 None 时由 add_event 跳过） - 三处 span.record_exception 换成 record_error，失败的错误码与可重试位跟着落下来 - 失败页面快照带上 url 与 profile，省得回头对是哪条通道的哪个地址 测试：test_scrape_telemetry.py 补一条会话层用例，用 MockTransport 让所有请求都回 挑战页，断言三次尝试各留一条 event（而不是只剩最后一次）、升级路径依次是 rewarm_on_home → browser，以及浏览器那一级的 outcome=failed 与 reason。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -29,10 +29,14 @@ from app.shared.errors import (
|
||||
UpstreamBlockedError,
|
||||
UpstreamRequestError,
|
||||
)
|
||||
from app.shared.telemetry import snapshot
|
||||
from app.shared.telemetry import add_event, record_error, snapshot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 逐次尝试 event 里错误串的截断长度：httpx 的异常消息可能很长（带完整 URL 与
|
||||
# 底层 socket 错误），而这里只是用来区分「这次是怎么失败的」，前半句就够
|
||||
_ERROR_MAX_CHARS = 200
|
||||
|
||||
|
||||
class RakumaSession:
|
||||
"""ラクマ 站点抓取会话，管理并发限流与失败重试"""
|
||||
@@ -108,7 +112,7 @@ class RakumaSession:
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
err = ResourceBusyError()
|
||||
span.record_exception(err)
|
||||
record_error(span, err)
|
||||
span.set_attribute("scrape.fail_reason", "ResourceBusyError")
|
||||
raise err from exc
|
||||
|
||||
@@ -119,6 +123,13 @@ class RakumaSession:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
# 每次尝试各记一条 event：属性同名后写覆盖先写，三次尝试
|
||||
# 跑完只剩最后一次的状态，中间那两次为什么失败全被盖掉
|
||||
add_event(span, "scrape.attempt", {
|
||||
"attempt": attempt,
|
||||
"outcome": "http_error",
|
||||
"error": last_error[:_ERROR_MAX_CHARS],
|
||||
})
|
||||
logger.warning(
|
||||
"ラクマ 抓取请求异常：url=%s attempt=%s/%s err=%s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
@@ -127,7 +138,7 @@ class RakumaSession:
|
||||
|
||||
if response.status_code == 404:
|
||||
err = ItemNotFoundError(f"Page not found: {url}")
|
||||
span.record_exception(err)
|
||||
record_error(span, err)
|
||||
span.set_attribute("scrape.fail_reason", "ItemNotFoundError")
|
||||
raise err
|
||||
|
||||
@@ -145,6 +156,12 @@ class RakumaSession:
|
||||
last_text = response.text
|
||||
span.set_attribute("scrape.last_status_code", response.status_code)
|
||||
span.set_attribute("scrape.html_bytes", len(response.text))
|
||||
add_event(span, "scrape.attempt", {
|
||||
"attempt": attempt,
|
||||
"outcome": "bad_status",
|
||||
"status_code": response.status_code,
|
||||
"error": last_error[:_ERROR_MAX_CHARS],
|
||||
})
|
||||
logger.warning(
|
||||
"ラクマ 抓取结果异常：url=%s attempt=%s/%s %s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
@@ -154,9 +171,13 @@ class RakumaSession:
|
||||
err = UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
else:
|
||||
err = UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
|
||||
span.record_exception(err)
|
||||
record_error(span, err)
|
||||
span.set_attribute("scrape.fail_reason", type(err).__name__)
|
||||
snapshot(span, "scrape.failed_html", last_text, self._settings.otel_snapshot_max_bytes)
|
||||
snapshot(
|
||||
span, "scrape.failed_html", last_text,
|
||||
self._settings.otel_snapshot_max_bytes,
|
||||
extra={"scrape.url": url, "scrape.profile": "rakuma"},
|
||||
)
|
||||
raise err
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
|
||||
@@ -38,10 +38,14 @@ from app.shared.errors import (
|
||||
)
|
||||
from app.scraping.parsers.state import PageValidator, require_state_marker
|
||||
from app.scraping.services.browser_fallback import BrowserFallback
|
||||
from app.shared.telemetry import snapshot
|
||||
from app.shared.telemetry import add_event, record_error, snapshot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 逐次尝试 event 里错误串的截断长度：httpx 异常消息可能很长（带完整 URL 与底层
|
||||
# socket 错误），这里只用来区分「这次是怎么失败的」，前半句够了
|
||||
_ERROR_MAX_CHARS = 200
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FetchedPage:
|
||||
@@ -191,7 +195,7 @@ class SiteSession:
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
err = ResourceBusyError()
|
||||
span.record_exception(err)
|
||||
record_error(span, err)
|
||||
span.set_attribute("scrape.fail_reason", "ResourceBusyError")
|
||||
raise err from exc
|
||||
|
||||
@@ -204,6 +208,14 @@ class SiteSession:
|
||||
response = await profile.client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
# 每次尝试各记一条 event。属性同名后写覆盖先写，三次尝试跑完
|
||||
# 只剩最后一次的状态码，前两次为什么失败、升级到哪一级全被
|
||||
# 盖掉——而这条链路的关键信息恰恰是「升级路径」。
|
||||
add_event(span, "scrape.attempt", {
|
||||
"attempt": attempt,
|
||||
"outcome": "http_error",
|
||||
"error": last_error[:_ERROR_MAX_CHARS],
|
||||
})
|
||||
logger.warning(
|
||||
"抓取请求异常：url=%s profile=%s attempt=%s/%s err=%s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
@@ -216,7 +228,7 @@ class SiteSession:
|
||||
|
||||
if response.status_code == 404:
|
||||
err = ItemNotFoundError(f"Page not found: {url}")
|
||||
span.record_exception(err)
|
||||
record_error(span, err)
|
||||
span.set_attribute("scrape.fail_reason", "ItemNotFoundError")
|
||||
raise err
|
||||
|
||||
@@ -232,11 +244,21 @@ class SiteSession:
|
||||
if reason is None:
|
||||
return FetchedPage(html=text, url=final_url)
|
||||
last_error = self._refine_failure(reason, text)
|
||||
if last_error.startswith("challenge page detected"):
|
||||
challenged = last_error.startswith("challenge page detected")
|
||||
if challenged:
|
||||
span.set_attribute("scrape.challenge_detected", True)
|
||||
outcome = "challenge" if challenged else "validate_failed"
|
||||
else:
|
||||
last_error = self._describe_error_status(response.status_code)
|
||||
outcome = "bad_status"
|
||||
last_text = text
|
||||
add_event(span, "scrape.attempt", {
|
||||
"attempt": attempt,
|
||||
"outcome": outcome,
|
||||
"status_code": response.status_code,
|
||||
"html_bytes": len(text),
|
||||
"error": last_error[:_ERROR_MAX_CHARS],
|
||||
})
|
||||
logger.warning(
|
||||
"抓取结果异常：url=%s profile=%s attempt=%s/%s %s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
@@ -250,20 +272,39 @@ class SiteSession:
|
||||
if attempt == 1:
|
||||
await self._rewarm_on_home(profile)
|
||||
span.set_attribute("scrape.rewarmed_on_home", True)
|
||||
add_event(span, "scrape.escalate", {
|
||||
"attempt": attempt, "to": "rewarm_on_home",
|
||||
})
|
||||
else:
|
||||
page = await self._escalate_to_browser(profile, url, validate)
|
||||
if page is not None:
|
||||
span.set_attribute("scrape.fell_back_to_browser", True)
|
||||
span.set_attribute("scrape.final_url", page.url)
|
||||
add_event(span, "scrape.escalate", {
|
||||
"attempt": attempt, "to": "browser", "outcome": "recovered",
|
||||
})
|
||||
return page
|
||||
# 兜底没救回来（浏览器不可用，或取回的页面仍不合格）。不记
|
||||
# 这条的话链路里只看得到「最终失败」，看不出浏览器这一级到底
|
||||
# 试过没有——而「没装 playwright」和「装了也被挡」要分开查。
|
||||
add_event(span, "scrape.escalate", {
|
||||
"attempt": attempt,
|
||||
"to": "browser",
|
||||
"outcome": "failed",
|
||||
"reason": self._browser.unavailable_reason,
|
||||
})
|
||||
|
||||
if self._is_server_error(last_error):
|
||||
err = UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
else:
|
||||
err = UpstreamBlockedError(f"Blocked while fetching {url}: {last_error}")
|
||||
span.record_exception(err)
|
||||
record_error(span, err)
|
||||
span.set_attribute("scrape.fail_reason", type(err).__name__)
|
||||
snapshot(span, "scrape.failed_html", last_text, self._settings.otel_snapshot_max_bytes)
|
||||
snapshot(
|
||||
span, "scrape.failed_html", last_text,
|
||||
self._settings.otel_snapshot_max_bytes,
|
||||
extra={"scrape.url": url, "scrape.profile": profile.name},
|
||||
)
|
||||
raise err
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
|
||||
Reference in New Issue
Block a user