q792602257andClaude Opus 5
865bbe3724
重试链路的关键信息是「升级路径」，而属性表达不了过程：同名属性后写覆盖先写， 三次尝试跑完只剩最后一次的状态码，前两次为什么失败、升到哪一级全被盖掉。 - site_session / rakuma_session：每次尝试各记一条 scrape.attempt event，带 attempt / outcome / status_code / 截断后的错误串；outcome 区分 http_error、 bad_status、challenge、validate_failed - site_session 的每次升级各记一条 scrape.escalate：rewarm_on_home、浏览器兜底 recovered / failed。兜底失败也要记——不然链路里只剩「最终失败」，看不出浏览器 这一级试过没有，而「没装 playwright」和「装了也被挡」是两个查法（reason 取 BrowserFallback.unavailable_reason，为 None 时由 add_event 跳过） - 三处 span.record_exception 换成 record_error，失败的错误码与可重试位跟着落下来 - 失败页面快照带上 url 与 profile，省得回头对是哪条通道的哪个地址 测试：test_scrape_telemetry.py 补一条会话层用例，用 MockTransport 让所有请求都回 挑战页，断言三次尝试各留一条 event（而不是只剩最后一次）、升级路径依次是 rewarm_on_home → browser，以及浏览器那一级的 outcome=failed 与 reason。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
297 lines
13 KiB
Python
297 lines
13 KiB
Python
"""抓取客户端失败分支的埋点测试：失败原因必须能从 span 上读出来
|
|
|
|
这里钉住的核心是一个真实存在过的 bug：失败分支里写的是无参
|
|
`span.record_exception()`，而该方法的 `exception` 是必填位置参数——**每一次抓取
|
|
失败都会在记录异常时抛 TypeError**，把真正的失败原因（反爬阻断 / 解析失败 /
|
|
404）整个替换掉。链路上只剩「调用发生过」，异常怎么来的完全看不到，失败页面
|
|
快照也永远落不下来（抛错发生在 snapshot 之前）。
|
|
|
|
用独立的 InMemory provider 断言 span，不碰全局 provider——OTel 的全局 provider
|
|
只允许设置一次，测试间共享会互相污染（与 test_telemetry.py 同一套思路）。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
import httpx
|
|
import pytest
|
|
from opentelemetry.sdk.trace import TracerProvider
|
|
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
|
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
|
from opentelemetry.trace import StatusCode
|
|
|
|
from app.scraping.models.scrape import (
|
|
ItemDetailRequest,
|
|
RakumaSearchRequest,
|
|
SearchRequest,
|
|
ShopItemsRequest,
|
|
)
|
|
from app.scraping.services import rakuma_client as rakuma_module
|
|
from app.scraping.services import rakuten_client as rakuten_module
|
|
from app.scraping.services import site_session as session_module
|
|
from app.scraping.services.rakuma_client import RakumaClient
|
|
from app.scraping.services.rakuten_client import RakutenClient
|
|
from app.scraping.services.site_session import SiteSession
|
|
from app.shared.config import Settings
|
|
from app.shared.errors import AppError, ScrapeParseError, UpstreamBlockedError
|
|
|
|
|
|
@pytest.fixture
|
|
def spans(monkeypatch) -> InMemorySpanExporter:
|
|
"""把两个抓取客户端模块级 tracer 换成写内存的，用于断言 span"""
|
|
exporter = InMemorySpanExporter()
|
|
provider = TracerProvider()
|
|
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
|
tracer = provider.get_tracer("test")
|
|
monkeypatch.setattr(rakuten_module, "tracer", tracer)
|
|
monkeypatch.setattr(rakuma_module, "tracer", tracer)
|
|
return exporter
|
|
|
|
|
|
def _settings() -> Settings:
|
|
return Settings(_env_file=None, otel_snapshot_max_bytes=2_000_000)
|
|
|
|
|
|
class _StubSession:
|
|
"""按需返回 HTML 或抛异常的会话桩；两站的 fetch_html 签名差异在这里吸收"""
|
|
|
|
def __init__(self, *, html: str = "", error: Exception | None = None):
|
|
self._html = html
|
|
self._error = error
|
|
|
|
async def fetch_html(self, url: str, *, mobile: bool = False) -> str:
|
|
if self._error is not None:
|
|
raise self._error
|
|
return self._html
|
|
|
|
async def fetch(self, url: str, *, mobile: bool = False, validator=None):
|
|
if self._error is not None:
|
|
raise self._error
|
|
raise AssertionError("本测试不该走到 fetch 的成功分支")
|
|
|
|
|
|
def _span(exporter: InMemorySpanExporter, name: str):
|
|
return next(s for s in exporter.get_finished_spans() if s.name == name)
|
|
|
|
|
|
async def test_parse_failure_preserves_original_exception(spans):
|
|
"""回归：解析失败要原样抛出业务异常，不能被埋点自身的 TypeError 顶替
|
|
|
|
这是原 bug 最直接的后果——上游拿到的不再是 4001（解析失败），而是一个
|
|
TypeError 兜底成的 500，错误码表整个失效。
|
|
"""
|
|
client = RakutenClient(_settings(), _StubSession(html="<html>no state</html>"))
|
|
|
|
with pytest.raises(ScrapeParseError) as excinfo:
|
|
await client.search(SearchRequest(keyword="switch"))
|
|
# 是业务异常而不是埋点炸出来的 TypeError
|
|
assert isinstance(excinfo.value, AppError)
|
|
assert excinfo.value.err_code == 4001
|
|
|
|
|
|
async def test_parse_failure_records_reason_and_page_snapshot(spans):
|
|
"""页面取回来了但解析不出：stage=parse，且失败页面 HTML 落成 event
|
|
|
|
「站点改版了」只能靠当时那份 HTML 判断，所以快照必须真的落下来——原 bug 里
|
|
记异常那步先抛了，snapshot 这行永远执行不到。
|
|
"""
|
|
html = "<html>changed layout</html>"
|
|
client = RakutenClient(_settings(), _StubSession(html=html))
|
|
|
|
with pytest.raises(ScrapeParseError):
|
|
await client.search(SearchRequest(keyword="switch"))
|
|
|
|
span = _span(spans, "parse.rakuten.search")
|
|
assert span.status.status_code is StatusCode.ERROR
|
|
assert span.attributes["parse.stage"] == "parse"
|
|
# 真实异常类名，不是一律写死的 "parse_error"
|
|
assert span.attributes["parse.fail_reason"] == "ScrapeParseError"
|
|
assert span.attributes["error.code"] == 4001
|
|
assert span.attributes["error.retryable"] is False
|
|
|
|
snapshot = next(e for e in span.events if e.name == "parse.failed_html")
|
|
assert snapshot.attributes["snapshot.html"] == html
|
|
# 快照要能对上是哪个地址的页面
|
|
assert "search.rakuten.co.jp" in snapshot.attributes["parse.url"]
|
|
|
|
|
|
async def test_fetch_failure_marks_fetch_stage_without_snapshot(spans):
|
|
"""页面根本没取回来：stage=fetch，没有快照可落
|
|
|
|
与 parse 阶段的排查方向相反（通道/反爬/上游 5xx，而不是站点改版），
|
|
所以要能直接按 parse.stage 分流，而不是对着有没有 HTML 猜。
|
|
"""
|
|
blocked = UpstreamBlockedError("Blocked while fetching: challenge page detected")
|
|
client = RakutenClient(_settings(), _StubSession(error=blocked))
|
|
|
|
with pytest.raises(UpstreamBlockedError):
|
|
await client.search(SearchRequest(keyword="switch"))
|
|
|
|
span = _span(spans, "parse.rakuten.search")
|
|
assert span.attributes["parse.stage"] == "fetch"
|
|
assert span.attributes["parse.fail_reason"] == "UpstreamBlockedError"
|
|
# 反爬阻断是可重试的，这一位直接决定上游要不要重来
|
|
assert span.attributes["error.retryable"] is True
|
|
assert not [e for e in span.events if e.name == "parse.failed_html"]
|
|
|
|
|
|
async def test_item_detail_failure_records_reason(spans):
|
|
"""详情接口走的是 fetch()（带校验器）而非 fetch_html，失败分支同样要记全"""
|
|
blocked = UpstreamBlockedError("challenge page detected")
|
|
client = RakutenClient(_settings(), _StubSession(error=blocked))
|
|
|
|
with pytest.raises(UpstreamBlockedError):
|
|
await client.item_detail(
|
|
ItemDetailRequest(shop_code="someshop", item_code="10000001")
|
|
)
|
|
|
|
span = _span(spans, "parse.rakuten.item_detail")
|
|
assert span.status.status_code is StatusCode.ERROR
|
|
assert span.attributes["parse.fail_reason"] == "UpstreamBlockedError"
|
|
|
|
|
|
async def test_shop_items_marks_delegate_stage(spans):
|
|
"""shop_items 自己不抓页面：失败在转调的 shop_detail / search 里
|
|
|
|
按 html 推断会得出「fetch 失败」的错误结论（它手里从来没有 html），
|
|
所以这里显式标 delegate，且不落快照——真正的现场在被转调那个 span 上。
|
|
"""
|
|
blocked = UpstreamBlockedError("challenge page detected")
|
|
client = RakutenClient(_settings(), _StubSession(error=blocked))
|
|
|
|
with pytest.raises(UpstreamBlockedError):
|
|
await client.shop_items(ShopItemsRequest(shop_code="someshop"))
|
|
|
|
span = _span(spans, "parse.rakuten.shop_items")
|
|
assert span.attributes["parse.stage"] == "delegate"
|
|
assert not [e for e in span.events if e.name == "parse.failed_html"]
|
|
# 被转调的那一步才是现场所在，它自己落了快照
|
|
inner = _span(spans, "parse.rakuten.shop_detail")
|
|
assert inner.attributes["parse.fail_reason"] == "UpstreamBlockedError"
|
|
|
|
|
|
async def test_rakuma_parse_failure_records_reason_and_snapshot(spans):
|
|
"""ラクマ 侧五个接口是同一套失败分支，同样要能读出原因与现场"""
|
|
html = "<html>rakuma changed</html>"
|
|
client = RakumaClient(_settings(), _StubSession(html=html))
|
|
|
|
with pytest.raises(AppError):
|
|
await client.search(RakumaSearchRequest(keyword="switch"))
|
|
|
|
span = _span(spans, "parse.rakuma.search")
|
|
assert span.status.status_code is StatusCode.ERROR
|
|
assert span.attributes["parse.stage"] == "parse"
|
|
assert "error.code" in span.attributes
|
|
snapshot = next(e for e in span.events if e.name == "parse.failed_html")
|
|
assert snapshot.attributes["snapshot.html"] == html
|
|
|
|
|
|
# ---- 会话层：逐次尝试与升级路径 ----
|
|
#
|
|
# 这一层的关键信息是「升级路径」：换 cookie → 浏览器兜底 → 放弃。属性表达不了过程
|
|
# （同名后写覆盖先写，三次尝试跑完只剩最后一次），所以每次尝试与每次升级各记一条
|
|
# event。site_session / rakuma_session 都在 fetch 内部 `trace.get_tracer(__name__)`，
|
|
# 所以这里替 `trace.get_tracer` 本身（与 test_telemetry.py 的 spans 夹具同一手法）。
|
|
|
|
|
|
@pytest.fixture
|
|
def session_spans(monkeypatch) -> InMemorySpanExporter:
|
|
exporter = InMemorySpanExporter()
|
|
provider = TracerProvider()
|
|
provider.add_span_processor(SimpleSpanProcessor(exporter))
|
|
monkeypatch.setattr(session_module.trace, "get_tracer", provider.get_tracer)
|
|
return exporter
|
|
|
|
|
|
class _FakeBrowser:
|
|
"""浏览器兜底替身：visit 恒不可用，让升级链走完整条路"""
|
|
|
|
def __init__(self) -> None:
|
|
self.unavailable_reason = "playwright is not installed"
|
|
|
|
async def visit(self, url: str, *, mobile: bool):
|
|
return None
|
|
|
|
async def close(self) -> None:
|
|
pass
|
|
|
|
|
|
async def _blocked_session() -> SiteSession:
|
|
"""所有请求都回 Akamai 挑战页的会话，用 MockTransport 拦截"""
|
|
settings = Settings(
|
|
_env_file=None, http_max_attempts=3, request_timeout_seconds=5.0,
|
|
max_site_concurrency=4,
|
|
)
|
|
session = SiteSession(settings, _FakeBrowser())
|
|
await session.start()
|
|
for profile in session._profiles.values():
|
|
await profile.client.aclose()
|
|
profile.client = httpx.AsyncClient(
|
|
transport=httpx.MockTransport(
|
|
lambda _r: httpx.Response(
|
|
200, text="<html>Access Denied. Reference #18.abc</html>"
|
|
)
|
|
),
|
|
follow_redirects=True,
|
|
)
|
|
return session
|
|
|
|
|
|
async def test_retry_attempts_and_escalation_are_recorded_as_events(session_spans):
|
|
"""每次尝试与每次升级各留一条 event，链路上能读出完整升级路径
|
|
|
|
这些信息无法用属性表达：`scrape.attempts` 只剩最后一次的值，前两次为什么失败、
|
|
换过 cookie 没有、浏览器兜底试过没有全被覆盖掉。
|
|
"""
|
|
session = await _blocked_session()
|
|
try:
|
|
with pytest.raises(UpstreamBlockedError):
|
|
await session.fetch_html(
|
|
"https://search.rakuten.co.jp/search/mall/x/", mobile=False
|
|
)
|
|
finally:
|
|
await session.close()
|
|
|
|
span = _span(session_spans, "scrape.fetch")
|
|
assert span.status.status_code is StatusCode.ERROR
|
|
assert span.attributes["scrape.fail_reason"] == "UpstreamBlockedError"
|
|
assert span.attributes["scrape.challenge_detected"] is True
|
|
|
|
# 三次尝试都留下了自己的记录，而不是只剩最后一次
|
|
attempts = [e for e in span.events if e.name == "scrape.attempt"]
|
|
assert [e.attributes["attempt"] for e in attempts] == [1, 2, 3]
|
|
assert {e.attributes["outcome"] for e in attempts} == {"challenge"}
|
|
|
|
# 升级路径：第 1 次失败后换 cookie，第 2 次失败后动用浏览器（本例不可用）
|
|
escalations = [e for e in span.events if e.name == "scrape.escalate"]
|
|
assert [e.attributes["to"] for e in escalations] == ["rewarm_on_home", "browser"]
|
|
browser_step = escalations[-1]
|
|
assert browser_step.attributes["outcome"] == "failed"
|
|
# 「没装 playwright」与「装了也被挡」要能分开查
|
|
assert browser_step.attributes["reason"] == "playwright is not installed"
|
|
|
|
# 最终失败页面的快照带上来源，便于确认是哪条通道的哪个地址
|
|
failed = next(e for e in span.events if e.name == "scrape.failed_html")
|
|
assert failed.attributes["scrape.profile"] == "pc"
|
|
assert "search.rakuten.co.jp" in failed.attributes["scrape.url"]
|
|
|
|
|
|
async def test_successful_scrape_leaves_span_ok(spans, search_state):
|
|
"""成功路径不被误标 ERROR，不落失败快照，且照常记结果指标
|
|
|
|
作为上面那些失败断言的对照：证明 ERROR 状态与 parse.fail_reason 是真的由失败
|
|
触发的，不是每条 span 都长这样。
|
|
"""
|
|
html = f"<script>window.__INITIAL_STATE__ = {json.dumps(search_state)};</script>"
|
|
client = RakutenClient(_settings(), _StubSession(html=html))
|
|
|
|
result = await client.search(SearchRequest(keyword="switch"))
|
|
|
|
span = _span(spans, "parse.rakuten.search")
|
|
assert span.status.status_code is not StatusCode.ERROR
|
|
assert "parse.fail_reason" not in span.attributes
|
|
assert "error.type" not in span.attributes
|
|
assert not [e for e in span.events if e.name == "parse.failed_html"]
|
|
# 成功时记的是结果指标，与失败侧属性互不重叠
|
|
assert span.attributes["parse.items"] == len(result.items)
|