feat(observability): 失败响应与网关信封进链路，手工埋点尊重 suppressed

失败此前在 trace 里近乎不可见：异常处理器把异常吃掉换成信封响应，自动 instrumentation 只看到一个 HTTP 状态码，而 AppError 默认 400、信封里 success=false，跟正常返回分不出来。 - api.py：四个异常处理器（对外失败的唯一出口）各记一次 span；兜底处理器额外 record_error——对外只回一句无信息量的错误文案，异常类型与栈只在本地日志里 - telemetry.py：新增 record_envelope / record_parse_failure / span_unless_suppressed；record_error 补 error.message / retryable / status_code；snapshot 支持 extra 带上「这份 HTML 是哪来的」 - worker/client.py：_request 自建 span，活到解信封之后。httpx 那个 CLIENT span 在 request() 返回时就结束，此时信封还没解——success=false code=6002（租约无效）在它看来是完成的 200 请求 - span_unless_suppressed：suppress_instrumentation 只被 instrumentation 库尊重，手工 span 不看它，lease 空转长轮询会从这个口子把孤立 trace 放回来 - 两个 scraping client 的解析失败分支收拢到 record_parse_failure；shop_items 显式标 stage=delegate 且不落快照（它自己不抓页面，按 html 推断只会得出「fetch 失败」的错误结论） Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-28 16:07:00 +08:00
parent 4cc30bc058
commit 8381896eeb
6 changed files with 451 additions and 72 deletions
@@ -5,6 +5,9 @@
 2. enabled=true + endpoint 时 setup 注册真实 TracerProvider；shutdown 复位。
 3. instrument_app 在 setup 之前调用也要真的装上中间件（三个服务都是导入期打桩）。
 4. traced / set_attributes / record_error 的行为。
+5. 「采集返回结果」这一侧：record_envelope（信封失败在 HTTP 层看不出来）、
+   record_parse_failure（失败在 fetch 还是 parse）、snapshot（失败页面快照），
+   以及 span_unless_suppressed 与 suppressed() 的配套关系。

 不打真实网络：OTLPSpanExporter 创建时不发请求，BatchSpanProcessor 异步批量
 上报在没有 span 产生时也不会触发。span 断言用独立的 InMemory provider，不碰
@@ -27,7 +30,7 @@ from opentelemetry.util.http import parse_excluded_urls

 from app.shared import telemetry
 from app.shared.config import Settings
-from app.shared.errors import OrderGuardError
+from app.shared.errors import OrderGuardError, ScrapeParseError, UpstreamBlockedError
 from app.shared.telemetry import is_initialized, setup_telemetry, shutdown_telemetry


@@ -189,6 +192,153 @@ def test_set_attributes_skips_none(spans):
    assert "b" not in attributes


+def test_record_error_keeps_app_error_fields(spans):
+    """AppError 的排查字段（错误码/可重试/状态码/消息）都要落到属性上
+
+    只落 error.type 不够：上游看到的是错误码，「该不该重试」看 retryable，
+    而 record_exception 记的 event 在多数观测后台里要展开才看得到、列表页筛不出来。
+    """
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.err") as span:
+        telemetry.record_error(span, UpstreamBlockedError("Akamai 挑战页"))
+
+    attributes = spans.get_finished_spans()[0].attributes
+    assert attributes["error.type"] == "UpstreamBlockedError"
+    assert attributes["error.code"] == UpstreamBlockedError().err_code
+    assert attributes["error.retryable"] is True
+    assert attributes["error.status_code"] == 400
+    assert "Akamai 挑战页" in attributes["error.message"]
+
+
+def test_record_error_on_plain_exception_omits_app_error_fields(spans):
+    """非 AppError 不应凭空长出 error.code / error.retryable 属性"""
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.plain") as span:
+        telemetry.record_error(span, RuntimeError("boom"))
+
+    attributes = spans.get_finished_spans()[0].attributes
+    assert attributes["error.type"] == "RuntimeError"
+    assert "error.code" not in attributes
+    assert "error.retryable" not in attributes
+
+
+def test_record_envelope_failure_marks_span_error(spans):
+    """信封失败要置 ERROR 并落错误码——HTTP 层看不出这次调用失败了"""
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.envelope") as span:
+        telemetry.record_envelope(
+            span, success=False, err_code=6002, msg="租约无效", status_code=409
+        )
+
+    finished = spans.get_finished_spans()[0]
+    assert finished.status.status_code is StatusCode.ERROR
+    assert finished.attributes["api.success"] is False
+    assert finished.attributes["api.code"] == 6002
+    assert finished.attributes["api.status_code"] == 409
+    assert finished.attributes["api.msg"] == "租约无效"
+
+
+def test_record_envelope_success_leaves_status_ok(spans):
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.envelope_ok") as span:
+        telemetry.record_envelope(span, success=True, status_code=200)
+
+    finished = spans.get_finished_spans()[0]
+    assert finished.status.status_code is not StatusCode.ERROR
+    assert finished.attributes["api.success"] is True
+
+
+def test_snapshot_truncates_and_carries_extra(spans):
+    """超限 HTML 截断并标注，附带的来源信息（URL 等）也要落在同一条 event 上"""
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.snapshot") as span:
+        telemetry.snapshot(
+            span, "parse.failed_html", "x" * 100, 10,
+            extra={"parse.url": "https://example.com/a", "parse.ignored": None},
+        )
+
+    event = spans.get_finished_spans()[0].events[0]
+    assert event.name == "parse.failed_html"
+    assert event.attributes["snapshot.html"] == "x" * 10
+    assert event.attributes["snapshot.original_bytes"] == 100
+    assert event.attributes["snapshot.truncated"] is True
+    assert event.attributes["parse.url"] == "https://example.com/a"
+    assert "parse.ignored" not in event.attributes
+
+
+def test_snapshot_skips_empty_html(spans):
+    """页面根本没取回来时不记空 event"""
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.snapshot_empty") as span:
+        telemetry.snapshot(span, "parse.failed_html", None, 100)
+        telemetry.snapshot(span, "parse.failed_html", "", 100)
+
+    assert spans.get_finished_spans()[0].events == ()
+
+
+def test_record_parse_failure_distinguishes_fetch_from_parse(spans):
+    """失败阶段按「HTML 有没有拿到」区分：两者排查方向相反
+
+    html 为空＝页面没取回来（通道/反爬/上游 5xx）；非空＝取回了但解析不出
+    （多半站点改版）。fail_reason 要给真实异常类名，不能一律写死 parse_error。
+    """
+    tracer = telemetry.trace.get_tracer(__name__)
+    with tracer.start_as_current_span("unit.fetch_fail") as span:
+        telemetry.record_parse_failure(
+            span, UpstreamBlockedError("挑战页"), html=None, max_bytes=100,
+            url="https://example.com/a",
+        )
+    with tracer.start_as_current_span("unit.parse_fail") as span:
+        telemetry.record_parse_failure(
+            span, ScrapeParseError("没有 state"), html="<html/>", max_bytes=100,
+            url="https://example.com/b",
+        )
+    with tracer.start_as_current_span("unit.delegate_fail") as span:
+        telemetry.record_parse_failure(
+            span, ScrapeParseError("转调失败"), stage="delegate",
+        )
+
+    by_name = {s.name: s for s in spans.get_finished_spans()}
+    fetch = by_name["unit.fetch_fail"]
+    assert fetch.attributes["parse.stage"] == "fetch"
+    assert fetch.attributes["parse.fail_reason"] == "UpstreamBlockedError"
+    # 页面没取回来，没有快照可落
+    assert [e.name for e in fetch.events] == ["exception"]
+
+    parsed = by_name["unit.parse_fail"]
+    assert parsed.attributes["parse.stage"] == "parse"
+    assert parsed.attributes["parse.fail_reason"] == "ScrapeParseError"
+    snapshot_event = next(e for e in parsed.events if e.name == "parse.failed_html")
+    assert snapshot_event.attributes["snapshot.html"] == "<html/>"
+    assert snapshot_event.attributes["parse.url"] == "https://example.com/b"
+
+    # 显式 stage 覆盖推断：编排方法的失败既不在自己的 fetch 也不在自己的 parse
+    assert by_name["unit.delegate_fail"].attributes["parse.stage"] == "delegate"
+
+
+def test_span_unless_suppressed_is_noop_inside_suppressed(spans):
+    """`suppressed()` 里手工埋点必须退化成 noop，否则空转长轮询绕开抑制刷满后台
+
+    `suppress_instrumentation` 只被 instrumentation 库尊重，手工
+    `start_as_current_span` 不看它——worker 的 lease 正是在 suppressed() 里调
+    GatewayClient._request 的。
+    """
+    tracer = telemetry.trace.get_tracer(__name__)
+
+    with telemetry.suppressed():
+        with telemetry.span_unless_suppressed(tracer, "unit.suppressed") as span:
+            # 属性/异常写在 noop span 上不能报错，调用方不必分支
+            telemetry.set_attributes(span, {"a": 1})
+            telemetry.record_error(span, RuntimeError("boom"))
+            assert not span.is_recording()
+
+    assert spans.get_finished_spans() == ()
+
+    with telemetry.span_unless_suppressed(tracer, "unit.not_suppressed") as span:
+        assert span.is_recording()
+    assert [s.name for s in spans.get_finished_spans()] == ["unit.not_suppressed"]
+
+
 def test_enabled_initializes_and_shutdown_releases():
    """enabled=true 时 setup 注册 TracerProvider，shutdown 后 _provider 复位"""
    settings = Settings(