feat(observability): 失败响应与网关信封进链路,手工埋点尊重 suppressed
失败此前在 trace 里近乎不可见:异常处理器把异常吃掉换成信封响应,自动 instrumentation 只看到一个 HTTP 状态码,而 AppError 默认 400、信封里 success=false,跟正常返回分不出来。 - api.py:四个异常处理器(对外失败的唯一出口)各记一次 span;兜底处理器额外 record_error——对外只回一句无信息量的错误文案,异常类型与栈只在本地日志里 - telemetry.py:新增 record_envelope / record_parse_failure / span_unless_suppressed;record_error 补 error.message / retryable / status_code;snapshot 支持 extra 带上「这份 HTML 是哪来的」 - worker/client.py:_request 自建 span,活到解信封之后。httpx 那个 CLIENT span 在 request() 返回时就结束,此时信封还没解——success=false code=6002(租约 无效)在它看来是完成的 200 请求 - span_unless_suppressed:suppress_instrumentation 只被 instrumentation 库尊重, 手工 span 不看它,lease 空转长轮询会从这个口子把孤立 trace 放回来 - 两个 scraping client 的解析失败分支收拢到 record_parse_failure;shop_items 显式标 stage=delegate 且不落快照(它自己不抓页面,按 html 推断只会得出 「fetch 失败」的错误结论) Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -15,15 +15,25 @@ import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.trace import SpanKind
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import AppError
|
||||
from app.shared.proxy import httpx_client_options
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
from app.shared.telemetry import (
|
||||
record_envelope,
|
||||
record_error,
|
||||
set_attributes,
|
||||
span_unless_suppressed,
|
||||
)
|
||||
from app.trading.worker.models import LeaseTask, QueryTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
tracer = trace.get_tracer(__name__)
|
||||
|
||||
|
||||
class GatewayClient:
|
||||
"""网关 HTTP 客户端
|
||||
@@ -49,27 +59,58 @@ class GatewayClient:
|
||||
# ---- 基础封装 ----
|
||||
|
||||
async def _request(self, method: str, path: str, **kwargs: Any) -> dict[str, Any]:
|
||||
"""发起请求并解信封。失败(success=False)抛 AppError"""
|
||||
response = await self._client.request(method, path, **kwargs)
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as exc:
|
||||
raise AppError(
|
||||
message=f"网关响应不是合法 JSON:HTTP {response.status_code}",
|
||||
code="GATEWAY_BAD_BODY",
|
||||
err_code=3001,
|
||||
retryable=True,
|
||||
) from exc
|
||||
"""发起请求并解信封。失败(success=False)抛 AppError
|
||||
|
||||
if not body.get("success"):
|
||||
raise AppError(
|
||||
message=body.get("msg", "网关返回失败"),
|
||||
code="GATEWAY_ERROR",
|
||||
err_code=int(body.get("code", 1500)),
|
||||
retryable=False,
|
||||
status_code=response.status_code,
|
||||
整段套一个自己的 span,而不是依赖 httpx 自动 instrumentation 那个 CLIENT
|
||||
span:**网关的失败在信封里,不在 HTTP 状态码上**。httpx 那个 span 在
|
||||
`request()` 返回时就结束了,此时信封还没解——一次 `success=false, code=6002`
|
||||
(租约无效)的调用在它看来是完成的 200 请求,链路里跟成功毫无区别。
|
||||
本 span 活到解信封之后,所以能把「这次调用的结论」记下来。
|
||||
"""
|
||||
with span_unless_suppressed(
|
||||
tracer,
|
||||
f"gateway.{path.strip('/').replace('/', '.')}",
|
||||
kind=SpanKind.CLIENT,
|
||||
) as span:
|
||||
set_attributes(span, {"gateway.method": method, "gateway.path": path})
|
||||
response = await self._client.request(method, path, **kwargs)
|
||||
try:
|
||||
body = response.json()
|
||||
except ValueError as exc:
|
||||
err = AppError(
|
||||
message=f"网关响应不是合法 JSON:HTTP {response.status_code}",
|
||||
code="GATEWAY_BAD_BODY",
|
||||
err_code=3001,
|
||||
retryable=True,
|
||||
)
|
||||
record_error(span, err)
|
||||
span.set_attribute("gateway.status_code", response.status_code)
|
||||
raise err from exc
|
||||
|
||||
if not body.get("success"):
|
||||
err_code = int(body.get("code", 1500))
|
||||
msg = body.get("msg", "网关返回失败")
|
||||
# 记成信封结果而不是只抛异常:上报被网关拒时,链路里能直接按
|
||||
# api.code 筛出是哪一类拒绝(6002 租约无效 / 6003 状态不允许…),
|
||||
# 不必回头翻 worker 日志。
|
||||
record_envelope(
|
||||
span,
|
||||
success=False,
|
||||
err_code=err_code,
|
||||
msg=msg,
|
||||
status_code=response.status_code,
|
||||
)
|
||||
raise AppError(
|
||||
message=msg,
|
||||
code="GATEWAY_ERROR",
|
||||
err_code=err_code,
|
||||
retryable=False,
|
||||
status_code=response.status_code,
|
||||
)
|
||||
record_envelope(
|
||||
span, success=True, status_code=response.status_code
|
||||
)
|
||||
return body
|
||||
return body
|
||||
|
||||
# ---- 接口 ----
|
||||
|
||||
|
||||
Reference in New Issue
Block a user