feat(observability): /health 不再产生 server span
容器 HEALTHCHECK 每 30 秒探一次、上游也在轮询,这些请求各自是一条孤立 trace,量大且没有信息量——和 worker 空转长轮询同一个问题,把观测后台刷满 的正是它们。 instrument_app 传 excluded_urls,由新增的 RAKUTEN_OTEL_EXCLUDED_URLS 控制 (默认 /health$)。按 search 匹配完整 URL,锚点保证不误伤 /api/health-*; 留空时传 None,回落到 OTel 自己的 OTEL_PYTHON_FASTAPI_EXCLUDED_URLS。 三个服务共用 instrument_app,因此抓取/交易/网关一并生效。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -77,6 +77,10 @@ RAKUTEN_OTEL_SERVICE_NAME=
|
||||
RAKUTEN_OTEL_HEADERS=
|
||||
# 失败 HTML 快照上限字节(默认 2MB,超出截断并标注 truncated=true)
|
||||
RAKUTEN_OTEL_SNAPSHOT_MAX_BYTES=2000000
|
||||
# 不上报 server span 的路径(逗号分隔正则,按 search 匹配完整 URL)。
|
||||
# 默认排掉 /health:容器 HEALTHCHECK 每 30 秒探一次、上游也在轮询,
|
||||
# 这些请求各自是一条孤立 trace,量大且没有信息量。留空表示不排除。
|
||||
RAKUTEN_OTEL_EXCLUDED_URLS=/health$
|
||||
|
||||
# ---- 以下仅交易服务使用 ----
|
||||
# 人工登录后落盘的 cookie 目录(相对项目根目录)。
|
||||
|
||||
@@ -679,7 +679,9 @@ FastAPI 与 httpx 走自动埋点,但那只覆盖「收到 HTTP 请求」与
|
||||
- `account_query`:一张只读查询单的根 span,带 `query.outcome`
|
||||
|
||||
worker 空转的长轮询(每 30 秒问一次网关有没有活干)刻意不埋点——它们没有信息量,
|
||||
量却极大,会把观测后台刷满。
|
||||
量却极大,会把观测后台刷满。`/health` 同理:容器 HEALTHCHECK 每 30 秒探一次、上游
|
||||
也在轮询,默认由 `RAKUTEN_OTEL_EXCLUDED_URLS`(默认 `/health$`)挡在 server span
|
||||
之外。该项是逗号分隔的正则、按 search 匹配完整 URL,留空则不排除任何路径。
|
||||
|
||||
> 解析失败时页面 HTML 会作为 span event 上报(`RAKUTEN_OTEL_SNAPSHOT_MAX_BYTES`
|
||||
> 控制上限,默认 2MB),用于事后复现「抓到的内容为什么解析不出预期字段」。
|
||||
@@ -699,7 +701,7 @@ worker 空转的长轮询(每 30 秒问一次网关有没有活干)刻意不
|
||||
- 抓取:`RAKUTEN_MAX_SITE_CONCURRENCY`(默认 `8`,两站各自独立计数)、`RAKUTEN_HTTP_MAX_ATTEMPTS`(默认 `3`)、`RAKUTEN_SESSION_TTL_SECONDS`(默认 `1800`,仅乐天)
|
||||
- 浏览器兜底(仅乐天):`RAKUTEN_BROWSER_FALLBACK_ENABLED`、`RAKUTEN_BROWSER_HEADLESS`、`RAKUTEN_BROWSER_CHANNEL`
|
||||
- 代理(需日本 IP 时):`RAKUTEN_PROXY_SERVER`、`RAKUTEN_PROXY_USERNAME`、`RAKUTEN_PROXY_PASSWORD`、`RAKUTEN_PROXY_BYPASS`
|
||||
- 链路追踪(可选,默认关闭;三个服务共用):`RAKUTEN_OTEL_ENABLED`、`RAKUTEN_OTEL_ENDPOINT`、`RAKUTEN_OTEL_HEADERS`、`RAKUTEN_OTEL_SNAPSHOT_MAX_BYTES`
|
||||
- 链路追踪(可选,默认关闭;三个服务共用):`RAKUTEN_OTEL_ENABLED`、`RAKUTEN_OTEL_ENDPOINT`、`RAKUTEN_OTEL_HEADERS`、`RAKUTEN_OTEL_SNAPSHOT_MAX_BYTES`、`RAKUTEN_OTEL_EXCLUDED_URLS`(默认 `/health$`)
|
||||
|
||||
> 从中国大陆直连实测可用、无需代理;`RAKUTEN_PROXY_SERVER` 留空即可。设置后,所有面向
|
||||
> 外部站点的 HTTPX 与 Playwright 流量都会经代理;本机和 Docker 服务间地址由
|
||||
|
||||
@@ -103,6 +103,11 @@ class Settings(BaseSettings):
|
||||
otel_service_name: str = "rakuten" # 仅作兜底;实际值由两侧 main.py 显式覆盖
|
||||
otel_headers: str | None = None # OTLP 鉴权头,形如 "k=v,k=v";当前 endpoint 裸跑,留空
|
||||
otel_export_interval_ms: int = 5000
|
||||
# 不产生 server span 的路径(逗号分隔正则,按 search 匹配完整 URL)。
|
||||
# 默认排掉 /health:容器 HEALTHCHECK 每 30 秒探一次、上游也在轮询,这些请求
|
||||
# 各自成为一条孤立 trace,量大且没有信息量——和 worker 空转长轮询同一个问题
|
||||
# (见 telemetry.py::suppressed)。留空表示不排除任何路径。
|
||||
otel_excluded_urls: str = "/health$"
|
||||
# 解析失败时把页面 HTML 作为 span event 上报的上限字节;超出截断并标注。
|
||||
# 单个搜索页 HTML 可达 200KB-2MB,调高时同步关注 OTLP 单次请求大小限制。
|
||||
otel_snapshot_max_bytes: int = 2_000_000
|
||||
|
||||
@@ -121,10 +121,17 @@ def instrument_app(app: "FastAPI") -> None:
|
||||
照样产生 span),所以顺序不构成问题。
|
||||
|
||||
otel 关闭时跳过:省掉一层用不上的中间件。
|
||||
|
||||
`excluded_urls` 把健康检查挡在 server span 之外(默认 `/health$`):容器
|
||||
HEALTHCHECK 每 30 秒探一次、上游也在轮询,这些请求各自是一条孤立 trace,
|
||||
量大且没有信息量。配置留空时传 None,让 OTel 回落到它自己的
|
||||
`OTEL_PYTHON_FASTAPI_EXCLUDED_URLS` 环境变量。
|
||||
"""
|
||||
if not get_settings().otel_enabled:
|
||||
settings = get_settings()
|
||||
if not settings.otel_enabled:
|
||||
return
|
||||
FastAPIInstrumentor.instrument_app(app)
|
||||
excluded = settings.otel_excluded_urls.strip() or None
|
||||
FastAPIInstrumentor.instrument_app(app, excluded_urls=excluded)
|
||||
|
||||
|
||||
def shutdown_telemetry() -> None:
|
||||
|
||||
@@ -15,12 +15,15 @@ from __future__ import annotations
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from opentelemetry import trace
|
||||
from opentelemetry.instrumentation import fastapi as otel_fastapi
|
||||
from opentelemetry.instrumentation.asgi import OpenTelemetryMiddleware
|
||||
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
|
||||
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
|
||||
from opentelemetry.sdk.trace import TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.trace import StatusCode
|
||||
from opentelemetry.util.http import parse_excluded_urls
|
||||
|
||||
from app.shared import telemetry
|
||||
from app.shared.config import Settings
|
||||
@@ -70,6 +73,59 @@ def test_instrument_app_works_before_setup_telemetry(monkeypatch):
|
||||
FastAPIInstrumentor.uninstrument_app(app)
|
||||
|
||||
|
||||
def _otel_middleware(app: FastAPI) -> OpenTelemetryMiddleware:
|
||||
"""从构建好的中间件栈里挖出 ASGI 打桩中间件,用于断言它的排除规则"""
|
||||
node = app.build_middleware_stack()
|
||||
while node is not None:
|
||||
if isinstance(node, OpenTelemetryMiddleware):
|
||||
return node
|
||||
node = getattr(node, "app", None)
|
||||
raise AssertionError("中间件栈里没有 OpenTelemetryMiddleware")
|
||||
|
||||
|
||||
def test_health_excluded_from_server_spans(monkeypatch):
|
||||
"""/health 不产生 server span,其它路径照常
|
||||
|
||||
容器 HEALTHCHECK 每 30 秒探一次、上游也在轮询,这些请求各自是一条孤立 trace,
|
||||
量大且没有信息量。排除规则按 search 匹配完整 URL,所以要钉住两件事:/health
|
||||
真的被挡掉,且 `$` 锚点没有顺手把 /api/* 一起挡掉。
|
||||
"""
|
||||
monkeypatch.setattr(
|
||||
telemetry, "get_settings", lambda: Settings(_env_file=None, otel_enabled=True)
|
||||
)
|
||||
app = FastAPI()
|
||||
try:
|
||||
telemetry.instrument_app(app)
|
||||
excluded = _otel_middleware(app).excluded_urls
|
||||
assert excluded.url_disabled("http://127.0.0.1:31108/health")
|
||||
assert not excluded.url_disabled("http://127.0.0.1:31108/api/cart/add")
|
||||
# 前缀匹配会误伤的反例:锚点保证只有 /health 本身被排除
|
||||
assert not excluded.url_disabled("http://127.0.0.1:31108/api/health-detail")
|
||||
finally:
|
||||
FastAPIInstrumentor.uninstrument_app(app)
|
||||
|
||||
|
||||
def test_excluded_urls_empty_falls_back_to_env(monkeypatch):
|
||||
"""配置留空时传 None,让 OTel 回落到它自己的环境变量而不是排除空字符串"""
|
||||
monkeypatch.setattr(
|
||||
telemetry,
|
||||
"get_settings",
|
||||
lambda: Settings(_env_file=None, otel_enabled=True, otel_excluded_urls=" "),
|
||||
)
|
||||
# OTel 那份环境变量在模块导入时就解析成常量了,setenv 已经晚了,只能直接替常量
|
||||
monkeypatch.setattr(
|
||||
otel_fastapi, "_excluded_urls_from_env", parse_excluded_urls("/metrics")
|
||||
)
|
||||
app = FastAPI()
|
||||
try:
|
||||
telemetry.instrument_app(app)
|
||||
excluded = _otel_middleware(app).excluded_urls
|
||||
assert excluded.url_disabled("http://127.0.0.1:31108/metrics")
|
||||
assert not excluded.url_disabled("http://127.0.0.1:31108/health")
|
||||
finally:
|
||||
FastAPIInstrumentor.uninstrument_app(app)
|
||||
|
||||
|
||||
def test_instrument_app_skipped_when_otel_disabled(monkeypatch):
|
||||
"""otel 关闭时不装中间件,省掉一层用不上的开销"""
|
||||
monkeypatch.setattr(
|
||||
|
||||
Reference in New Issue
Block a user