q792602257andClaude Opus 4.6
2b7db521c4
抓取-解析链路原本只有日志，出现"抓回内容但解析不出预期字段"时定位慢。 接入 OpenTelemetry traces（FastAPI/httpx 自动 + 手写 fetch/parse span）， 解析失败时把页面 HTML 作为 span event 上报，便于事后复现。 Dockerfile 默认开启，镜像一启动即导出到自建 OTLP endpoint。 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
103 lines
3.7 KiB
Python
103 lines
3.7 KiB
Python
"""抓取服务入口：FastAPI 应用创建与生命周期管理
|
|
|
|
职责：
|
|
- 构建抓取服务容器（依赖注入）
|
|
- 管理应用生命周期（启动/关闭抓取会话与兜底浏览器）
|
|
- 注册抓取路由和全局异常处理器
|
|
|
|
这个进程全程匿名、无状态，可按需要多开实例。需要账号登录态的加购、下单与
|
|
订单监控在交易服务里（`python -m app.trading.main`），两者独立部署。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from contextlib import asynccontextmanager
|
|
|
|
from fastapi import FastAPI
|
|
|
|
from app.scraping.api.routes.health import router as health_router
|
|
from app.scraping.api.routes.rakuma import router as rakuma_router
|
|
from app.scraping.api.routes.scrape import router as scrape_router
|
|
from app.scraping.container import ScrapingContainer
|
|
from app.scraping.services.browser_fallback import BrowserFallback
|
|
from app.scraping.services.rakuma_client import RakumaClient
|
|
from app.scraping.services.rakuma_session import RakumaSession
|
|
from app.scraping.services.rakuten_client import RakutenClient
|
|
from app.scraping.services.site_session import SiteSession
|
|
from app.shared.api import register_exception_handlers
|
|
from app.shared.config import get_settings
|
|
from app.shared.logging_setup import configure_logging
|
|
from app.shared.telemetry import instrument_app, setup_telemetry, shutdown_telemetry
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def build_container() -> ScrapingContainer:
|
|
"""构建抓取服务容器，组装所有依赖"""
|
|
settings = get_settings()
|
|
browser_fallback = BrowserFallback(settings)
|
|
site_session = SiteSession(settings, browser_fallback)
|
|
rakuten_client = RakutenClient(settings, site_session)
|
|
rakuma_session = RakumaSession(settings)
|
|
rakuma_client = RakumaClient(settings, rakuma_session)
|
|
return ScrapingContainer(
|
|
settings=settings,
|
|
browser_fallback=browser_fallback,
|
|
site_session=site_session,
|
|
rakuten_client=rakuten_client,
|
|
rakuma_session=rakuma_session,
|
|
rakuma_client=rakuma_client,
|
|
)
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
"""应用生命周期管理：启动时初始化各服务，关闭时释放资源"""
|
|
container = build_container()
|
|
app.state.container = container
|
|
|
|
configure_logging(container.settings)
|
|
# telemetry 必须在创建任何 httpx.AsyncClient 之前完成 setup，否则 httpx
|
|
# 自动 instrumentation 抓不到现有客户端的请求。
|
|
setup_telemetry(container.settings, service_name="rakuten-scraping")
|
|
logger.info("抓取服务启动：%s:%s", container.settings.app_host, container.settings.app_port)
|
|
logger.info("日志级别：%s", container.settings.log_level)
|
|
logger.info("当前环境：%s", container.settings.app_env)
|
|
await container.site_session.start()
|
|
await container.rakuma_session.start()
|
|
try:
|
|
yield
|
|
finally:
|
|
await container.rakuma_session.close()
|
|
await container.site_session.close()
|
|
await container.browser_fallback.close()
|
|
shutdown_telemetry()
|
|
|
|
|
|
def create_app() -> FastAPI:
|
|
"""创建 FastAPI 应用实例，注册路由和异常处理器"""
|
|
app = FastAPI(title="Rakuten Scraper Service", lifespan=lifespan)
|
|
app.include_router(health_router)
|
|
app.include_router(scrape_router)
|
|
app.include_router(rakuma_router)
|
|
register_exception_handlers(app)
|
|
instrument_app(app)
|
|
return app
|
|
|
|
|
|
app = create_app()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
|
|
settings = get_settings()
|
|
configure_logging(settings)
|
|
uvicorn.run(
|
|
"app.scraping.main:app",
|
|
host=settings.app_host,
|
|
port=settings.app_port,
|
|
log_config=None,
|
|
timeout_keep_alive=120,
|
|
)
|