Init
This commit is contained in:
@@ -0,0 +1 @@
|
||||
"""Core configuration and infrastructure helpers."""
|
||||
@@ -0,0 +1,131 @@
|
||||
"""应用配置：通过环境变量和 .env 文件加载所有配置项
|
||||
|
||||
配置项统一使用 SURUGAYA_ 前缀，例如 SURUGAYA_APP_PORT=31106。
|
||||
支持 .env 文件自动加载。
|
||||
"""
|
||||
from functools import lru_cache
|
||||
from typing import Literal
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from pathlib import Path
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""应用全局配置
|
||||
|
||||
所有配置项均可通过环境变量覆盖，前缀为 SURUGAYA_。
|
||||
例如：SURUGAYA_APP_PORT=31106 对应 app_port 配置项。
|
||||
"""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=str(BASE_DIR / ".env"), # 使用绝对路径
|
||||
env_file_encoding="utf-8",
|
||||
env_prefix="SURUGAYA_",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# ---- 服务基本配置 ----
|
||||
app_name: str = "Surugaya Scraper Service"
|
||||
app_env: Literal["dev", "prod", "test"] = "dev"
|
||||
app_host: str = "0.0.0.0"
|
||||
app_port: int = 31106
|
||||
|
||||
# ---- 日志配置 ----
|
||||
log_level: str = "DEBUG"
|
||||
log_to_file: bool | None = None # None 表示根据环境自动决定
|
||||
log_dir: str = "logs"
|
||||
log_rotation: str = "10 MB"
|
||||
log_retention: str = "14 days"
|
||||
log_compression: str = "zip"
|
||||
log_format: str = "{time:YYYY-MM-DD HH:mm:ss} {level} {message}"
|
||||
log_enqueue: bool = True
|
||||
|
||||
# ---- 鉴权配置 ----
|
||||
bearer_token: str = "NilsO8Il8yoT1wazJFs2eeOmlPRfWuSx"
|
||||
|
||||
# ---- 浏览器配置 ----
|
||||
browser_enabled: bool = True
|
||||
browser_headless: bool | None = None # None 表示根据环境自动决定
|
||||
browser_keep_open: bool | None = None # None 表示根据环境自动决定
|
||||
browser_keep_open_on_error: bool | None = None # None 表示根据环境自动决定
|
||||
browser_channel: str | None = "chrome"
|
||||
browser_pool_size: int = 3
|
||||
max_site_concurrency: int = 5 # 最大并发页面数
|
||||
page_acquire_timeout_seconds: float = 60.0 # 获取页面槽位超时
|
||||
request_timeout_seconds: float = 120.0 # 页面请求超时
|
||||
cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时
|
||||
session_ttl_seconds: int = 900 # 会话过期时间（秒）
|
||||
browser_locale: str = "ja-JP"
|
||||
browser_timezone_id: str = "Asia/Tokyo"
|
||||
browser_user_agent: str = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式
|
||||
|
||||
# ---- Redis 配置（可选，未配置时使用进程内存）----
|
||||
redis_host: str = ""
|
||||
redis_port: int = 6379
|
||||
redis_password: str | None = None
|
||||
redis_db: int = 0
|
||||
session_namespace: str = "jp_surugaya"
|
||||
|
||||
# ---- 代理配置（可选，用于日本 IP）----
|
||||
proxy_server: str | None = None
|
||||
proxy_username: str | None = None
|
||||
proxy_password: str | None = None
|
||||
|
||||
# ---- 目标站点 ----
|
||||
base_url: str = "https://www.suruga-ya.jp"
|
||||
|
||||
@property
|
||||
def browser_headless_effective(self) -> bool:
|
||||
"""浏览器无头模式：显式配置优先，否则开发环境使用有头模式方便调试"""
|
||||
if self.browser_headless is not None:
|
||||
return self.browser_headless
|
||||
if self.app_env == "dev":
|
||||
return False
|
||||
return True
|
||||
|
||||
@property
|
||||
def browser_keep_open_on_error_effective(self) -> bool:
|
||||
"""出错时是否保留浏览器窗口：仅在开发环境有头模式下保留"""
|
||||
if self.browser_keep_open_on_error is not None:
|
||||
return self.browser_keep_open_on_error
|
||||
return self.app_env == "dev" and not self.browser_headless_effective
|
||||
|
||||
@property
|
||||
def browser_keep_open_effective(self) -> bool:
|
||||
"""成功后是否保留浏览器窗口：仅在开发环境有头模式下保留"""
|
||||
if self.browser_keep_open is not None:
|
||||
return self.browser_keep_open
|
||||
return self.app_env == "dev" and not self.browser_headless_effective
|
||||
|
||||
@property
|
||||
def log_to_file_effective(self) -> bool:
|
||||
"""是否写日志文件：显式配置优先，否则生产环境默认写文件"""
|
||||
if self.log_to_file is not None:
|
||||
return self.log_to_file
|
||||
return self.app_env == "prod"
|
||||
|
||||
@property
|
||||
def proxy(self) -> dict[str, str] | None:
|
||||
"""构建 Playwright 代理配置字典"""
|
||||
if not self.proxy_server:
|
||||
return None
|
||||
|
||||
proxy: dict[str, str] = {"server": self.proxy_server}
|
||||
if self.proxy_username:
|
||||
proxy["username"] = self.proxy_username
|
||||
if self.proxy_password:
|
||||
proxy["password"] = self.proxy_password
|
||||
return proxy
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
"""获取全局配置单例"""
|
||||
return Settings()
|
||||
@@ -0,0 +1,25 @@
|
||||
"""服务容器：集中管理所有服务实例，用于依赖注入"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.browser_pool import BrowserPool
|
||||
from app.services.cloudflare_session import CloudflareSessionManager
|
||||
from app.services.session_store import SessionStore
|
||||
from app.services.surugaya_client import SurugayaClient
|
||||
from app.services.cargo_service import CargoService
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ServiceContainer:
|
||||
"""服务容器，持有所有核心服务实例
|
||||
|
||||
通过 FastAPI 的 app.state.container 在请求间共享，
|
||||
各路由通过依赖注入获取容器中的服务。
|
||||
"""
|
||||
|
||||
settings: Settings
|
||||
session_store: SessionStore
|
||||
browser_pool: BrowserPool
|
||||
session_manager: CloudflareSessionManager
|
||||
surugaya_client: SurugayaClient
|
||||
cargo_service: CargoService
|
||||
@@ -0,0 +1,78 @@
|
||||
"""应用异常定义：所有业务异常均继承自 AppError
|
||||
|
||||
错误码规范：
|
||||
- 1xxx: 请求/鉴权错误
|
||||
- 2xxx: 浏览器资源错误
|
||||
- 3xxx: Cloudflare/反爬相关错误
|
||||
- 4xxx: 页面解析错误
|
||||
"""
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
"""应用基础异常，携带错误码、HTTP 状态码和是否可重试信息"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
code: str,
|
||||
err_code: int,
|
||||
retryable: bool = False,
|
||||
status_code: int = 400,
|
||||
headers: dict[str, str] | None = None,
|
||||
):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code
|
||||
self.err_code = err_code
|
||||
self.retryable = retryable
|
||||
self.status_code = status_code
|
||||
self.headers = headers
|
||||
|
||||
|
||||
class AuthenticationError(AppError):
|
||||
"""鉴权失败（Bearer Token 无效或缺失）"""
|
||||
|
||||
def __init__(self, message: str = "Invalid credentials"):
|
||||
super().__init__(
|
||||
message=message,
|
||||
code="AUTH_INVALID",
|
||||
err_code=1001,
|
||||
retryable=False,
|
||||
status_code=401,
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
class BrowserUnavailableError(AppError):
|
||||
"""浏览器不可用（未启用或启动失败）"""
|
||||
|
||||
def __init__(self, message: str = "Browser pool is unavailable"):
|
||||
super().__init__(message=message, code="BROWSER_UNAVAILABLE", err_code=2001, retryable=True)
|
||||
|
||||
|
||||
class ResourceBusyError(AppError):
|
||||
"""浏览器资源繁忙（获取页面槽位超时）"""
|
||||
|
||||
def __init__(self, message: str = "Timed out while waiting for a browser slot"):
|
||||
super().__init__(message=message, code="RESOURCE_BUSY", err_code=2002, retryable=True)
|
||||
|
||||
|
||||
class CloudflareChallengeError(AppError):
|
||||
"""Cloudflare 挑战未在规定时间内通过"""
|
||||
|
||||
def __init__(self, message: str = "Cloudflare challenge did not clear in time"):
|
||||
super().__init__(message=message, code="CLOUDFLARE_CHALLENGE", err_code=3001, retryable=True)
|
||||
|
||||
|
||||
class UpstreamBlockedError(AppError):
|
||||
"""上游请求被 Cloudflare 阻断（如 1020/403 错误）"""
|
||||
|
||||
def __init__(self, message: str = "The upstream request was blocked"):
|
||||
super().__init__(message=message, code="UPSTREAM_BLOCKED", err_code=3002, retryable=True)
|
||||
|
||||
|
||||
class ScrapeParseError(AppError):
|
||||
"""页面解析失败"""
|
||||
|
||||
def __init__(self, message: str = "Failed to parse the target page"):
|
||||
super().__init__(message=message, code="PARSE_ERROR", err_code=4001, retryable=False)
|
||||
@@ -0,0 +1,82 @@
|
||||
"""日志配置：使用 loguru 替代标准 logging，统一管理日志输出
|
||||
|
||||
- 控制台输出：始终启用
|
||||
- 文件输出：生产环境默认启用，支持日志轮转、压缩和保留策略
|
||||
- 拦截标准 logging：将 uvicorn 等第三方库的日志转发到 loguru
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import FrameType
|
||||
|
||||
from loguru import logger as loguru_logger
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
class InterceptHandler(logging.Handler):
|
||||
"""拦截标准 logging 日志，转发到 loguru"""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
try:
|
||||
level: str | int = loguru_logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = record.levelno
|
||||
|
||||
frame: FrameType | None = logging.currentframe()
|
||||
depth = 2
|
||||
while frame and frame.f_code.co_filename == logging.__file__:
|
||||
frame = frame.f_back
|
||||
depth += 1
|
||||
|
||||
loguru_logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||
|
||||
|
||||
def configure_logging(settings: Settings) -> None:
|
||||
"""根据配置初始化日志系统
|
||||
|
||||
- 移除 loguru 默认 handler
|
||||
- 添加控制台输出
|
||||
- 生产环境添加文件输出（轮转、压缩、保留策略）
|
||||
- 拦截标准 logging，统一走 loguru 输出
|
||||
"""
|
||||
loguru_logger.remove()
|
||||
loguru_logger.add(
|
||||
sys.stdout,
|
||||
level=settings.log_level.upper(),
|
||||
format=settings.log_format,
|
||||
enqueue=settings.log_enqueue,
|
||||
backtrace=False,
|
||||
diagnose=False,
|
||||
)
|
||||
|
||||
if settings.log_to_file_effective:
|
||||
log_dir = Path(settings.log_dir)
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_path = log_dir / f"app_{settings.app_env}.log"
|
||||
loguru_logger.add(
|
||||
str(log_path),
|
||||
level=settings.log_level.upper(),
|
||||
format=settings.log_format,
|
||||
rotation=settings.log_rotation,
|
||||
retention=settings.log_retention,
|
||||
compression=settings.log_compression,
|
||||
enqueue=settings.log_enqueue,
|
||||
encoding="utf-8",
|
||||
backtrace=False,
|
||||
diagnose=False,
|
||||
)
|
||||
|
||||
intercept = InterceptHandler()
|
||||
logging.basicConfig(handlers=[intercept], level=getattr(logging, settings.log_level.upper(), logging.INFO), force=True)
|
||||
|
||||
# 将 uvicorn 的日志也转发到 loguru
|
||||
for name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
|
||||
logger = logging.getLogger(name)
|
||||
logger.handlers = [intercept]
|
||||
logger.propagate = False
|
||||
|
||||
# 降低 asyncio 的日志级别，避免过多输出
|
||||
logging.getLogger("asyncio").setLevel(logging.WARNING)
|
||||
Reference in New Issue
Block a user