119 lines
4.1 KiB
Python
119 lines
4.1 KiB
Python
"""应用配置：通过环境变量和 .env 文件加载所有配置项
|
|
|
|
配置项统一使用 RAKUTEN_ 前缀，例如 RAKUTEN_APP_PORT=31107。
|
|
支持 .env 文件自动加载。
|
|
"""
|
|
from functools import lru_cache
|
|
from pathlib import Path
|
|
from typing import Literal
|
|
|
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
|
|
from app.core import site
|
|
|
|
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
|
|
|
|
|
class Settings(BaseSettings):
|
|
"""应用全局配置
|
|
|
|
所有配置项均可通过环境变量覆盖，前缀为 RAKUTEN_。
|
|
例如：RAKUTEN_APP_PORT=31107 对应 app_port 配置项。
|
|
"""
|
|
|
|
model_config = SettingsConfigDict(
|
|
env_file=str(BASE_DIR / ".env"),
|
|
env_file_encoding="utf-8",
|
|
env_prefix="RAKUTEN_",
|
|
extra="ignore",
|
|
)
|
|
|
|
# ---- 服务基本配置 ----
|
|
app_name: str = "Rakuten Scraper Service"
|
|
app_env: Literal["dev", "prod", "test"] = "dev"
|
|
app_host: str = "0.0.0.0"
|
|
app_port: int = 31107
|
|
|
|
# ---- 日志配置 ----
|
|
log_level: str = "INFO"
|
|
log_to_file: bool | None = None # None 表示根据环境自动决定
|
|
log_dir: str = "logs"
|
|
log_rotation: str = "100 MB"
|
|
log_retention: str = "14 days"
|
|
log_compression: str = "zip"
|
|
log_format: str = "{time:YYYY-MM-DD HH:mm:ss} {level} {message}"
|
|
log_enqueue: bool = True
|
|
|
|
# ---- 鉴权配置 ----
|
|
bearer_token: str = "REPLACE_WITH_TOKEN_32CHARS"
|
|
|
|
# ---- HTTP 抓取配置 ----
|
|
request_timeout_seconds: float = 30.0
|
|
max_site_concurrency: int = 8 # 对站点的最大并发请求数
|
|
http_max_attempts: int = 3 # 单次抓取的最大尝试次数（含首次）
|
|
session_ttl_seconds: float = 1800.0 # Akamai cookie 会话最长复用时长，超时后重新预热
|
|
|
|
# ---- 浏览器兜底配置 ----
|
|
# 纯 HTTP 被 Akamai 拦截时，用 Playwright 打开页面取回 cookie 再回灌给
|
|
# HTTP 客户端重试。日常流量不会触发；未安装 playwright 时自动降级为不兜底。
|
|
browser_fallback_enabled: bool = True
|
|
browser_headless: bool | None = None # None 表示根据环境自动决定
|
|
browser_channel: str | None = None # 例如 chrome；留空使用 bundled chromium
|
|
browser_launch_timeout_seconds: float = 60.0
|
|
browser_nav_timeout_seconds: float = 60.0
|
|
|
|
# ---- 代理配置（可选，用于日本 IP）----
|
|
proxy_server: str | None = None
|
|
proxy_username: str | None = None
|
|
proxy_password: str | None = None
|
|
|
|
# ---- 目标站点 ----
|
|
home_url: str = site.HOME_URL
|
|
|
|
@property
|
|
def browser_headless_effective(self) -> bool:
|
|
"""浏览器无头模式：显式配置优先，否则开发环境使用有头模式方便调试"""
|
|
if self.browser_headless is not None:
|
|
return self.browser_headless
|
|
return self.app_env != "dev"
|
|
|
|
@property
|
|
def log_to_file_effective(self) -> bool:
|
|
"""是否写日志文件：显式配置优先，否则生产环境默认写文件"""
|
|
if self.log_to_file is not None:
|
|
return self.log_to_file
|
|
return self.app_env == "prod"
|
|
|
|
@property
|
|
def playwright_proxy(self) -> dict[str, str] | None:
|
|
"""构建 Playwright 代理配置字典"""
|
|
if not self.proxy_server:
|
|
return None
|
|
|
|
proxy: dict[str, str] = {"server": self.proxy_server}
|
|
if self.proxy_username:
|
|
proxy["username"] = self.proxy_username
|
|
if self.proxy_password:
|
|
proxy["password"] = self.proxy_password
|
|
return proxy
|
|
|
|
@property
|
|
def httpx_proxy(self) -> str | None:
|
|
"""构建 httpx 代理 URL（含认证信息）"""
|
|
if not self.proxy_server:
|
|
return None
|
|
if not self.proxy_username:
|
|
return self.proxy_server
|
|
|
|
scheme, _, rest = self.proxy_server.partition("://")
|
|
if not rest:
|
|
return self.proxy_server
|
|
credentials = f"{self.proxy_username}:{self.proxy_password or ''}"
|
|
return f"{scheme}://{credentials}@{rest}"
|
|
|
|
|
|
@lru_cache(maxsize=1)
|
|
def get_settings() -> Settings:
|
|
"""获取全局配置单例"""
|
|
return Settings()
|