拆分抓取与交易服务
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,260 @@
|
||||
"""登录态会话：持有已登录账号的 cookie，供加购与下单链路使用
|
||||
|
||||
与抓取链路（app/scraping 的 site_session / rakuma_session）不只是分模块，
|
||||
而是分进程运行，原因：
|
||||
|
||||
- 抓取是**匿名**的，cookie 只用来过 Akamai 限速，丢了重新预热即可，无状态可言。
|
||||
- 加购下单必须**带账号**，cookie 一旦失效不能自动恢复——乐天与 ラクマ 登录都有
|
||||
reCAPTCHA 与设备验证，只能由人重新登录一次。因此这里的失效处理是「明确报错让
|
||||
上游停下」，而不是像抓取那样静默重试。
|
||||
- 这份登录态在整个系统里只能有一份。跟抓取同进程的话，抓取一扩容就会复制出 N 份
|
||||
登录态与 N 个订单轮询，同一个账号被并发操作。
|
||||
|
||||
登录态来源是 Playwright 的 storage_state：由 `scripts/login.py` 起一个有头浏览器
|
||||
让人工登录一次后落盘，本服务只读取、不生产。账号密码不经过本服务，也不写日志。
|
||||
|
||||
cookie 会被同时喂给两处：
|
||||
- httpx 客户端 —— 加购这类纯表单提交走 HTTP 更快
|
||||
- Playwright context —— 下单确认页有 JS 参与，必要时用浏览器走完
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import NotLoggedInError, UpstreamRequestError
|
||||
from app.trading.core import auth_site
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AuthStatus:
|
||||
"""一个站点的登录态快照"""
|
||||
|
||||
site: str
|
||||
state_file_exists: bool
|
||||
logged_in: bool | None # None 表示尚未探测过
|
||||
checked_at: float | None = None
|
||||
detail: str = ""
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"site": self.site,
|
||||
"state_file_exists": self.state_file_exists,
|
||||
"logged_in": self.logged_in,
|
||||
"checked_age_seconds": (
|
||||
round(time.monotonic() - self.checked_at, 1) if self.checked_at else None
|
||||
),
|
||||
"detail": self.detail,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _SiteAuth:
|
||||
"""一个站点的登录通道：独立的 httpx 客户端与登录态缓存"""
|
||||
|
||||
name: str
|
||||
client: httpx.AsyncClient
|
||||
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
logged_in: bool | None = None
|
||||
checked_at: float | None = None
|
||||
detail: str = ""
|
||||
|
||||
|
||||
class AuthSession:
|
||||
"""持有两站登录态的会话
|
||||
|
||||
职责边界：只负责「有没有登录态、cookie 是什么、还有效吗」，
|
||||
具体加购/下单的业务请求由各自的 client 组装后借这里的 HTTP 客户端发出。
|
||||
"""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self._settings = settings
|
||||
self._sites: dict[str, _SiteAuth] = {}
|
||||
|
||||
# ---- 生命周期 ----
|
||||
|
||||
async def start(self) -> None:
|
||||
"""为两站创建带登录 cookie 的 HTTP 客户端
|
||||
|
||||
登录态文件不存在时同样创建客户端（只是没有 cookie），这样 /health 与
|
||||
/api/auth/status 能如实回报「未登录」，而不是整个服务起不来。
|
||||
"""
|
||||
for name, profile in auth_site.PROFILES.items():
|
||||
if name in self._sites:
|
||||
continue
|
||||
client = httpx.AsyncClient(
|
||||
headers=profile.headers(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
proxy=self._settings.httpx_proxy,
|
||||
http2=True,
|
||||
)
|
||||
self._sites[name] = _SiteAuth(name=name, client=client)
|
||||
loaded = self._load_cookies(name)
|
||||
logger.info("登录通道已就绪：site=%s cookies=%s", name, loaded)
|
||||
|
||||
async def close(self) -> None:
|
||||
for auth in self._sites.values():
|
||||
try:
|
||||
await auth.client.aclose()
|
||||
except Exception:
|
||||
logger.debug("关闭登录 HTTP 客户端失败：site=%s", auth.name, exc_info=True)
|
||||
self._sites.clear()
|
||||
|
||||
# ---- 登录态文件 ----
|
||||
|
||||
def state_path(self, site: str) -> Path:
|
||||
"""某站点 storage_state 文件的落盘路径"""
|
||||
return self._settings.auth_state_path / auth_site.profile(site).state_filename
|
||||
|
||||
def _load_cookies(self, site: str) -> int:
|
||||
"""把 storage_state 里的 cookie 灌进该站的 httpx 客户端，返回条数"""
|
||||
path = self.state_path(site)
|
||||
if not path.exists():
|
||||
return 0
|
||||
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError) as exc:
|
||||
logger.warning("登录态文件读取失败：site=%s path=%s err=%s", site, path, exc)
|
||||
return 0
|
||||
|
||||
auth = self._sites[site]
|
||||
auth.client.cookies.clear()
|
||||
count = 0
|
||||
for cookie in state.get("cookies", []):
|
||||
name = cookie.get("name")
|
||||
value = cookie.get("value")
|
||||
if not name or value is None:
|
||||
continue
|
||||
auth.client.cookies.set(
|
||||
name,
|
||||
value,
|
||||
domain=cookie.get("domain") or "",
|
||||
path=cookie.get("path") or "/",
|
||||
)
|
||||
count += 1
|
||||
return count
|
||||
|
||||
def reload(self, site: str) -> int:
|
||||
"""重新从磁盘加载登录态（人工登录完成后调用），返回 cookie 条数"""
|
||||
auth = self._sites.get(site)
|
||||
if auth is None:
|
||||
raise ValueError(f"未知站点：{site}")
|
||||
count = self._load_cookies(site)
|
||||
auth.logged_in = None
|
||||
auth.checked_at = None
|
||||
auth.detail = "已重新加载登录态，尚未探测"
|
||||
logger.info("登录态已重新加载：site=%s cookies=%s", site, count)
|
||||
return count
|
||||
|
||||
# ---- 登录态探测 ----
|
||||
|
||||
async def check(self, site: str) -> AuthStatus:
|
||||
"""探测某站登录态是否仍然有效
|
||||
|
||||
每次都真实打一次请求——登录态过期没有可靠的本地判据（cookie 的 expires
|
||||
与服务端会话不是一回事），只能问站点。
|
||||
"""
|
||||
auth = self._require_site(site)
|
||||
async with auth.lock:
|
||||
try:
|
||||
if site == "rakuten":
|
||||
logged_in, detail = await self._check_rakuten(auth)
|
||||
else:
|
||||
logged_in, detail = await self._check_rakuma(auth)
|
||||
except httpx.HTTPError as exc:
|
||||
raise UpstreamRequestError(
|
||||
f"登录态探测请求失败：site={site} err={type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
|
||||
auth.logged_in = logged_in
|
||||
auth.checked_at = time.monotonic()
|
||||
auth.detail = detail
|
||||
logger.info("登录态探测：site=%s logged_in=%s detail=%s", site, logged_in, detail)
|
||||
return self.status(site)
|
||||
|
||||
async def _check_rakuten(self, auth: _SiteAuth) -> tuple[bool, str]:
|
||||
"""购物车页含「現在ログインしていません」即未登录"""
|
||||
response = await auth.client.get(auth_site.PROFILES["rakuten"].probe_url)
|
||||
if response.status_code >= 400:
|
||||
return False, f"购物车页返回 status {response.status_code}"
|
||||
if not auth_site.is_logged_in(
|
||||
"rakuten", final_url=str(response.url), body=response.text
|
||||
):
|
||||
return False, "购物车页显示未登录"
|
||||
return True, "购物车页未出现未登录标记"
|
||||
|
||||
async def _check_rakuma(self, auth: _SiteAuth) -> tuple[bool, str]:
|
||||
"""/mypage 被重定向到 /users/sign_in 即未登录"""
|
||||
response = await auth.client.get(auth_site.PROFILES["rakuma"].probe_url)
|
||||
if response.status_code >= 400:
|
||||
return False, f"mypage 返回 status {response.status_code}"
|
||||
if not auth_site.is_logged_in(
|
||||
"rakuma", final_url=str(response.url), body=response.text
|
||||
):
|
||||
return False, "mypage 被重定向至登录页"
|
||||
return True, "mypage 正常返回"
|
||||
|
||||
async def require_logged_in(self, site: str) -> None:
|
||||
"""确保某站处于登录态，否则抛错
|
||||
|
||||
加购与下单前的统一入口。登录态失效时不做任何自动恢复尝试——两站登录都需要
|
||||
人工过验证码，只能让上游停下来重新跑一次 scripts/login.py。
|
||||
"""
|
||||
status = await self.check(site)
|
||||
if not status.logged_in:
|
||||
raise NotLoggedInError(site=site, detail=status.detail)
|
||||
|
||||
# ---- 对外访问 ----
|
||||
|
||||
@property
|
||||
def sites(self) -> tuple[str, ...]:
|
||||
"""已初始化的站点名"""
|
||||
return tuple(self._sites)
|
||||
|
||||
def client(self, site: str) -> httpx.AsyncClient:
|
||||
"""取该站带登录 cookie 的 HTTP 客户端"""
|
||||
return self._require_site(site).client
|
||||
|
||||
def cookies_for_browser(self, site: str) -> list[dict[str, Any]]:
|
||||
"""导出 cookie 供 Playwright context 使用"""
|
||||
path = self.state_path(site)
|
||||
if not path.exists():
|
||||
return []
|
||||
try:
|
||||
state = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return []
|
||||
return list(state.get("cookies", []))
|
||||
|
||||
def status(self, site: str) -> AuthStatus:
|
||||
"""该站登录态快照（不触发探测，用缓存结果）"""
|
||||
auth = self._require_site(site)
|
||||
return AuthStatus(
|
||||
site=site,
|
||||
state_file_exists=self.state_path(site).exists(),
|
||||
logged_in=auth.logged_in,
|
||||
checked_at=auth.checked_at,
|
||||
detail=auth.detail,
|
||||
)
|
||||
|
||||
def status_all(self) -> dict[str, dict[str, Any]]:
|
||||
"""两站登录态快照，供健康检查展示"""
|
||||
return {name: self.status(name).to_dict() for name in self._sites}
|
||||
|
||||
def _require_site(self, site: str) -> _SiteAuth:
|
||||
auth = self._sites.get(site)
|
||||
if auth is None:
|
||||
raise ValueError(f"未知站点或登录会话未初始化：{site}")
|
||||
return auth
|
||||
Reference in New Issue
Block a user