This commit is contained in:
2026-07-27 21:29:36 +08:00
parent 18cf7ae079
commit 6247d68fb5
10 changed files with 849 additions and 128 deletions
+97 -9
View File
@@ -4,14 +4,18 @@
而是分进程运行,原因:
- 抓取是**匿名**的,cookie 只用来过 Akamai 限速,丢了重新预热即可,无状态可言。
- 加购下单必须**带账号**,cookie 一旦失效不能自动恢复——乐天与 ラクマ 登录都有
reCAPTCHA 与设备验证,只能由人重新登录一次。因此这里的失效处理是「明确报错让
上游停下」,而不是像抓取那样静默重试
- 加购下单必须**带账号**,cookie 一旦失效需要重登——乐天与 ラクマ 登录都有
reCAPTCHA 与设备验证,全自动不可靠。本服务支持「失效时自动重登 + 撞验证码弹
有头浏览器让人工接管」的降级路径(依赖 account.yaml 与 relogin_enabled)
- 这份登录态在整个系统里只能有一份。跟抓取同进程的话,抓取一扩容就会复制出 N 份
登录态与 N 个订单轮询,同一个账号被并发操作。
登录态来源是 Playwright 的 storage_state:由 `scripts/login.py` 起一个有头浏览器
让人工登录一次后落盘,本服务只读取、不生产。账号密码不经过本服务,也不写日志。
登录态来源是 Playwright 的 storage_state:
- 首次登录由 scripts/login.py 完成;
- 运行时失效且 relogin_enabled=true 时,本模块自动触发 login_runner.login_one()
重登(同账号串行,site 级 asyncio.Lock 防并发)。
账号密码不在本模块持有——try_relogin() 时从 account.yaml 临时读取,调用结束即
被 GC。日志只打 account.id,不打 username/password。
cookie 会被同时喂给两处:
- httpx 客户端 —— 加购这类纯表单提交走 HTTP 更快
@@ -80,6 +84,8 @@ class AuthSession:
def __init__(self, settings: Settings):
self._settings = settings
self._sites: dict[str, _SiteAuth] = {}
# 自动重登的 site 级互斥锁:同账号同时只能一个登录流程(user_data_dir 被锁)
self._relogin_locks: dict[str, asyncio.Lock] = {}
# ---- 生命周期 ----
@@ -209,12 +215,94 @@ class AuthSession:
async def require_logged_in(self, site: str) -> None:
"""确保某站处于登录态,否则抛错
加购与下单前的统一入口。登录态失效时不做任何自动恢复尝试——两站登录都需要
人工过验证码,只能让上游停下来重新跑一次 scripts/login.py。
加购与下单前的统一入口。失效时的处理顺序:
1. 检测到未登录 → 若 relogin_enabled,尝试 try_relogin(site)
2. 重登成功 → 重新 check 一次,登录态转好即放行
3. 重登失败 / 未启用 / account.yaml 缺失 → 抛 NotLoggedInError 让 worker 转 needs_human
重登只在这一层(任务前置检查)触发;任务执行过程中失效不重试,
避免脏状态(cart 已提交但响应后 cookie 失效等场景)。
"""
status = await self.check(site)
if not status.logged_in:
raise NotLoggedInError(site=site, detail=status.detail)
if status.logged_in:
return
# 尝试自动重登(relogin_enabled=False 或无 account.yaml 时返回 False)
if await self.try_relogin(site):
status = await self.check(site)
if status.logged_in:
return
detail = f"重登后仍判定未登录:{status.detail}"
else:
detail = status.detail
raise NotLoggedInError(site=site, detail=detail)
# ---- 自动重登 ----
async def try_relogin(self, site: str) -> bool:
"""触发自动重登:读 account.yaml 默认账号 → 跑 login_one → reload cookie
- relogin_enabled=False 直接返回 False(不抛错,让调用方走原降级路径)
- account.yaml 缺失或该站没账号 → 返回 False,仅记日志
- 同 site 串行(asyncio.Lock),并发请求只跑一次登录流程
- 账号密码不进日志:只打 account.id;login_runner 内部完成填表
成功返回前已完成 self.reload(site),httpx 客户端拿到新 cookie;
Playwright 侧(SiteInteractor)通过 storage_state 文件 mtime 自检刷新。
"""
if not self._settings.relogin_enabled:
return False
# site 级锁:同账号同 user_data_dir,并发重登会撞锁
lock = self._relogin_locks.setdefault(site, asyncio.Lock())
async with lock:
# 拿锁后再 check 一次——可能别的协程刚重登过
status = self.status(site)
if status.logged_in:
return True
# 读 account.yaml(失败时降级,不抛错)
try:
from app.trading.services import login_runner
accounts_by_site = login_runner.load_accounts()
except FileNotFoundError as exc:
logger.warning("自动重登跳过:%s", exc)
return False
except ValueError as exc:
logger.warning("自动重登跳过:account.yaml 格式错误:%s", exc)
return False
account = login_runner.default_account_for(site, accounts_by_site)
if account is None:
logger.warning("自动重登跳过:account.yaml 没有 %s 的账号", site)
return False
logger.info(
"自动重登启动:site=%s account_id=%s timeout=%ss",
site, account.id, self._settings.relogin_timeout_seconds,
)
try:
ok = await login_runner.login_one(
account,
self._settings,
timeout_seconds=self._settings.relogin_timeout_seconds,
progress=None, # 用 logger,不打到 stdout
)
except Exception:
logger.exception("自动重登异常:site=%s account_id=%s", site, account.id)
return False
if not ok:
logger.warning("自动重登失败:site=%s account_id=%s", site, account.id)
return False
# 重登成功:reload cookie 到 httpx 客户端
count = self.reload(site)
logger.info(
"自动重登成功:site=%s account_id=%s cookies=%s",
site, account.id, count,
)
return True
# ---- 对外访问 ----
+455
View File
@@ -0,0 +1,455 @@
"""账号密码自动登录的运行时核心,供 scripts/login.py(CLI)与 AuthSession(运行时重登)共用
设计要点:
- **凭据安全**:Account 在内存中只存到 login_one() 返回;日志只打 account.id,绝不打
username/password。account.yaml 由调用方负责(默认项目根,已在 .gitignore 排除)。
- **持久化浏览器**:用 launch_persistent_context,user_data_dir 来自 account.yaml,
cookie/localStorage/缓存跨重启保留。
- **人工 fallback**:自动填账号密码后撞 reCAPTCHA / OTP 时,浏览器保持有头模式,
由人完成验证,脚本继续轮询登录态;超时则失败返回。
- **登录态判据**:访问 profile.login_url 看落地 URL 是否被踢到 SSO。旧 marker
「現在ログインしていません」在新版 SP cart SPA 上不可靠(详见
project://jp-rakuten/checkout-flow-probe-findings §2),本模块不用。
"""
from __future__ import annotations
import asyncio
import logging
import sys
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import yaml
from app.shared.config import BASE_DIR, Settings
from app.trading.core import auth_site
logger = logging.getLogger(__name__)
# 自动填表后通常秒级;fallback 人工要给足过验证码、收短信的时间
_DEFAULT_TIMEOUT_SECONDS = 600
_POLL_INTERVAL_SECONDS = 5
_ACCOUNTS_FILE = BASE_DIR / "account.yaml"
@dataclass(slots=True)
class Account:
"""account.yaml 中的一条账号记录"""
site: str
id: str
username: str
password: str
user_data_dir: Path
state_filename: str
default: bool
# ---- YAML 读取与账号选择 ----
def accounts_file_path() -> Path:
"""account.yaml 默认在项目根"""
return _ACCOUNTS_FILE
def _resolve_user_data_dir(site: str, raw: dict[str, Any], account_id: str) -> Path:
"""从 YAML 取 user_data_dir,未配置则回退默认"""
raw_dir = raw.get("user_data_dir") or f".browser-data/{site}-{account_id}"
p = Path(raw_dir)
if not p.is_absolute():
p = BASE_DIR / p
return p
def _resolve_state_filename(site: str, raw: dict[str, Any], is_first: bool) -> str:
"""state_filename 解析:显式 > 首项默认 <site>_state.json(兼容 AuthSession)> 按 id 派生"""
explicit = raw.get("state_filename")
if explicit:
return str(explicit)
if is_first:
return f"{site}_state.json"
return f"{site}_state_{raw.get('id', 'account')}.json"
def load_accounts() -> dict[str, list[Account]]:
"""读 account.yaml,按站点分组返回所有账号
文件不存在时抛 FileNotFoundError,调用方决定如何降级。
"""
path = accounts_file_path()
if not path.exists():
raise FileNotFoundError(
f"未找到 {path}。请复制 account.yaml.example 为 account.yaml 并填入凭据。"
)
raw = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
result: dict[str, list[Account]] = {}
for site in auth_site.SITES:
records = raw.get(site) or []
if not isinstance(records, list):
raise ValueError(f"account.yaml: {site} 应为列表,收到 {type(records).__name__}")
accounts: list[Account] = []
for idx, rec in enumerate(records):
if not isinstance(rec, dict):
raise ValueError(f"account.yaml: {site}[{idx}] 应为字典")
for required in ("id", "username", "password"):
if not rec.get(required):
raise ValueError(
f"account.yaml: {site}[{idx}].{required} 必填"
)
account_id = str(rec["id"])
accounts.append(
Account(
site=site,
id=account_id,
username=str(rec["username"]),
password=str(rec["password"]),
user_data_dir=_resolve_user_data_dir(site, rec, account_id),
state_filename=_resolve_state_filename(site, rec, is_first=(idx == 0)),
default=bool(rec.get("default", False)),
)
)
result[site] = accounts
return result
def select_accounts(
site: str,
accounts_by_site: dict[str, list[Account]],
*,
account_id: str | None,
all_accounts: bool,
) -> list[Account]:
"""按 --account / --all-accounts 选择要登录的账号列表
`site="all"` 时跳过 account.yaml 未配置的站点;若所有站点都没配则报错。
"""
if site == "all":
targets: list[Account] = []
for s in auth_site.SITES:
pool = accounts_by_site.get(s, [])
if not pool:
continue
targets.extend(
select_accounts(s, accounts_by_site, account_id=account_id, all_accounts=all_accounts)
)
if not targets:
raise ValueError("account.yaml 没有任何站点的账号记录")
return targets
pool = accounts_by_site.get(site, [])
if not pool:
raise ValueError(f"account.yaml 没有 {site} 的账号记录")
if all_accounts:
return list(pool)
if account_id:
for acc in pool:
if acc.id == account_id:
return [acc]
raise ValueError(f"account.yaml 中 {site} 没有 id={account_id} 的账号")
# 默认:标 default 的;无标记用首项
for acc in pool:
if acc.default:
return [acc]
return [pool[0]]
def default_account_for(site: str, accounts_by_site: dict[str, list[Account]]) -> Account | None:
"""取某站的默认账号(标 default 或首项),用于自动重登。无配置返回 None"""
pool = accounts_by_site.get(site, [])
if not pool:
return None
for acc in pool:
if acc.default:
return acc
return pool[0]
# ---- 登录态判定(URL 判据,不依赖旧 marker)----
async def _is_logged_in(page, site: str) -> bool:
"""访问 profile.login_url 看落地是否被踢到 SSO
- rakuten:未登录会被重定向到 login.account.rakuten.com 或 /login 路径
- rakuma:未登录会被重定向到 /users/sign_in
"""
profile = auth_site.profile(site)
await page.goto(profile.login_url, wait_until="domcontentloaded", timeout=60_000)
final_url = page.url
if site == "rakuten":
return "login.account.rakuten.com" not in final_url and "/login" not in final_url
return auth_site.RAKUMA_SIGN_IN_PATH not in final_url
# ---- 自动填表(启发式选择器)----
async def _try_autofill(page, account: Account) -> bool:
"""启发式填账号密码并提交
Returns:
True 表示已尝试自动填表(无论是否撞验证码);
False 表示页面没有密码框(已经登录或落地异常),交给调用方判定。
"""
try:
await page.wait_for_selector(
"input[type='password']", state="visible", timeout=10_000
)
except Exception:
return False
# 用户名框:按常见命名启发式找
username_selectors = [
"input[type='email']",
"input[name='username']",
"input[name='user_id']",
"input[name*='email' i]",
"input[name*='userid' i]",
"input[name*='login' i]",
"input[autocomplete='username']",
"input[type='text']:visible",
"input[type='tel']:visible",
]
username_input = None
for sel in username_selectors:
username_input = await page.query_selector(sel)
if username_input and await username_input.is_visible():
break
username_input = None
if username_input:
await username_input.fill(account.username)
pwd = await page.query_selector("input[type='password']:visible")
if pwd:
await pwd.fill(account.password)
# 提交按钮:按文案与类型启发式
submit_selectors = [
"button[type='submit']:visible",
"input[type='submit']:visible",
"button:has-text('ログイン')",
"button:has-text('Sign in')",
"button:has-text('Login')",
"[data-role='submit']:visible",
]
for sel in submit_selectors:
btn = await page.query_selector(sel)
if btn and await btn.is_visible():
try:
await btn.click()
except Exception:
continue
break
return True
async def _detect_human_challenge(page) -> str | None:
"""检测页面是否出现需要人工处理的验证(reCAPTCHA / OTP / 设备验证)"""
if await page.query_selector("iframe[title*='reCAPTCHA' i], iframe[src*='recaptcha']"):
return "reCAPTCHA"
if await page.query_selector("iframe[src*='hcaptcha']"):
return "hCaptcha"
if await page.query_selector(
"input[name*='otp' i], input[name*='code' i], input[autocomplete='one-time-code']"
):
return "短信/邮箱 OTP"
if await page.query_selector(
"input[name*='verification' i], input[name*='captcha' i]"
):
return "设备验证/图形验证码"
return None
# ---- 单账号登录主流程 ----
async def login_one(
account: Account,
settings: Settings,
*,
timeout_seconds: int = _DEFAULT_TIMEOUT_SECONDS,
progress: Callable[[str], None] | None = None,
) -> bool:
"""对一个账号执行自动登录,成功则保存 storage_state
Args:
account: 账号记录(含明文凭据,调用方负责保密)
settings: 全局配置(取 auth_state_path / browser_channel / playwright_proxy)
timeout_seconds: 整个登录流程的最长等待(含人工 fallback)
progress: 可选的进度回调,默认用 logger.info;CLI 传 print
Returns:
True 表示登录成功且 storage_state 已落盘;False 表示超时或失败
"""
from playwright.async_api import async_playwright
log = progress or (lambda msg: logger.info(msg))
profile = auth_site.profile(account.site)
state_path = settings.auth_state_path / account.state_filename
account.user_data_dir.mkdir(parents=True, exist_ok=True)
log(f"=== {profile.label}{account.site}/{account.id})登录 ===")
log(f"持久化目录:{account.user_data_dir}")
log(f"登录入口:{profile.login_url}")
async with async_playwright() as playwright:
context = await playwright.chromium.launch_persistent_context(
user_data_dir=str(account.user_data_dir),
headless=False, # 自动登录仍需有头:撞验证码要人工接管
channel=settings.browser_channel or None,
proxy=settings.playwright_proxy,
user_agent=profile.user_agent,
locale="ja-JP",
timezone_id="Asia/Tokyo",
viewport=(
{"width": 390, "height": 844}
if profile.mobile
else {"width": 1440, "height": 900}
),
is_mobile=profile.mobile,
has_touch=profile.mobile,
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
)
try:
page = context.pages[0] if context.pages else await context.new_page()
await page.goto(profile.login_url, wait_until="domcontentloaded", timeout=60_000)
# 已经登录则直接保存
if await _is_logged_in(page, account.site):
log(f"已处于登录态,跳过填表:{account.id}")
else:
filled = await _try_autofill(page, account)
if not filled:
log("未找到登录表单且未登录,请在浏览器里手动完成登录")
else:
# 不打账号/密码值,只标识已提交
log(f"已自动填写凭据并提交:account_id={account.id}")
await page.wait_for_load_state("domcontentloaded", timeout=30_000)
challenge = await _detect_human_challenge(page)
if challenge:
log(
f"检测到 {challenge}:请在浏览器窗口里人工完成验证。"
f"最长等待 {timeout_seconds // 60} 分钟。"
)
# 轮询登录态
waited = 0
ok = False
while waited < timeout_seconds:
try:
if await _is_logged_in(page, account.site):
ok = True
break
except Exception as exc:
log(f"探测中({waited}s):{type(exc).__name__}")
await asyncio.sleep(_POLL_INTERVAL_SECONDS)
waited += _POLL_INTERVAL_SECONDS
if waited % 30 == 0:
log(f"等待登录完成…({waited}s)")
if not ok:
log(f"等待超时,未检测到登录态:{account.id}")
return False
import json
state = await context.storage_state()
state_path.write_text(
json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8"
)
log(f"登录成功,已保存 {len(state.get('cookies', []))} 条 cookie → {state_path}")
return True
finally:
await context.close()
# ---- CLI 入口(供 scripts/login.py 调用)----
async def cli_main(argv: list[str] | None = None) -> int:
"""命令行入口:argparse + 调用 login_one
返回进程退出码。stdout 用 print(给人看),不打账号密码值。
"""
import argparse
parser = argparse.ArgumentParser(description="按 account.yaml 自动登录乐天/ラクマ")
parser.add_argument(
"--site",
choices=[*auth_site.SITES, "all"],
default="all",
help="要登录的站点,默认 all",
)
parser.add_argument(
"--account",
help="account.yaml 中的账号 id;不传则用 default=true 或首项",
)
parser.add_argument(
"--all-accounts",
action="store_true",
help="登录该站点 account.yaml 里的所有账号(按顺序串行)",
)
parser.add_argument(
"--log-level",
default="INFO",
choices=["DEBUG", "INFO", "WARNING", "ERROR"],
)
args = parser.parse_args(argv)
logging.basicConfig(
level=args.log_level,
format="{asctime} {levelname} {message}",
style="{",
)
try:
accounts_by_site = load_accounts()
except (FileNotFoundError, ValueError) as exc:
print(f"配置错误:{exc}", file=sys.stderr)
return 2
try:
targets = select_accounts(
args.site,
accounts_by_site,
account_id=args.account,
all_accounts=args.all_accounts,
)
except ValueError as exc:
print(f"账号选择失败:{exc}", file=sys.stderr)
return 2
if not targets:
print("没有匹配的账号可登录", file=sys.stderr)
return 2
print(
f"将登录 {len(targets)} 个账号:"
+ ", ".join(f"{a.site}/{a.id}" for a in targets)
)
settings = Settings()
results: dict[str, bool] = {}
for acc in targets:
key = f"{acc.site}/{acc.id}"
try:
results[key] = await login_one(acc, settings, progress=print)
except Exception as exc:
print(f"{key} 登录异常:{type(exc).__name__}: {exc}", file=sys.stderr)
results[key] = False
print("\n=== 结果 ===")
for key, ok in results.items():
print(f" {key}: {'已登录' if ok else '失败'}")
if all(results.values()):
print("\n登录态已就绪,可启动服务并用 POST /api/auth/status 复核。")
return 0
return 1
+46
View File
@@ -91,6 +91,9 @@ class SiteInteractor:
self._browser = None
self._context = None # playwright BrowserContext
self._per_task_state = {}
# 上次 build context 时读到的 storage_state 文件 mtime。
# 用于在任务间检测 AuthSession 重登后产生的新 storage_state,触发 context 重建。
self._state_mtime: float | None = None
# ---- 生命周期 ----
@@ -109,6 +112,8 @@ class SiteInteractor:
"登录态文件不存在:site_interactor 以无 cookie 状态启动,"
"加购请求会被站点拒认"
)
else:
self._state_mtime = state_path.stat().st_mtime
self._playwright = await async_playwright().start()
self._browser = await self._playwright.chromium.launch(
@@ -127,6 +132,41 @@ class SiteInteractor:
)
logger.info("SiteInteractor 已就绪:storage_state=%s", storage_state or "(none)")
async def _refresh_context_if_stale(self) -> None:
"""检查 storage_state 文件 mtime,变化则重建 context
AuthSession.try_relogin 成功后会重写 storage_state 文件。本 context 启动时
用快照式 storage_state 创建,cookie 不会自动同步——必须关掉旧 context、
用新文件重建。在 add_to_cart / verify_cart 开头各调一次,开销可接受
(只在 mtime 变了才重建)。
"""
state_path = self._settings.auth_state_path / auth_site.profile("rakuten").state_filename
if not state_path.exists():
return
mtime = state_path.stat().st_mtime
if mtime == self._state_mtime:
return
logger.info(
"storage_state 文件变化(mtime %s%s),重建 context",
self._state_mtime, mtime,
)
if self._context is not None:
try:
await self._context.close()
except Exception:
logger.debug("关闭旧 context 失败", exc_info=True)
self._context = await self._browser.new_context(
storage_state=str(state_path),
user_agent=auth_site.RAKUTEN_USER_AGENT,
locale="ja-JP",
timezone_id="Asia/Tokyo",
viewport={"width": 390, "height": 844},
is_mobile=True,
has_touch=True,
)
self._state_mtime = mtime
async def close(self) -> None:
"""关闭 context、browser、playwright,吞掉单个 close 异常"""
for resource, name in (
@@ -171,6 +211,9 @@ class SiteInteractor:
await self._auth_session.require_logged_in("rakuten")
# 任务开始前刷新 context:AuthSession 可能刚自动重登过,storage_state 变了
await self._refresh_context_if_stale()
page = await self._context.new_page()
try:
try:
@@ -261,6 +304,9 @@ class SiteInteractor:
"""
await self._auth_session.require_logged_in("rakuten")
# 同 add_to_cart:刷新 context 以反映可能的自动重登结果
await self._refresh_context_if_stale()
per_task = self._per_task_state.get(task.task_id, {})
item_id = (task.intent or {}).get("item_id") or per_task.get("item_id")
if not item_id: