Compare commits
25
Commits
55c01ae4f5
...
6b6b243132
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6b6b243132 | ||
|
|
f31e127ba7 | ||
|
|
086ab82614 | ||
|
|
daa5555fd4 | ||
|
|
48c6f2a28f | ||
|
|
d54af141eb | ||
|
|
df93bc1e63 | ||
|
|
f6c3976c0a | ||
|
|
6b77f3bcc3 | ||
|
|
3b53b8f28b | ||
|
|
c03158488b | ||
|
|
b01c9659d1 | ||
|
|
6c38089234 | ||
|
|
912e10086f | ||
|
|
c263793fb2 | ||
|
|
fabca9510d | ||
|
|
3dc2aeb3a2 | ||
|
|
63c41b61e7 | ||
|
|
e2875f0c00 | ||
|
|
295fc7ad69 | ||
|
|
e5f7da09a9 | ||
|
|
8b2fd74336 | ||
|
|
93c1a83406 | ||
|
|
51e2538438 | ||
|
|
5f7921b788 |
@@ -14,10 +14,18 @@ __pycache__/
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
# 这三样都不进镜像(运行时用挂载提供),也不该被送进 build context:
|
||||
# account.yaml 是明文密码+卡号,.auth 是可直接冒充账号的 cookie,
|
||||
# .browser-data 是带登录态的浏览器 profile。
|
||||
account.yaml
|
||||
.auth/
|
||||
.browser-data/
|
||||
|
||||
# 运行时产物
|
||||
logs/
|
||||
.probe/
|
||||
# 订单库、证据快照(含真实姓名地址等 PII),只走挂载
|
||||
data/
|
||||
|
||||
# 测试不进镜像(Jenkinsfile 里单独跑)
|
||||
tests/
|
||||
@@ -32,6 +40,7 @@ README.md
|
||||
|
||||
# CI / 部署文件本身
|
||||
Dockerfile
|
||||
Dockerfile.*
|
||||
.dockerignore
|
||||
Jenkinsfile
|
||||
docker-compose*.yml
|
||||
|
||||
+41
-1
@@ -60,6 +60,9 @@ RAKUTEN_PROXY_SERVER=
|
||||
RAKUTEN_PROXY_USERNAME=
|
||||
# 代理密码(如代理需要认证则填写)
|
||||
RAKUTEN_PROXY_PASSWORD=
|
||||
# 不经代理的主机名(逗号分隔,支持 *.internal 这类通配符)。默认覆盖本机与 Docker
|
||||
# 服务间通信;内部网关或抓取服务使用其它域名时,将其显式加入此列表。
|
||||
RAKUTEN_PROXY_BYPASS=localhost,127.0.0.1,::1,rakuten-api,rakuten-trading,rakuten-gateway
|
||||
|
||||
# ---- OpenTelemetry traces(可选;默认关闭)----
|
||||
# 启用后把抓取-解析链路以 span 导出到 OTLP/HTTP endpoint,
|
||||
@@ -81,10 +84,29 @@ RAKUTEN_AUTH_STATE_DIR=.auth
|
||||
# 下单金额上限(日元):实际应付超过该值直接拒绝提交,防止解析出错或页面改版
|
||||
# 导致买到远超预期的订单。设为 0 表示不设上限(不建议)。
|
||||
RAKUTEN_ORDER_MAX_TOTAL_YEN=30000
|
||||
# 付款后监控(订单列表页轮询)间隔(秒)与最大轮询次数。间隔不宜太短——同一账号
|
||||
# 频繁访问订单页有被风控盯上的风险。默认 3 小时一次,最多 80 次(约 10 天)。
|
||||
RAKUTEN_ORDER_MONITOR_POLL_INTERVAL_SECONDS=10800
|
||||
RAKUTEN_ORDER_MONITOR_MAX_CHECKS=80
|
||||
|
||||
# 登录态失效时是否自动重登(需要项目根有 account.yaml,含明文密码)。
|
||||
# 关闭时登录态一掉就抛 5001 / 转 needs_human,需要人工跑 scripts/login.py。
|
||||
RAKUTEN_RELOGIN_ENABLED=true
|
||||
# 自动登录(含撞验证码时等人工接管)的最长总耗时(秒)。无人值守的机器可以调小
|
||||
# (如 60)以尽快失败转 needs_human。
|
||||
RAKUTEN_RELOGIN_TIMEOUT_SECONDS=300
|
||||
# 启动时是否自动登录一次:起服务即按 account.yaml 把登录态准备好,不必先在宿主机跑
|
||||
# scripts/login.py。后台执行不阻塞端口,失败只记日志。
|
||||
# **刻意留成注释**:裸机跑不开(代码默认 false,别在开发机上一启动就弹浏览器),
|
||||
# 容器部署由 docker-compose.yml 给 true。这里一旦写成显式值,compose 里的
|
||||
# `${RAKUTEN_AUTO_LOGIN_ON_START:-true}` 就会读到 .env 的值而失效——compose 的变量
|
||||
# 插值和 env_file 读的是同一个 ./.env。要手动覆盖时才取消注释。
|
||||
# RAKUTEN_AUTO_LOGIN_ON_START=false
|
||||
|
||||
# ---- 以下仅下单任务网关使用 ----
|
||||
# 任务队列 SQLite 文件路径(相对项目根目录)。务必放在持久化卷上,丢了等于
|
||||
# 丢了一批下单任务。详见 docs/order-gateway.md。
|
||||
# 丢了一批下单任务。查询通道(account_queries 表)在同一份库文件里。
|
||||
# 详见 docs/order-gateway.md。
|
||||
RAKUTEN_GATEWAY_DB_PATH=data/gateway.db
|
||||
# 任务租约 TTL(秒)。worker 领取后必须在此时间内首次 report 或 renew,否则
|
||||
# 任务被置为 stale(**绝不自动重投**,需要人工 reclaim)。
|
||||
@@ -94,6 +116,24 @@ RAKUTEN_LEASE_MAX_WAIT_SECONDS=60
|
||||
# worker 心跳超时阈值(秒)。超过即视为失联,/health 报 degraded。
|
||||
RAKUTEN_WORKER_OFFLINE_ALERT_SECONDS=300
|
||||
|
||||
# ---- 账号只读查询通道(网关与本地 worker 共享),详见 docs/order-gateway.md §11 ----
|
||||
# 查询单租约 TTL(秒)。worker 领走后需在此期间回结果,否则自动重投回丢列。
|
||||
# 与下单任务的关键区别:**只读操作可以安全重投**。
|
||||
RAKUTEN_QUERY_LEASE_TTL_SECONDS=180
|
||||
# 查询单整体存活上限(秒)。从创建算起,超过仍未完成即置 expired(多半本地 worker 不在线)。
|
||||
RAKUTEN_QUERY_TTL_SECONDS=900
|
||||
# 同一张查询单最多被领取几次。重投累计到此仍无结果即置 failed。
|
||||
RAKUTEN_QUERY_MAX_ATTEMPTS=3
|
||||
# 终态查询单保留时长(秒)。结果里带站点原始 JSON,超期由 sweep 清理。
|
||||
RAKUTEN_QUERY_RETENTION_SECONDS=604800
|
||||
# worker 侧单次站点读取上限(秒)。查询在等账号锁(下单正跑)时也会超时失败,
|
||||
# 上游重发即可——只读,重发没有副作用。
|
||||
RAKUTEN_ACCOUNT_QUERY_TIMEOUT_SECONDS=120
|
||||
# order_list 查询缺省翻页上限。上游可用 params.max_pages 覆盖(1..20)。
|
||||
RAKUTEN_ACCOUNT_QUERY_DEFAULT_MAX_PAGES=3
|
||||
# 回报结果 JSON 体积上限(字节)。超限先丢站点原始 JSON,仍超限判失败让上游缩小窗口。
|
||||
RAKUTEN_QUERY_RESULT_MAX_BYTES=1048576
|
||||
|
||||
# ---- 以下仅交易服务内的下单 worker 使用 ----
|
||||
# 网关 URL。**留空则不启动 worker**,交易服务只跑登录态接口。
|
||||
# 部署形态:本地机(NAT 后无公网入口)通过出站长轮询从这里领任务。
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
# Windows 签出(core.autocrlf)会把文本文件转成 CRLF。
|
||||
# Dockerfile.trading 内嵌 heredoc 生成 shell 入口脚本,CRLF 会让 shebang
|
||||
# 变成 /bin/sh\r,容器启动报 "No such file or directory"。以下文件强制 LF:
|
||||
Dockerfile* text eol=lf
|
||||
*.sh text eol=lf
|
||||
*.py text eol=lf
|
||||
*.yaml text eol=lf
|
||||
*.yml text eol=lf
|
||||
*.toml text eol=lf
|
||||
*.json text eol=lf
|
||||
@@ -18,3 +18,6 @@ openapi.json
|
||||
account.yaml
|
||||
.browser-data/
|
||||
.claude
|
||||
|
||||
# 结账研究证据:含真实姓名/地址等 PII(卡号已打码),绝不可提交
|
||||
data/
|
||||
|
||||
+6
-3
@@ -47,6 +47,9 @@ ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
VIRTUAL_ENV=/app/.venv \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
# rakuten 用户是 --no-create-home 建的;Chromium(浏览器兜底)启动要写
|
||||
# $HOME/.config、~/.pki,HOME 指向不存在的目录时会 CHECK 失败 SIGTRAP 自杀
|
||||
HOME=/home/rakuten \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
|
||||
# Playwright 浏览器二进制走 npmmirror 镜像,避免从 azureedge 拉取缓慢
|
||||
PLAYWRIGHT_DOWNLOAD_HOST=https://cdn.npmmirror.com/binaries/playwright \
|
||||
@@ -108,9 +111,9 @@ WORKDIR /app
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY --from=builder /app/app /app/app
|
||||
|
||||
# 可写目录:logs(Loguru)、浏览器二进制(~300MB)
|
||||
RUN mkdir -p /app/logs /ms-playwright \
|
||||
&& chown -R rakuten:rakuten /app /ms-playwright
|
||||
# 可写目录:logs(Loguru)、浏览器二进制(~300MB)、/home/rakuten(Chromium 的 ~/.config 等)
|
||||
RUN mkdir -p /app/logs /ms-playwright /home/rakuten \
|
||||
&& chown -R rakuten:rakuten /app /ms-playwright /home/rakuten
|
||||
|
||||
USER rakuten
|
||||
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
#
|
||||
# 有状态端镜像:交易服务(下单购买 + 订单查询/监控),默认 python -m app.trading.main,:31108。
|
||||
#
|
||||
# 为什么不复用根目录 Dockerfile(抓取服务镜像):
|
||||
# 1. 下单/结算/加购必须用**有头** Chromium——site_interact.py 里 headless 是硬编码
|
||||
# False(2026-08-14 实测:无头会让购物车/结算 SPA 失灵)。抓取镜像里没有任何 X
|
||||
# 显示,交易进程一 launch 就崩。这里内置 Xvfb 虚拟显示解决。
|
||||
# 2. 这条链路上有设计好的人工接管点(登录验证码、3DS/OTP、needs_human),远程主机上
|
||||
# 必须能真的看见那个浏览器窗口 → 内置可选 x11vnc(默认关闭)。
|
||||
# 3. 页面是日文,证据截图要有日文字形 → fonts-ipafont(抓取侧只取 HTML,不需要)。
|
||||
#
|
||||
# 同一镜像也能跑下单任务网关(python -m app.gateway.main,:31109):网关不碰浏览器,
|
||||
# 用 RAKUTEN_XVFB_ENABLED=false 关掉虚拟显示即可。
|
||||
#
|
||||
# 交易服务与网关都**只能单实例**:登录态 cookie 全局唯一、订单监控是常驻轮询、
|
||||
# SQLite 单连接。不要 --scale,也不要在前面挂多副本负载均衡。
|
||||
|
||||
# ----------------------------- builder -----------------------------
|
||||
FROM registry.jerryyan.top/library/python:3.13-slim AS builder
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
# 国内 PyPI 镜像:避免从官方 PyPI 拉包缓慢
|
||||
UV_INDEX_URL=https://mirrors.aliyun.com/pypi/simple/
|
||||
|
||||
# 与根 Dockerfile 相同来源:python3.13-* 标签的 uv 在 /usr/local/bin/
|
||||
COPY --from=registry-ghcr.jerryyan.top/astral-sh/uv:python3.13-bookworm-slim /usr/local/bin/uv /usr/local/bin/uvx /usr/local/bin/
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# 只装运行期依赖:playwright / aiosqlite 都在 [project].dependencies 主表里,
|
||||
# 交易服务不需要 --extra browser(那是抓取侧的可选兜底)。
|
||||
COPY uv.lock pyproject.toml ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-dev
|
||||
|
||||
COPY app ./app
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --frozen --no-dev
|
||||
|
||||
# ----------------------------- runtime -----------------------------
|
||||
FROM registry.jerryyan.top/library/python:3.13-slim AS runtime
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
VIRTUAL_ENV=/app/.venv \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
# rakuten 用户是 --no-create-home 建的;Chromium 启动要写 $HOME/.config、~/.pki,
|
||||
# HOME 指向不存在的目录时会 CHECK 失败直接 SIGTRAP(int3)自杀
|
||||
HOME=/home/rakuten \
|
||||
PLAYWRIGHT_BROWSERS_PATH=/ms-playwright \
|
||||
PLAYWRIGHT_DOWNLOAD_HOST=https://cdn.npmmirror.com/binaries/playwright \
|
||||
# 落库时间戳全是 UTC aware(local_db/task_queue 用 datetime.now(timezone.utc)),
|
||||
# 这里设日本时区只影响日志与调试快照文件名的可读性,不改变任何持久化语义。
|
||||
TZ=Asia/Tokyo \
|
||||
# 虚拟显示:Chromium 有头模式必须有 DISPLAY。跑网关时置 false 可完全跳过 Xvfb。
|
||||
DISPLAY=:99 \
|
||||
RAKUTEN_XVFB_ENABLED=true \
|
||||
RAKUTEN_XVFB_SCREEN=1280x1024x24 \
|
||||
# 远程接管(验证码 / 3DS / needs_human 时肉眼看浏览器):默认关闭。
|
||||
# 开启前务必设 RAKUTEN_VNC_PASSWORD,且端口只对内网/SSH 隧道开放——
|
||||
# 这个屏幕上是已登录的真实账号与结算页。
|
||||
RAKUTEN_VNC_ENABLED=false \
|
||||
RAKUTEN_VNC_PORT=5900 \
|
||||
RAKUTEN_TRADING_HOST=0.0.0.0 \
|
||||
RAKUTEN_TRADING_PORT=31108 \
|
||||
# 与根镜像保持一致:默认导出 traces 到自建 OTLP(裸跑无鉴权),
|
||||
# 服务名由 app/trading/main.py 显式设为 rakuten-trading。
|
||||
RAKUTEN_OTEL_ENABLED=true \
|
||||
RAKUTEN_OTEL_ENDPOINT=https://oltp.jerryyan.top/v1/traces
|
||||
|
||||
# 有头 Chromium 运行依赖 + Xvfb + 日文字体。
|
||||
# 显式列包而不用 `playwright install-deps`(后者装的是一整套 apt 源里的当前版本,
|
||||
# 构建结果随源漂移)。Debian t64 过渡把一批库改名(libasound2 → libasound2t64 等),
|
||||
# 所以先整表装一次,失败再逐包回退试 ${pkg}t64,让镜像在 bookworm/trixie 基础镜像上都能建起来。
|
||||
RUN set -eu; \
|
||||
{ \
|
||||
if [ -f /etc/apt/sources.list ]; then \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list; \
|
||||
fi; \
|
||||
if [ -f /etc/apt/sources.list.d/debian.sources ]; then \
|
||||
sed -i 's|deb.debian.org|mirrors.aliyun.com|g; s|security.debian.org|mirrors.aliyun.com|g' /etc/apt/sources.list.d/debian.sources; \
|
||||
fi; \
|
||||
}; \
|
||||
PKGS="ca-certificates curl tzdata \
|
||||
xvfb x11vnc \
|
||||
fonts-liberation fonts-ipafont-gothic fonts-ipafont-mincho \
|
||||
libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 \
|
||||
libcairo2 libcairo-gobject2 libcups2 libdbus-1-3 libdrm2 libgbm1 \
|
||||
libgdk-pixbuf-2.0-0 libglib2.0-0 libgtk-3-0 libnspr4 libnss3 libpango-1.0-0 \
|
||||
libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 \
|
||||
libxkbcommon0 libxrandr2 libxshmfence1 xdg-utils"; \
|
||||
apt-get update; \
|
||||
if ! apt-get install -y --no-install-recommends $PKGS; then \
|
||||
for pkg in $PKGS; do \
|
||||
apt-get install -y --no-install-recommends "$pkg" \
|
||||
|| apt-get install -y --no-install-recommends "${pkg}t64"; \
|
||||
done; \
|
||||
fi; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 非 root 账号(uid/gid 与根镜像一致,方便共用宿主机上的 .auth / data 目录属主)
|
||||
RUN groupadd --system --gid 10001 rakuten \
|
||||
&& useradd --system --uid 10001 --gid rakuten --no-create-home --shell /usr/sbin/nologin rakuten
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY --from=builder /app/.venv /app/.venv
|
||||
COPY --from=builder /app/app /app/app
|
||||
# 只带 login.py:容器里首次产出登录态要用它(配合 VNC 人工过验证码),
|
||||
# scripts/ 下其余都是探针脚本,属开发工具,不进镜像。
|
||||
COPY scripts/login.py /app/scripts/login.py
|
||||
|
||||
# 入口:拉起 Xvfb(可选 x11vnc)后 exec 真正的进程,保证信号直达 PID 1。
|
||||
COPY --chmod=0755 <<'ENTRYPOINT_SH' /usr/local/bin/rakuten-entrypoint.sh
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
if [ "${RAKUTEN_XVFB_ENABLED:-true}" = "true" ]; then
|
||||
screen_num="${DISPLAY#:}"
|
||||
socket="/tmp/.X11-unix/X${screen_num%%.*}"
|
||||
Xvfb "$DISPLAY" -screen 0 "${RAKUTEN_XVFB_SCREEN:-1280x1024x24}" -nolisten tcp &
|
||||
|
||||
# 等 X socket 就绪再放行;否则 Chromium 会以 "Missing X server" 直接失败
|
||||
waited=0
|
||||
while [ ! -e "$socket" ]; do
|
||||
waited=$((waited + 1))
|
||||
if [ "$waited" -ge 100 ]; then
|
||||
echo "Xvfb 10 秒内未就绪($socket 不存在),放弃启动" >&2
|
||||
exit 1
|
||||
fi
|
||||
sleep 0.1
|
||||
done
|
||||
echo "Xvfb 已就绪:DISPLAY=$DISPLAY screen=${RAKUTEN_XVFB_SCREEN:-1280x1024x24}"
|
||||
|
||||
if [ "${RAKUTEN_VNC_ENABLED:-false}" = "true" ]; then
|
||||
# 有密码就用密码,没有则明文无鉴权——后者只允许在 SSH 隧道/内网里用
|
||||
if [ -n "${RAKUTEN_VNC_PASSWORD:-}" ]; then
|
||||
mkdir -p /tmp/.vnc
|
||||
x11vnc -storepasswd "$RAKUTEN_VNC_PASSWORD" /tmp/.vnc/passwd >/dev/null 2>&1
|
||||
auth_args="-rfbauth /tmp/.vnc/passwd"
|
||||
else
|
||||
echo "警告:RAKUTEN_VNC_ENABLED=true 但未设 RAKUTEN_VNC_PASSWORD,VNC 无鉴权" >&2
|
||||
auth_args="-nopw"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
x11vnc -display "$DISPLAY" -forever -shared -bg -quiet \
|
||||
-rfbport "${RAKUTEN_VNC_PORT:-5900}" $auth_args
|
||||
echo "x11vnc 已启动:端口 ${RAKUTEN_VNC_PORT:-5900}(屏幕上是真实已登录账号,勿暴露公网)"
|
||||
fi
|
||||
fi
|
||||
|
||||
exec "$@"
|
||||
ENTRYPOINT_SH
|
||||
|
||||
# 可写目录:logs(Loguru)、data(订单 SQLite + 证据)、.auth(登录态 cookie)、
|
||||
# .browser-data(login.py 的持久化浏览器目录)、/ms-playwright(Chromium 二进制)、
|
||||
# /tmp/.X11-unix(Xvfb socket,非 root 也要能建)、/home/rakuten(Chromium 的 ~/.config 等)
|
||||
RUN mkdir -p /app/logs /app/data /app/.auth /app/.browser-data /ms-playwright /tmp/.X11-unix /home/rakuten \
|
||||
&& chmod 1777 /tmp/.X11-unix \
|
||||
&& chown -R rakuten:rakuten /app /ms-playwright /home/rakuten
|
||||
|
||||
USER rakuten
|
||||
|
||||
RUN /app/.venv/bin/playwright install chromium
|
||||
|
||||
# /health 只读缓存登录态,不触发站点请求,适合高频探活。
|
||||
# 端口取 RAKUTEN_HEALTH_PORT,未设时用交易端口;跑网关的容器把它设成 31109。
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=45s --retries=3 \
|
||||
CMD curl -fsS http://127.0.0.1:${RAKUTEN_HEALTH_PORT:-${RAKUTEN_TRADING_PORT}}/health || exit 1
|
||||
|
||||
# 31108 交易服务,31109 下单任务网关(同镜像换 command),5900 可选 VNC
|
||||
EXPOSE 31108 31109 5900
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/rakuten-entrypoint.sh"]
|
||||
CMD ["python", "-m", "app.trading.main"]
|
||||
Vendored
+65
-12
@@ -1,6 +1,12 @@
|
||||
// Jenkinsfile —— jp-rakuten(乐天 / ラクマ 抓取服务)
|
||||
// Jenkinsfile —— jp-rakuten(抓取服务 + 有状态端)
|
||||
//
|
||||
// 一次构建出两个镜像,两者用途不同、不能互相替代:
|
||||
// git.jerryyan.net/jp/rakuten-api ← Dockerfile 抓取服务(无状态可多开)
|
||||
// git.jerryyan.net/jp/rakuten-trading ← Dockerfile.trading 有状态端(下单购买+订单查询)
|
||||
// 有状态端单独出镜像是因为下单/结算必须用**有头** Chromium(headless 会让结算 SPA
|
||||
// 失灵),镜像里要带 Xvfb + 日文字体 + 可选 x11vnc,抓取镜像没有这些。
|
||||
// 网关(app.gateway.main)复用 rakuten-trading 镜像,只是换 command。
|
||||
//
|
||||
// 默认 registry: git.jerryyan.net/jp/rakuten-api
|
||||
// 构建参数说明见各 stage 顶部;首次使用前请在 Jenkins 里配置凭据:
|
||||
// - git-credentials : 拉代码的 SSH / HTTPS 凭据
|
||||
// - gitea-registry : docker login git.jerryyan.net 的凭据(username/password)
|
||||
@@ -18,10 +24,19 @@ pipeline {
|
||||
parameters {
|
||||
string(name: 'IMAGE_NAME',
|
||||
defaultValue: 'git.jerryyan.net/jp/rakuten-api',
|
||||
description: '镜像完整名(含 registry host)')
|
||||
description: '抓取服务镜像完整名(含 registry host)')
|
||||
string(name: 'TRADING_IMAGE_NAME',
|
||||
defaultValue: 'git.jerryyan.net/jp/rakuten-trading',
|
||||
description: '有状态端(下单+订单查询,含网关)镜像完整名')
|
||||
string(name: 'IMAGE_TAG',
|
||||
defaultValue: '',
|
||||
description: '自定义 tag;留空则用 <BUILD_NUMBER>-<git short sha>')
|
||||
description: '自定义 tag;留空则用 <BUILD_NUMBER>-<git short sha>。两个镜像共用同一 tag')
|
||||
booleanParam(name: 'BUILD_SCRAPING',
|
||||
defaultValue: true,
|
||||
description: '构建并推送抓取服务镜像')
|
||||
booleanParam(name: 'BUILD_TRADING',
|
||||
defaultValue: true,
|
||||
description: '构建并推送有状态端镜像(体积大:含 Chromium + Xvfb,约 +700MB)')
|
||||
booleanParam(name: 'SKIP_TEST',
|
||||
defaultValue: false,
|
||||
description: '跳过单测(紧急发版用,正常构建不要勾)')
|
||||
@@ -82,7 +97,10 @@ pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
stage('Build image') {
|
||||
stage('Build image: scraping') {
|
||||
when {
|
||||
expression { return params.BUILD_SCRAPING }
|
||||
}
|
||||
steps {
|
||||
sh """
|
||||
docker build \
|
||||
@@ -95,18 +113,49 @@ pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
stage('Push image') {
|
||||
stage('Build image: trading') {
|
||||
when {
|
||||
expression { return params.BUILD_TRADING }
|
||||
}
|
||||
// 有状态端镜像:Chromium + Xvfb + 日文字体,比抓取镜像明显大也明显慢,
|
||||
// 首次构建(无缓存)拉 Chromium 二进制约 300MB
|
||||
steps {
|
||||
sh """
|
||||
docker build \
|
||||
-t ${params.TRADING_IMAGE_NAME}:${env.IMAGE_TAG} \
|
||||
-t ${params.TRADING_IMAGE_NAME}:latest \
|
||||
--label org.opencontainers.image.revision=${env.GIT_SHA} \
|
||||
--label org.opencontainers.image.version=${env.IMAGE_TAG} \
|
||||
-f Dockerfile.trading .
|
||||
"""
|
||||
}
|
||||
}
|
||||
|
||||
stage('Push images') {
|
||||
when {
|
||||
expression { return params.BUILD_SCRAPING || params.BUILD_TRADING }
|
||||
}
|
||||
steps {
|
||||
withCredentials([usernamePassword(
|
||||
credentialsId: 'gitea-registry',
|
||||
usernameVariable: 'REGISTRY_USER',
|
||||
passwordVariable: 'REGISTRY_PASS',
|
||||
)]) {
|
||||
sh """
|
||||
echo "\$REGISTRY_PASS" | docker login git.jerryyan.net -u "\$REGISTRY_USER" --password-stdin
|
||||
docker push ${env.IMAGE_NAME}:${env.IMAGE_TAG}
|
||||
docker push ${env.IMAGE_NAME}:latest
|
||||
"""
|
||||
sh 'echo "$REGISTRY_PASS" | docker login git.jerryyan.net -u "$REGISTRY_USER" --password-stdin'
|
||||
script {
|
||||
if (params.BUILD_SCRAPING) {
|
||||
sh """
|
||||
docker push ${env.IMAGE_NAME}:${env.IMAGE_TAG}
|
||||
docker push ${env.IMAGE_NAME}:latest
|
||||
"""
|
||||
}
|
||||
if (params.BUILD_TRADING) {
|
||||
sh """
|
||||
docker push ${params.TRADING_IMAGE_NAME}:${env.IMAGE_TAG}
|
||||
docker push ${params.TRADING_IMAGE_NAME}:latest
|
||||
"""
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -118,10 +167,14 @@ pipeline {
|
||||
sh """
|
||||
docker rmi -f ${env.IMAGE_NAME}:${env.IMAGE_TAG} 2>/dev/null || true
|
||||
docker rmi -f ${env.IMAGE_NAME}:latest 2>/dev/null || true
|
||||
docker rmi -f ${params.TRADING_IMAGE_NAME}:${env.IMAGE_TAG} 2>/dev/null || true
|
||||
docker rmi -f ${params.TRADING_IMAGE_NAME}:latest 2>/dev/null || true
|
||||
"""
|
||||
}
|
||||
success {
|
||||
echo "构建成功:${env.IMAGE_NAME}:${env.IMAGE_TAG}"
|
||||
echo "构建成功:tag=${env.IMAGE_TAG}" +
|
||||
(params.BUILD_SCRAPING ? " ${env.IMAGE_NAME}" : "") +
|
||||
(params.BUILD_TRADING ? " ${params.TRADING_IMAGE_NAME}" : "")
|
||||
}
|
||||
failure {
|
||||
echo "构建失败,请查看上方日志"
|
||||
|
||||
@@ -127,7 +127,12 @@ PC UA 在搜索页、详情页、店铺页上都能拿到完整模板。因此
|
||||
| --- | --- |
|
||||
| `GET /health` | 健康检查,含乐天账号登录态(只读缓存,不打站点) |
|
||||
| `POST /api/auth/status` | 查询登录态,默认真实探测一次 |
|
||||
| `POST /api/auth/login` | 按 `account.yaml` 自动登录(已登录则跳过;撞验证码要人工接管) |
|
||||
| `POST /api/auth/reload` | 人工重新登录后免重启换上新 cookie |
|
||||
| `POST /api/cart/add` | 加购(`item_url` + 数量/规格/选项) |
|
||||
| `POST /api/cart/status` | 购物车件数与登录态(轻量,不渲染整页) |
|
||||
| `POST /api/cart/clear` | 清空购物车 |
|
||||
| `POST /api/cart/remove` | 删除指定 `item_id` |
|
||||
|
||||
下单任务网关(:31109):
|
||||
|
||||
@@ -141,12 +146,31 @@ PC UA 在搜索页、详情页、店铺页上都能拿到完整模板。因此
|
||||
| `POST /api/orders/{id}/reclaim` | 把 stale 任务重新租给 worker(**绝不自动重投**) |
|
||||
| `GET /api/orders/{id}` | 任务详情 + 完整状态历史 |
|
||||
| `GET /api/orders` | 任务列表(运维与上游对账用) |
|
||||
| `POST /api/account/queries` | 提交**账号只读查询**(订单列表 / 单笔详情,幂等) |
|
||||
| `GET /api/account/queries/lease` | 本地 worker 领取查询单(可安全重投,与下单任务相反) |
|
||||
| `POST /api/account/queries/{id}/result` | 本地回报查询结果(站点真实订单数据) |
|
||||
| `GET /api/account/queries/{id}` | 取查询单状态与结果 |
|
||||
| `GET /api/account/queries` | 查询单列表(运维排查) |
|
||||
|
||||
网关的契约与状态机详见 [docs/order-gateway.md](docs/order-gateway.md)
|
||||
(账号只读查询通道见该文档 §11,用于「从已登录账号拉取真实订单」)。
|
||||
|
||||
网关的契约与状态机详见 [docs/order-gateway.md](docs/order-gateway.md)。
|
||||
|
||||
三个服务共用同一个 Bearer Token,错误码表也是同一份。
|
||||
启动后分别在 `http://127.0.0.1:31107/docs`、`:31108/docs`、`:31109/docs` 查看 OpenAPI 文档。
|
||||
|
||||
要一份能直接导入 Apifox / Postman 的合并文档,用仓库根的 `openapi.json`:三个服务的接口都在里面,
|
||||
每条接口带 operation 级 `servers`(导入后不必手动切端口)与 Bearer 鉴权声明。它是 gitignore 的本地
|
||||
生成物(不进版本库),由脚本产出,别手改:
|
||||
|
||||
```bash
|
||||
.venv/Scripts/python.exe scripts/export_openapi.py # 重新导出
|
||||
.venv/Scripts/python.exe scripts/export_openapi.py --check # 校验是否已最新
|
||||
```
|
||||
|
||||
**改了接口记得手动重新导出**:因为文件不在版本库里,单测无法校验它是否过期(`tests/test_openapi_export.py`
|
||||
只在内存里检查合并逻辑本身:三服务覆盖、operation 级 servers、鉴权标注、operationId 唯一、`$ref` 可解析)。
|
||||
|
||||
## 安装
|
||||
|
||||
```bash
|
||||
@@ -182,6 +206,43 @@ uv sync --extra dev --extra browser
|
||||
cookie 落在 `.auth/`(已 gitignore,内含可直接冒充账号的凭据,不要提交或外传)。
|
||||
后续重新登录后调 `POST /api/auth/reload` 换上新 cookie,不必重启服务。
|
||||
|
||||
配了 `account.yaml` 的话不用每次手动登:`RAKUTEN_RELOGIN_ENABLED=true`(默认)时登录态失效会自动重登,
|
||||
`RAKUTEN_AUTO_LOGIN_ON_START=true` 时服务启动就自己登一次,也可以随时调 `POST /api/auth/login` 触发。
|
||||
撞 reCAPTCHA / 设备验证时流程会停在有头浏览器上等人工完成,等不到就超时失败——自动登录只是代填
|
||||
`account.yaml` 里的凭据,不绕过站点校验。
|
||||
|
||||
## 容器部署
|
||||
|
||||
两个镜像,用途不能互换:
|
||||
|
||||
| 镜像 | Dockerfile | 跑什么 |
|
||||
| --- | --- | --- |
|
||||
| `rakuten-api` | `Dockerfile` | 抓取服务(无状态,可多开) |
|
||||
| `rakuten-trading` | `Dockerfile.trading` | 有状态端:交易服务(默认)与下单网关(换 `command`) |
|
||||
|
||||
有状态端单独出镜像不是为了整洁:下单/结算必须用**有头** Chromium(`headless=True` 会让结算 SPA 失灵),
|
||||
镜像里要带 Xvfb 虚拟显示、日文字体,以及给人工接管用的可选 x11vnc,抓取镜像没有这些。
|
||||
|
||||
```bash
|
||||
# 同机联调:起抓取 API 与交易服务
|
||||
docker compose up -d
|
||||
|
||||
# 正式拓扑:服务器侧只起抓取 API,本地机侧只起交易服务
|
||||
docker compose up -d scraping
|
||||
docker compose up -d trading
|
||||
|
||||
# 额外起下单网关(正式拓扑里它在服务器侧)
|
||||
docker compose --profile gateway up -d
|
||||
```
|
||||
|
||||
部署前置与挂载说明写在 `docker-compose.yml` 文件头:至少要有 `.env` 与 `account.yaml`,
|
||||
`.auth/`、`data/`、`logs/` 走挂载持久化。Jenkins 上两个镜像由同一条流水线产出
|
||||
(`BUILD_SCRAPING` / `BUILD_TRADING` 两个开关控制)。
|
||||
|
||||
抓取服务默认使用 `git.jerryyan.net/jp/rakuten-api:latest`,可通过
|
||||
`RAKUTEN_SCRAPING_IMAGE`、`RAKUTEN_SCRAPING_TAG` 与 `RAKUTEN_SCRAPING_BIND`
|
||||
分别覆盖镜像名、tag 和宿主机监听地址;默认只绑定 `127.0.0.1:31107`。
|
||||
|
||||
## 测试
|
||||
|
||||
```bash
|
||||
@@ -546,6 +607,7 @@ cookie 落在 `.auth/`(已 gitignore,内含可直接冒充账号的凭据,
|
||||
|
||||
- `POST /api/cart/add` — 加购,入参 `{item_url, quantity?, variant_id?, choice?}`
|
||||
- `POST /api/cart/status` — 调 cart count API,返回购物车商品件数
|
||||
(空车是正常结果:`count=0, raw_status="101"`;获取失败才报错 5002)
|
||||
- `POST /api/cart/clear` — 清空购物车(UI 点击 `button[aria-label="削除"]`)
|
||||
- `POST /api/cart/remove` — 删除指定 `item_id`
|
||||
|
||||
@@ -581,12 +643,16 @@ trading 加购时的字段选择策略:多规格挑第一个非售罄的 varia
|
||||
| 6002 | 租约无效:不是持有者、已过期或任务已终结(网关) | 409 |
|
||||
| 6003 | 任务状态不允许该操作(如对已终结任务 reclaim)(网关) | 409 |
|
||||
| 6004 | 已有任务在执行中,本次不发放(正常返回空,仅诊断用) | 200 |
|
||||
| 6005 | 查询单不存在(网关,账号只读查询通道) | 404 |
|
||||
| 6006 | 查询单租约无效(网关,账号只读查询通道) | 409 |
|
||||
|
||||
错误码在两站、三个服务之间通用。ラクマ 链路不会出现 `3002`(无反爬拦截行为)
|
||||
与 `4002`(无子站跳转);`5xxx` 只会来自交易服务——抓取服务全程匿名,不会有登录态问题。
|
||||
`5001` 与 `5004` 都标记为不可重试:前者要人工重新登录,后者要调用方改入参。
|
||||
`6xxx` 只会来自网关,全部标记为不可重试——任务编排侧重试无意义,部分场景
|
||||
(如租约过期)重试可能变成重复下单。
|
||||
`6xxx` 只会来自网关:`6001`–`6004`(下单任务通道)全部标记为不可重试——任务编排侧
|
||||
重试无意义,部分场景(如租约过期)重试可能变成重复下单;`6005`–`6006`
|
||||
(账号只读查询通道)**可以重试**——查询只读,重发没有副作用
|
||||
(见 [docs/order-gateway.md §11](docs/order-gateway.md#11-账号只读查询通道))。
|
||||
|
||||
## 常用环境变量
|
||||
|
||||
@@ -597,10 +663,13 @@ trading 加购时的字段选择策略:多规格挑第一个非售罄的 varia
|
||||
- 抓取服务:`RAKUTEN_APP_HOST`、`RAKUTEN_APP_PORT`(默认 `31107`)、`RAKUTEN_APP_ENV`
|
||||
- 交易服务:`RAKUTEN_TRADING_HOST`、`RAKUTEN_TRADING_PORT`(默认 `31108`)、`RAKUTEN_AUTH_STATE_DIR`(默认 `.auth`)、`RAKUTEN_ORDER_MAX_TOTAL_YEN`(默认 `30000`)
|
||||
- 下单任务网关:`RAKUTEN_GATEWAY_HOST`、`RAKUTEN_GATEWAY_PORT`(默认 `31109`)、`RAKUTEN_GATEWAY_DB_PATH`(默认 `data/gateway.db`)、`RAKUTEN_LEASE_TTL_SECONDS`(默认 `300`)、`RAKUTEN_WORKER_OFFLINE_ALERT_SECONDS`(默认 `300`)
|
||||
- 账号只读查询通道(网关 + 本地 worker 两侧共用):`RAKUTEN_QUERY_LEASE_TTL_SECONDS`(默认 `180`)、`RAKUTEN_QUERY_TTL_SECONDS`(默认 `900`)、`RAKUTEN_QUERY_MAX_ATTEMPTS`(默认 `3`)、`RAKUTEN_QUERY_RETENTION_SECONDS`(默认 `604800`)、`RAKUTEN_ACCOUNT_QUERY_TIMEOUT_SECONDS`(默认 `120`)、`RAKUTEN_ACCOUNT_QUERY_DEFAULT_MAX_PAGES`(默认 `3`)、`RAKUTEN_QUERY_RESULT_MAX_BYTES`(默认 `1048576`)
|
||||
- 本地下单 worker(在交易服务内,按 `RAKUTEN_ORDER_GATEWAY_URL` 是否配置决定是否启动):`RAKUTEN_ORDER_GATEWAY_URL`、`RAKUTEN_WORKER_ID`、`RAKUTEN_TRADING_DB_PATH`(默认 `data/trading.db`)、`RAKUTEN_EVIDENCE_DIR`(默认 `data/evidence`)、`RAKUTEN_SCRAPER_BASE_URL`
|
||||
- 鉴权:`RAKUTEN_BEARER_TOKEN`(三个服务共用)
|
||||
- 抓取:`RAKUTEN_MAX_SITE_CONCURRENCY`(默认 `8`,两站各自独立计数)、`RAKUTEN_HTTP_MAX_ATTEMPTS`(默认 `3`)、`RAKUTEN_SESSION_TTL_SECONDS`(默认 `1800`,仅乐天)
|
||||
- 浏览器兜底(仅乐天):`RAKUTEN_BROWSER_FALLBACK_ENABLED`、`RAKUTEN_BROWSER_HEADLESS`、`RAKUTEN_BROWSER_CHANNEL`
|
||||
- 代理(需日本 IP 时):`RAKUTEN_PROXY_SERVER`、`RAKUTEN_PROXY_USERNAME`、`RAKUTEN_PROXY_PASSWORD`
|
||||
- 代理(需日本 IP 时):`RAKUTEN_PROXY_SERVER`、`RAKUTEN_PROXY_USERNAME`、`RAKUTEN_PROXY_PASSWORD`、`RAKUTEN_PROXY_BYPASS`
|
||||
|
||||
> 从中国大陆直连实测可用、无需代理;`RAKUTEN_PROXY_SERVER` 留空即可。
|
||||
> 从中国大陆直连实测可用、无需代理;`RAKUTEN_PROXY_SERVER` 留空即可。设置后,所有面向
|
||||
> 外部站点的 HTTPX 与 Playwright 流量都会经代理;本机和 Docker 服务间地址由
|
||||
> `RAKUTEN_PROXY_BYPASS` 直连。该设置不影响 OTel 上报。
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
"""账号订单编目与定时下派触发路由(docs/order-gateway.md §12)
|
||||
|
||||
collector 周期性把账号真实订单沉淀进 `account_orders` 编目,这里是让上游与运维
|
||||
读取编目、以及手动触发一轮扫描的入口:
|
||||
|
||||
- GET /api/account/orders 列出已发现的账号订单(分页、可按状态过滤)
|
||||
- GET /api/account/orders/{order_number} 单笔编目订单
|
||||
- POST /api/account/discovery/trigger 立即派一轮 order_list 扫描
|
||||
|
||||
与 `/api/account/queries` 是两回事:那边是上游主动提交的一次性查询单;这里是
|
||||
网关自己周期盘点产生的持久编目。编目行只是规范化字段(订单号/店铺/日期/配送
|
||||
状态),不含站点原始 JSON——要看细节去拿对应订单的 order_detail 查询单。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from app.gateway.container import GatewayContainer
|
||||
from app.gateway.models import (
|
||||
CatalogOrderListData,
|
||||
CatalogedOrder,
|
||||
TriggerDiscoveryData,
|
||||
)
|
||||
from app.shared.api import ApiResponse, get_container, require_bearer_token
|
||||
from app.shared.errors import CatalogOrderNotFoundError
|
||||
|
||||
router = APIRouter(prefix="/api/account", tags=["account-orders"])
|
||||
|
||||
|
||||
@router.get(
|
||||
"/orders",
|
||||
response_model=ApiResponse[CatalogOrderListData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def list_cataloged_orders(
|
||||
state: str | None = Query(default=None, description="按映射后的 OrderState 过滤"),
|
||||
limit: int = Query(default=50, ge=1, le=500),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[CatalogOrderListData]:
|
||||
"""列出网关编目里已发现的账号订单,按最近出现时间倒序"""
|
||||
rows, total = await container.db.list_account_orders(
|
||||
state=state, limit=limit, offset=offset
|
||||
)
|
||||
items = [_row_to_model(r) for r in rows]
|
||||
return ApiResponse[CatalogOrderListData](
|
||||
success=True, msg="success",
|
||||
data=CatalogOrderListData(items=items, total=total, limit=limit, offset=offset),
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/orders/{order_number}",
|
||||
response_model=ApiResponse[CatalogedOrder],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def get_cataloged_order(
|
||||
order_number: str,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[CatalogedOrder]:
|
||||
"""取单笔编目订单;编目里没有则 6005(与查询单同号语义)"""
|
||||
row = await container.db.get_account_order(order_number)
|
||||
if row is None:
|
||||
raise CatalogOrderNotFoundError(order_number)
|
||||
return ApiResponse[CatalogedOrder](
|
||||
success=True, msg="success", data=_row_to_model(row), code=0
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/discovery/trigger",
|
||||
response_model=ApiResponse[TriggerDiscoveryData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def trigger_discovery(
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[TriggerDiscoveryData]:
|
||||
"""手动立即派一轮 order_list 扫描,不等定时器到点
|
||||
|
||||
返回本轮下派的单号,结果稍后看对应订单的 order_detail 查询单或编目。
|
||||
"""
|
||||
data = await container.collector.trigger_list_sweep()
|
||||
return ApiResponse[TriggerDiscoveryData](
|
||||
success=True, msg="success", data=TriggerDiscoveryData(**data), code=0
|
||||
)
|
||||
|
||||
|
||||
def _row_to_model(row) -> CatalogedOrder:
|
||||
return CatalogedOrder(
|
||||
order_number=row.order_number,
|
||||
shop_id=row.shop_id,
|
||||
shop_name=row.shop_name,
|
||||
order_date=row.order_date,
|
||||
delivery_status=row.delivery_status,
|
||||
order_state=row.order_state,
|
||||
discovered_at=row.discovered_at,
|
||||
detail_fetched_at=row.detail_fetched_at,
|
||||
last_seen_at=row.last_seen_at,
|
||||
)
|
||||
@@ -16,8 +16,11 @@ async def health(
|
||||
|
||||
只做规格 §4.7 的两条兜底告警:worker 失联、任务长时间无人领。
|
||||
付款期限监控不在网关——本地机 7×24 在线,那套逻辑放本地。
|
||||
另外带上账号只读查询的积压数量:它不参与告警判定(查询有自己的 TTL 会自动
|
||||
expired),只是给运维一个可见信号。
|
||||
"""
|
||||
data = await container.task_queue.health_snapshot()
|
||||
data.queued_query_count = await container.query_queue.queued_count()
|
||||
return ApiResponse[GatewayHealthData](
|
||||
success=True, msg="success", data=data, code=0
|
||||
)
|
||||
|
||||
@@ -45,13 +45,27 @@ async def submit_order(
|
||||
payload: SubmitOrderRequest,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[SubmitOrderData]:
|
||||
"""上游提交下单意图
|
||||
"""上游提交下单意图,返回 task_id
|
||||
|
||||
重复提交同一 task_id 不新建任务,返回既有任务且 created=false——上游重发
|
||||
不会变成两单。
|
||||
网关只做任务队列与状态镜像:intent 原文存库,由本地 worker 出站长轮询领走
|
||||
(GET /api/orders/lease),真正的加购、下单、付款都在本地机完成。
|
||||
|
||||
幂等:同一个 task_id 重复提交不新建任务,返回既有任务且 created=false——
|
||||
上游重发不会变成两单。下单不可逆,这是防重复下单的第一道闸。
|
||||
|
||||
可带 callback_url:任务到达终态(succeeded / failed / needs_human)或被置
|
||||
stale 时,网关向该地址 POST 一条 JSON 通知(best-effort 单次投递,§4.8)。
|
||||
幂等重发不会更新既有任务的回调地址。
|
||||
|
||||
提交后任务为 queued。进度跟踪用 GET /api/orders/{task_id}(任务状态 +
|
||||
完整状态历史);任务状态词汇表:queued / leased / running / succeeded /
|
||||
failed / needs_human / stale。
|
||||
"""
|
||||
data = await container.task_queue.submit(
|
||||
task_id=payload.task_id, site=payload.site, intent=payload.intent
|
||||
task_id=payload.task_id,
|
||||
site=payload.site,
|
||||
intent=payload.intent,
|
||||
callback_url=payload.callback_url,
|
||||
)
|
||||
return ApiResponse[SubmitOrderData](
|
||||
success=True, msg="success", data=data, code=0
|
||||
@@ -64,15 +78,22 @@ async def submit_order(
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def lease_order(
|
||||
worker_id: str = Query(...),
|
||||
wait: int = Query(default=30, ge=0, le=300),
|
||||
site: str | None = Query(default=None),
|
||||
worker_id: str = Query(..., description="worker 标识,每次调用刷新心跳"),
|
||||
wait: int = Query(
|
||||
default=30, ge=0, le=300,
|
||||
description="无任务时的挂起秒数。服务端上限 RAKUTEN_LEASE_MAX_WAIT_SECONDS(默认 60)",
|
||||
),
|
||||
site: str | None = Query(default=None, description="限定站点(rakuten);不传不限"),
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[LeaseData | None]:
|
||||
"""本地长轮询领取
|
||||
"""本地长轮询领取下单任务(worker 专用)
|
||||
|
||||
无可领任务时挂起到 wait 秒后返回 data: null,HTTP 仍 200。已有 leased/running
|
||||
任务时立即返回空(全局并发度 1)。
|
||||
- 每次调用刷新 worker 心跳(workers.last_seen_at),不另建心跳接口。
|
||||
- **全局并发度 1**:已存在 leased / running 任务时立即返回空,绝不发第二个任务。
|
||||
- 有可领任务时原子置 queued → leased,写入 lease_owner 与过期时间
|
||||
(TTL 由 RAKUTEN_LEASE_TTL_SECONDS 控制,默认 300 秒),lease_count += 1。
|
||||
- 无任务时挂起至多 wait 秒再返回,HTTP 仍 200、data 为 null。
|
||||
- 响应 lease_count > 1 只可能来自 reclaim(正常 lease 拿不到 stale 任务)。
|
||||
"""
|
||||
data = await container.task_queue.lease(
|
||||
worker_id=worker_id,
|
||||
@@ -89,13 +110,21 @@ async def lease_order(
|
||||
"/{task_id}/renew",
|
||||
response_model=ApiResponse[RenewData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
responses={
|
||||
404: {"description": "任务不存在(错误码 6001)"},
|
||||
409: {"description": "租约无效:不是持有者或任务已终结(错误码 6002)"},
|
||||
},
|
||||
)
|
||||
async def renew_order(
|
||||
task_id: str,
|
||||
payload: RenewRequest,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[RenewData]:
|
||||
"""续租。worker 在长任务中每 60 秒调一次,避免租约 TTL 误判"""
|
||||
"""续租(worker 专用)
|
||||
|
||||
执行时间可能超过租约 TTL(下单页面慢、付款要等),worker 在长任务中每 60 秒
|
||||
调一次,避免租约过期被误判成 stale。
|
||||
"""
|
||||
data = await container.task_queue.renew(task_id, payload.worker_id)
|
||||
return ApiResponse[RenewData](success=True, msg="success", data=data, code=0)
|
||||
|
||||
@@ -104,15 +133,27 @@ async def renew_order(
|
||||
"/{task_id}/report",
|
||||
response_model=ApiResponse[ReportData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
responses={
|
||||
404: {"description": "任务不存在(错误码 6001)"},
|
||||
409: {"description": "租约无效或终态冲突(错误码 6002 / 6003)"},
|
||||
},
|
||||
)
|
||||
async def report_order(
|
||||
task_id: str,
|
||||
payload: ReportRequest,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[ReportData]:
|
||||
"""本地回报订单状态
|
||||
"""本地回报订单状态(worker 专用)
|
||||
|
||||
同一 (task_id, state) 重复上报幂等。terminal=true 时释放租约并推到终态。
|
||||
worker 每一步遵循「动作 → 落证据 → 写本地库 → 回报」,先落证据再回报,
|
||||
保证服务器上看到的状态一定有本地证据可查。
|
||||
|
||||
- worker_id 必须是当前租约持有者,否则 6002。
|
||||
- 首次 report 把任务从 leased 推到 running。
|
||||
- 同一 (task_id, state) 重复上报幂等(覆盖同一行,recorded=false),
|
||||
网络抖动重发安全。
|
||||
- terminal=true 时释放租约并推到终态:terminal_status 显式指定优先,否则按
|
||||
state 推断(paid→succeeded,cancelled→failed,其余→needs_human)。
|
||||
"""
|
||||
data = await container.task_queue.report(
|
||||
task_id,
|
||||
@@ -133,16 +174,27 @@ async def report_order(
|
||||
"/{task_id}/reclaim",
|
||||
response_model=ApiResponse[ReclaimData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
responses={
|
||||
404: {"description": "任务不存在(错误码 6001)"},
|
||||
409: {"description": "任务不是 stale 状态(错误码 6003)"},
|
||||
},
|
||||
)
|
||||
async def reclaim_order(
|
||||
task_id: str,
|
||||
payload: ReclaimRequest,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[ReclaimData]:
|
||||
"""把 stale 任务重新租给 worker(恢复领取)
|
||||
"""把 stale 任务重新租给 worker(恢复领取,绝不自动重投)
|
||||
|
||||
返回体的 lease_count 必然 > 1,worker 必须先核对站点订单列表再决定是否
|
||||
继续执行——核对不出结论时报 needs_human,绝不重新提交。
|
||||
租约过期后任务置 stale 并告警,**不回 queued、不会被正常 lease 取到**——本地
|
||||
可能已经下单成功、只是回报那一步断网,自动重投等于再买一次。恢复只能显式
|
||||
调本接口。
|
||||
|
||||
返回体的 lease_count 必然 > 1 且带 known_state(租约丢失前的最新订单状态)。
|
||||
worker 执行前**必须先核对站点订单列表**(用 intent 里的商品 + 时间窗口比对):
|
||||
确认已下单则直接补报状态,核对不出结论报 needs_human,绝不重新提交。
|
||||
|
||||
仅 stale 状态可 reclaim;其他状态报 6003。
|
||||
"""
|
||||
data = await container.task_queue.reclaim(task_id, payload.worker_id)
|
||||
return ApiResponse[ReclaimData](success=True, msg="success", data=data, code=0)
|
||||
@@ -152,12 +204,18 @@ async def reclaim_order(
|
||||
"/{task_id}",
|
||||
response_model=ApiResponse[TaskDetail],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
responses={404: {"description": "任务不存在(错误码 6001)"}},
|
||||
)
|
||||
async def get_order(
|
||||
task_id: str,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[TaskDetail]:
|
||||
"""任务详情 + 完整状态历史"""
|
||||
"""任务详情 + 完整状态历史
|
||||
|
||||
返回任务本体(status / 租约信息 / intent 原文)、latest_state(最新一次上报的
|
||||
订单状态)与 reports(append-only 状态历史,含金额、付款期限、站点订单号、
|
||||
证据路径)。任务不存在报 6001。
|
||||
"""
|
||||
data = await container.task_queue.get_task_detail(task_id)
|
||||
return ApiResponse[TaskDetail](success=True, msg="success", data=data, code=0)
|
||||
|
||||
@@ -168,13 +226,16 @@ async def get_order(
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def list_orders(
|
||||
status: str | None = Query(default=None),
|
||||
site: str | None = Query(default=None),
|
||||
limit: int = Query(default=50, ge=1, le=500),
|
||||
offset: int = Query(default=0, ge=0),
|
||||
status: str | None = Query(
|
||||
default=None,
|
||||
description="按任务状态筛选:queued / leased / running / succeeded / failed / needs_human / stale",
|
||||
),
|
||||
site: str | None = Query(default=None, description="按站点筛选(rakuten)"),
|
||||
limit: int = Query(default=50, ge=1, le=500, description="分页大小"),
|
||||
offset: int = Query(default=0, ge=0, description="分页偏移"),
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[TaskListData]:
|
||||
"""任务列表(运维与上游对账用)"""
|
||||
"""任务列表,按创建时间倒序(运维与上游对账用)"""
|
||||
data = await container.task_queue.list_tasks(
|
||||
status=status, site=site, limit=limit, offset=offset
|
||||
)
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
"""账号只读查询通道路由(docs/order-gateway.md §11)
|
||||
|
||||
上游要的是「已登录账号在站点上的真实订单」,而不是网关自己记录的任务状态镜像;
|
||||
但本地机在 NAT 后没有公网入口,网关推不进去。所以这条通道与下单任务同构:
|
||||
上游把查询意图放进队列,本地 worker 出站长轮询领走、去站点上真读一次、回结果。
|
||||
|
||||
- POST /api/account/queries 上游提交查询单(幂等)
|
||||
- GET /api/account/queries/lease 本地长轮询领取
|
||||
- POST /api/account/queries/{id}/result 本地回结果
|
||||
- GET /api/account/queries/{id} 取查询单状态与结果
|
||||
- GET /api/account/queries 查询单列表(运维排查)
|
||||
|
||||
**纯异步**:提交只拿单号,结果去 GET 取。网关不提供「挂起等结果」的接口——
|
||||
下单任务可能占着账号锁跑好几分钟,同步等待会把上游的连接一起卡住。
|
||||
|
||||
注意路由声明顺序:`/lease` 必须在 `/{query_id}` 之前,否则 lease 会被当成 query_id。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
|
||||
from app.gateway.container import GatewayContainer
|
||||
from app.gateway.models import (
|
||||
QueryDetail,
|
||||
QueryLeaseData,
|
||||
QueryListData,
|
||||
QueryResultData,
|
||||
QueryResultRequest,
|
||||
SubmitQueryData,
|
||||
SubmitQueryRequest,
|
||||
)
|
||||
from app.shared.api import ApiResponse, get_container, require_bearer_token
|
||||
|
||||
router = APIRouter(prefix="/api/account/queries", tags=["account-queries"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"",
|
||||
response_model=ApiResponse[SubmitQueryData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def submit_query(
|
||||
payload: SubmitQueryRequest,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[SubmitQueryData]:
|
||||
"""提交一张账号只读查询单(纯异步:这里只拿单号)
|
||||
|
||||
上游要的是「已登录账号在站点上的真实订单」,而不是网关的任务状态镜像——两者
|
||||
会不一致(站点侧被商家取消、走别的渠道下的单,网关镜像都看不见)。查询单进
|
||||
队列后由本地 worker 出站领走、去站点上真读一次;结果去
|
||||
GET /api/account/queries/{query_id} 取。
|
||||
|
||||
幂等:同一 query_id 重复提交返回既有单且 created=false。查询只读,重发无
|
||||
副作用,幂等键的意义是上游重发时不会拿到两个单号。
|
||||
|
||||
kind=order_list:按时间窗口翻页扫描订单列表(params 支持 since / max_pages)。
|
||||
kind=order_detail:读单笔注文番号详情(params 必填 order_number)。
|
||||
kind 非法在提交时就 422,不会等 worker 领走才失败。
|
||||
"""
|
||||
data = await container.query_queue.submit(
|
||||
query_id=payload.query_id,
|
||||
site=payload.site,
|
||||
kind=payload.kind.value,
|
||||
params=payload.params,
|
||||
)
|
||||
return ApiResponse[SubmitQueryData](success=True, msg="success", data=data, code=0)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/lease",
|
||||
response_model=ApiResponse[QueryLeaseData | None],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def lease_query(
|
||||
worker_id: str = Query(..., description="查询 worker 标识(不刷下单 worker 心跳)"),
|
||||
wait: int = Query(
|
||||
default=30, ge=0, le=300,
|
||||
description="无单可领时的挂起秒数。服务端上限 RAKUTEN_LEASE_MAX_WAIT_SECONDS(默认 60)",
|
||||
),
|
||||
site: str | None = Query(default=None, description="限定站点(rakuten);不传不限"),
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[QueryLeaseData | None]:
|
||||
"""本地长轮询领取查询单(查询 worker 专用)
|
||||
|
||||
- 与下单 lease 不同:**没有**「已有任务在执行就一律返回空」的闸门,多张查询单
|
||||
可以同时在飞;账号级串行由本地 worker 的账号锁保证,下单跑着时查询排队等锁。
|
||||
- 不刷下单 worker 心跳——心跳的语义是「下单 worker 还活着」,查询循环活着不
|
||||
代表它活着。
|
||||
- 无单可领时挂起至多 wait 秒再返回,HTTP 仍 200、data 为 null。
|
||||
- 租约过期自动重投(查询只读,重投安全,与下单「绝不自动重投」相反):
|
||||
attempt 递增,用尽(RAKUTEN_QUERY_MAX_ATTEMPTS,默认 3)置 failed;
|
||||
超过查询单整体 TTL(RAKUTEN_QUERY_TTL_SECONDS,默认 900 秒)置 expired。
|
||||
"""
|
||||
data = await container.query_queue.lease(
|
||||
worker_id=worker_id,
|
||||
wait=wait,
|
||||
site=site,
|
||||
max_wait=container.settings.lease_max_wait_seconds,
|
||||
)
|
||||
return ApiResponse[QueryLeaseData | None](success=True, msg="success", data=data, code=0)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/{query_id}/result",
|
||||
response_model=ApiResponse[QueryResultData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
responses={
|
||||
404: {"description": "查询单不存在(错误码 6005)"},
|
||||
409: {"description": "查询单租约无效:不是持有者、已被重投或已终结(错误码 6006)"},
|
||||
},
|
||||
)
|
||||
async def submit_query_result(
|
||||
query_id: str,
|
||||
payload: QueryResultRequest,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[QueryResultData]:
|
||||
"""本地回报查询结果(查询 worker 专用)
|
||||
|
||||
- success=true 时 result 必填;false 时 error_message 必填,error_code 沿用
|
||||
统一错误码表(如 5001 掉登录),原样带给上游按同一张表分支。
|
||||
- 租约不是自己的(多半是执行超时、单子已被重投给下一轮)时报 6006——
|
||||
**丢弃本次结果即可,不要重发**,否则迟到的旧结果会覆盖新一轮的结果。
|
||||
- result 体积上限 RAKUTEN_QUERY_RESULT_MAX_BYTES(默认 1MB):超限由 worker 侧
|
||||
先丢 raw / raw_pages(置 raw_omitted)再判失败,让上游缩小窗口重来。
|
||||
"""
|
||||
data = await container.query_queue.submit_result(
|
||||
query_id,
|
||||
worker_id=payload.worker_id,
|
||||
success=payload.success,
|
||||
result=payload.result,
|
||||
error_code=payload.error_code,
|
||||
error_message=payload.error_message,
|
||||
)
|
||||
return ApiResponse[QueryResultData](success=True, msg="success", data=data, code=0)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/{query_id}",
|
||||
response_model=ApiResponse[QueryDetail],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
responses={404: {"description": "查询单不存在或已过保留期被清理(错误码 6005)"}},
|
||||
)
|
||||
async def get_query(
|
||||
query_id: str,
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[QueryDetail]:
|
||||
"""取查询单状态与结果
|
||||
|
||||
仍在 queued / leased 就是还没读到,稍后再来;failed / expired 看 error;
|
||||
status=succeeded 时 result 为「规范化字段 + 站点原始 JSON」,结构按 kind:
|
||||
|
||||
**kind=order_list**:`{kind, since, max_pages, window_fully_covered, orders[], raw_pages[]}`
|
||||
— orders[] 含 order_number / order_date / shop_id / shop_name / items[](稳定契约字段);
|
||||
raw_pages 是站点 __INITIAL_STATE__.orderListData 原文(按翻页顺序,可能因体积上限
|
||||
被省略并置 raw_omitted=true)。
|
||||
**window_fully_covered=false 时,「列表里没有某笔订单」不等于「账号里没有」**——
|
||||
翻页在覆盖完窗口前就停了(命中 max_pages、页面改版、掉登录),请缩小窗口重查或
|
||||
交人工,不要当成确定性的否定结论。
|
||||
|
||||
**kind=order_detail**:`{kind, order_number, found, stage_label, order_state,
|
||||
delivery_status, raw, raw_available}`
|
||||
— found=false 不是错误:站点自己说下单后要约 10 分钟才反映;stage_label 是站点
|
||||
原文(如「ご注文確認中」),order_state 是映射后的订单状态(映射不到为 null),
|
||||
delivery_status 是站点结构化配送状态码(原样带给上游对账);金额、收货地址、
|
||||
付款方式等在 raw.orderData 里原样透传,本服务不抽取。
|
||||
|
||||
终态查询单保留 RAKUTEN_QUERY_RETENTION_SECONDS(默认 7 天),过期被清理后查也报 6005。
|
||||
"""
|
||||
data = await container.query_queue.get_detail(query_id)
|
||||
return ApiResponse[QueryDetail](success=True, msg="success", data=data, code=0)
|
||||
|
||||
|
||||
@router.get(
|
||||
"",
|
||||
response_model=ApiResponse[QueryListData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def list_queries(
|
||||
status: str | None = Query(
|
||||
default=None,
|
||||
description="按状态筛选:queued / leased / succeeded / failed / expired",
|
||||
),
|
||||
kind: str | None = Query(default=None, description="按类型筛选:order_list / order_detail"),
|
||||
limit: int = Query(default=50, ge=1, le=500, description="分页大小"),
|
||||
offset: int = Query(default=0, ge=0, description="分页偏移"),
|
||||
container: GatewayContainer = Depends(get_container),
|
||||
) -> ApiResponse[QueryListData]:
|
||||
"""查询单列表,按创建时间倒序(运维排查用)
|
||||
|
||||
注意定时下派通道(collector)自主派发的 discover-* 查询单也会出现在这里。
|
||||
"""
|
||||
data = await container.query_queue.list_queries(
|
||||
status=status, kind=kind, limit=limit, offset=offset
|
||||
)
|
||||
return ApiResponse[QueryListData](success=True, msg="success", data=data, code=0)
|
||||
@@ -0,0 +1,119 @@
|
||||
"""终结类事件回调:把任务结局 POST 到上游登记的通知地址
|
||||
|
||||
触发时机只有两类(docs/order-gateway.md §4.8):
|
||||
|
||||
- worker 的 terminal report 把任务推入终态(succeeded / failed / needs_human)
|
||||
- 租约过期被 sweep 置 stale(任务卡住,需要人工 reclaim)
|
||||
|
||||
投递是 best-effort:单次尝试,失败只记日志、不重试。回调只是「省轮询」的提示,
|
||||
权威状态仍以 GET /api/orders/{task_id} 为准。任何异常都不允许逃出发送路径,
|
||||
不能影响 report / sweep 主流程。
|
||||
|
||||
出站遵循统一代理策略(app/shared/proxy.py):回调地址逐任务不同,而代理 bypass
|
||||
是按目标 host 判定的,所以客户端按发送逐次构造,不能在建通知器时定死。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.shared.proxy import httpx_client_options
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.gateway.db import ReportRow, TaskRow
|
||||
from app.shared.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def terminal_payload(task: TaskRow, report: ReportRow, final_status: str) -> dict[str, Any]:
|
||||
"""terminal report 触发的通知:任务终态 + 本次上报的字段(金额/单号/证据路径等)"""
|
||||
return {
|
||||
"task_id": task.task_id,
|
||||
"site": task.site,
|
||||
"event": "terminal",
|
||||
"status": final_status,
|
||||
"state": report.state,
|
||||
"payable_yen": report.payable_yen,
|
||||
"pay_deadline": report.pay_deadline,
|
||||
"site_order_id": report.site_order_id,
|
||||
"evidence_ref": report.evidence_ref,
|
||||
"detail": report.detail,
|
||||
"reported_at": report.reported_at,
|
||||
}
|
||||
|
||||
|
||||
def stale_payload(task: TaskRow) -> dict[str, Any]:
|
||||
"""租约过期触发的通知:任务卡住,需人工 reclaim(绝不自动重投,见规格 §5)"""
|
||||
return {
|
||||
"task_id": task.task_id,
|
||||
"site": task.site,
|
||||
"event": "stale",
|
||||
"status": "stale",
|
||||
"detail": "租约过期,任务已置 stale,需人工 reclaim",
|
||||
}
|
||||
|
||||
|
||||
class CallbackNotifier:
|
||||
"""把终结类事件 POST 到上游 callback_url(发后不管,失败只记日志)
|
||||
|
||||
`notify` 只做调度,真正的 HTTP 发送在后台任务里完成,调用方(report / sweep)
|
||||
不被回调的耗时阻塞。在途任务用 `self._pending` 持有强引用,避免事件循环只持
|
||||
弱引用导致任务被提前回收;`aclose` 时会等这些在途通知发完(各自有超时兜底)。
|
||||
|
||||
`client` 仅供测试注入(如 MockTransport);生产路径为 None,按回调地址
|
||||
逐次构造带代理策略的临时客户端(回调是低频事件,代价可忽略)。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
settings: Settings,
|
||||
timeout_seconds: float,
|
||||
client: httpx.AsyncClient | None = None,
|
||||
):
|
||||
self._settings = settings
|
||||
self._timeout = timeout_seconds
|
||||
self._client = client
|
||||
self._pending: set[asyncio.Task[None]] = set()
|
||||
|
||||
def notify(self, callback_url: str, payload: dict[str, Any]) -> None:
|
||||
"""调度一次后台发送。调用方不等待,回调慢/挂不阻塞任务主流程"""
|
||||
task = asyncio.create_task(self._send(callback_url, payload))
|
||||
self._pending.add(task)
|
||||
task.add_done_callback(self._pending.discard)
|
||||
|
||||
async def _send(self, callback_url: str, payload: dict[str, Any]) -> None:
|
||||
task_id = payload.get("task_id")
|
||||
try:
|
||||
if self._client is not None:
|
||||
response = await self._client.post(callback_url, json=payload)
|
||||
else:
|
||||
async with httpx.AsyncClient(
|
||||
timeout=self._timeout,
|
||||
**httpx_client_options(self._settings, target_url=callback_url),
|
||||
) as client:
|
||||
response = await client.post(callback_url, json=payload)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
# 回调失败不允许影响任务主流程:记日志后丢弃,上游可靠轮询对账兜底
|
||||
logger.warning(
|
||||
"回调通知发送失败(已丢弃):task_id=%s url=%s err=%s", task_id, callback_url, exc
|
||||
)
|
||||
return
|
||||
if response.status_code >= 400:
|
||||
logger.warning(
|
||||
"回调通知返回非成功状态(已丢弃):task_id=%s url=%s status=%s",
|
||||
task_id, callback_url, response.status_code,
|
||||
)
|
||||
return
|
||||
logger.info("回调通知已送达:task_id=%s url=%s", task_id, callback_url)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
"""关停:等在途通知发完(各次发送有超时兜底),再关 HTTP 客户端"""
|
||||
if self._pending:
|
||||
await asyncio.gather(*self._pending, return_exceptions=True)
|
||||
if self._client is not None:
|
||||
await self._client.aclose()
|
||||
@@ -0,0 +1,353 @@
|
||||
"""定时下派通道:周期性发现账号真实订单并沉淀到网关编目(docs/order-gateway.md §12)
|
||||
|
||||
**它解决什么问题**:上游经「账号只读查询通道」(§11)要订单详情,得**自己先知道
|
||||
order_number 再下派**。可账号在站点上可能走了别的渠道下单、被商家改状态,网关的
|
||||
状态镜像根本看不到。本模块让网关**自己定期去盘点账号**:派一张 order_list 查询
|
||||
(走 §11 那条队列,本地 worker 出站真读),拿到列表后把「网关编目里还没有的订单」
|
||||
回写进 `account_orders` 表,再逐笔派 order_detail 查询把配送阶段沉淀下来。之后
|
||||
上游直接查 `/api/account/orders` 就能拿到账号里真实有哪些订单,不必再自己记。
|
||||
|
||||
**实现里最要紧的一条边界**:collector 只读 `query_queue` 查询单结果里的
|
||||
**规范化字段**(`orders[].order_number/order_date/shop_id/shop_name` 与
|
||||
`delivery_status/order_state`——这些是 worker 在 query_runner 里专门为对外契约
|
||||
抽好的稳定字段,见 §11.3),**绝不解析结果里的 `raw` / `raw_pages` 站点原始 JSON**。
|
||||
「网关不解释 query 的 params/result」这条原则照旧适用于上游自提的查询单;collector
|
||||
自主派发的这批单是网关自己的编排数据,消费自己的规范化契约不违反它。
|
||||
|
||||
**状态模型刻意保持无痕(不新增跟踪表)**:
|
||||
- collector 只在内存里记「本轮派出去但还没处理结果」的 query_id(`_pending`),
|
||||
进程重启即清空。重启后既有的 discover-* 查询单要么被 worker 消费完留在 DB 里
|
||||
(无主结果,7 天保留期后被 sweep 清掉),要么过期/失败;下一轮 tick 会派一张
|
||||
全新的 list 扫描,结果幂等 upsert 进编目,不会重复。
|
||||
- `order_detail` 的 query_id 用 `discover-detail-<order_number>-<日期>`
|
||||
**按天分段**:同一天内对同一笔订单重复 submit 会命中幂等返回同一张单(不会
|
||||
因每轮 tick 都满足「详情已过期」就反复创建);跨天自然生成新单刷新详情。
|
||||
- 编目 upsert 以 `order_number` 为主键,重复采集只刷新,不产生重复行。
|
||||
|
||||
**谁负责执行**:本模块只负责派单与沉淀结果;实际的订单列表/详情抓取仍然由本地
|
||||
worker 的 `query_runner` 经 §11 队列执行,本模块不碰浏览器。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.gateway.db import AccountOrderRow, GatewayDB
|
||||
from app.gateway.query_queue import QueryQueue
|
||||
from app.shared.task_state import (
|
||||
AccountQueryKind,
|
||||
QueryStatus,
|
||||
TERMINAL_STATUSES,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.shared.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# collector 主循环的节拍(秒):常驻时定时去收割结果与检查排队状态。派单间隔
|
||||
# 由 settings.account_discovery_interval_seconds / account_detail_refresh_seconds
|
||||
# 决定,不靠这个节拍驱动。
|
||||
_TICK_SECONDS = 60
|
||||
|
||||
# 查询单终态集合(字符串形式):SUCCEEDED/FAILED/EXPIRED。收割时据此判断「单子
|
||||
# 出结果了没」——仍在 queued/leased 的一律不碰,留到下一轮。
|
||||
_TERMINAL = tuple(s.value for s in TERMINAL_STATUSES)
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def _day_key(dt: datetime) -> str:
|
||||
"""给 order_detail 的 query_id 按天分桶用:如 20260816"""
|
||||
return dt.astimezone(timezone.utc).strftime("%Y%m%d")
|
||||
|
||||
|
||||
def _coerce_str(value: Any) -> str | None:
|
||||
"""把 result 里的值安全地转成字符串;缺失/非法返回 None,不报错"""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
return str(value)
|
||||
|
||||
|
||||
def _parse_list_result(result: dict[str, Any] | None) -> list[dict[str, Any]]:
|
||||
"""从 order_list 查询结果里抽规范化订单行,形状不符则保守地返回空
|
||||
|
||||
只取 `orders` 数组里每个元素的 order_number 这几个规范化字段;`raw_pages`
|
||||
原地透传的站点原始 JSON **一律不碰**。这里不抛错——collector 是后台任务,
|
||||
某个字段缺了只会导致这笔订单索引不到,不该让整轮扫描崩溃。
|
||||
"""
|
||||
if not result:
|
||||
return []
|
||||
orders = result.get("orders")
|
||||
if not isinstance(orders, list):
|
||||
logger.warning("order_list 结果里没有 orders 数组,本轮不沉淀:result_keys=%s", list(result))
|
||||
return []
|
||||
return [o for o in orders if isinstance(o, dict) and o.get("order_number")]
|
||||
|
||||
|
||||
class OrderDiscoveryCollector:
|
||||
"""网关侧的周期性订单发现器
|
||||
|
||||
`run()` 是常驻后台循环(与 `_sweep_loop` 并列),每 `_TICK_SECONDS` 跑一次
|
||||
`tick()`。tick 做三件事:
|
||||
|
||||
1. 收割:把 `_pending` 里已到终态的查询单结果消化掉,写进 `account_orders`;
|
||||
2. 派列表:距上次 list 扫描超过 `list_interval_seconds` 且没有在飞的 list 单时,
|
||||
派一张 order_list 查询(整段账号历史,`max_pages` 由设置给);
|
||||
3. 派详情:把「还没成功取过详情」或「详情已过 detail_refresh_seconds」的编目
|
||||
订单各派一张 order_detail 查询。
|
||||
|
||||
生命周期由 `GatewayContainer` 持有、`main.lifespan` 起停。被禁用时(
|
||||
`settings.account_discovery_enabled=False`)`run()` 直接返回,不做事。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
settings: "Settings",
|
||||
db: GatewayDB,
|
||||
query_queue: QueryQueue,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
self._db = db
|
||||
self._queue = query_queue
|
||||
self._running = False
|
||||
# 本进程派出去、还没处理结果的 discover-* 单:query_id → kind。
|
||||
# 用 dict 而不只存 id,是为了在「是否有一张 list 单还在飞」的判断上不再
|
||||
# 靠前缀猜——list 单的 id 是自动生成的 q-...,detail 单是 discover-detail-...,
|
||||
# 混在一个 set 里光看前缀容易错。
|
||||
self._pending: dict[str, str] = {}
|
||||
# 本进程已为哪些订单派过当天的详情查询(order_number 集合)。防止「详情
|
||||
# found=False 导致订单一直算过期、每轮 tick 都重发一张同天详情单」的重复
|
||||
# 提交——同日内每笔订单只派一张详情,跨天由《日期分段》的 query_id 自然刷新。
|
||||
self._detail_dispatched_this_day: set[str] = set()
|
||||
self._last_list_submitted: datetime | None = None
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return self._settings.account_discovery_enabled
|
||||
|
||||
# ---- 主循环 ----
|
||||
|
||||
async def run(self) -> None:
|
||||
"""常驻循环:被启用时每 _TICK_SECONDS 跑一次 tick"""
|
||||
if not self.enabled:
|
||||
logger.info("定时下派通道未启用(RAKUTEN_ACCOUNT_DISCOVERY_ENABLED=false),不启动")
|
||||
return
|
||||
self._running = True
|
||||
logger.info(
|
||||
"定时下派通道启动:list 扫描间隔 %ss,详情刷新间隔 %ss",
|
||||
self._settings.account_discovery_interval_seconds,
|
||||
self._settings.account_detail_refresh_seconds,
|
||||
)
|
||||
while self._running:
|
||||
try:
|
||||
await self.tick()
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
# 后台任务不能因为偶发错误退出,否则编目再也不会更新
|
||||
logger.exception("定时下派 tick 异常,继续下一轮")
|
||||
await asyncio.sleep(_TICK_SECONDS)
|
||||
|
||||
# ---- 单轮 ----
|
||||
|
||||
async def tick(self) -> None:
|
||||
await self._collect_results()
|
||||
await self._dispatch_list_if_due()
|
||||
await self._dispatch_stale_details()
|
||||
|
||||
async def _collect_results(self) -> None:
|
||||
"""把 _pending 里到终态的查询单结果消化进编目"""
|
||||
pending = list(self._pending)
|
||||
for query_id in pending:
|
||||
try:
|
||||
detail = await self._queue.get_detail(query_id)
|
||||
except Exception: # noqa: BLE001 (6005:查询单已过保留期被清)
|
||||
self._pending.pop(query_id, None)
|
||||
logger.warning("discover 查询单已不存在(可能被清理),丢弃:query_id=%s", query_id)
|
||||
continue
|
||||
if detail.status not in _TERMINAL:
|
||||
continue # 还在飞,下一轮再看
|
||||
kind = self._pending.pop(query_id, None)
|
||||
try:
|
||||
if detail.status == QueryStatus.SUCCEEDED:
|
||||
if kind == AccountQueryKind.ORDER_LIST.value:
|
||||
await self._ingest_list_result(detail.result, query_id=query_id)
|
||||
elif kind == AccountQueryKind.ORDER_DETAIL.value:
|
||||
await self._ingest_detail_result(detail.result)
|
||||
else:
|
||||
logger.warning(
|
||||
"discover 查询单未成功:query_id=%s kind=%s status=%s error=%s",
|
||||
query_id, kind, detail.status,
|
||||
(detail.error.message if detail.error else ""),
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("消化 discover 查询单结果失败:query_id=%s", query_id)
|
||||
|
||||
async def _ingest_list_result(self, result: dict[str, Any] | None, *, query_id: str) -> None:
|
||||
"""把 order_list 结果里每笔订单 upsert 进编目(回写网关)
|
||||
|
||||
这是「采集到网关中不存在的订单就写回」的核心。只写规范化字段;订单是否
|
||||
真的覆盖完整窗口已由 worker 的 `window_fully_covered` 标明,否则这里到的
|
||||
只是部分窗口——仍然先沉淀拿到的那部分,日志里注明。
|
||||
"""
|
||||
orders = _parse_list_result(result)
|
||||
seen = 0
|
||||
for entry in orders:
|
||||
order_number = _coerce_str(entry.get("order_number")) or ""
|
||||
existing = await self._db.get_account_order(order_number) if order_number else None
|
||||
order = AccountOrderRow(
|
||||
order_number=order_number,
|
||||
shop_id=_coerce_str(entry.get("shop_id")),
|
||||
shop_name=_coerce_str(entry.get("shop_name")) or "",
|
||||
order_date=_coerce_str(entry.get("order_date")),
|
||||
# 列表结果不带配送阶段/详情时间。订单若已编目过(详情取过),这里
|
||||
# 保留既有数据,别拿 None 把详情阶段悄悄抹掉;新订单则为 None,等
|
||||
# 后面的 detail 下派来填。
|
||||
delivery_status=existing.delivery_status if existing else None,
|
||||
order_state=existing.order_state if existing else None,
|
||||
discovered_at=existing.discovered_at if existing else _iso(utcnow()),
|
||||
detail_fetched_at=existing.detail_fetched_at if existing else None,
|
||||
last_seen_at=_iso(utcnow()),
|
||||
)
|
||||
inserted = await self._db.upsert_account_order(order)
|
||||
seen += 1
|
||||
if inserted:
|
||||
logger.info(
|
||||
"定时下派:发现网关编目中不存在的新订单并回写:order_number=%s shop=%s",
|
||||
order.order_number, order.shop_name,
|
||||
)
|
||||
covered = bool(result and result.get("window_fully_covered"))
|
||||
logger.info(
|
||||
"discover list 收割完成:query_id=%s orders=%s window_fully_covered=%s",
|
||||
query_id, seen, covered,
|
||||
)
|
||||
|
||||
async def _ingest_detail_result(self, result: dict[str, Any] | None) -> None:
|
||||
"""把 order_detail 结果里规范化出的配送阶段写进编目对应订单
|
||||
|
||||
只更新 order_number 命中编目的行:detail 查询是我们为编目订单派出的,
|
||||
正常情况下一定命中。found=False 是「站点说这单还没反映出来」,不覆盖已有
|
||||
数据,只刷新 detail_fetched_at 免得每轮都重派。
|
||||
"""
|
||||
if not result:
|
||||
return
|
||||
order_number = _coerce_str(result.get("order_number"))
|
||||
if not order_number:
|
||||
logger.warning("order_detail 结果缺 order_number,跳过:result_keys=%s", list(result))
|
||||
return
|
||||
row = await self._db.get_account_order(order_number)
|
||||
if row is None:
|
||||
# 理论上不该发生:detail 单都是为编目订单派出的。可能编目被删或并发
|
||||
# 重置,保守处理:只记日志。
|
||||
logger.warning("discover detail 拿到了编目不存在的订单,忽略:order_number=%s", order_number)
|
||||
return
|
||||
now = _iso(utcnow())
|
||||
found = bool(result.get("found"))
|
||||
# found=False 是「站点说这单还没反映出来」:不动已有的配送阶段/详情时间,
|
||||
# 免得一次瞬时抖动把编目里已沉淀的状态抹掉(结果里这些字段是 null)。
|
||||
upsert = AccountOrderRow(
|
||||
order_number=row.order_number,
|
||||
shop_id=row.shop_id,
|
||||
shop_name=row.shop_name,
|
||||
order_date=row.order_date,
|
||||
delivery_status=(
|
||||
_coerce_str(result.get("delivery_status")) if found else row.delivery_status
|
||||
),
|
||||
order_state=_coerce_str(result.get("order_state")) if found else row.order_state,
|
||||
discovered_at=row.discovered_at,
|
||||
detail_fetched_at=now if found else row.detail_fetched_at,
|
||||
last_seen_at=row.last_seen_at,
|
||||
)
|
||||
await self._db.upsert_account_order(upsert)
|
||||
if found:
|
||||
logger.info(
|
||||
"discover detail 已沉淀:order_number=%s delivery_status=%s order_state=%s",
|
||||
order_number, upsert.delivery_status, upsert.order_state,
|
||||
)
|
||||
|
||||
# ---- 派单 ----
|
||||
|
||||
async def _dispatch_list_if_due(self) -> None:
|
||||
"""距上次 list 扫描够久且没有在飞的 list 单时,派一张 order_list 查询"""
|
||||
interval = self._settings.account_discovery_interval_seconds
|
||||
due = self._last_list_submitted is None or (
|
||||
utcnow() - self._last_list_submitted >= timedelta(seconds=interval)
|
||||
)
|
||||
has_inflight_list = any(
|
||||
kind == AccountQueryKind.ORDER_LIST.value for kind in self._pending.values()
|
||||
)
|
||||
if not due or has_inflight_list:
|
||||
return
|
||||
params = {"max_pages": self._settings.account_discovery_max_pages}
|
||||
data = await self._queue.submit(
|
||||
query_id=None, site="rakuten",
|
||||
kind=AccountQueryKind.ORDER_LIST.value, params=params,
|
||||
)
|
||||
self._last_list_submitted = utcnow()
|
||||
self._pending[data.query_id] = AccountQueryKind.ORDER_LIST.value
|
||||
logger.info("定时下派:已派 order_list 扫描,query_id=%s", data.query_id)
|
||||
|
||||
async def _dispatch_stale_details(self) -> None:
|
||||
"""把「没取过详情」或「详情已过刷新期」的编目订单各派一张 order_detail 查询
|
||||
|
||||
当天已派过详情的订单跳过(`_detail_dispatched_this_day`),避免因
|
||||
found=False 一直算「过期」而每轮 tick 重复提交同一张同天详情单。
|
||||
"""
|
||||
cutoff = _iso(utcnow() - timedelta(seconds=self._settings.account_detail_refresh_seconds))
|
||||
stale = await self._db.list_account_orders_needing_detail(cutoff)
|
||||
for order in stale:
|
||||
if order.order_number in self._detail_dispatched_this_day:
|
||||
continue
|
||||
await self._dispatch_one_detail(order.order_number)
|
||||
self._detail_dispatched_this_day.add(order.order_number)
|
||||
|
||||
async def _dispatch_one_detail(self, order_number: str) -> None:
|
||||
"""派一张 order_detail 查询;同一天内重复派命中幂等返回既有单
|
||||
|
||||
返回的 query_id 无论新旧都放进 _pending,保证它到终态时会被收割——
|
||||
旧单若早已终态,下一轮 _collect_results 收割即丢弃,无副作用。
|
||||
"""
|
||||
# 按天分段,跨天自然刷新,同一天内不重复派
|
||||
qid = f"discover-detail-{order_number}-{_day_key(utcnow())}"
|
||||
data = await self._queue.submit(
|
||||
query_id=qid, site="rakuten",
|
||||
kind=AccountQueryKind.ORDER_DETAIL.value,
|
||||
params={"order_number": order_number},
|
||||
)
|
||||
self._pending[data.query_id] = AccountQueryKind.ORDER_DETAIL.value
|
||||
logger.info("定时下派:已派 order_detail,order_number=%s query_id=%s", order_number, data.query_id)
|
||||
|
||||
# ---- 手动触发 ----
|
||||
|
||||
async def trigger_list_sweep(self) -> dict[str, Any]:
|
||||
"""立即派一轮 order_list 扫描(POST /api/account/discovery/trigger 用)
|
||||
|
||||
返回 submit 的 data 以表示本轮是否新建。因为不重记 last_list_submitted,
|
||||
触发后到下一个整间隔前,定时器不会重复派(仍受「有在飞 list 单就不派」保护)。
|
||||
"""
|
||||
params = {"max_pages": self._settings.account_discovery_max_pages}
|
||||
data = await self._queue.submit(
|
||||
query_id=None, site="rakuten",
|
||||
kind=AccountQueryKind.ORDER_LIST.value, params=params,
|
||||
)
|
||||
self._pending[data.query_id] = AccountQueryKind.ORDER_LIST.value
|
||||
return {
|
||||
"query_id": data.query_id,
|
||||
"status": data.status.value,
|
||||
"created": data.created,
|
||||
}
|
||||
|
||||
|
||||
def _iso(dt: datetime) -> str:
|
||||
return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
@@ -3,9 +3,10 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
from app.gateway.db import GatewayDB
|
||||
from app.gateway.query_queue import QueryQueue
|
||||
from app.gateway.task_queue import TaskQueue
|
||||
from app.shared.config import Settings
|
||||
|
||||
@@ -19,11 +20,25 @@ class GatewayContainer:
|
||||
与抓取/交易容器同样的依赖注入风格,但持有的是任务队列与 SQLite 连接,
|
||||
生命周期由 main.lifespan 管理:start 时打开 DB,close 时关闭。
|
||||
|
||||
`sweep_task` 是常驻后台扫描,把过期的 leased/running 推到 stale。
|
||||
即使没有 lease 请求,过期的任务也会被及时发现(规格 §5 的关键约束)。
|
||||
两条通道各有一个队列对象,共用同一个 GatewayDB:
|
||||
- `task_queue` 下单任务(不可逆写,租约过期只置 stale)
|
||||
- `query_queue` 账号只读查询(可安全重投,见 query_queue.py 顶部对照表)
|
||||
|
||||
`sweep_task` 是常驻后台扫描,把过期的 leased/running 推到 stale,并顺带扫
|
||||
一轮查询单(重投 / 过期 / 清理过保留期的结果)。即使没有 lease 请求,过期的
|
||||
任务也会被及时发现(规格 §5 的关键约束)。
|
||||
|
||||
`collector` 是定时下派通道(§12):周期性派 order_list / order_detail 查询、
|
||||
把账户真实订单沉淀进 `account_orders` 编目。`collector_task` 是它的后台任务,
|
||||
在 lifespan 里与 `sweep_task` 一起启动/取消。
|
||||
"""
|
||||
|
||||
settings: Settings
|
||||
db: GatewayDB
|
||||
task_queue: TaskQueue
|
||||
query_queue: QueryQueue
|
||||
collector: object | None = field(default=None) # app.gateway.collector.OrderDiscoveryCollector
|
||||
# 终结类事件回调(§4.8):有 callback_url 的任务到达终态/变 stale 时 POST 通知上游
|
||||
notifier: object | None = field(default=None) # app.gateway.callback.CallbackNotifier
|
||||
sweep_task: asyncio.Task | None = None
|
||||
collector_task: asyncio.Task | None = None
|
||||
|
||||
+374
-6
@@ -1,10 +1,11 @@
|
||||
"""网关 SQLite 访问层:三张表 + 原生 SQL,无业务逻辑
|
||||
"""网关 SQLite 访问层:四张表 + 原生 SQL,无业务逻辑
|
||||
|
||||
业务规则(状态机、并发度、幂等)放在 `task_queue.py`,本模块只负责持久化与查询,
|
||||
业务规则(状态机、并发度、幂等)放在 `task_queue.py`(下单任务)与
|
||||
`query_queue.py`(账号只读查询),本模块只负责持久化与查询,
|
||||
返回 dataclass 行。所有时间戳以 ISO8601 UTC 字符串存储(以 `Z` 结尾),便于
|
||||
跨进程对账;时间戳运算在 task_queue 层完成。
|
||||
跨进程对账;时间戳运算在上层完成。
|
||||
|
||||
写操作由 task_queue 的 asyncio.Lock 串行化(详见该模块),本层不重复加锁,
|
||||
写操作由上层的 asyncio.Lock 串行化(详见对应模块),本层不重复加锁,
|
||||
因此**调用方必须确保写操作在外层锁的保护下进行**。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
@@ -24,6 +25,7 @@ CREATE TABLE IF NOT EXISTS tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
site TEXT NOT NULL,
|
||||
intent_json TEXT NOT NULL,
|
||||
callback_url TEXT, -- 终结类事件通知地址,见 docs/order-gateway.md §4.8
|
||||
status TEXT NOT NULL,
|
||||
lease_owner TEXT,
|
||||
lease_expires_at TEXT,
|
||||
@@ -49,6 +51,44 @@ CREATE TABLE IF NOT EXISTS workers (
|
||||
worker_id TEXT PRIMARY KEY,
|
||||
last_seen_at TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- 账号只读查询单(见 docs/order-gateway.md §11)。与 tasks 分表,不是洁癖:
|
||||
-- 两者的安全约束相反(只读可重投 vs 写操作绝不重投),共表会让「租约过期怎么办」
|
||||
-- 这个最关键的分支变成一个 if,迟早被改错。
|
||||
CREATE TABLE IF NOT EXISTS account_queries (
|
||||
query_id TEXT PRIMARY KEY, -- 幂等键,语义同 task_id
|
||||
site TEXT NOT NULL,
|
||||
kind TEXT NOT NULL, -- order_list / order_detail
|
||||
params_json TEXT NOT NULL, -- 查询参数原文,网关不解释内容
|
||||
status TEXT NOT NULL, -- 见 shared.task_state.QueryStatus
|
||||
lease_owner TEXT,
|
||||
lease_expires_at TEXT,
|
||||
attempts INTEGER NOT NULL DEFAULT 0, -- 被领取过几次
|
||||
result_json TEXT, -- worker 回的结果原文
|
||||
error_code INTEGER,
|
||||
error_message TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL,
|
||||
completed_at TEXT
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_queries_pending ON account_queries(status, created_at);
|
||||
|
||||
-- 定时下派通道(docs/order-gateway.md §12):账号里真实订单的「已发现」目录。
|
||||
-- collector 周期性派 order_list / order_detail 查询,把 worker 回结果里的
|
||||
-- **规范化字段**(订单号/店铺/日期/配送状态)沉淀到这里——这是网关自己的编目,
|
||||
-- 不是上游查询单的原样透传。只消费规范化契约,绝不碰结果里的站点原始 JSON。
|
||||
CREATE TABLE IF NOT EXISTS account_orders (
|
||||
order_number TEXT PRIMARY KEY,
|
||||
shop_id TEXT,
|
||||
shop_name TEXT,
|
||||
order_date TEXT, -- 站点 ISO8601 字符串
|
||||
delivery_status TEXT, -- 最近一次 detail 查询带回的站点码(如 CHECKING_ORDER)
|
||||
order_state TEXT, -- 映射后的 OrderState(如 delivered)
|
||||
discovered_at TEXT NOT NULL, -- 首次经 account 采集到该订单的时间
|
||||
detail_fetched_at TEXT, -- 最近一次成功取到详情页的时间
|
||||
last_seen_at TEXT NOT NULL -- 最近一次出现在订单列表里的时间
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_account_orders_last_seen ON account_orders(last_seen_at);
|
||||
"""
|
||||
|
||||
|
||||
@@ -59,6 +99,7 @@ class TaskRow:
|
||||
task_id: str
|
||||
site: str
|
||||
intent_json: str
|
||||
callback_url: str | None
|
||||
status: str
|
||||
lease_owner: str | None
|
||||
lease_expires_at: str | None
|
||||
@@ -93,11 +134,55 @@ class WorkerRow:
|
||||
last_seen_at: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class QueryRow:
|
||||
"""account_queries 表一行的强类型视图"""
|
||||
|
||||
query_id: str
|
||||
site: str
|
||||
kind: str
|
||||
params_json: str
|
||||
status: str
|
||||
lease_owner: str | None
|
||||
lease_expires_at: str | None
|
||||
attempts: int
|
||||
result_json: str | None
|
||||
error_code: int | None
|
||||
error_message: str | None
|
||||
created_at: str
|
||||
updated_at: str
|
||||
completed_at: str | None
|
||||
|
||||
@property
|
||||
def params(self) -> dict[str, Any]:
|
||||
return json.loads(self.params_json)
|
||||
|
||||
@property
|
||||
def result(self) -> dict[str, Any] | None:
|
||||
return json.loads(self.result_json) if self.result_json else None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AccountOrderRow:
|
||||
"""account_orders 表一行的强类型视图(定时下派通道的编目,见 §12)"""
|
||||
|
||||
order_number: str
|
||||
shop_id: str | None
|
||||
shop_name: str
|
||||
order_date: str | None
|
||||
delivery_status: str | None
|
||||
order_state: str | None
|
||||
discovered_at: str
|
||||
detail_fetched_at: str | None
|
||||
last_seen_at: str
|
||||
|
||||
|
||||
def _row_to_task(row: aiosqlite.Row) -> TaskRow:
|
||||
return TaskRow(
|
||||
task_id=row["task_id"],
|
||||
site=row["site"],
|
||||
intent_json=row["intent_json"],
|
||||
callback_url=row["callback_url"],
|
||||
status=row["status"],
|
||||
lease_owner=row["lease_owner"],
|
||||
lease_expires_at=row["lease_expires_at"],
|
||||
@@ -124,6 +209,39 @@ def _row_to_worker(row: aiosqlite.Row) -> WorkerRow:
|
||||
return WorkerRow(worker_id=row["worker_id"], last_seen_at=row["last_seen_at"])
|
||||
|
||||
|
||||
def _row_to_query(row: aiosqlite.Row) -> QueryRow:
|
||||
return QueryRow(
|
||||
query_id=row["query_id"],
|
||||
site=row["site"],
|
||||
kind=row["kind"],
|
||||
params_json=row["params_json"],
|
||||
status=row["status"],
|
||||
lease_owner=row["lease_owner"],
|
||||
lease_expires_at=row["lease_expires_at"],
|
||||
attempts=row["attempts"],
|
||||
result_json=row["result_json"],
|
||||
error_code=row["error_code"],
|
||||
error_message=row["error_message"],
|
||||
created_at=row["created_at"],
|
||||
updated_at=row["updated_at"],
|
||||
completed_at=row["completed_at"],
|
||||
)
|
||||
|
||||
|
||||
def _row_to_account_order(row: aiosqlite.Row) -> AccountOrderRow:
|
||||
return AccountOrderRow(
|
||||
order_number=row["order_number"],
|
||||
shop_id=row["shop_id"],
|
||||
shop_name=row["shop_name"] or "",
|
||||
order_date=row["order_date"],
|
||||
delivery_status=row["delivery_status"],
|
||||
order_state=row["order_state"],
|
||||
discovered_at=row["discovered_at"],
|
||||
detail_fetched_at=row["detail_fetched_at"],
|
||||
last_seen_at=row["last_seen_at"],
|
||||
)
|
||||
|
||||
|
||||
class GatewayDB:
|
||||
"""网关 SQLite 访问对象
|
||||
|
||||
@@ -141,9 +259,18 @@ class GatewayDB:
|
||||
self._conn = await aiosqlite.connect(str(self._db_path))
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
await self._conn.executescript(SCHEMA)
|
||||
await self._migrate()
|
||||
await self._conn.commit()
|
||||
logger.info("网关 DB 已就绪:%s", self._db_path)
|
||||
|
||||
async def _migrate(self) -> None:
|
||||
"""给既有库补新列(CREATE TABLE IF NOT EXISTS 不会改老表)"""
|
||||
async with self.conn.execute("PRAGMA table_info(tasks)") as cur:
|
||||
columns = {row["name"] for row in await cur.fetchall()}
|
||||
if "callback_url" not in columns:
|
||||
await self.conn.execute("ALTER TABLE tasks ADD COLUMN callback_url TEXT")
|
||||
logger.info("迁移:tasks 表补充 callback_url 列")
|
||||
|
||||
async def close(self) -> None:
|
||||
if self._conn is not None:
|
||||
await self._conn.close()
|
||||
@@ -166,13 +293,14 @@ class GatewayDB:
|
||||
"""插入新任务。返回 True=新建,False=task_id 已存在(幂等命中)"""
|
||||
try:
|
||||
await self.conn.execute(
|
||||
"INSERT INTO tasks (task_id, site, intent_json, status, "
|
||||
"INSERT INTO tasks (task_id, site, intent_json, callback_url, status, "
|
||||
"lease_owner, lease_expires_at, lease_count, created_at, updated_at) "
|
||||
"VALUES (?, ?, ?, ?, NULL, NULL, 0, ?, ?)",
|
||||
"VALUES (?, ?, ?, ?, ?, NULL, NULL, 0, ?, ?)",
|
||||
(
|
||||
row.task_id,
|
||||
row.site,
|
||||
row.intent_json,
|
||||
row.callback_url,
|
||||
row.status,
|
||||
row.created_at,
|
||||
row.updated_at,
|
||||
@@ -354,3 +482,243 @@ class GatewayDB:
|
||||
(status,),
|
||||
) as cur:
|
||||
return (await cur.fetchone())[0]
|
||||
|
||||
# ---- account_queries ----
|
||||
|
||||
async def get_query(self, query_id: str) -> QueryRow | None:
|
||||
async with self.conn.execute(
|
||||
"SELECT * FROM account_queries WHERE query_id = ?", (query_id,)
|
||||
) as cur:
|
||||
row = await cur.fetchone()
|
||||
return _row_to_query(row) if row else None
|
||||
|
||||
async def insert_query(self, row: QueryRow) -> bool:
|
||||
"""插入新查询单。返回 True=新建,False=query_id 已存在(幂等命中)"""
|
||||
try:
|
||||
await self.conn.execute(
|
||||
"INSERT INTO account_queries (query_id, site, kind, params_json, status, "
|
||||
"lease_owner, lease_expires_at, attempts, result_json, error_code, "
|
||||
"error_message, created_at, updated_at, completed_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, NULL, NULL, 0, NULL, NULL, NULL, ?, ?, NULL)",
|
||||
(
|
||||
row.query_id,
|
||||
row.site,
|
||||
row.kind,
|
||||
row.params_json,
|
||||
row.status,
|
||||
row.created_at,
|
||||
row.updated_at,
|
||||
),
|
||||
)
|
||||
await self.conn.commit()
|
||||
return True
|
||||
except aiosqlite.IntegrityError:
|
||||
return False
|
||||
|
||||
async def update_query(
|
||||
self,
|
||||
query_id: str,
|
||||
*,
|
||||
status: str | None = None,
|
||||
lease_owner: str | None = None,
|
||||
lease_expires_at: str | None = None,
|
||||
attempts: int | None = None,
|
||||
result_json: str | None = None,
|
||||
error_code: int | None = None,
|
||||
error_message: str | None = None,
|
||||
updated_at: str | None = None,
|
||||
completed_at: str | None = None,
|
||||
clear_lease: bool = False,
|
||||
) -> None:
|
||||
"""更新查询单字段。clear_lease=True 时把 lease_owner/expires_at 置 NULL"""
|
||||
sets: list[str] = []
|
||||
params: list[Any] = []
|
||||
for column, value in (
|
||||
("status", status),
|
||||
("lease_owner", lease_owner),
|
||||
("lease_expires_at", lease_expires_at),
|
||||
("attempts", attempts),
|
||||
("result_json", result_json),
|
||||
("error_code", error_code),
|
||||
("error_message", error_message),
|
||||
("updated_at", updated_at),
|
||||
("completed_at", completed_at),
|
||||
):
|
||||
if value is not None:
|
||||
sets.append(f"{column} = ?")
|
||||
params.append(value)
|
||||
if clear_lease:
|
||||
sets.append("lease_owner = NULL")
|
||||
sets.append("lease_expires_at = NULL")
|
||||
if not sets:
|
||||
return
|
||||
params.append(query_id)
|
||||
await self.conn.execute(
|
||||
f"UPDATE account_queries SET {', '.join(sets)} WHERE query_id = ?",
|
||||
params,
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def pick_queued_query(self, site: str | None) -> QueryRow | None:
|
||||
"""取最早的 queued 查询单,可选按站点过滤"""
|
||||
if site:
|
||||
sql = (
|
||||
"SELECT * FROM account_queries WHERE status = ? AND site = ? "
|
||||
"ORDER BY created_at ASC LIMIT 1"
|
||||
)
|
||||
params: tuple[Any, ...] = ("queued", site)
|
||||
else:
|
||||
sql = "SELECT * FROM account_queries WHERE status = ? ORDER BY created_at ASC LIMIT 1"
|
||||
params = ("queued",)
|
||||
async with self.conn.execute(sql, params) as cur:
|
||||
row = await cur.fetchone()
|
||||
return _row_to_query(row) if row else None
|
||||
|
||||
async def list_queries(
|
||||
self,
|
||||
*,
|
||||
status: str | None = None,
|
||||
kind: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[QueryRow], int]:
|
||||
"""分页列出查询单,按 created_at 降序(查询是即时行为,最近的先看)"""
|
||||
where = []
|
||||
params: list[Any] = []
|
||||
if status:
|
||||
where.append("status = ?")
|
||||
params.append(status)
|
||||
if kind:
|
||||
where.append("kind = ?")
|
||||
params.append(kind)
|
||||
clause = f"WHERE {' AND '.join(where)}" if where else ""
|
||||
|
||||
async with self.conn.execute(
|
||||
f"SELECT COUNT(*) FROM account_queries {clause}", params
|
||||
) as cur:
|
||||
total = (await cur.fetchone())[0]
|
||||
|
||||
sql = (
|
||||
f"SELECT * FROM account_queries {clause} "
|
||||
"ORDER BY created_at DESC LIMIT ? OFFSET ?"
|
||||
)
|
||||
async with self.conn.execute(sql, [*params, limit, offset]) as cur:
|
||||
rows = await cur.fetchall()
|
||||
return [_row_to_query(r) for r in rows], total
|
||||
|
||||
async def list_queries_in_statuses(self, statuses: tuple[str, ...]) -> list[QueryRow]:
|
||||
"""取处于给定状态集合的全部查询单(sweep 与健康检查用)"""
|
||||
if not statuses:
|
||||
return []
|
||||
placeholders = ", ".join("?" for _ in statuses)
|
||||
sql = (
|
||||
f"SELECT * FROM account_queries WHERE status IN ({placeholders}) "
|
||||
"ORDER BY created_at ASC"
|
||||
)
|
||||
async with self.conn.execute(sql, statuses) as cur:
|
||||
rows = await cur.fetchall()
|
||||
return [_row_to_query(r) for r in rows]
|
||||
|
||||
async def count_queries_by_status(self, status: str) -> int:
|
||||
async with self.conn.execute(
|
||||
"SELECT COUNT(*) FROM account_queries WHERE status = ?", (status,)
|
||||
) as cur:
|
||||
return (await cur.fetchone())[0]
|
||||
|
||||
async def delete_queries_completed_before(self, cutoff: str) -> int:
|
||||
"""清理 completed_at 早于 cutoff 的终态查询单,返回删除条数
|
||||
|
||||
结果里带站点原始 JSON,不清会一直涨。只删已完成的,进行中的一律不动。
|
||||
"""
|
||||
cursor = await self.conn.execute(
|
||||
"DELETE FROM account_queries WHERE completed_at IS NOT NULL AND completed_at < ?",
|
||||
(cutoff,),
|
||||
)
|
||||
await self.conn.commit()
|
||||
return cursor.rowcount or 0
|
||||
|
||||
# ---- account_orders(定时下派通道的编目,见 docs/order-gateway.md §12)----
|
||||
|
||||
async def upsert_account_order(self, row: AccountOrderRow) -> bool:
|
||||
"""把一笔账号订单写进编目(回写)。存在则刷新,返回 True=新写入
|
||||
|
||||
这正是「采集到网关中不存在的订单就写回」的落点:主键是 order_number,
|
||||
重复采集同一笔订单只是刷新 last_seen 等字段,不会产生重复行。
|
||||
"""
|
||||
async with self.conn.execute(
|
||||
"SELECT 1 FROM account_orders WHERE order_number = ?", (row.order_number,)
|
||||
) as cur:
|
||||
existed = await cur.fetchone() is not None
|
||||
await self.conn.execute(
|
||||
"INSERT INTO account_orders (order_number, shop_id, shop_name, order_date, "
|
||||
"delivery_status, order_state, discovered_at, detail_fetched_at, last_seen_at) "
|
||||
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) "
|
||||
"ON CONFLICT(order_number) DO UPDATE SET "
|
||||
"shop_id = excluded.shop_id, shop_name = excluded.shop_name, "
|
||||
"order_date = excluded.order_date, delivery_status = excluded.delivery_status, "
|
||||
"order_state = excluded.order_state, detail_fetched_at = excluded.detail_fetched_at, "
|
||||
"last_seen_at = excluded.last_seen_at",
|
||||
(
|
||||
row.order_number,
|
||||
row.shop_id,
|
||||
row.shop_name,
|
||||
row.order_date,
|
||||
row.delivery_status,
|
||||
row.order_state,
|
||||
row.discovered_at,
|
||||
row.detail_fetched_at,
|
||||
row.last_seen_at,
|
||||
),
|
||||
)
|
||||
await self.conn.commit()
|
||||
return not existed
|
||||
|
||||
async def get_account_order(self, order_number: str) -> AccountOrderRow | None:
|
||||
async with self.conn.execute(
|
||||
"SELECT * FROM account_orders WHERE order_number = ?", (order_number,)
|
||||
) as cur:
|
||||
row = await cur.fetchone()
|
||||
return _row_to_account_order(row) if row else None
|
||||
|
||||
async def list_account_orders(
|
||||
self,
|
||||
*,
|
||||
state: str | None = None,
|
||||
limit: int = 50,
|
||||
offset: int = 0,
|
||||
) -> tuple[list[AccountOrderRow], int]:
|
||||
"""分页列出编目订单,按 last_seen(最近出现)倒序"""
|
||||
where = []
|
||||
params: list[Any] = []
|
||||
if state:
|
||||
where.append("order_state = ?")
|
||||
params.append(state)
|
||||
clause = f"WHERE {' AND '.join(where)}" if where else ""
|
||||
|
||||
async with self.conn.execute(
|
||||
f"SELECT COUNT(*) FROM account_orders {clause}", params
|
||||
) as cur:
|
||||
total = (await cur.fetchone())[0]
|
||||
|
||||
sql = (
|
||||
f"SELECT * FROM account_orders {clause} "
|
||||
"ORDER BY last_seen_at DESC LIMIT ? OFFSET ?"
|
||||
)
|
||||
async with self.conn.execute(sql, [*params, limit, offset]) as cur:
|
||||
rows = await cur.fetchall()
|
||||
return [_row_to_account_order(r) for r in rows], total
|
||||
|
||||
async def list_account_orders_needing_detail(self, cutoff_iso: str) -> list[AccountOrderRow]:
|
||||
"""取「还没成功取过详情」或「详情已过刷新期(detail_fetched_at < cutoff)」的编目订单
|
||||
|
||||
collector 据此派新一批 order_detail 查询。校验用 detail_fetched_at
|
||||
IS NULL OR < cutoff,保证不会对同一笔订单反复派(详情查询结果回来后才
|
||||
落 detail_fetched_at)。
|
||||
"""
|
||||
async with self.conn.execute(
|
||||
"SELECT * FROM account_orders WHERE detail_fetched_at IS NULL "
|
||||
"OR detail_fetched_at < ? ORDER BY last_seen_at ASC",
|
||||
(cutoff_iso,),
|
||||
) as cur:
|
||||
rows = await cur.fetchall()
|
||||
return [_row_to_account_order(r) for r in rows]
|
||||
|
||||
+52
-3
@@ -14,10 +14,15 @@ from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from app.gateway.api.routes.account import router as account_router
|
||||
from app.gateway.api.routes.health import router as health_router
|
||||
from app.gateway.api.routes.orders import router as orders_router
|
||||
from app.gateway.api.routes.queries import router as queries_router
|
||||
from app.gateway.callback import CallbackNotifier
|
||||
from app.gateway.collector import OrderDiscoveryCollector
|
||||
from app.gateway.container import GatewayContainer
|
||||
from app.gateway.db import GatewayDB
|
||||
from app.gateway.query_queue import QueryQueue
|
||||
from app.gateway.task_queue import TaskQueue
|
||||
from app.shared.api import register_exception_handlers
|
||||
from app.shared.config import get_settings
|
||||
@@ -29,19 +34,40 @@ SWEEP_INTERVAL_SECONDS = 60
|
||||
|
||||
|
||||
def build_container() -> GatewayContainer:
|
||||
"""构建网关容器:DB + 任务队列"""
|
||||
"""构建网关容器:DB + 下单任务队列 + 账号只读查询队列 + 回调通知器"""
|
||||
settings = get_settings()
|
||||
db = GatewayDB(settings.gateway_db_path_resolved)
|
||||
# 终结类事件回调(§4.8):best-effort 单次投递,失败只记日志
|
||||
notifier = CallbackNotifier(
|
||||
settings=settings, timeout_seconds=settings.callback_timeout_seconds
|
||||
)
|
||||
task_queue = TaskQueue(
|
||||
db,
|
||||
lease_ttl_seconds=settings.lease_ttl_seconds,
|
||||
worker_offline_alert_seconds=settings.worker_offline_alert_seconds,
|
||||
notifier=notifier,
|
||||
)
|
||||
query_queue = QueryQueue(
|
||||
db,
|
||||
lease_ttl_seconds=settings.query_lease_ttl_seconds,
|
||||
query_ttl_seconds=settings.query_ttl_seconds,
|
||||
max_attempts=settings.query_max_attempts,
|
||||
retention_seconds=settings.query_retention_seconds,
|
||||
)
|
||||
collector = OrderDiscoveryCollector(
|
||||
settings=settings, db=db, query_queue=query_queue
|
||||
)
|
||||
return GatewayContainer(
|
||||
settings=settings, db=db, task_queue=task_queue,
|
||||
query_queue=query_queue, collector=collector, notifier=notifier,
|
||||
)
|
||||
return GatewayContainer(settings=settings, db=db, task_queue=task_queue)
|
||||
|
||||
|
||||
async def _sweep_loop(container: GatewayContainer) -> None:
|
||||
"""常驻后台任务:每 60 秒扫一次过期租约,把 leased/running 推到 stale
|
||||
"""常驻后台任务:每 60 秒扫一次过期租约
|
||||
|
||||
下单侧把 leased/running 推到 stale;查询侧重投超时的、置 expired 超 TTL 的、
|
||||
清掉过保留期的结果。
|
||||
|
||||
lease 请求本身也会顺带扫一次,但 worker 退场后没人来 lease,必须靠这个
|
||||
兜底——否则过期的任务永远停在 active 状态,健康检查看不到,stale 任务也
|
||||
@@ -53,6 +79,9 @@ async def _sweep_loop(container: GatewayContainer) -> None:
|
||||
swept = await container.task_queue.sweep()
|
||||
if swept:
|
||||
logger.info("sweep 把 %s 个过期任务置为 stale", swept)
|
||||
query_swept = await container.query_queue.sweep()
|
||||
if query_swept:
|
||||
logger.info("sweep 处理了 %s 张超时/过期的查询单", query_swept)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception: # noqa: BLE001
|
||||
@@ -78,14 +107,29 @@ async def lifespan(app: FastAPI):
|
||||
startup_swept = await container.task_queue.sweep()
|
||||
if startup_swept:
|
||||
logger.warning("启动时把 %s 个遗留过期任务置为 stale", startup_swept)
|
||||
startup_queries = await container.query_queue.sweep()
|
||||
if startup_queries:
|
||||
logger.warning("启动时处理了 %s 张遗留的超时/过期查询单", startup_queries)
|
||||
|
||||
container.sweep_task = asyncio.create_task(
|
||||
_sweep_loop(container), name="gateway-sweep"
|
||||
)
|
||||
# 定时下派通道(§12):被启用时 collector.run() 里自带常驻循环,禁用则直接返回
|
||||
assert container.collector is not None
|
||||
container.collector_task = asyncio.create_task(
|
||||
container.collector.run(), name="gateway-collector"
|
||||
)
|
||||
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if container.collector_task is not None:
|
||||
container.collector_task.cancel()
|
||||
try:
|
||||
await container.collector_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
container.collector_task = None
|
||||
if container.sweep_task is not None:
|
||||
container.sweep_task.cancel()
|
||||
try:
|
||||
@@ -93,6 +137,9 @@ async def lifespan(app: FastAPI):
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
container.sweep_task = None
|
||||
if container.notifier is not None:
|
||||
# 等在途回调发完(各次发送有超时兜底),避免关停时静默丢通知
|
||||
await container.notifier.aclose()
|
||||
await container.db.close()
|
||||
|
||||
|
||||
@@ -101,6 +148,8 @@ def create_app() -> FastAPI:
|
||||
app = FastAPI(title="Rakuten Order Gateway", lifespan=lifespan)
|
||||
app.include_router(health_router)
|
||||
app.include_router(orders_router)
|
||||
app.include_router(queries_router)
|
||||
app.include_router(account_router)
|
||||
register_exception_handlers(app)
|
||||
return app
|
||||
|
||||
|
||||
+343
-73
@@ -2,14 +2,16 @@
|
||||
|
||||
intent 字段刻意保留成 `dict[str, Any]`——网关不解释下单意图,结构由 trading 侧
|
||||
定义。网关只负责把它存下来、原样吐给 worker,避免业务规则悄悄渗进任务队列。
|
||||
账号只读查询的 `params` / `result` 同理。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
from app.shared.task_state import AccountQueryKind, OrderState, QueryStatus, TaskStatus
|
||||
|
||||
|
||||
# ---- POST /api/orders ----
|
||||
@@ -22,17 +24,58 @@ class SubmitOrderRequest(BaseModel):
|
||||
不新建任务,返回既有任务且 created=false。
|
||||
"""
|
||||
|
||||
task_id: str | None = None
|
||||
site: str
|
||||
intent: dict[str, Any]
|
||||
task_id: str | None = Field(
|
||||
default=None,
|
||||
description="幂等键。不传则服务端生成;同一 task_id 重复提交不新建任务,"
|
||||
"返回既有任务(created=false)——上游重发不会变成两单",
|
||||
examples=["po-20260727-0001"],
|
||||
)
|
||||
site: str = Field(
|
||||
description="站点标识。交易服务只覆盖乐天市场,固定 rakuten",
|
||||
examples=["rakuten"],
|
||||
)
|
||||
intent: dict[str, Any] = Field(
|
||||
description=(
|
||||
"下单意图原文。网关不解释内容,原样存库并透传给本地 worker,结构由 trading 侧定义:"
|
||||
"item_url(必填,商品页 URL);quantity(可选,默认 1);"
|
||||
"variant_id(多规格商品必填,取自 /api/item_detail 的 variants,不传时 worker 自动选第一个非售罄规格);"
|
||||
"choice(可选,商品选项,如 \"颜色:赤\",可传字符串或字符串列表);"
|
||||
"max_total_yen(可选,本次金额上限:确认页实际应付超过即中止并报 needs_human,"
|
||||
"缺省用服务端 RAKUTEN_ORDER_MAX_TOTAL_YEN)"
|
||||
),
|
||||
examples=[{
|
||||
"item_url": "https://item.rakuten.co.jp/shop/code/",
|
||||
"quantity": 1,
|
||||
"variant_id": "1001",
|
||||
"max_total_yen": 30000,
|
||||
}],
|
||||
)
|
||||
callback_url: str | None = Field(
|
||||
default=None,
|
||||
description="终结类事件的异步通知地址(http/https)。任务到达终态(succeeded / "
|
||||
"failed / needs_human)或被置 stale 时,网关向该地址 POST 一条 JSON 通知,"
|
||||
"上游可免去轮询。注意:幂等重发(created=false)不会更新既有任务的回调地址",
|
||||
examples=["https://upstream.example.com/hooks/rakuten-order"],
|
||||
)
|
||||
|
||||
@field_validator("callback_url")
|
||||
@classmethod
|
||||
def _validate_callback_url(cls, value: str | None) -> str | None:
|
||||
"""只接受 http/https 且带 host 的绝对 URL,其余当场 422"""
|
||||
if value is None:
|
||||
return None
|
||||
parts = urlsplit(value)
|
||||
if parts.scheme not in ("http", "https") or not parts.netloc:
|
||||
raise ValueError("callback_url 必须是 http/https 绝对 URL")
|
||||
return value
|
||||
|
||||
|
||||
class SubmitOrderData(BaseModel):
|
||||
"""提交响应"""
|
||||
|
||||
task_id: str
|
||||
status: TaskStatus
|
||||
created: bool # True=本次新建,False=命中既有任务(幂等)
|
||||
task_id: str = Field(description="任务 ID(幂等键),后续查询与 worker 回报都用它")
|
||||
status: TaskStatus = Field(description="任务状态,新建恒为 queued")
|
||||
created: bool = Field(description="True=本次新建,False=命中既有任务(幂等重发)")
|
||||
|
||||
|
||||
# ---- GET /api/orders/lease ----
|
||||
@@ -45,12 +88,20 @@ class LeaseData(BaseModel):
|
||||
领取(stale → reclaim),worker 必须先核对站点订单列表,见 runner 主循环。
|
||||
"""
|
||||
|
||||
task_id: str
|
||||
site: str
|
||||
intent: dict[str, Any]
|
||||
lease_expires_at: str # ISO8601 UTC
|
||||
lease_count: int
|
||||
known_state: OrderState | None # 之前上报过的最新订单状态;首次领取为 null
|
||||
task_id: str = Field(description="任务 ID(幂等键)")
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
intent: dict[str, Any] = Field(description="下单意图原文,与提交时一致")
|
||||
lease_expires_at: str = Field(
|
||||
description="租约过期时间(ISO8601 UTC)。TTL 由 RAKUTEN_LEASE_TTL_SECONDS 控制"
|
||||
"(默认 300 秒);执行中每 60 秒调 renew 续租"
|
||||
)
|
||||
lease_count: int = Field(
|
||||
description="该任务被领取过的次数。>1 表示恢复领取(stale → reclaim):worker 执行前"
|
||||
"必须先核对站点订单列表确认这单下没下,核对不出结论报 needs_human,绝不直接重新下单"
|
||||
)
|
||||
known_state: OrderState | None = Field(
|
||||
description="之前上报过的最新订单状态;首次领取为 null。恢复领取时是核对的重要线索"
|
||||
)
|
||||
|
||||
|
||||
# ---- POST /api/orders/{id}/renew ----
|
||||
@@ -59,15 +110,15 @@ class LeaseData(BaseModel):
|
||||
class RenewRequest(BaseModel):
|
||||
"""续租请求"""
|
||||
|
||||
worker_id: str
|
||||
worker_id: str = Field(description="worker 标识,必须是当前租约持有者,否则 6002")
|
||||
|
||||
|
||||
class RenewData(BaseModel):
|
||||
"""续租响应"""
|
||||
|
||||
task_id: str
|
||||
lease_expires_at: str
|
||||
lease_count: int
|
||||
task_id: str = Field(description="任务 ID")
|
||||
lease_expires_at: str = Field(description="续租后的新过期时间(ISO8601 UTC)")
|
||||
lease_count: int = Field(description="领取次数,续租不改变")
|
||||
|
||||
|
||||
# ---- POST /api/orders/{id}/report ----
|
||||
@@ -80,23 +131,39 @@ class ReportRequest(BaseModel):
|
||||
不产生第二条记录。terminal=true 时释放租约并把任务推到终态。
|
||||
"""
|
||||
|
||||
worker_id: str
|
||||
state: OrderState
|
||||
payable_yen: int | None = None
|
||||
pay_deadline: str | None = None # ISO8601
|
||||
site_order_id: str | None = None
|
||||
evidence_ref: str | None = None # 本地相对路径,不含扩展名
|
||||
detail: str = ""
|
||||
terminal: bool = False
|
||||
terminal_status: TaskStatus | None = None # terminal=true 时指定终态,缺省按 state 推断
|
||||
worker_id: str = Field(description="worker 标识,必须是当前租约持有者,否则 6002")
|
||||
state: OrderState = Field(
|
||||
description="本次上报的订单状态:created / in_cart / ordered / awaiting_payment / "
|
||||
"paid / shipped / delivered / cancelled"
|
||||
)
|
||||
payable_yen: int | None = Field(default=None, description="实际应付金额(日元),进入确认页后上报")
|
||||
pay_deadline: str | None = Field(default=None, description="付款期限(ISO8601),コンビニ払い 常见三天")
|
||||
site_order_id: str | None = Field(default=None, description="站点侧订单号(注文番号),下单成功后上报")
|
||||
evidence_ref: str | None = Field(
|
||||
default=None,
|
||||
description="本地证据相对路径,如 po-20260727-0001/03-order-confirm(不含扩展名)。"
|
||||
"证据原文(HTML/截图)留在本地,不回传",
|
||||
)
|
||||
detail: str = Field(default="", description="补充说明,如付款方式与单号、失败原因")
|
||||
terminal: bool = Field(
|
||||
default=False,
|
||||
description="True 表示任务到此结束:释放租约并推到终态。缺省按 state 推断终态:"
|
||||
"paid→succeeded,cancelled→failed,其余(如 awaiting_payment 搁置)→needs_human",
|
||||
)
|
||||
terminal_status: TaskStatus | None = Field(
|
||||
default=None,
|
||||
description="terminal=true 时显式指定终态(succeeded / failed / needs_human),缺省按 state 推断",
|
||||
)
|
||||
|
||||
|
||||
class ReportData(BaseModel):
|
||||
"""回报响应"""
|
||||
|
||||
task_id: str
|
||||
status: TaskStatus
|
||||
recorded: bool # False=同 (task_id, state) 已存在,本次为幂等覆盖;True=新写入
|
||||
task_id: str = Field(description="任务 ID")
|
||||
status: TaskStatus = Field(description="回报后的任务状态")
|
||||
recorded: bool = Field(
|
||||
description="True=新写入一条状态记录;False=同 (task_id, state) 已存在,本次为幂等覆盖"
|
||||
)
|
||||
|
||||
|
||||
# ---- POST /api/orders/{id}/reclaim ----
|
||||
@@ -105,58 +172,250 @@ class ReportData(BaseModel):
|
||||
class ReclaimRequest(BaseModel):
|
||||
"""把 stale 任务重新租给 worker"""
|
||||
|
||||
worker_id: str
|
||||
worker_id: str = Field(
|
||||
description="worker 标识。reclaim 是显式恢复动作——正常 lease 永远拿不到 stale 任务"
|
||||
)
|
||||
|
||||
|
||||
class ReclaimData(BaseModel):
|
||||
"""reclaim 响应,结构与 LeaseData 一致,但 lease_count 必然 > 1"""
|
||||
|
||||
task_id: str
|
||||
site: str
|
||||
intent: dict[str, Any]
|
||||
lease_expires_at: str
|
||||
lease_count: int
|
||||
known_state: OrderState | None
|
||||
task_id: str = Field(description="任务 ID(幂等键)")
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
intent: dict[str, Any] = Field(description="下单意图原文,与提交时一致")
|
||||
lease_expires_at: str = Field(description="新租约的过期时间(ISO8601 UTC)")
|
||||
lease_count: int = Field(
|
||||
description="必然 > 1:worker 执行前必须先核对站点订单列表,确认这单到底下没下"
|
||||
)
|
||||
known_state: OrderState | None = Field(
|
||||
description="租约丢失前上报过的最新订单状态,恢复核对时的重要线索"
|
||||
)
|
||||
|
||||
|
||||
# ---- GET /api/orders/{id} 与 GET /api/orders ----
|
||||
|
||||
|
||||
class ReportEntry(BaseModel):
|
||||
"""task_reports 单行"""
|
||||
"""task_reports 单行:一次状态上报(append-only,最新一条即当前状态)"""
|
||||
|
||||
state: OrderState
|
||||
payable_yen: int | None = None
|
||||
pay_deadline: str | None = None
|
||||
site_order_id: str | None = None
|
||||
evidence_ref: str | None = None
|
||||
detail: str = ""
|
||||
reported_at: str
|
||||
state: OrderState = Field(description="上报的订单状态")
|
||||
payable_yen: int | None = Field(default=None, description="实际应付金额(日元)")
|
||||
pay_deadline: str | None = Field(default=None, description="付款期限(ISO8601)")
|
||||
site_order_id: str | None = Field(default=None, description="站点侧订单号(注文番号)")
|
||||
evidence_ref: str | None = Field(default=None, description="本地证据相对路径(不含扩展名)")
|
||||
detail: str = Field(default="", description="补充说明")
|
||||
reported_at: str = Field(description="上报时间(ISO8601 UTC)")
|
||||
|
||||
|
||||
class TaskDetail(BaseModel):
|
||||
"""单个任务详情"""
|
||||
|
||||
task_id: str
|
||||
site: str
|
||||
intent: dict[str, Any]
|
||||
status: TaskStatus
|
||||
lease_owner: str | None = None
|
||||
lease_expires_at: str | None = None
|
||||
lease_count: int = 0
|
||||
created_at: str
|
||||
updated_at: str
|
||||
latest_state: OrderState | None = None
|
||||
reports: list[ReportEntry] = Field(default_factory=list)
|
||||
task_id: str = Field(description="任务 ID(幂等键)")
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
intent: dict[str, Any] = Field(description="下单意图原文")
|
||||
callback_url: str | None = Field(
|
||||
default=None, description="提交时登记的终结类事件通知地址;未登记为 null"
|
||||
)
|
||||
status: TaskStatus = Field(
|
||||
description="任务状态:queued / leased / running / succeeded / failed / needs_human / stale"
|
||||
)
|
||||
lease_owner: str | None = Field(default=None, description="当前租约持有者(worker_id);无租约为 null")
|
||||
lease_expires_at: str | None = Field(default=None, description="租约过期时间(ISO8601 UTC);无租约为 null")
|
||||
lease_count: int = Field(default=0, description="被领取过的次数,>1 即发生过恢复")
|
||||
created_at: str = Field(description="创建时间(ISO8601 UTC)")
|
||||
updated_at: str = Field(description="最近更新时间(ISO8601 UTC)")
|
||||
latest_state: OrderState | None = Field(default=None, description="最新一次上报的订单状态;从未上报为 null")
|
||||
reports: list[ReportEntry] = Field(default_factory=list, description="完整状态历史(append-only)")
|
||||
|
||||
|
||||
class TaskListData(BaseModel):
|
||||
"""任务列表"""
|
||||
|
||||
items: list[TaskDetail]
|
||||
total: int
|
||||
limit: int
|
||||
offset: int
|
||||
items: list[TaskDetail] = Field(description="任务详情列表,按创建时间倒序")
|
||||
total: int = Field(description="符合筛选条件的总条数")
|
||||
limit: int = Field(description="本次分页大小")
|
||||
offset: int = Field(description="本次分页偏移")
|
||||
|
||||
|
||||
# ---- 账号只读查询通道(见 docs/order-gateway.md §11)----
|
||||
|
||||
|
||||
class SubmitQueryRequest(BaseModel):
|
||||
"""上游提交一张账号只读查询单
|
||||
|
||||
query_id 语义同 task_id:上游自带的幂等键,重复提交返回既有单。
|
||||
params 由网关原样透传给 worker,结构由 trading 侧定义(见 §11.2):
|
||||
- order_list:`{"since": ISO8601?, "max_pages": int?}`
|
||||
- order_detail:`{"order_number": "306087-20260813-0863947697"}`
|
||||
"""
|
||||
|
||||
query_id: str | None = Field(
|
||||
default=None,
|
||||
description="幂等键。不传则服务端生成;同一 query_id 重复提交返回既有单(created=false)",
|
||||
examples=["aq-20260816-0001"],
|
||||
)
|
||||
site: str = Field(default="rakuten", description="站点标识,固定 rakuten")
|
||||
kind: AccountQueryKind = Field(
|
||||
description="查询类型:order_list=按时间窗口翻页扫描订单列表;"
|
||||
"order_detail=读单笔注文番号的详情(配送阶段等)"
|
||||
)
|
||||
params: dict[str, Any] = Field(
|
||||
default_factory=dict,
|
||||
description=(
|
||||
"查询参数,网关原样透传给 worker。order_list:since(可选,ISO8601 窗口下界,"
|
||||
"缺省不设下界)、max_pages(可选,1..20,缺省用服务端 "
|
||||
"RAKUTEN_ACCOUNT_QUERY_DEFAULT_MAX_PAGES=3);"
|
||||
"order_detail:order_number(必填,站点注文番号)"
|
||||
),
|
||||
examples=[{"order_number": "306087-20260813-0863947697"}],
|
||||
)
|
||||
|
||||
|
||||
class SubmitQueryData(BaseModel):
|
||||
"""提交响应。纯异步:这里只拿到单号,结果去 GET /api/account/queries/{id} 取"""
|
||||
|
||||
query_id: str = Field(description="查询单 ID(幂等键),取结果时用它")
|
||||
status: QueryStatus = Field(description="查询单状态,新建恒为 queued")
|
||||
created: bool = Field(description="True=本次新建,False=命中既有单(幂等重发)")
|
||||
|
||||
|
||||
class QueryLeaseData(BaseModel):
|
||||
"""GET /api/account/queries/lease 的响应
|
||||
|
||||
无可领查询单时整个 data 为 null(HTTP 仍 200)。没有 known_state 之类的字段——
|
||||
查询是无状态的一次性动作,attempt > 1 只是说明上一轮超时被重投了,worker
|
||||
不需要为此改变行为(只读,重跑安全)。
|
||||
"""
|
||||
|
||||
query_id: str = Field(description="查询单 ID")
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
kind: AccountQueryKind = Field(description="查询类型(order_list / order_detail)")
|
||||
params: dict[str, Any] = Field(description="查询参数原文,与提交时一致")
|
||||
lease_expires_at: str = Field(
|
||||
description="租约过期时间(ISO8601 UTC)。TTL 由 RAKUTEN_QUERY_LEASE_TTL_SECONDS 控制"
|
||||
"(默认 180 秒),过期自动重投"
|
||||
)
|
||||
attempt: int = Field(description="第几次被领取。>1 表示上一轮超时被重投(只读,重跑安全)")
|
||||
|
||||
|
||||
class QueryResultRequest(BaseModel):
|
||||
"""worker 回报查询结果
|
||||
|
||||
success=true 时 result 必填;false 时 error_message 必填、error_code 可选
|
||||
(沿用 shared.errors 的错误码,便于上游按同一张表分支)。
|
||||
"""
|
||||
|
||||
worker_id: str = Field(description="worker 标识,必须是当前租约持有者,否则 6006")
|
||||
success: bool = Field(description="站点读取是否成功")
|
||||
result: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="success=true 时必填:规范化字段 + 站点原始 JSON,结构按 kind 见 "
|
||||
"GET /api/account/queries/{id} 的说明。体积上限 RAKUTEN_QUERY_RESULT_MAX_BYTES(默认 1MB)",
|
||||
)
|
||||
error_code: int | None = Field(
|
||||
default=None, description="success=false 时的错误码,沿用统一错误码表(如 5001 掉登录)"
|
||||
)
|
||||
error_message: str = Field(default="", description="success=false 时必填,失败原因")
|
||||
|
||||
|
||||
class QueryResultData(BaseModel):
|
||||
"""回报响应"""
|
||||
|
||||
query_id: str = Field(description="查询单 ID")
|
||||
status: QueryStatus = Field(description="回报后的查询单状态(succeeded / failed)")
|
||||
|
||||
|
||||
class QueryError(BaseModel):
|
||||
"""查询失败的原因"""
|
||||
|
||||
code: int | None = Field(default=None, description="错误码(沿用统一错误码表),可能为 null")
|
||||
message: str = Field(default="", description="失败原因")
|
||||
|
||||
|
||||
class QueryDetail(BaseModel):
|
||||
"""单张查询单的完整视图
|
||||
|
||||
result 是 worker 回的原文(站点原始 JSON + 规范化字段),网关不解释内容。
|
||||
"""
|
||||
|
||||
query_id: str = Field(description="查询单 ID(幂等键)")
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
kind: AccountQueryKind = Field(description="查询类型(order_list / order_detail)")
|
||||
params: dict[str, Any] = Field(description="查询参数原文")
|
||||
status: QueryStatus = Field(
|
||||
description="查询单状态:queued / leased / succeeded / failed / expired"
|
||||
)
|
||||
lease_owner: str | None = Field(default=None, description="当前租约持有者(worker_id);无租约为 null")
|
||||
lease_expires_at: str | None = Field(default=None, description="租约过期时间(ISO8601 UTC);无租约为 null")
|
||||
attempts: int = Field(default=0, description="已被领取的次数,用尽(默认 3 次)置 failed")
|
||||
created_at: str = Field(description="创建时间(ISO8601 UTC)")
|
||||
updated_at: str = Field(description="最近更新时间(ISO8601 UTC)")
|
||||
completed_at: str | None = Field(default=None, description="到达终态的时间(ISO8601 UTC);未终结为 null")
|
||||
result: dict[str, Any] | None = Field(
|
||||
default=None,
|
||||
description="status=succeeded 时的结果:规范化字段 + 站点原始 JSON,结构按 kind "
|
||||
"见接口说明;未成功为 null",
|
||||
)
|
||||
error: QueryError | None = Field(default=None, description="status=failed/expired 时的失败原因;否则为 null")
|
||||
|
||||
|
||||
class QueryListData(BaseModel):
|
||||
"""查询单列表(运维排查用)"""
|
||||
|
||||
items: list[QueryDetail] = Field(description="查询单详情列表,按创建时间倒序")
|
||||
total: int = Field(description="符合筛选条件的总条数")
|
||||
limit: int = Field(description="本次分页大小")
|
||||
offset: int = Field(description="本次分页偏移")
|
||||
|
||||
|
||||
# ---- 定时下派通道:账号订单编目(docs/order-gateway.md §12)----
|
||||
|
||||
|
||||
class CatalogedOrder(BaseModel):
|
||||
"""account_orders 编目的单笔订单
|
||||
|
||||
这是网关 collector 收到 worker 回结果后**自己缩写出来的目录行**(只读规范化
|
||||
字段:订单号/店铺/日期/配送状态),不是查询单 result 的原样透传——`params`
|
||||
与 `result` 仍保持「网关不解释内容」,唯独 collector 消费的是 worker 那边
|
||||
已经规范化了的稳定契约字段(见 §12 说明)。
|
||||
"""
|
||||
|
||||
order_number: str = Field(description="站点注文番号,编目与下单任务表对账的天然键")
|
||||
shop_id: str | None = Field(default=None, description="店铺 ID;列表扫描未给出为 null")
|
||||
shop_name: str = Field(default="", description="店铺名(站点原文)")
|
||||
order_date: str | None = Field(default=None, description="下单日期(站点原文);列表扫描未给出为 null")
|
||||
delivery_status: str | None = Field(
|
||||
default=None,
|
||||
description="站点配送阶段的结构化枚举码(如 CHECKING_ORDER),原样带给上游对账;"
|
||||
"未识别时为 null 或 stepper 兜底文本",
|
||||
)
|
||||
order_state: str | None = Field(
|
||||
default=None, description="worker 规范化上报的订单状态(OrderState 词汇表);映射不到为 null"
|
||||
)
|
||||
discovered_at: str = Field(description="首次编目时间(ISO8601 UTC)")
|
||||
detail_fetched_at: str | None = Field(
|
||||
default=None, description="最近一次取到订单详情的时间(ISO8601 UTC);从未取过为 null"
|
||||
)
|
||||
last_seen_at: str = Field(description="最近一次列表扫描见到该订单的时间(ISO8601 UTC)")
|
||||
|
||||
|
||||
class CatalogOrderListData(BaseModel):
|
||||
"""编目订单列表(GET /api/account/orders)"""
|
||||
|
||||
items: list[CatalogedOrder] = Field(description="编目订单列表,按最近出现(last_seen_at)倒序")
|
||||
total: int = Field(description="符合筛选条件的总条数")
|
||||
limit: int = Field(description="本次分页大小")
|
||||
offset: int = Field(description="本次分页偏移")
|
||||
|
||||
|
||||
class TriggerDiscoveryData(BaseModel):
|
||||
"""手动触发一轮下派的响应"""
|
||||
|
||||
query_id: str = Field(description="本轮 order_list 下派的查询单号")
|
||||
status: QueryStatus = Field(description="该查询单当前状态")
|
||||
created: bool = Field(
|
||||
description="本轮扫描是否新建了一张 order_list 单(False=已有一张在飞/已入队)"
|
||||
)
|
||||
|
||||
|
||||
# ---- GET /health ----
|
||||
@@ -165,18 +424,18 @@ class TaskListData(BaseModel):
|
||||
class WorkerHealthEntry(BaseModel):
|
||||
"""单个 worker 的健康指标"""
|
||||
|
||||
worker_id: str
|
||||
last_seen_at: str
|
||||
last_seen_seconds: int
|
||||
worker_id: str = Field(description="worker 标识")
|
||||
last_seen_at: str = Field(description="最近一次心跳时间(ISO8601 UTC)")
|
||||
last_seen_seconds: int = Field(description="距上次心跳的秒数,超过 300 秒(5 分钟)判失联")
|
||||
|
||||
|
||||
class QueuedAlertEntry(BaseModel):
|
||||
"""长时间无人领的任务告警项"""
|
||||
|
||||
task_id: str
|
||||
site: str
|
||||
created_at: str
|
||||
age_seconds: int
|
||||
task_id: str = Field(description="任务 ID")
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
created_at: str = Field(description="任务创建时间(ISO8601 UTC)")
|
||||
age_seconds: int = Field(description="从创建到现在的秒数,超过阈值即视为长时间无人领")
|
||||
|
||||
|
||||
class GatewayHealthData(BaseModel):
|
||||
@@ -186,9 +445,20 @@ class GatewayHealthData(BaseModel):
|
||||
不在网关——本地机 7×24 在线,那套逻辑放本地。
|
||||
"""
|
||||
|
||||
status: str # "ok" 或 "degraded"
|
||||
queued_count: int
|
||||
active_tasks: list[TaskDetail] = Field(default_factory=list)
|
||||
workers: list[WorkerHealthEntry] = Field(default_factory=list)
|
||||
offline_workers: list[WorkerHealthEntry] = Field(default_factory=list)
|
||||
stale_queued_tasks: list[QueuedAlertEntry] = Field(default_factory=list)
|
||||
status: str = Field(description="整体健康状态:ok=无告警,degraded=有 worker 失联或任务长时间无人领")
|
||||
queued_count: int = Field(description="等待领取的下单任务数量(status=queued)")
|
||||
# 等待本地 worker 领取的账号只读查询单数量。查询没有「无人领即告警」这条
|
||||
# 规则(它有自己的 TTL 会自动 expired),这里只是给运维一个可见的积压信号。
|
||||
queued_query_count: int = Field(default=0, description="等待领取的账号只读查询单数量(积压信号,不参与告警)")
|
||||
active_tasks: list[TaskDetail] = Field(
|
||||
default_factory=list, description="当前 leased/running 的下单任务详情列表"
|
||||
)
|
||||
workers: list[WorkerHealthEntry] = Field(
|
||||
default_factory=list, description="全部已知 worker 的健康指标"
|
||||
)
|
||||
offline_workers: list[WorkerHealthEntry] = Field(
|
||||
default_factory=list, description="失联 worker(距上次心跳超过 300 秒),非空即 degraded"
|
||||
)
|
||||
stale_queued_tasks: list[QueuedAlertEntry] = Field(
|
||||
default_factory=list, description="长时间无人领的 queued 任务告警项,非空即 degraded"
|
||||
)
|
||||
|
||||
@@ -0,0 +1,357 @@
|
||||
"""账号只读查询通道的业务逻辑:状态机、租约、重投、过期与保留期清理
|
||||
|
||||
与 `task_queue.py` 是**刻意分开的两套语义**,不是重复代码(见 docs/order-gateway.md §11):
|
||||
|
||||
| | 下单任务队列 | 本模块(只读查询) |
|
||||
| --- | --- | --- |
|
||||
| 操作性质 | 不可逆写 | 只读 |
|
||||
| 全局并发度 1 | 是(同账号写操作必须串行) | 否(账号级串行由本地 SiteInteractor 的锁兜底) |
|
||||
| 租约过期 | 置 stale,**绝不自动重投**,等人工 reclaim | 回 queued **自动重投**,重跑无副作用 |
|
||||
| 续租 | 有(下单要几分钟) | 无(超时即重投,比续租简单且安全) |
|
||||
| 心跳 | lease 兼作心跳 | **不刷心跳**(见 `lease` 注释) |
|
||||
|
||||
这两张表共用一个 `GatewayDB` 连接,但各自持有自己的锁:查询的长轮询等待不该
|
||||
把下单任务的 lease/report 一起挂住。SQLite 单连接本身是串行的,两把锁只是各自
|
||||
保护「检查 + 写入」的原子序列。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import secrets
|
||||
from datetime import datetime, timedelta, timezone
|
||||
|
||||
from app.gateway.db import GatewayDB, QueryRow
|
||||
from app.gateway.models import (
|
||||
QueryDetail,
|
||||
QueryError,
|
||||
QueryLeaseData,
|
||||
QueryListData,
|
||||
QueryResultData,
|
||||
SubmitQueryData,
|
||||
)
|
||||
from app.shared.errors import QueryLeaseInvalidError, QueryNotFoundError
|
||||
from app.shared.task_state import QUERY_TERMINAL_STATUSES, QueryStatus
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def to_iso(dt: datetime) -> str:
|
||||
"""统一时间戳格式:ISO8601 UTC,秒精度,Z 结尾(与 task_queue 一致)"""
|
||||
return dt.astimezone(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
||||
|
||||
|
||||
def parse_iso(s: str) -> datetime:
|
||||
if s.endswith("Z"):
|
||||
s = s[:-1] + "+00:00"
|
||||
return datetime.fromisoformat(s)
|
||||
|
||||
|
||||
def _generate_query_id() -> str:
|
||||
"""服务端生成的 query_id:与 task_id 用不同前缀,日志里一眼能分清两条通道"""
|
||||
return f"q-{to_iso(utcnow())[:10].replace('-', '')}-{secrets.token_hex(4)}"
|
||||
|
||||
|
||||
def _row_to_detail(row: QueryRow) -> QueryDetail:
|
||||
error = None
|
||||
if row.error_code is not None or row.error_message:
|
||||
error = QueryError(code=row.error_code, message=row.error_message or "")
|
||||
return QueryDetail(
|
||||
query_id=row.query_id,
|
||||
site=row.site,
|
||||
kind=row.kind, # type: ignore[arg-type]
|
||||
params=row.params,
|
||||
status=QueryStatus(row.status),
|
||||
lease_owner=row.lease_owner,
|
||||
lease_expires_at=row.lease_expires_at,
|
||||
attempts=row.attempts,
|
||||
created_at=row.created_at,
|
||||
updated_at=row.updated_at,
|
||||
completed_at=row.completed_at,
|
||||
result=row.result,
|
||||
error=error,
|
||||
)
|
||||
|
||||
|
||||
class QueryQueue:
|
||||
"""账号只读查询单队列
|
||||
|
||||
生命周期与 TaskQueue 一致:由 GatewayContainer 持有,共用同一个 GatewayDB。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
db: GatewayDB,
|
||||
*,
|
||||
lease_ttl_seconds: int,
|
||||
query_ttl_seconds: int,
|
||||
max_attempts: int,
|
||||
retention_seconds: int,
|
||||
):
|
||||
self._db = db
|
||||
self._lease_ttl = lease_ttl_seconds
|
||||
self._query_ttl = query_ttl_seconds
|
||||
self._max_attempts = max_attempts
|
||||
self._retention = retention_seconds
|
||||
self._lock = asyncio.Lock()
|
||||
self._cond = asyncio.Condition(self._lock)
|
||||
|
||||
# ---- 提交 ----
|
||||
|
||||
async def submit(
|
||||
self, *, query_id: str | None, site: str, kind: str, params: dict
|
||||
) -> SubmitQueryData:
|
||||
"""上游提交查询单。query_id 缺省时服务端生成;重复提交幂等"""
|
||||
qid = query_id or _generate_query_id()
|
||||
now = to_iso(utcnow())
|
||||
row = QueryRow(
|
||||
query_id=qid,
|
||||
site=site,
|
||||
kind=kind,
|
||||
params_json=json.dumps(params, ensure_ascii=False),
|
||||
status=QueryStatus.QUEUED.value,
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
attempts=0,
|
||||
result_json=None,
|
||||
error_code=None,
|
||||
error_message=None,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
completed_at=None,
|
||||
)
|
||||
async with self._lock:
|
||||
created = await self._db.insert_query(row)
|
||||
if created:
|
||||
self._cond.notify_all()
|
||||
else:
|
||||
existing = await self._db.get_query(qid)
|
||||
if existing is None:
|
||||
raise RuntimeError(f"查询单 {qid} 既未新建也无法读取,状态异常")
|
||||
row = existing
|
||||
return SubmitQueryData(
|
||||
query_id=row.query_id, status=QueryStatus(row.status), created=created
|
||||
)
|
||||
|
||||
# ---- 领取 ----
|
||||
|
||||
async def lease(
|
||||
self, *, worker_id: str, wait: int, site: str | None, max_wait: int
|
||||
) -> QueryLeaseData | None:
|
||||
"""本地长轮询领取一张查询单
|
||||
|
||||
与下单 lease 的三点不同:
|
||||
1. **没有全局并发度 1 的闸门**——只读操作允许多张单同时在飞;真正的账号级
|
||||
串行由本地 SiteInteractor 那把锁保证,网关不必也不该重复实现。
|
||||
2. **不刷 workers.last_seen_at**。心跳的语义是「下单 worker 还活着」,
|
||||
如果查询循环活着就刷心跳,下单主循环挂了也看不出来,告警会失真。
|
||||
3. 领走前先 sweep:把超时的重投回 queued、过期的置 expired。
|
||||
"""
|
||||
effective_wait = max(0, min(wait, max_wait))
|
||||
deadline = utcnow() + timedelta(seconds=effective_wait)
|
||||
|
||||
async with self._lock:
|
||||
await self._sweep_locked()
|
||||
|
||||
while True:
|
||||
row = await self._db.pick_queued_query(site)
|
||||
if row is not None:
|
||||
return await self._lease_to_locked(row, worker_id)
|
||||
|
||||
remaining = (deadline - utcnow()).total_seconds()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
try:
|
||||
await asyncio.wait_for(self._cond.wait(), timeout=remaining)
|
||||
except asyncio.TimeoutError:
|
||||
return None
|
||||
|
||||
async def _lease_to_locked(self, row: QueryRow, worker_id: str) -> QueryLeaseData:
|
||||
"""把一张 queued 查询单租给 worker。调用方必须持有 self._lock"""
|
||||
now = utcnow()
|
||||
expires = to_iso(now + timedelta(seconds=self._lease_ttl))
|
||||
attempt = row.attempts + 1
|
||||
await self._db.update_query(
|
||||
row.query_id,
|
||||
status=QueryStatus.LEASED.value,
|
||||
lease_owner=worker_id,
|
||||
lease_expires_at=expires,
|
||||
attempts=attempt,
|
||||
updated_at=to_iso(now),
|
||||
)
|
||||
return QueryLeaseData(
|
||||
query_id=row.query_id,
|
||||
site=row.site,
|
||||
kind=row.kind, # type: ignore[arg-type]
|
||||
params=row.params,
|
||||
lease_expires_at=expires,
|
||||
attempt=attempt,
|
||||
)
|
||||
|
||||
# ---- 回结果 ----
|
||||
|
||||
async def submit_result(
|
||||
self,
|
||||
query_id: str,
|
||||
*,
|
||||
worker_id: str,
|
||||
success: bool,
|
||||
result: dict | None,
|
||||
error_code: int | None,
|
||||
error_message: str,
|
||||
) -> QueryResultData:
|
||||
"""worker 回报结果
|
||||
|
||||
校验 `worker_id == lease_owner` 且查询单未终结,否则 6006——最典型的场景是
|
||||
执行超时、单子已被重投给下一轮,迟到的结果必须丢弃,不能覆盖新结果。
|
||||
"""
|
||||
async with self._lock:
|
||||
row = await self._db.get_query(query_id)
|
||||
if row is None:
|
||||
raise QueryNotFoundError(query_id)
|
||||
status = QueryStatus(row.status)
|
||||
if status in QUERY_TERMINAL_STATUSES:
|
||||
raise QueryLeaseInvalidError(
|
||||
f"查询单 {query_id} 已是终态 {status.value},不再接受结果"
|
||||
)
|
||||
if row.lease_owner != worker_id:
|
||||
raise QueryLeaseInvalidError(
|
||||
f"worker {worker_id} 不是查询单 {query_id} 的租约持有者"
|
||||
f"(当前持有者 {row.lease_owner})"
|
||||
)
|
||||
|
||||
now = to_iso(utcnow())
|
||||
if success:
|
||||
await self._db.update_query(
|
||||
query_id,
|
||||
status=QueryStatus.SUCCEEDED.value,
|
||||
result_json=json.dumps(result or {}, ensure_ascii=False),
|
||||
updated_at=now,
|
||||
completed_at=now,
|
||||
clear_lease=True,
|
||||
)
|
||||
final = QueryStatus.SUCCEEDED
|
||||
else:
|
||||
await self._db.update_query(
|
||||
query_id,
|
||||
status=QueryStatus.FAILED.value,
|
||||
error_code=error_code,
|
||||
error_message=error_message or "worker 未给出失败原因",
|
||||
updated_at=now,
|
||||
completed_at=now,
|
||||
clear_lease=True,
|
||||
)
|
||||
final = QueryStatus.FAILED
|
||||
logger.warning(
|
||||
"查询单失败:query_id=%s kind=%s code=%s msg=%s",
|
||||
query_id, row.kind, error_code, error_message,
|
||||
)
|
||||
return QueryResultData(query_id=query_id, status=final)
|
||||
|
||||
# ---- 清扫:重投 / 过期 / 保留期 ----
|
||||
|
||||
async def sweep(self) -> int:
|
||||
"""扫一轮:租约超时重投、整体超时置 expired、过保留期的终态单清理
|
||||
|
||||
返回本轮发生状态变更的查询单条数(删除的不计入)。
|
||||
"""
|
||||
async with self._lock:
|
||||
changed = await self._sweep_locked()
|
||||
purged = await self._db.delete_queries_completed_before(
|
||||
to_iso(utcnow() - timedelta(seconds=self._retention))
|
||||
)
|
||||
if purged:
|
||||
logger.info("清理了 %s 张过保留期的查询单", purged)
|
||||
return changed
|
||||
|
||||
async def _sweep_locked(self) -> int:
|
||||
"""锁内清扫。调用方必须持有 self._lock"""
|
||||
now = utcnow()
|
||||
changed = 0
|
||||
rows = await self._db.list_queries_in_statuses(
|
||||
(QueryStatus.QUEUED.value, QueryStatus.LEASED.value)
|
||||
)
|
||||
for row in rows:
|
||||
# 整体 TTL 优先:单子已经没有意义了,再重投也只是白跑一趟
|
||||
if parse_iso(row.created_at) + timedelta(seconds=self._query_ttl) < now:
|
||||
await self._db.update_query(
|
||||
row.query_id,
|
||||
status=QueryStatus.EXPIRED.value,
|
||||
error_code=None,
|
||||
error_message=(
|
||||
f"查询单超过 {self._query_ttl} 秒仍未完成"
|
||||
f"(attempts={row.attempts},多半是本地 worker 不在线)"
|
||||
),
|
||||
updated_at=to_iso(now),
|
||||
completed_at=to_iso(now),
|
||||
clear_lease=True,
|
||||
)
|
||||
changed += 1
|
||||
logger.warning("查询单过期:query_id=%s kind=%s", row.query_id, row.kind)
|
||||
continue
|
||||
|
||||
if row.status != QueryStatus.LEASED.value:
|
||||
continue
|
||||
if not row.lease_expires_at or parse_iso(row.lease_expires_at) >= now:
|
||||
continue
|
||||
|
||||
if row.attempts >= self._max_attempts:
|
||||
await self._db.update_query(
|
||||
row.query_id,
|
||||
status=QueryStatus.FAILED.value,
|
||||
error_message=(
|
||||
f"已被领取 {row.attempts} 次仍未回结果,不再重投"
|
||||
"(可能是站点页面卡住或本地 worker 反复崩溃)"
|
||||
),
|
||||
updated_at=to_iso(now),
|
||||
completed_at=to_iso(now),
|
||||
clear_lease=True,
|
||||
)
|
||||
logger.warning(
|
||||
"查询单重投次数用尽,置 failed:query_id=%s attempts=%s",
|
||||
row.query_id, row.attempts,
|
||||
)
|
||||
else:
|
||||
# 只读操作可以安全重投——这正是查询与下单任务最大的语义差别
|
||||
await self._db.update_query(
|
||||
row.query_id,
|
||||
status=QueryStatus.QUEUED.value,
|
||||
updated_at=to_iso(now),
|
||||
clear_lease=True,
|
||||
)
|
||||
logger.info(
|
||||
"查询单租约过期,重投回队列:query_id=%s attempts=%s",
|
||||
row.query_id, row.attempts,
|
||||
)
|
||||
changed += 1
|
||||
|
||||
if changed:
|
||||
self._cond.notify_all()
|
||||
return changed
|
||||
|
||||
# ---- 查询 ----
|
||||
|
||||
async def get_detail(self, query_id: str) -> QueryDetail:
|
||||
row = await self._db.get_query(query_id)
|
||||
if row is None:
|
||||
raise QueryNotFoundError(query_id)
|
||||
return _row_to_detail(row)
|
||||
|
||||
async def list_queries(
|
||||
self, *, status: str | None, kind: str | None, limit: int, offset: int
|
||||
) -> QueryListData:
|
||||
rows, total = await self._db.list_queries(
|
||||
status=status, kind=kind, limit=limit, offset=offset
|
||||
)
|
||||
return QueryListData(
|
||||
items=[_row_to_detail(r) for r in rows], total=total, limit=limit, offset=offset
|
||||
)
|
||||
|
||||
async def queued_count(self) -> int:
|
||||
"""等待领取的查询单数量(/health 用)"""
|
||||
return await self._db.count_queries_by_status(QueryStatus.QUEUED.value)
|
||||
+64
-20
@@ -25,6 +25,7 @@ from app.shared.task_state import (
|
||||
TERMINAL_STATUSES,
|
||||
TaskStatus,
|
||||
)
|
||||
from app.gateway.callback import CallbackNotifier, stale_payload, terminal_payload
|
||||
from app.gateway.db import GatewayDB, ReportRow, TaskRow
|
||||
from app.gateway.models import (
|
||||
GatewayHealthData,
|
||||
@@ -74,6 +75,7 @@ def _task_to_detail(task: TaskRow, latest_state: str | None, reports: list[Repor
|
||||
task_id=task.task_id,
|
||||
site=task.site,
|
||||
intent=task.intent,
|
||||
callback_url=task.callback_url,
|
||||
status=TaskStatus(task.status),
|
||||
lease_owner=task.lease_owner,
|
||||
lease_expires_at=task.lease_expires_at,
|
||||
@@ -92,17 +94,37 @@ class TaskQueue:
|
||||
保护下;长轮询等待通过 `self._cond` 唤醒。
|
||||
"""
|
||||
|
||||
def __init__(self, db: GatewayDB, *, lease_ttl_seconds: int, worker_offline_alert_seconds: int):
|
||||
def __init__(
|
||||
self,
|
||||
db: GatewayDB,
|
||||
*,
|
||||
lease_ttl_seconds: int,
|
||||
worker_offline_alert_seconds: int,
|
||||
notifier: CallbackNotifier | None = None,
|
||||
):
|
||||
self._db = db
|
||||
self._lease_ttl = lease_ttl_seconds
|
||||
self._worker_offline_alert_seconds = worker_offline_alert_seconds
|
||||
# 终结类事件通知(§4.8)。None 表示完全关闭回调;测试可替换为记录型假实现
|
||||
self.notifier = notifier
|
||||
self._lock = asyncio.Lock()
|
||||
self._cond = asyncio.Condition(self._lock)
|
||||
|
||||
# ---- 提交 ----
|
||||
|
||||
async def submit(self, *, task_id: str | None, site: str, intent: dict) -> SubmitOrderData:
|
||||
"""上游提交下单意图。task_id 缺省时服务端生成;重复提交幂等"""
|
||||
async def submit(
|
||||
self,
|
||||
*,
|
||||
task_id: str | None,
|
||||
site: str,
|
||||
intent: dict,
|
||||
callback_url: str | None = None,
|
||||
) -> SubmitOrderData:
|
||||
"""上游提交下单意图。task_id 缺省时服务端生成;重复提交幂等
|
||||
|
||||
callback_url 只在首次创建时写入;幂等重发(created=false)返回既有任务,
|
||||
不更新任何字段。
|
||||
"""
|
||||
import json
|
||||
|
||||
tid = task_id or _generate_task_id()
|
||||
@@ -111,6 +133,7 @@ class TaskQueue:
|
||||
task_id=tid,
|
||||
site=site,
|
||||
intent_json=json.dumps(intent, ensure_ascii=False),
|
||||
callback_url=callback_url,
|
||||
status=TaskStatus.QUEUED.value,
|
||||
lease_owner=None,
|
||||
lease_expires_at=None,
|
||||
@@ -152,7 +175,7 @@ class TaskQueue:
|
||||
|
||||
async with self._lock:
|
||||
# 先把过期租约扫到 stale,避免「卡死的任务」堵住新任务
|
||||
await self._sweep_locked()
|
||||
self._notify_stale(await self._sweep_locked())
|
||||
|
||||
await self._db.upsert_worker(worker_id, to_iso(utcnow()))
|
||||
|
||||
@@ -272,18 +295,17 @@ class TaskQueue:
|
||||
)
|
||||
|
||||
now = to_iso(utcnow())
|
||||
inserted = await self._db.upsert_report(
|
||||
ReportRow(
|
||||
task_id=task_id,
|
||||
state=state,
|
||||
payable_yen=payable_yen,
|
||||
pay_deadline=pay_deadline,
|
||||
site_order_id=site_order_id,
|
||||
evidence_ref=evidence_ref,
|
||||
detail=detail,
|
||||
reported_at=now,
|
||||
)
|
||||
report_row = ReportRow(
|
||||
task_id=task_id,
|
||||
state=state,
|
||||
payable_yen=payable_yen,
|
||||
pay_deadline=pay_deadline,
|
||||
site_order_id=site_order_id,
|
||||
evidence_ref=evidence_ref,
|
||||
detail=detail,
|
||||
reported_at=now,
|
||||
)
|
||||
inserted = await self._db.upsert_report(report_row)
|
||||
|
||||
final_status = status.value
|
||||
if terminal:
|
||||
@@ -298,6 +320,10 @@ class TaskQueue:
|
||||
)
|
||||
# 任务终结可能让并发度 1 的闸门放开,唤醒等着的 lease
|
||||
self._cond.notify_all()
|
||||
# 终结类事件回调(§4.8):仅「本次真正把任务推入终态」才通知——
|
||||
# 已终结任务的后续 report(如付款后监控的 shipped/delivered)不重复通知
|
||||
if status not in TERMINAL_STATUSES:
|
||||
self._notify_terminal(task, report_row, final_status)
|
||||
|
||||
return ReportData(task_id=task_id, status=TaskStatus(final_status), recorded=inserted)
|
||||
|
||||
@@ -355,13 +381,15 @@ class TaskQueue:
|
||||
async def sweep(self) -> int:
|
||||
"""把过期的 leased/running 任务置 stale。返回清扫条数"""
|
||||
async with self._lock:
|
||||
return await self._sweep_locked()
|
||||
swept = await self._sweep_locked()
|
||||
self._notify_stale(swept)
|
||||
return len(swept)
|
||||
|
||||
async def _sweep_locked(self) -> int:
|
||||
"""锁内执行 sweep。调用方必须持有 self._lock"""
|
||||
async def _sweep_locked(self) -> list[TaskRow]:
|
||||
"""锁内执行 sweep。调用方必须持有 self._lock。返回本次被置 stale 的任务"""
|
||||
active = await self._db.list_tasks_in_statuses(tuple(s.value for s in ACTIVE_STATUSES))
|
||||
now = utcnow()
|
||||
swept = 0
|
||||
swept: list[TaskRow] = []
|
||||
for task in active:
|
||||
if task.lease_expires_at and parse_iso(task.lease_expires_at) < now:
|
||||
await self._db.update_task(
|
||||
@@ -370,7 +398,7 @@ class TaskQueue:
|
||||
clear_lease=True,
|
||||
updated_at=to_iso(now),
|
||||
)
|
||||
swept += 1
|
||||
swept.append(task)
|
||||
logger.warning(
|
||||
"租约过期,任务置 stale(不自动重投):task_id=%s site=%s",
|
||||
task.task_id,
|
||||
@@ -382,6 +410,22 @@ class TaskQueue:
|
||||
self._cond.notify_all()
|
||||
return swept
|
||||
|
||||
# ---- 终结类事件回调(§4.8)----
|
||||
# notify 只是调度后台发送任务,持锁调用是安全的:真正的 HTTP 发送不碰这把锁
|
||||
|
||||
def _notify_terminal(self, task: TaskRow, report_row: ReportRow, final_status: str) -> None:
|
||||
if self.notifier is not None and task.callback_url:
|
||||
self.notifier.notify(
|
||||
task.callback_url, terminal_payload(task, report_row, final_status)
|
||||
)
|
||||
|
||||
def _notify_stale(self, swept: list[TaskRow]) -> None:
|
||||
if self.notifier is None:
|
||||
return
|
||||
for task in swept:
|
||||
if task.callback_url:
|
||||
self.notifier.notify(task.callback_url, stale_payload(task))
|
||||
|
||||
# ---- 查询 ----
|
||||
|
||||
async def get_task_detail(self, task_id: str) -> TaskDetail:
|
||||
|
||||
+326
-294
@@ -46,26 +46,29 @@ class SearchRequest(BaseModel):
|
||||
3. 只传 genre_id:抓取该分类下的商品
|
||||
"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = Field(default=1, ge=1, le=150) # 站点侧最多约 150 页(subset 6750 / 45)
|
||||
sort: SortOption = SortOption.STANDARD
|
||||
genre_id: str | None = None # 乐天分类 ID,如 565950
|
||||
min_price: int | None = Field(default=None, ge=0)
|
||||
max_price: int | None = Field(default=None, ge=0)
|
||||
shop_id: int | None = None # 限定店铺(对应 `sid` 参数,取搜索结果的 shop.shop_id)
|
||||
exclude_keyword: str | None = None # 排除词(`nitem`)
|
||||
title_only: bool = False # 仅在商品标题中匹配(`sf=1`)
|
||||
or_query: bool = False # 关键词之间用 OR 而非 AND(`st=O`)
|
||||
min_review_score: int | None = Field(default=None, ge=1, le=5) # 最低评分
|
||||
condition: ItemCondition | None = None # 新品 / 中古 / 租赁
|
||||
include_sold_out: bool = False # 包含售罄商品
|
||||
free_shipping: bool = False # 仅免运费
|
||||
has_review: bool = False # 仅有评论
|
||||
next_day_delivery: bool = False # 仅次日达
|
||||
super_deal: bool = False # 仅 SuperDEAL
|
||||
tags: list[str] = Field(default_factory=list) # 站点标签 ID(`tg`)
|
||||
search_url: HttpUrl | None = None
|
||||
exclude_ads: bool = True # 剔除搜索结果中混入的 CPC 广告位
|
||||
keyword: str = Field(default="", description="搜索关键词")
|
||||
page: int = Field(default=1, ge=1, le=150, description="页码,站点侧最多约 150 页(subset 6750 / 45)")
|
||||
sort: SortOption = Field(default=SortOption.STANDARD, description="排序方式,对应搜索页 `s=` 参数")
|
||||
genre_id: str | None = Field(default=None, description="乐天分类 ID,如 565950")
|
||||
min_price: int | None = Field(default=None, ge=0, description="最低价格(日元)")
|
||||
max_price: int | None = Field(default=None, ge=0, description="最高价格(日元)")
|
||||
shop_id: int | None = Field(default=None, description="限定店铺(对应 `sid` 参数,取搜索结果的 shop.shop_id)")
|
||||
exclude_keyword: str | None = Field(default=None, description="排除词(`nitem`)")
|
||||
title_only: bool = Field(default=False, description="仅在商品标题中匹配(`sf=1`)")
|
||||
or_query: bool = Field(default=False, description="关键词之间用 OR 而非 AND(`st=O`)")
|
||||
min_review_score: int | None = Field(default=None, ge=1, le=5, description="最低评分")
|
||||
condition: ItemCondition | None = Field(default=None, description="商品成色筛选:新品 / 中古 / 租赁")
|
||||
include_sold_out: bool = Field(default=False, description="包含售罄商品")
|
||||
free_shipping: bool = Field(default=False, description="仅免运费")
|
||||
has_review: bool = Field(default=False, description="仅有评论")
|
||||
next_day_delivery: bool = Field(default=False, description="仅次日达")
|
||||
super_deal: bool = Field(default=False, description="仅 SuperDEAL")
|
||||
tags: list[str] = Field(default_factory=list, description="站点标签 ID(`tg`)")
|
||||
search_url: HttpUrl | None = Field(
|
||||
default=None,
|
||||
description="直接透传一条乐天搜索页 URL,服务端原样抓取;此时除 page 与 exclude_ads 外的筛选字段全部忽略",
|
||||
)
|
||||
exclude_ads: bool = Field(default=True, description="剔除搜索结果中混入的 CPC 广告位")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_search_target(self) -> SearchRequest:
|
||||
@@ -84,8 +87,8 @@ class SearchRequest(BaseModel):
|
||||
class ShopDetailRequest(BaseModel):
|
||||
"""乐天商家详情请求参数:传店铺代码(店铺 URL 的路径段),或直接传店铺页 URL"""
|
||||
|
||||
shop_code: str | None = None # 店铺代码,如 edion
|
||||
shop_url: HttpUrl | None = None
|
||||
shop_code: str | None = Field(default=None, description="店铺代码,如 edion")
|
||||
shop_url: HttpUrl | None = Field(default=None, description="店铺页 URL;与 shop_code 二选一")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> ShopDetailRequest:
|
||||
@@ -104,18 +107,18 @@ class ShopItemsRequest(BaseModel):
|
||||
换出 shop_id,多花一次请求,能直接给 shop_id 时优先给。
|
||||
"""
|
||||
|
||||
shop_id: int | None = None # 取自搜索结果或商家详情的 shop.shop_id
|
||||
shop_code: str | None = None # 店铺代码,如 edion
|
||||
keyword: str = "" # 在店铺内按关键词过滤
|
||||
page: int = Field(default=1, ge=1, le=150)
|
||||
sort: SortOption = SortOption.STANDARD
|
||||
genre_id: str | None = None
|
||||
min_price: int | None = Field(default=None, ge=0)
|
||||
max_price: int | None = Field(default=None, ge=0)
|
||||
condition: ItemCondition | None = None
|
||||
include_sold_out: bool = False
|
||||
free_shipping: bool = False
|
||||
exclude_ads: bool = True
|
||||
shop_id: int | None = Field(default=None, description="店铺 ID,取自搜索结果或商家详情的 shop.shop_id")
|
||||
shop_code: str | None = Field(default=None, description="店铺代码,如 edion")
|
||||
keyword: str = Field(default="", description="在店铺内按关键词过滤")
|
||||
page: int = Field(default=1, ge=1, le=150, description="页码")
|
||||
sort: SortOption = Field(default=SortOption.STANDARD, description="排序方式")
|
||||
genre_id: str | None = Field(default=None, description="乐天分类 ID,如 565950")
|
||||
min_price: int | None = Field(default=None, ge=0, description="最低价格(日元)")
|
||||
max_price: int | None = Field(default=None, ge=0, description="最高价格(日元)")
|
||||
condition: ItemCondition | None = Field(default=None, description="商品成色筛选")
|
||||
include_sold_out: bool = Field(default=False, description="包含售罄商品")
|
||||
free_shipping: bool = Field(default=False, description="仅免运费")
|
||||
exclude_ads: bool = Field(default=True, description="剔除搜索结果中混入的 CPC 广告位")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> ShopItemsRequest:
|
||||
@@ -149,30 +152,32 @@ class ShopItemsRequest(BaseModel):
|
||||
class ShopDetailData(BaseModel):
|
||||
"""乐天商家详情数据"""
|
||||
|
||||
shop_id: int | None = None
|
||||
shop_code: str = ""
|
||||
shop_name: str = ""
|
||||
shop_url: str = ""
|
||||
introduction: str = "" # 店铺简介
|
||||
signboard_url: str = "" # 店铺招牌图
|
||||
logo_url: str = ""
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
shop_id: int | None = Field(default=None, description="店铺 ID")
|
||||
shop_code: str = Field(default="", description="店铺代码,如 edion")
|
||||
shop_name: str = Field(default="", description="店铺名")
|
||||
shop_url: str = Field(default="", description="店铺页地址")
|
||||
introduction: str = Field(default="", description="店铺简介")
|
||||
signboard_url: str = Field(default="", description="店铺招牌图地址")
|
||||
logo_url: str = Field(default="", description="店铺 Logo 图地址")
|
||||
review_score: float = Field(default=0.0, description="综合评分")
|
||||
review_count: int = Field(default=0, description="评价数")
|
||||
# 站点在评价数过少时不展示评分;此时 review_score 不可信
|
||||
review_displayed: bool = False
|
||||
is_39_shop: bool = False # 39ショップ(满 3980 日元免运费)
|
||||
age_verification_required: bool = False
|
||||
status: int | None = None # 站点店铺状态码,1 = 营业中
|
||||
holidays: list[str] = Field(default_factory=list) # 店铺休息日
|
||||
review_displayed: bool = Field(default=False, description="站点是否展示了评分;未展示时 review_score 不可信")
|
||||
is_39_shop: bool = Field(default=False, description="39ショップ(满 3980 日元免运费)")
|
||||
age_verification_required: bool = Field(default=False, description="购买该店铺商品需要年龄确认")
|
||||
status: int | None = Field(default=None, description="站点店铺状态码,1 = 营业中")
|
||||
holidays: list[str] = Field(default_factory=list, description="店铺休息日")
|
||||
|
||||
|
||||
class ItemDetailRequest(BaseModel):
|
||||
"""商品详情请求参数:传 shop_code + item_code,或直接传商品页 URL"""
|
||||
|
||||
shop_code: str | None = None # 店铺代码,如 edion(商品 URL 的第一段)
|
||||
item_code: str | None = None # 店铺内商品编号,如 4902370549263(商品 URL 的第二段)
|
||||
item_url: HttpUrl | None = None
|
||||
include_sku_variants: bool = True # SKU 组合可能多达数百条,不需要时可关闭
|
||||
shop_code: str | None = Field(default=None, description="店铺代码,如 edion(商品 URL 的第一段)")
|
||||
item_code: str | None = Field(default=None, description="店铺内商品编号,如 4902370549263(商品 URL 的第二段)")
|
||||
item_url: HttpUrl | None = Field(default=None, description="商品页 URL;不传时需同时提供 shop_code 与 item_code")
|
||||
include_sku_variants: bool = Field(
|
||||
default=True, description="是否返回 SKU 组合明细;SKU 组合可能多达数百条,不需要时可关闭"
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_item_target(self) -> ItemDetailRequest:
|
||||
@@ -187,164 +192,169 @@ class GenreRequest(BaseModel):
|
||||
不传 genre_id 时返回 39 个顶层分类;传入时返回该分类的信息、祖先路径与直接子分类。
|
||||
"""
|
||||
|
||||
genre_id: str | None = None
|
||||
genre_id: str | None = Field(
|
||||
default=None,
|
||||
description="分类 ID。不传返回 39 个顶层分类;传入返回该分类的信息、祖先路径与直接子分类",
|
||||
)
|
||||
|
||||
|
||||
class GenreNode(BaseModel):
|
||||
"""分类树上的一个节点"""
|
||||
|
||||
genre_id: str = ""
|
||||
name: str = ""
|
||||
genre_id: str = Field(default="", description="分类 ID")
|
||||
name: str = Field(default="", description="分类名")
|
||||
# 该分类下的商品数。顶层列表不返回该值:站点给出的是「当前查询在该分类下的
|
||||
# 命中数」,与分类自身的商品总量不是一回事,避免误用。
|
||||
item_count: int | None = None
|
||||
shortcut: str = "" # 站点分类短代码,如 game / flower
|
||||
is_leaf: bool = False # 叶子分类,没有下级
|
||||
url: str = "" # 分类页地址
|
||||
item_count: int | None = Field(default=None, description="该分类下的商品数;顶层列表不返回该值")
|
||||
shortcut: str = Field(default="", description="站点分类短代码,如 game / flower")
|
||||
is_leaf: bool = Field(default=False, description="叶子分类,没有下级")
|
||||
url: str = Field(default="", description="分类页地址")
|
||||
|
||||
|
||||
class GenreData(BaseModel):
|
||||
"""分类查询结果"""
|
||||
|
||||
genre_id: str = "" # 空串表示顶层
|
||||
name: str = ""
|
||||
full_name: str = "" # 站点给出的完整分类名,仅分类页有
|
||||
description: str = "" # 站点分类描述,仅分类页有
|
||||
is_leaf: bool = False
|
||||
url: str = ""
|
||||
ancestors: list[GenreNode] = Field(default_factory=list) # 从顶层到父级,不含自身
|
||||
children: list[GenreNode] = Field(default_factory=list) # 直接子分类
|
||||
genre_id: str = Field(default="", description="分类 ID,空串表示顶层")
|
||||
name: str = Field(default="", description="分类名")
|
||||
full_name: str = Field(default="", description="站点给出的完整分类名,仅分类页有")
|
||||
description: str = Field(default="", description="站点分类描述,仅分类页有")
|
||||
is_leaf: bool = Field(default=False, description="叶子分类,没有下级")
|
||||
url: str = Field(default="", description="分类页地址")
|
||||
ancestors: list[GenreNode] = Field(default_factory=list, description="祖先分类路径,从顶层到父级,不含自身")
|
||||
children: list[GenreNode] = Field(default_factory=list, description="直接子分类")
|
||||
|
||||
|
||||
class ShopSummary(BaseModel):
|
||||
"""店铺信息"""
|
||||
|
||||
shop_id: int | None = None
|
||||
shop_code: str = "" # 店铺 URL 代码,如 edion;与 item_code 一起可定位商品
|
||||
shop_name: str = ""
|
||||
shop_url: str = ""
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
shop_id: int | None = Field(default=None, description="店铺 ID")
|
||||
shop_code: str = Field(default="", description="店铺 URL 代码,如 edion;与 item_code 一起可定位商品")
|
||||
shop_name: str = Field(default="", description="店铺名")
|
||||
shop_url: str = Field(default="", description="店铺页地址")
|
||||
review_score: float = Field(default=0.0, description="综合评分")
|
||||
review_count: int = Field(default=0, description="评价数")
|
||||
|
||||
|
||||
class ReviewSummary(BaseModel):
|
||||
"""评价信息"""
|
||||
|
||||
score: float = 0.0
|
||||
count: int = 0
|
||||
url: str = ""
|
||||
score: float = Field(default=0.0, description="评分")
|
||||
count: int = Field(default=0, description="评价数")
|
||||
url: str = Field(default="", description="评价页地址")
|
||||
|
||||
|
||||
class SearchItem(BaseModel):
|
||||
"""搜索结果中的单个商品"""
|
||||
|
||||
item_id: str = "" # 乐天内部商品 ID(搜索结果的 code 字段)
|
||||
item_code: str = "" # 商品 URL 第二段,调详情接口用
|
||||
item_name: str = ""
|
||||
item_url: str = "" # 真实商品页地址;广告位已还原为 originalItemUrl
|
||||
catch_copy: str = "" # 商品副标题
|
||||
price: int = 0
|
||||
price_range: str = "" # 多 SKU 时的价格区间,如 "1000~2000"
|
||||
has_price_range: bool = False
|
||||
image_url: str = ""
|
||||
image_urls: list[str] = Field(default_factory=list)
|
||||
shop: ShopSummary = Field(default_factory=ShopSummary)
|
||||
review: ReviewSummary = Field(default_factory=ReviewSummary)
|
||||
genre_id: str = ""
|
||||
genre_path: str = "" # 形如 /0/101205/565950/566404
|
||||
genre_names: list[str] = Field(default_factory=list)
|
||||
shipping_fee: int | None = None # 站点未给出时为 null
|
||||
delivery_message: str = ""
|
||||
point_count: int = 0
|
||||
is_sold_out: bool = False
|
||||
is_ad: bool = False # CPC 广告位
|
||||
has_multi_sku: bool = False
|
||||
variant_id: str = ""
|
||||
item_options: dict[str, Any] = Field(default_factory=dict) # 站点 itemOptions 原样透出
|
||||
item_id: str = Field(default="", description="乐天内部商品 ID(搜索结果的 code 字段)")
|
||||
item_code: str = Field(default="", description="商品 URL 第二段,调详情接口用")
|
||||
item_name: str = Field(default="", description="商品名")
|
||||
item_url: str = Field(default="", description="真实商品页地址;广告位已还原为 originalItemUrl")
|
||||
catch_copy: str = Field(default="", description="商品副标题")
|
||||
price: int = Field(default=0, description="价格(日元)")
|
||||
price_range: str = Field(default="", description="多 SKU 时的价格区间,如「1000~2000」")
|
||||
has_price_range: bool = Field(default=False, description="是否为多 SKU 价格区间")
|
||||
image_url: str = Field(default="", description="主图地址")
|
||||
image_urls: list[str] = Field(default_factory=list, description="图片地址列表")
|
||||
shop: ShopSummary = Field(default_factory=ShopSummary, description="所属店铺信息")
|
||||
review: ReviewSummary = Field(default_factory=ReviewSummary, description="商品评价信息")
|
||||
genre_id: str = Field(default="", description="分类 ID")
|
||||
genre_path: str = Field(default="", description="分类路径,形如 /0/101205/565950/566404")
|
||||
genre_names: list[str] = Field(default_factory=list, description="分类名列表")
|
||||
shipping_fee: int | None = Field(default=None, description="运费(日元);站点未给出时为 null")
|
||||
delivery_message: str = Field(default="", description="配送说明")
|
||||
point_count: int = Field(default=0, description="乐天积分倍率(站点 point.count 原值)")
|
||||
is_sold_out: bool = Field(default=False, description="已售罄")
|
||||
is_ad: bool = Field(default=False, description="CPC 广告位")
|
||||
has_multi_sku: bool = Field(default=False, description="有多个 SKU")
|
||||
variant_id: str = Field(default="", description="SKU 组合 ID(站点 variantId 原值)")
|
||||
item_options: dict[str, Any] = Field(default_factory=dict, description="站点 itemOptions 原样透出")
|
||||
|
||||
|
||||
class SearchResultData(BaseModel):
|
||||
"""搜索结果数据"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = 1
|
||||
page_size: int = 0
|
||||
total_count: int = 0 # 站点声明的命中总数
|
||||
reachable_count: int = 0 # 实际可翻页取到的上限(站点 subset,随查询条件变化)
|
||||
has_more: bool = False
|
||||
keyword: str = Field(default="", description="搜索关键词")
|
||||
page: int = Field(default=1, description="当前页码")
|
||||
page_size: int = Field(default=0, description="每页条数")
|
||||
total_count: int = Field(default=0, description="站点声明的命中总数")
|
||||
reachable_count: int = Field(default=0, description="实际可翻页取到的上限(站点 subset,随查询条件变化)")
|
||||
has_more: bool = Field(default=False, description="是否还有下一页")
|
||||
# 请求页码超出 reachable_count 对应的页数。站点此时不会返回空列表,而是
|
||||
# 静默回绕到第 1 页;这里识别出来并把 items 置空,避免上游把重复数据当新数据。
|
||||
out_of_range: bool = False
|
||||
ad_count: int = 0 # 本页被识别出的广告位数量(exclude_ads=true 时已从 items 剔除)
|
||||
request_url: str = "" # 实际抓取的乐天页面地址,便于排查
|
||||
items: list[SearchItem] = Field(default_factory=list)
|
||||
out_of_range: bool = Field(
|
||||
default=False, description="请求页码超出可翻页范围;此时 items 置空(站点会静默回绕到第 1 页,已识别)"
|
||||
)
|
||||
ad_count: int = Field(default=0, description="本页被识别出的广告位数量(exclude_ads=true 时已从 items 剔除)")
|
||||
request_url: str = Field(default="", description="实际抓取的乐天页面地址,便于排查")
|
||||
items: list[SearchItem] = Field(default_factory=list, description="本页商品列表")
|
||||
|
||||
|
||||
class SkuAttribute(BaseModel):
|
||||
"""SKU 属性项"""
|
||||
|
||||
title: str = ""
|
||||
value: str = ""
|
||||
title: str = Field(default="", description="属性名")
|
||||
value: str = Field(default="", description="属性值")
|
||||
|
||||
|
||||
class SkuAxisValue(BaseModel):
|
||||
"""SKU 选择轴上的一个取值"""
|
||||
|
||||
value: str = ""
|
||||
label: str = ""
|
||||
is_sold_out: bool = False
|
||||
value: str = Field(default="", description="取值原值")
|
||||
label: str = Field(default="", description="取值展示名")
|
||||
is_sold_out: bool = Field(default=False, description="该取值已售罄")
|
||||
|
||||
|
||||
class SkuAxis(BaseModel):
|
||||
"""SKU 选择轴,如「颜色」「尺码」"""
|
||||
|
||||
key: str = ""
|
||||
label: str = ""
|
||||
values: list[SkuAxisValue] = Field(default_factory=list)
|
||||
key: str = Field(default="", description="轴标识")
|
||||
label: str = Field(default="", description="轴展示名")
|
||||
values: list[SkuAxisValue] = Field(default_factory=list, description="可选取值列表")
|
||||
|
||||
|
||||
class SkuVariant(BaseModel):
|
||||
"""一个具体的 SKU 组合"""
|
||||
|
||||
variant_id: str = ""
|
||||
selector_values: list[str] = Field(default_factory=list) # 与 axis 顺序对应的取值
|
||||
price: int = 0
|
||||
quantity: int = 0
|
||||
is_sold_out: bool = False
|
||||
delivery_message: str = ""
|
||||
attributes: list[SkuAttribute] = Field(default_factory=list)
|
||||
variant_id: str = Field(default="", description="SKU 组合 ID")
|
||||
selector_values: list[str] = Field(default_factory=list, description="与 axis 顺序对应的取值")
|
||||
price: int = Field(default=0, description="价格(日元)")
|
||||
quantity: int = Field(default=0, description="库存数量")
|
||||
is_sold_out: bool = Field(default=False, description="已售罄")
|
||||
delivery_message: str = Field(default="", description="配送说明")
|
||||
attributes: list[SkuAttribute] = Field(default_factory=list, description="SKU 属性列表")
|
||||
|
||||
|
||||
class SkuInfo(BaseModel):
|
||||
"""商品 SKU 信息"""
|
||||
|
||||
inventory_type: str = "" # single / multiple
|
||||
quantity: int = 0
|
||||
show_inventory: bool = False
|
||||
delivery_message: str = ""
|
||||
attributes: list[SkuAttribute] = Field(default_factory=list)
|
||||
axis: list[SkuAxis] = Field(default_factory=list)
|
||||
variants: list[SkuVariant] = Field(default_factory=list) # include_sku_variants=false 时为空
|
||||
variant_count: int = 0 # 不受 include_sku_variants 影响,始终为真实组合数
|
||||
inventory_type: str = Field(default="", description="库存类型:single / multiple")
|
||||
quantity: int = Field(default=0, description="库存数量")
|
||||
show_inventory: bool = Field(default=False, description="站点是否展示库存")
|
||||
delivery_message: str = Field(default="", description="配送说明")
|
||||
attributes: list[SkuAttribute] = Field(default_factory=list, description="SKU 属性列表")
|
||||
axis: list[SkuAxis] = Field(default_factory=list, description="SKU 选择轴,如「颜色」「尺码」")
|
||||
variants: list[SkuVariant] = Field(default_factory=list, description="SKU 组合明细;include_sku_variants=false 时为空")
|
||||
variant_count: int = Field(default=0, description="SKU 组合总数;不受 include_sku_variants 影响,始终为真实组合数")
|
||||
|
||||
|
||||
class ShippingInfo(BaseModel):
|
||||
"""配送与运费信息"""
|
||||
|
||||
shipping_fee: int | None = None
|
||||
is_shipping_free: bool = False
|
||||
is_asuraku: bool = False # あす楽(次日达)
|
||||
is_next_day_delivery: bool = False
|
||||
free_shipping_threshold: int | None = None
|
||||
prefecture_id: int | None = None # 站点默认收货地(13 = 东京都)
|
||||
delivery_message: str = ""
|
||||
shipping_fee: int | None = Field(default=None, description="运费(日元);站点未给出时为 null")
|
||||
is_shipping_free: bool = Field(default=False, description="免运费")
|
||||
is_asuraku: bool = Field(default=False, description="あす楽(次日达)")
|
||||
is_next_day_delivery: bool = Field(default=False, description="次日达")
|
||||
free_shipping_threshold: int | None = Field(default=None, description="免运费门槛(日元)")
|
||||
prefecture_id: int | None = Field(default=None, description="站点默认收货地(13 = 东京都)")
|
||||
delivery_message: str = Field(default="", description="配送说明")
|
||||
|
||||
|
||||
class Breadcrumb(BaseModel):
|
||||
"""分类面包屑"""
|
||||
|
||||
name: str = ""
|
||||
url: str = ""
|
||||
name: str = Field(default="", description="分类名")
|
||||
url: str = Field(default="", description="分类页地址")
|
||||
|
||||
|
||||
class ItemDetailData(BaseModel):
|
||||
@@ -358,28 +368,28 @@ class ItemDetailData(BaseModel):
|
||||
(purchase_condition / is_sold_out)、规格(sku.variants)、起订单位等。
|
||||
"""
|
||||
|
||||
source: str = "ichiba" # ichiba / books / brandavenue / biccamera
|
||||
source_url: str = "" # 实际解析的页面地址;跳转时与 item_url 不同
|
||||
item_id: str = ""
|
||||
item_code: str = ""
|
||||
item_name: str = ""
|
||||
catch_copy: str = ""
|
||||
description: str = "" # 店铺自填的商品说明,含 HTML
|
||||
item_url: str = ""
|
||||
price: int = 0 # 最低售价(含税,多 SKU 时为最低价)
|
||||
pre_tax_price: int = 0
|
||||
tax_flag: bool = False
|
||||
tax_rate: float = 0.0
|
||||
purchase_condition: str = "" # 站点原值,enabled 表示可购买
|
||||
is_sold_out: bool = False
|
||||
purchase_unit: int = 0 # 起订单位
|
||||
images: list[str] = Field(default_factory=list)
|
||||
shop: ShopSummary = Field(default_factory=ShopSummary)
|
||||
review: ReviewSummary = Field(default_factory=ReviewSummary)
|
||||
genre_id: str = ""
|
||||
breadcrumbs: list[Breadcrumb] = Field(default_factory=list)
|
||||
shipping: ShippingInfo = Field(default_factory=ShippingInfo)
|
||||
sku: SkuInfo = Field(default_factory=SkuInfo)
|
||||
source: str = Field(default="ichiba", description="数据来源站点:ichiba / books / brandavenue / biccamera")
|
||||
source_url: str = Field(default="", description="实际解析的页面地址;跳转时与 item_url 不同")
|
||||
item_id: str = Field(default="", description="乐天内部商品 ID")
|
||||
item_code: str = Field(default="", description="店铺内商品编号(商品 URL 第二段)")
|
||||
item_name: str = Field(default="", description="商品名")
|
||||
catch_copy: str = Field(default="", description="商品副标题")
|
||||
description: str = Field(default="", description="店铺自填的商品说明,含 HTML")
|
||||
item_url: str = Field(default="", description="商品页地址")
|
||||
price: int = Field(default=0, description="最低售价(日元,含税;多 SKU 时为最低价)")
|
||||
pre_tax_price: int = Field(default=0, description="税前价格(日元)")
|
||||
tax_flag: bool = Field(default=False, description="价格是否含税(站点 taxFlag 原值)")
|
||||
tax_rate: float = Field(default=0.0, description="税率")
|
||||
purchase_condition: str = Field(default="", description="站点原值,enabled 表示可购买")
|
||||
is_sold_out: bool = Field(default=False, description="已售罄")
|
||||
purchase_unit: int = Field(default=0, description="起订单位")
|
||||
images: list[str] = Field(default_factory=list, description="商品图片地址列表")
|
||||
shop: ShopSummary = Field(default_factory=ShopSummary, description="所属店铺信息")
|
||||
review: ReviewSummary = Field(default_factory=ReviewSummary, description="商品评价信息")
|
||||
genre_id: str = Field(default="", description="分类 ID")
|
||||
breadcrumbs: list[Breadcrumb] = Field(default_factory=list, description="分类面包屑")
|
||||
shipping: ShippingInfo = Field(default_factory=ShippingInfo, description="该商品的配送与运费信息")
|
||||
sku: SkuInfo = Field(default_factory=SkuInfo, description="SKU 信息")
|
||||
|
||||
|
||||
class HealthData(BaseModel):
|
||||
@@ -388,11 +398,11 @@ class HealthData(BaseModel):
|
||||
这里不含登录态——登录态属于交易服务,查它请打交易服务的 /health。
|
||||
"""
|
||||
|
||||
status: str
|
||||
browser_fallback_enabled: bool
|
||||
browser_fallback_ready: bool
|
||||
browser_fallback_error: str | None = None
|
||||
sessions: dict[str, Any] = Field(default_factory=dict)
|
||||
status: str = Field(description="服务状态,正常为 ok")
|
||||
browser_fallback_enabled: bool = Field(description="是否启用浏览器兜底抓取")
|
||||
browser_fallback_ready: bool = Field(description="浏览器兜底是否就绪")
|
||||
browser_fallback_error: str | None = Field(default=None, description="浏览器兜底不可用的原因;正常为 null")
|
||||
sessions: dict[str, Any] = Field(default_factory=dict, description="各站点抓取会话状态(rakuten / rakuma)")
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
@@ -449,21 +459,25 @@ class RakumaSearchRequest(BaseModel):
|
||||
keyword、category_id、brand_id、search_url 四者至少提供一个。
|
||||
"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = Field(default=1, ge=1, le=100) # 站点侧 page>100 直接 404
|
||||
sort: RakumaSortOption = RakumaSortOption.STANDARD
|
||||
category_id: str | None = None # ラクマ 分类 ID,如 788
|
||||
brand_id: str | None = None # ラクマ 品牌 ID,如 5296
|
||||
min_price: int | None = Field(default=None, ge=0)
|
||||
max_price: int | None = Field(default=None, ge=0)
|
||||
exclude_keyword: str | None = None # 排除词(站点 `excluded_query`,需与 keyword 同时使用)
|
||||
conditions: list[RakumaCondition] = Field(default_factory=list) # 可多选
|
||||
transaction: RakumaTransaction | None = None # 不传表示不限
|
||||
free_shipping: bool = False # 仅「送料込み」(卖家承担运费)
|
||||
anonymous_shipping: bool = False # 仅匿名配送
|
||||
except_for_no_brand: bool = False # 排除无品牌商品;与 brand_id 互斥
|
||||
authenticity_types: list[RakumaAuthenticity] = Field(default_factory=list)
|
||||
search_url: HttpUrl | None = None
|
||||
keyword: str = Field(default="", description="搜索关键词")
|
||||
page: int = Field(default=1, ge=1, le=100, description="页码,站点侧 page>100 直接 404")
|
||||
sort: RakumaSortOption = Field(default=RakumaSortOption.STANDARD, description="排序方式")
|
||||
category_id: str | None = Field(default=None, description="ラクマ 分类 ID,如 788")
|
||||
brand_id: str | None = Field(default=None, description="ラクマ 品牌 ID,如 5296")
|
||||
min_price: int | None = Field(default=None, ge=0, description="最低价格(日元)")
|
||||
max_price: int | None = Field(default=None, ge=0, description="最高价格(日元)")
|
||||
exclude_keyword: str | None = Field(
|
||||
default=None, description="排除词(站点 `excluded_query`,需与 keyword 同时使用)"
|
||||
)
|
||||
conditions: list[RakumaCondition] = Field(default_factory=list, description="商品状态筛选,可多选")
|
||||
transaction: RakumaTransaction | None = Field(default=None, description="售卖状态筛选,不传表示不限")
|
||||
free_shipping: bool = Field(default=False, description="仅「送料込み」(卖家承担运费)")
|
||||
anonymous_shipping: bool = Field(default=False, description="仅匿名配送")
|
||||
except_for_no_brand: bool = Field(default=False, description="排除无品牌商品;与 brand_id 互斥")
|
||||
authenticity_types: list[RakumaAuthenticity] = Field(default_factory=list, description="正品鉴定服务类型筛选,可多选")
|
||||
search_url: HttpUrl | None = Field(
|
||||
default=None, description="直接透传一条 fril.jp 搜索页 URL;此时除 page 外的筛选字段全部忽略"
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_search_target(self) -> RakumaSearchRequest:
|
||||
@@ -480,8 +494,8 @@ class RakumaSearchRequest(BaseModel):
|
||||
class RakumaItemDetailRequest(BaseModel):
|
||||
"""ラクマ 商品详情请求参数:传 item_id(商品 URL 的最后一段),或直接传商品页 URL"""
|
||||
|
||||
item_id: str | None = None # 商品页 hash,如 4aca1d6db3e422f3a251a8a8b61e1eff
|
||||
item_url: HttpUrl | None = None
|
||||
item_id: str | None = Field(default=None, description="商品页 hash,如 4aca1d6db3e422f3a251a8a8b61e1eff")
|
||||
item_url: HttpUrl | None = Field(default=None, description="商品页 URL;与 item_id 二选一")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_item_target(self) -> RakumaItemDetailRequest:
|
||||
@@ -493,10 +507,12 @@ class RakumaItemDetailRequest(BaseModel):
|
||||
class RakumaShopDetailRequest(BaseModel):
|
||||
"""ラクマ 卖家详情请求参数:传 shop_id(店铺 URL 的最后一段),或直接传店铺页 URL"""
|
||||
|
||||
shop_id: str | None = None # 店铺页 hash,如 422750cb7921557bc8dba2416915d968
|
||||
shop_url: HttpUrl | None = None
|
||||
shop_id: str | None = Field(default=None, description="店铺页 hash,如 422750cb7921557bc8dba2416915d968")
|
||||
shop_url: HttpUrl | None = Field(default=None, description="店铺页 URL;与 shop_id 二选一")
|
||||
# 评价明细在单独的 /review 页上,需要多打一次请求,默认不取
|
||||
include_reviews: bool = False
|
||||
include_reviews: bool = Field(
|
||||
default=False, description="是否返回评价明细;评价在单独的 /review 页,会多打一次请求"
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> RakumaShopDetailRequest:
|
||||
@@ -512,9 +528,9 @@ class RakumaShopItemsRequest(BaseModel):
|
||||
排序与筛选参数,因此这里只有页码。
|
||||
"""
|
||||
|
||||
shop_id: str | None = None
|
||||
shop_url: HttpUrl | None = None
|
||||
page: int = Field(default=1, ge=1)
|
||||
shop_id: str | None = Field(default=None, description="店铺页 hash")
|
||||
shop_url: HttpUrl | None = Field(default=None, description="店铺页 URL;与 shop_id 二选一")
|
||||
page: int = Field(default=1, ge=1, description="页码")
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> RakumaShopItemsRequest:
|
||||
@@ -531,10 +547,14 @@ class RakumaCategoryRequest(BaseModel):
|
||||
组织方式,不会多打请求。
|
||||
"""
|
||||
|
||||
category_id: str | None = None
|
||||
# 返回该分类下的完整子树(不止直接子级)。分类树共三层,
|
||||
# 顶层分类的子树可达上百条。
|
||||
include_descendants: bool = False
|
||||
category_id: str | None = Field(
|
||||
default=None,
|
||||
description="分类 ID。不传返回 14 个顶层分类;传入返回该分类的名称、祖先路径与直接子分类",
|
||||
)
|
||||
include_descendants: bool = Field(
|
||||
default=False,
|
||||
description="返回该分类下的完整子树(不止直接子级);分类树共三层,顶层分类的子树可达上百条",
|
||||
)
|
||||
|
||||
|
||||
class RakumaCategoryNode(BaseModel):
|
||||
@@ -544,73 +564,81 @@ class RakumaCategoryNode(BaseModel):
|
||||
商品数要逐个分类去抓 `/category/{id}` 页面,成本过高,本接口不提供。
|
||||
"""
|
||||
|
||||
category_id: str = ""
|
||||
name: str = ""
|
||||
parent_id: str = "" # "0" 表示顶层分类
|
||||
is_leaf: bool = False # 叶子分类,没有下级
|
||||
url: str = "" # 分类页地址
|
||||
# include_descendants=true 时填充下级分类,否则始终为空
|
||||
children: list[RakumaCategoryNode] = Field(default_factory=list)
|
||||
category_id: str = Field(default="", description="分类 ID")
|
||||
name: str = Field(default="", description="分类名")
|
||||
parent_id: str = Field(default="", description="父分类 ID,「0」表示顶层分类")
|
||||
is_leaf: bool = Field(default=False, description="叶子分类,没有下级")
|
||||
url: str = Field(default="", description="分类页地址")
|
||||
children: list[RakumaCategoryNode] = Field(
|
||||
default_factory=list, description="下级分类;include_descendants=true 时填充,否则始终为空"
|
||||
)
|
||||
|
||||
|
||||
class RakumaCategoryData(BaseModel):
|
||||
"""ラクマ 分类查询结果"""
|
||||
|
||||
category_id: str = "" # 空串表示顶层
|
||||
name: str = ""
|
||||
full_name: str = "" # 从顶层拼到自身的路径名,如「エンタメ/ホビー / ゲームソフト/ゲーム機本体 / 家庭用ゲームソフト」
|
||||
is_leaf: bool = False
|
||||
url: str = ""
|
||||
total_count: int = 0 # 站点分类树的节点总数,用于确认取到的是全量树
|
||||
ancestors: list[RakumaCategoryNode] = Field(default_factory=list) # 从顶层到父级,不含自身
|
||||
children: list[RakumaCategoryNode] = Field(default_factory=list) # 直接子分类;include_descendants=true 时带子树
|
||||
category_id: str = Field(default="", description="分类 ID,空串表示顶层")
|
||||
name: str = Field(default="", description="分类名")
|
||||
full_name: str = Field(
|
||||
default="",
|
||||
description="从顶层拼到自身的路径名,如「エンタメ/ホビー / ゲームソフト/ゲーム機本体 / 家庭用ゲームソフト」",
|
||||
)
|
||||
is_leaf: bool = Field(default=False, description="叶子分类,没有下级")
|
||||
url: str = Field(default="", description="分类页地址")
|
||||
total_count: int = Field(default=0, description="站点分类树的节点总数,用于确认取到的是全量树")
|
||||
ancestors: list[RakumaCategoryNode] = Field(
|
||||
default_factory=list, description="祖先分类路径,从顶层到父级,不含自身"
|
||||
)
|
||||
children: list[RakumaCategoryNode] = Field(
|
||||
default_factory=list, description="直接子分类;include_descendants=true 时带子树"
|
||||
)
|
||||
|
||||
|
||||
class RakumaSeller(BaseModel):
|
||||
"""ラクマ 卖家(出品者)摘要"""
|
||||
|
||||
shop_id: str = "" # 店铺页 hash,可直接用于 /api/rakuma/shop_detail
|
||||
user_id: str = "" # 站点内部数值用户 ID
|
||||
shop_name: str = "" # 店铺名,卖家可自定义
|
||||
user_name: str = "" # 用户昵称
|
||||
shop_url: str = ""
|
||||
icon_url: str = ""
|
||||
seller_type: str = "" # 站点原值,如 一般 / 事業者
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
is_verified: bool = False # 本人確認済
|
||||
shop_id: str = Field(default="", description="店铺页 hash,可直接用于 /api/rakuma/shop_detail")
|
||||
user_id: str = Field(default="", description="站点内部数值用户 ID")
|
||||
shop_name: str = Field(default="", description="店铺名,卖家可自定义")
|
||||
user_name: str = Field(default="", description="用户昵称")
|
||||
shop_url: str = Field(default="", description="店铺页地址")
|
||||
icon_url: str = Field(default="", description="头像地址")
|
||||
seller_type: str = Field(default="", description="卖家类型(站点原值),如 一般 / 事業者")
|
||||
review_score: float = Field(default=0.0, description="综合评分")
|
||||
review_count: int = Field(default=0, description="评价数")
|
||||
is_verified: bool = Field(default=False, description="本人確認済")
|
||||
|
||||
|
||||
class RakumaSearchItem(BaseModel):
|
||||
"""ラクマ 搜索结果中的单个商品"""
|
||||
|
||||
item_id: str = "" # 商品页 hash,调详情接口用
|
||||
item_number: str = "" # 站点内部数值商品 ID
|
||||
item_name: str = ""
|
||||
item_url: str = ""
|
||||
price: int = 0
|
||||
image_url: str = ""
|
||||
is_sold_out: bool = False
|
||||
brand_id: str = ""
|
||||
brand_name: str = ""
|
||||
category_id: str = ""
|
||||
category_names: list[str] = Field(default_factory=list)
|
||||
seller_user_id: str = "" # 卖家数值 ID;店铺 hash 需从详情页取
|
||||
seller_type: str = ""
|
||||
item_id: str = Field(default="", description="商品页 hash,调详情接口用")
|
||||
item_number: str = Field(default="", description="站点内部数值商品 ID")
|
||||
item_name: str = Field(default="", description="商品名")
|
||||
item_url: str = Field(default="", description="商品页地址")
|
||||
price: int = Field(default=0, description="价格(日元)")
|
||||
image_url: str = Field(default="", description="主图地址")
|
||||
is_sold_out: bool = Field(default=False, description="已售出")
|
||||
brand_id: str = Field(default="", description="品牌 ID")
|
||||
brand_name: str = Field(default="", description="品牌名")
|
||||
category_id: str = Field(default="", description="分类 ID")
|
||||
category_names: list[str] = Field(default_factory=list, description="分类名列表")
|
||||
seller_user_id: str = Field(default="", description="卖家数值 ID;店铺 hash 需从详情页取")
|
||||
seller_type: str = Field(default="", description="卖家类型(站点原值),如 一般 / 事業者")
|
||||
|
||||
|
||||
class RakumaSearchResultData(BaseModel):
|
||||
"""ラクマ 搜索结果数据"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = 1
|
||||
page_size: int = 0
|
||||
keyword: str = Field(default="", description="搜索关键词")
|
||||
page: int = Field(default=1, description="当前页码")
|
||||
page_size: int = Field(default=0, description="每页条数")
|
||||
# 站点声明的命中总数。页面上展示为「約1,190,000件」的四舍五入值,
|
||||
# 这里取的是埋点属性里的精确值。
|
||||
total_count: int = 0
|
||||
has_more: bool = False
|
||||
request_url: str = ""
|
||||
items: list[RakumaSearchItem] = Field(default_factory=list)
|
||||
total_count: int = Field(default=0, description="站点声明的命中总数(埋点属性里的精确值)")
|
||||
has_more: bool = Field(default=False, description="是否还有下一页")
|
||||
request_url: str = Field(default="", description="实际抓取的页面地址,便于排查")
|
||||
items: list[RakumaSearchItem] = Field(default_factory=list, description="本页商品列表")
|
||||
|
||||
|
||||
class RakumaItemDetailData(BaseModel):
|
||||
@@ -620,78 +648,82 @@ class RakumaItemDetailData(BaseModel):
|
||||
「規格」在站点上只体现为一个可选的尺码字段。
|
||||
"""
|
||||
|
||||
item_id: str = ""
|
||||
item_number: str = ""
|
||||
item_name: str = ""
|
||||
description: str = ""
|
||||
item_url: str = ""
|
||||
price: int = 0
|
||||
is_sold_out: bool = False
|
||||
images: list[str] = Field(default_factory=list)
|
||||
condition: str = "" # 商品の状態,站点原文如「目立った傷や汚れなし」
|
||||
size: str = "" # サイズ,无尺码时为空
|
||||
brand_id: str = ""
|
||||
brand_name: str = ""
|
||||
category_id: str = "" # 最具体的一级分类 ID
|
||||
breadcrumbs: list[Breadcrumb] = Field(default_factory=list)
|
||||
shipping_payer: str = "" # 配送料の負担,如「送料込」
|
||||
shipping_method: str = "" # 配送方法
|
||||
shipping_date_estimate: str = "" # 発送日の目安
|
||||
shipping_from: str = "" # 発送元の地域
|
||||
is_anonymous_shipping: bool = False # 匿名配送
|
||||
like_count: int = 0 # いいね数
|
||||
comment_count: int = 0
|
||||
posted_at: str = "" # 站点展示的相对时间,如「約1時間前」
|
||||
seller: RakumaSeller = Field(default_factory=RakumaSeller)
|
||||
item_id: str = Field(default="", description="商品页 hash")
|
||||
item_number: str = Field(default="", description="站点内部数值商品 ID")
|
||||
item_name: str = Field(default="", description="商品名")
|
||||
description: str = Field(default="", description="商品说明(卖家自填)")
|
||||
item_url: str = Field(default="", description="商品页地址")
|
||||
price: int = Field(default=0, description="价格(日元)")
|
||||
is_sold_out: bool = Field(default=False, description="已售出")
|
||||
images: list[str] = Field(default_factory=list, description="商品图片地址列表")
|
||||
condition: str = Field(default="", description="商品の状態,站点原文如「目立った傷や汚れなし」")
|
||||
size: str = Field(default="", description="サイズ,无尺码时为空")
|
||||
brand_id: str = Field(default="", description="品牌 ID")
|
||||
brand_name: str = Field(default="", description="品牌名")
|
||||
category_id: str = Field(default="", description="最具体的一级分类 ID")
|
||||
breadcrumbs: list[Breadcrumb] = Field(default_factory=list, description="分类面包屑")
|
||||
shipping_payer: str = Field(default="", description="配送料の負担,如「送料込」")
|
||||
shipping_method: str = Field(default="", description="配送方法")
|
||||
shipping_date_estimate: str = Field(default="", description="発送日の目安")
|
||||
shipping_from: str = Field(default="", description="発送元の地域")
|
||||
is_anonymous_shipping: bool = Field(default=False, description="匿名配送")
|
||||
like_count: int = Field(default=0, description="いいね数")
|
||||
comment_count: int = Field(default=0, description="评论数")
|
||||
posted_at: str = Field(default="", description="站点展示的相对时间,如「約1時間前」")
|
||||
seller: RakumaSeller = Field(default_factory=RakumaSeller, description="卖家(出品者)信息")
|
||||
|
||||
|
||||
class RakumaReview(BaseModel):
|
||||
"""ラクマ 卖家的一条交易评价"""
|
||||
|
||||
rating: str = "" # good / normal / bad
|
||||
title: str = "" # 站点原文,如「よい出品者です」
|
||||
comment: str = ""
|
||||
reviewer_name: str = ""
|
||||
reviewed_at: str = "" # 站点展示的日期,如 2026/05/04
|
||||
rating: str = Field(default="", description="评价档位:good / normal / bad")
|
||||
title: str = Field(default="", description="站点原文标题,如「よい出品者です」")
|
||||
comment: str = Field(default="", description="评价内容")
|
||||
reviewer_name: str = Field(default="", description="评价者昵称")
|
||||
reviewed_at: str = Field(default="", description="站点展示的日期,如 2026/05/04")
|
||||
|
||||
|
||||
class RakumaRatingBreakdown(BaseModel):
|
||||
"""评价数量分档"""
|
||||
|
||||
good: int = 0
|
||||
normal: int = 0
|
||||
bad: int = 0
|
||||
good: int = Field(default=0, description="好评数")
|
||||
normal: int = Field(default=0, description="中评数")
|
||||
bad: int = Field(default=0, description="差评数")
|
||||
|
||||
|
||||
class RakumaShopDetailData(BaseModel):
|
||||
"""ラクマ 卖家详情数据"""
|
||||
|
||||
shop_id: str = ""
|
||||
user_id: str = ""
|
||||
shop_name: str = ""
|
||||
user_name: str = ""
|
||||
shop_url: str = ""
|
||||
icon_url: str = ""
|
||||
cover_url: str = ""
|
||||
introduction: str = "" # プロフィール文
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
is_verified: bool = False # 本人確認済
|
||||
verification_label: str = "" # 站点原文,如「本人確認済」/「本人確認未完了」
|
||||
item_count: int = 0 # 该卖家在售 + 已售商品总数
|
||||
shop_id: str = Field(default="", description="店铺页 hash")
|
||||
user_id: str = Field(default="", description="站点内部数值用户 ID")
|
||||
shop_name: str = Field(default="", description="店铺名,卖家可自定义")
|
||||
user_name: str = Field(default="", description="用户昵称")
|
||||
shop_url: str = Field(default="", description="店铺页地址")
|
||||
icon_url: str = Field(default="", description="头像地址")
|
||||
cover_url: str = Field(default="", description="封面图地址")
|
||||
introduction: str = Field(default="", description="プロフィール文(自我介绍)")
|
||||
review_score: float = Field(default=0.0, description="综合评分")
|
||||
review_count: int = Field(default=0, description="评价数")
|
||||
is_verified: bool = Field(default=False, description="本人確認済")
|
||||
verification_label: str = Field(default="", description="站点原文,如「本人確認済」/「本人確認未完了」")
|
||||
item_count: int = Field(default=0, description="该卖家在售 + 已售商品总数")
|
||||
# 以下三项需 include_reviews=true 才会填充
|
||||
rating_breakdown: RakumaRatingBreakdown = Field(default_factory=RakumaRatingBreakdown)
|
||||
seller_rating_breakdown: RakumaRatingBreakdown = Field(default_factory=RakumaRatingBreakdown)
|
||||
reviews: list[RakumaReview] = Field(default_factory=list)
|
||||
rating_breakdown: RakumaRatingBreakdown = Field(
|
||||
default_factory=RakumaRatingBreakdown, description="评价数量分档(全部评价)"
|
||||
)
|
||||
seller_rating_breakdown: RakumaRatingBreakdown = Field(
|
||||
default_factory=RakumaRatingBreakdown, description="评价数量分档(仅作为卖家收到的评价)"
|
||||
)
|
||||
reviews: list[RakumaReview] = Field(default_factory=list, description="评价明细列表")
|
||||
|
||||
|
||||
class RakumaShopItemsData(BaseModel):
|
||||
"""ラクマ 卖家商品列表数据"""
|
||||
|
||||
shop_id: str = ""
|
||||
shop_name: str = ""
|
||||
page: int = 1
|
||||
total_count: int = 0 # 该卖家的商品总数
|
||||
has_more: bool = False
|
||||
request_url: str = ""
|
||||
items: list[RakumaSearchItem] = Field(default_factory=list)
|
||||
shop_id: str = Field(default="", description="店铺页 hash")
|
||||
shop_name: str = Field(default="", description="店铺名")
|
||||
page: int = Field(default=1, description="当前页码")
|
||||
total_count: int = Field(default=0, description="该卖家的商品总数")
|
||||
has_more: bool = Field(default=False, description="是否还有下一页")
|
||||
request_url: str = Field(default="", description="实际抓取的页面地址,便于排查")
|
||||
items: list[RakumaSearchItem] = Field(default_factory=list, description="本页商品列表")
|
||||
|
||||
@@ -16,6 +16,7 @@ from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.proxy import playwright_launch_proxy
|
||||
from app.scraping.core import site
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -100,7 +101,7 @@ class BrowserFallback:
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self._settings.browser_headless_effective,
|
||||
channel=self._settings.browser_channel or None,
|
||||
proxy=self._settings.playwright_proxy,
|
||||
proxy=playwright_launch_proxy(self._settings),
|
||||
timeout=self._settings.browser_launch_timeout_seconds * 1000,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
|
||||
@@ -22,6 +22,7 @@ from opentelemetry import trace
|
||||
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.config import Settings
|
||||
from app.shared.proxy import httpx_client_options
|
||||
from app.shared.errors import (
|
||||
ItemNotFoundError,
|
||||
ResourceBusyError,
|
||||
@@ -51,7 +52,7 @@ class RakumaSession:
|
||||
headers=site.default_headers(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
proxy=self._settings.httpx_proxy,
|
||||
**httpx_client_options(self._settings),
|
||||
http2=True,
|
||||
)
|
||||
logger.info(
|
||||
|
||||
@@ -23,6 +23,7 @@ from opentelemetry import trace
|
||||
|
||||
from app.scraping.core import site
|
||||
from app.shared.config import Settings
|
||||
from app.shared.proxy import httpx_client_options
|
||||
from app.shared.errors import (
|
||||
ItemNotFoundError,
|
||||
ResourceBusyError,
|
||||
@@ -91,7 +92,7 @@ class SiteSession:
|
||||
headers=site.default_headers(mobile=mobile),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
proxy=self._settings.httpx_proxy,
|
||||
**httpx_client_options(self._settings),
|
||||
http2=True,
|
||||
),
|
||||
)
|
||||
@@ -265,7 +266,7 @@ class SiteSession:
|
||||
return reason.startswith("upstream status") or reason.startswith("httpx") or "Error:" in reason
|
||||
|
||||
async def _ensure_warm(self, profile: _Profile) -> None:
|
||||
"""确保通道持有新鲜的 Akamai cookie;过期或缺失时访问首页预热"""
|
||||
"""确保通道有一次新鲜的首页预热;过期时重新访问首页"""
|
||||
if self._is_warm(profile):
|
||||
return
|
||||
|
||||
@@ -274,7 +275,10 @@ class SiteSession:
|
||||
return
|
||||
try:
|
||||
response = await profile.client.get(self._settings.home_url)
|
||||
profile.warmed_at = time.monotonic()
|
||||
# Akamai 不保证每次都下发 cookie;首页探测成功本身就是可复用的
|
||||
# 预热结果,cookie 只用于观测和失败升级时的回灌。
|
||||
if response.status_code < 400:
|
||||
profile.warmed_at = time.monotonic()
|
||||
logger.info(
|
||||
"会话预热完成:profile=%s status=%s cookies=%s",
|
||||
profile.name,
|
||||
@@ -284,14 +288,14 @@ class SiteSession:
|
||||
except httpx.HTTPError as exc:
|
||||
# 预热失败不阻断本次抓取:直连目标页仍可能成功,只是慢
|
||||
logger.warning("会话预热失败:profile=%s err=%s", profile.name, exc)
|
||||
profile.warmed_at = time.monotonic()
|
||||
profile.warmed_at = 0.0
|
||||
|
||||
def _is_warm(self, profile: _Profile) -> bool:
|
||||
if not profile.warmed_at:
|
||||
return False
|
||||
if time.monotonic() - profile.warmed_at > self._settings.session_ttl_seconds:
|
||||
return False
|
||||
return bool(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES))
|
||||
return True
|
||||
|
||||
async def _invalidate(self, profile: _Profile) -> None:
|
||||
"""清空通道 cookie 并强制下次重新预热"""
|
||||
|
||||
+8
-5
@@ -16,7 +16,7 @@ from typing import Any, Generic, TypeVar
|
||||
from fastapi import Depends, FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel, ValidationError
|
||||
from pydantic import BaseModel, Field, ValidationError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from app.shared.errors import AppError, AuthenticationError
|
||||
@@ -29,10 +29,13 @@ T = TypeVar("T")
|
||||
class ApiResponse(BaseModel, Generic[T]):
|
||||
"""统一 API 响应格式"""
|
||||
|
||||
success: bool
|
||||
msg: str
|
||||
data: T | None = None
|
||||
code: int
|
||||
success: bool = Field(description="请求是否成功")
|
||||
msg: str = Field(description="提示信息;失败时为错误原因")
|
||||
data: T | None = Field(default=None, description="响应数据;无数据或失败时为 null")
|
||||
code: int = Field(
|
||||
description="错误码:0 表示成功,其余见统一错误码表(如 1001 鉴权失败、"
|
||||
"1002 请求参数校验失败、5xxx 交易服务、6xxx 网关)"
|
||||
)
|
||||
|
||||
|
||||
# ---- 依赖注入 ----
|
||||
|
||||
+81
-1
@@ -8,9 +8,11 @@
|
||||
「通用 / 仅抓取 / 仅交易 / 仅网关 / 仅交易 worker」分区标注,各进程只读自己那部分。
|
||||
"""
|
||||
import socket
|
||||
from fnmatch import fnmatchcase
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
from pydantic import Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
@@ -88,6 +90,8 @@ class Settings(BaseSettings):
|
||||
proxy_server: str | None = None
|
||||
proxy_username: str | None = None
|
||||
proxy_password: str | None = None
|
||||
# 逗号分隔的直连主机名或通配符。服务间请求不应绕到公网代理。
|
||||
proxy_bypass: str = "localhost,127.0.0.1,::1,rakuten-api,rakuten-trading,rakuten-gateway"
|
||||
|
||||
# ---- OpenTelemetry traces(通用,可选;默认关闭)----
|
||||
# 启用后把抓取-解析链路以 span 导出到 OTLP/HTTP endpoint,用于排查"抓到
|
||||
@@ -108,6 +112,16 @@ class Settings(BaseSettings):
|
||||
# 下单金额上限(日元)。实际应付金额超过该值时拒绝提交,防止解析出错或
|
||||
# 页面改版导致买到远超预期的订单。设为 0 表示不设上限(不建议)。
|
||||
order_max_total_yen: int = 30000
|
||||
# 自动下单时在支付方式页选择的选项文案(按可见文案匹配,非 value/id)。
|
||||
# 2026-08-11 未经真实确认页验证:见 site_interact.py::_select_payment_method。
|
||||
order_payment_method: str = "クレジットカード"
|
||||
# 付款后监控(订单列表页轮询):间隔与最大轮询次数。2026-08-13 用真实订单号
|
||||
# 实测过 order.my.rakuten.co.jp 的结构,见 site_interact.py::check_order_status。
|
||||
# 间隔不宜太短——同一账号频繁访问订单页有被风控盯上的风险;默认 3 小时一次,
|
||||
# 最多轮询 80 次(约 10 天,覆盖绝大多数国内配送时长),到期仍未到「配達完了」
|
||||
# 就停止(不是失败,只是不再继续追踪,详见 runner.py::_monitor_order)。
|
||||
order_monitor_poll_interval_seconds: int = 3 * 3600
|
||||
order_monitor_max_checks: int = 80
|
||||
|
||||
# ---- 自动重登(仅交易服务,需要 account.yaml)----
|
||||
# 检测到登录态失效时,是否在 require_logged_in 内自动触发重登。需要项目根
|
||||
@@ -118,6 +132,11 @@ class Settings(BaseSettings):
|
||||
# worker 内触发时整个任务会被卡住这段时间,所以不宜过长;本地机无人值守时
|
||||
# 可以设小(如 60)尽快失败转 needs_human。
|
||||
relogin_timeout_seconds: int = 300
|
||||
# 启动时是否自动登录一次。容器部署用:镜像里没有落盘的 storage_state,
|
||||
# 开启后启动即按 account.yaml 自己登一次,不必先在宿主机跑 scripts/login.py。
|
||||
# 后台执行(不阻塞 HTTP 端口),失败只记日志——服务照常提供 /health 与
|
||||
# /api/auth/*,撞验证码可人工接管后调 /api/auth/login 重试。
|
||||
auto_login_on_start: bool = False
|
||||
|
||||
# ---- 下单任务网关(仅网关进程 app.gateway.main 使用)----
|
||||
# 网关的 SQLite 文件路径(相对项目根目录)。任务队列与状态镜像都在这里,
|
||||
@@ -131,6 +150,45 @@ class Settings(BaseSettings):
|
||||
# worker 心跳超时阈值(秒)。网关 /health 据此判断 worker 是否失联:
|
||||
# 正常 worker 每 30 秒来一次 lease,超过该阈值未来 lease 即视为异常。
|
||||
worker_offline_alert_seconds: int = 300
|
||||
# 终结类事件回调(callback_url)的单次 HTTP 超时(秒)。投递是 best-effort:
|
||||
# 超时/失败只记日志不重试,权威状态以 GET /api/orders/{task_id} 为准。
|
||||
callback_timeout_seconds: float = 10.0
|
||||
|
||||
# ---- 账号只读查询通道(网关 + worker 两侧共用,见 docs/order-gateway.md §11)----
|
||||
# 查询单租约 TTL(秒)。worker 领走后必须在此时间内回结果,否则网关把它
|
||||
# **重投回 queued**——只读操作重复执行没有副作用,与下单任务的 stale 语义相反。
|
||||
query_lease_ttl_seconds: int = 180
|
||||
# 查询单整体存活上限(秒)。从创建算起,超过仍未拿到结果即置 expired
|
||||
# (多半是本地 worker 不在线)。上游据此判断「这张单不用再等了」。
|
||||
query_ttl_seconds: int = 900
|
||||
# 同一张查询单最多被领取几次。租约过期重投累计到这个次数仍无结果即置 failed,
|
||||
# 避免某条查询让 worker 反复卡死在同一个页面上。
|
||||
query_max_attempts: int = 3
|
||||
# 终态查询单的保留时长(秒)。结果里带站点原始 JSON,体积不小,超期由 sweep
|
||||
# 清掉;默认 7 天,够上游对账与排查。
|
||||
query_retention_seconds: int = 7 * 24 * 3600
|
||||
# worker 侧单次站点读取的耗时上限(秒)。查询与下单共用同一把账号锁,正在
|
||||
# 跑的下单会让查询排队等待,这个上限保证查询不会一直挂着——超时按失败回报,
|
||||
# 上游重发即可(只读,重发无副作用)。
|
||||
account_query_timeout_seconds: int = 120
|
||||
# 订单列表查询缺省翻页上限。上游可在 params.max_pages 覆盖(1..20)。
|
||||
account_query_default_max_pages: int = 3
|
||||
# 回报给网关的结果 JSON 体积上限(字节)。超过时丢掉站点原始 JSON、只回
|
||||
# 规范化字段并标记 raw_omitted——避免把几 MB 的页面状态塞进网关 SQLite。
|
||||
query_result_max_bytes: int = 1024 * 1024
|
||||
|
||||
# ---- 定时下派通道(仅网关进程 app.gateway.main 使用,见 docs/order-gateway.md §12)----
|
||||
# 网关周期性派 order_list / order_detail 查询、把账号真实订单沉淀进编目
|
||||
# account_orders 表(回写网关)。关闭则 collector 完全不启动。
|
||||
account_discovery_enabled: bool = True
|
||||
# 两轮 order_list 扫描的间隔(秒)。间隔也是「该订单详情何时到期需刷新」的
|
||||
# 参照之一,默认 6 小时一次列表、24 小时刷新一次单笔详情,符合「周期性盘点
|
||||
# 账号」的节奏,也不至于频繁戳订单页触发风控。
|
||||
account_discovery_interval_seconds: int = 6 * 3600
|
||||
account_discovery_max_pages: int = 3
|
||||
# 已编目订单的详情刷新间隔(秒)。detail_fetched_at 距今超过这个值就再派一张
|
||||
# order_detail 查询更新配送阶段。
|
||||
account_detail_refresh_seconds: int = 24 * 3600
|
||||
|
||||
# ---- 本地下单 worker(仅交易服务内的 worker 子模块使用)----
|
||||
# 网关 URL。**留空则不启动 worker**,交易服务只跑登录态接口。
|
||||
@@ -177,6 +235,8 @@ class Settings(BaseSettings):
|
||||
proxy["username"] = self.proxy_username
|
||||
if self.proxy_password:
|
||||
proxy["password"] = self.proxy_password
|
||||
if self.proxy_bypass.strip():
|
||||
proxy["bypass"] = self.proxy_bypass
|
||||
return proxy
|
||||
|
||||
@property
|
||||
@@ -190,9 +250,29 @@ class Settings(BaseSettings):
|
||||
scheme, _, rest = self.proxy_server.partition("://")
|
||||
if not rest:
|
||||
return self.proxy_server
|
||||
credentials = f"{self.proxy_username}:{self.proxy_password or ''}"
|
||||
credentials = f"{quote(self.proxy_username, safe='')}:{quote(self.proxy_password or '', safe='')}"
|
||||
return f"{scheme}://{credentials}@{rest}"
|
||||
|
||||
def proxy_bypasses(self, url: str) -> bool:
|
||||
"""目标 URL 的主机是否应绕过外部代理。"""
|
||||
host = urlsplit(url).hostname
|
||||
if not host:
|
||||
return False
|
||||
|
||||
normalized_host = host.rstrip(".").lower()
|
||||
for raw_pattern in self.proxy_bypass.split(","):
|
||||
pattern = raw_pattern.strip().rstrip(".").lower()
|
||||
if not pattern:
|
||||
continue
|
||||
if pattern.startswith("."):
|
||||
suffix = pattern[1:]
|
||||
if normalized_host == suffix or normalized_host.endswith(f".{suffix}"):
|
||||
return True
|
||||
continue
|
||||
if fnmatchcase(normalized_host, pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
@property
|
||||
def auth_state_path(self) -> Path:
|
||||
"""登录态目录的绝对路径,不存在时创建"""
|
||||
|
||||
@@ -172,6 +172,23 @@ class OrderGuardError(AppError):
|
||||
super().__init__(message=message, code="ORDER_GUARD", err_code=5004, retryable=False)
|
||||
|
||||
|
||||
class CheckoutBlockedError(AppError):
|
||||
"""结算流程被站点风控拦截(session upgrade 二次验证 / 3DS / 短信验证等)
|
||||
|
||||
对应 docs/order-gateway.md §10.1:这类阻断不是本服务的 bug,也不是普通的请求失败,
|
||||
是站点主动要求人工验证。实测记录(data/evidence/checkout-research-20260811/NOTES.md):
|
||||
session upgrade 密码页多次自动提交均未能确认稳定通过,行为更像是站点对自动化环境
|
||||
的针对性降级,而非偶发网络问题——**不适合原地重试**,重试只会累积同账号的失败验证
|
||||
次数,增加被风控盯上/锁定的风险。
|
||||
|
||||
worker 收到这类错误应停在当前进度并上报 needs_human,交人工用有头浏览器接管当前
|
||||
登录态完成验证,而不是当成普通失败(retryable/FAILED)处理。
|
||||
"""
|
||||
|
||||
def __init__(self, message: str = "结算流程被站点风控拦截,需人工介入"):
|
||||
super().__init__(message=message, code="CHECKOUT_BLOCKED", err_code=5005, retryable=False)
|
||||
|
||||
|
||||
# ---- 下单任务编排(仅网关进程 app.gateway.main 使用)----
|
||||
# 下单不可逆,因此任务队列侧的错误一律标记 retryable=False——重复入队/重投
|
||||
# 都可能变成重复下单。详见 docs/order-gateway.md §8。
|
||||
@@ -224,3 +241,57 @@ class InvalidTaskStateError(AppError):
|
||||
retryable=False,
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
|
||||
# ---- 账号只读查询通道(仅网关进程使用,见 docs/order-gateway.md §11)----
|
||||
# 与下单任务不同:查询是只读的,**可以安全重投**,因此这两类错误标 retryable=True,
|
||||
# 上游重发一张查询单不会有任何副作用。
|
||||
|
||||
|
||||
class QueryNotFoundError(AppError):
|
||||
"""查询单不存在(或已过保留期被清理)"""
|
||||
|
||||
def __init__(self, query_id: str):
|
||||
super().__init__(
|
||||
message=f"查询单不存在:{query_id}",
|
||||
code="QUERY_NOT_FOUND",
|
||||
err_code=6005,
|
||||
retryable=False,
|
||||
status_code=404,
|
||||
)
|
||||
self.query_id = query_id
|
||||
|
||||
|
||||
class QueryLeaseInvalidError(AppError):
|
||||
"""查询单租约无效:不是持有者、已被重投给别人或已终结
|
||||
|
||||
最常见的触发场景是 worker 执行超时、查询单被 sweep 重投后,原 worker 才姗姗
|
||||
来迟地回结果——这时结果必须被拒绝,否则会覆盖掉新一轮的执行结果。
|
||||
"""
|
||||
|
||||
def __init__(self, message: str = "查询单租约无效"):
|
||||
super().__init__(
|
||||
message=message,
|
||||
code="QUERY_LEASE_INVALID",
|
||||
err_code=6006,
|
||||
retryable=True,
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
|
||||
class CatalogOrderNotFoundError(AppError):
|
||||
"""编目订单不存在:定时下派通道的 account_orders 里没有这笔订单
|
||||
|
||||
沿用 6005(与查询单同号的「不存在」语义),区别只在对象是编目里的订单而非
|
||||
查询单,便于上游按同一张错误码表分支。
|
||||
"""
|
||||
|
||||
def __init__(self, order_number: str):
|
||||
super().__init__(
|
||||
message=f"编目订单不存在:{order_number}",
|
||||
code="CATALOG_ORDER_NOT_FOUND",
|
||||
err_code=6005,
|
||||
retryable=False,
|
||||
status_code=404,
|
||||
)
|
||||
self.order_number = order_number
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
"""项目出站代理策略;OTel exporter 不使用这里的配置。"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.shared.config import Settings
|
||||
|
||||
|
||||
def httpx_client_options(
|
||||
settings: "Settings", *, target_url: str | None = None
|
||||
) -> dict[str, object]:
|
||||
"""返回 HTTPX 客户端统一的代理与环境变量策略。"""
|
||||
proxy = None if target_url and settings.proxy_bypasses(target_url) else settings.httpx_proxy
|
||||
options: dict[str, object] = {"trust_env": False}
|
||||
if proxy is not None:
|
||||
options["proxy"] = proxy
|
||||
return options
|
||||
|
||||
|
||||
def playwright_launch_proxy(settings: "Settings") -> dict[str, str] | None:
|
||||
"""返回 Playwright 启动参数使用的统一代理设置。"""
|
||||
return settings.playwright_proxy
|
||||
@@ -49,6 +49,43 @@ LEASABLE_STATUSES: frozenset[TaskStatus] = frozenset({TaskStatus.QUEUED})
|
||||
RECLAIMABLE_STATUSES: frozenset[TaskStatus] = frozenset({TaskStatus.STALE})
|
||||
|
||||
|
||||
class QueryStatus(StrEnum):
|
||||
"""账号只读查询单状态(网关权威,见 docs/order-gateway.md §11)
|
||||
|
||||
与 TaskStatus 刻意分成两套词汇表,因为安全约束正好相反:
|
||||
|
||||
- 下单是不可逆写操作 → 租约过期只能置 stale 等人工 reclaim,绝不自动重投
|
||||
- 查询是只读操作 → 租约过期直接回 queued 自动重投,重复执行没有副作用
|
||||
|
||||
没有 running:查询是「领走 → 一次性回结果」,中间没有需要单独表达的进行态,
|
||||
也不需要续租(超时就重投)。
|
||||
"""
|
||||
|
||||
QUEUED = "queued" # 已入队,等待 worker 领取
|
||||
LEASED = "leased" # 已被 worker 领走,等待回结果
|
||||
SUCCEEDED = "succeeded" # 终态:拿到结果
|
||||
FAILED = "failed" # 终态:worker 明确失败,或重投次数用尽
|
||||
EXPIRED = "expired" # 终态:超过查询单 TTL 仍未完成(多半是 worker 不在线)
|
||||
|
||||
|
||||
# 查询单终态集合:到达后 result 一律报 6006,sweep 也不再动它
|
||||
QUERY_TERMINAL_STATUSES: frozenset[QueryStatus] = frozenset(
|
||||
{QueryStatus.SUCCEEDED, QueryStatus.FAILED, QueryStatus.EXPIRED}
|
||||
)
|
||||
|
||||
|
||||
class AccountQueryKind(StrEnum):
|
||||
"""账号只读查询的种类
|
||||
|
||||
放在 shared 是因为它是网关与 worker 之间的 HTTP 契约的一部分:网关不解释
|
||||
`params` 的内容(与 intent 同样原样透传),但**校验 kind 合法**——不然上游
|
||||
拼错一个字符,要等 worker 领走、执行、回报失败才知道,比当场 422 差得多。
|
||||
"""
|
||||
|
||||
ORDER_LIST = "order_list" # 订单列表(可带时间窗与翻页上限)
|
||||
ORDER_DETAIL = "order_detail" # 单笔订单详情(按注文番号)
|
||||
|
||||
|
||||
class OrderState(StrEnum):
|
||||
"""订单状态(本地权威,网关只存镜像)
|
||||
|
||||
|
||||
@@ -1,17 +1,22 @@
|
||||
"""登录态路由:查询与重新加载账号登录态
|
||||
"""登录态路由:查询、自动登录与重新加载账号登录态
|
||||
|
||||
抓取接口全部匿名,只有加购与下单需要账号。登录本身不在这里做——乐天登录要过
|
||||
reCAPTCHA 与设备验证,由 `scripts/login.py` 起有头浏览器人工完成一次,
|
||||
本服务只读取落盘的 cookie。这里提供的是运维视角的两个动作:
|
||||
抓取接口全部匿名,只有加购与下单需要账号。这里是运维视角的三个动作:
|
||||
|
||||
- `/api/auth/status`:现在还登录着吗(默认真实打一次请求探测,不看缓存)
|
||||
- `/api/auth/login`:未登录时按 account.yaml 自动登录一次(撞验证码要人工接管)
|
||||
- `/api/auth/reload`:人工重新登录后,免重启服务重新读取登录态
|
||||
|
||||
自动登录不是「绕过校验」:账号密码是用户自己配在 account.yaml 里的,代填进站点
|
||||
自己的登录表单;撞 reCAPTCHA / 设备验证时流程会停在有头浏览器上等人工完成,
|
||||
等不到就超时失败(容器部署时用 VNC 接管,见 docker-compose.yml)。
|
||||
"""
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.shared.api import ApiResponse, get_container, require_bearer_token
|
||||
from app.trading.container import TradingContainer
|
||||
from app.trading.models import (
|
||||
AuthLoginData,
|
||||
AuthLoginRequest,
|
||||
AuthReloadData,
|
||||
AuthReloadRequest,
|
||||
AuthSiteStatus,
|
||||
@@ -43,8 +48,9 @@ async def auth_status(
|
||||
不需要这次网络往返时传 `refresh=false`,此时返回上一次探测的缓存结果
|
||||
(`logged_in` 为 null 表示从未探测过)。
|
||||
|
||||
`logged_in=false` 时,加购与下单接口会直接返回 5001,需要重新跑
|
||||
`scripts/login.py --site <site>` 后调用 `/api/auth/reload`。
|
||||
`logged_in=false` 时,加购与下单接口会直接返回 5001。恢复办法:调
|
||||
`/api/auth/login` 让服务自己登一次,或人工跑 `scripts/login.py --site <site>`
|
||||
后调 `/api/auth/reload`。
|
||||
"""
|
||||
sites = [payload.site.value] if payload.site else list(container.auth_session.sites)
|
||||
statuses = []
|
||||
@@ -64,6 +70,57 @@ async def auth_status(
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/login",
|
||||
response_model=ApiResponse[AuthLoginData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def auth_login(
|
||||
payload: AuthLoginRequest,
|
||||
container: TradingContainer = Depends(get_container),
|
||||
) -> ApiResponse[AuthLoginData]:
|
||||
"""按 account.yaml 自动登录(已登录则跳过)
|
||||
|
||||
动作顺序与加购/下单前的 `require_logged_in` 完全一致,只是把它单独暴露出来,
|
||||
便于部署后主动把登录态准备好、以及登录态掉了之后手动救一次:
|
||||
|
||||
1. 真实探测一次登录态;已登录直接返回(不会白起一次浏览器)
|
||||
2. 未登录 → 读 account.yaml 默认账号跑登录流程(同站点串行)
|
||||
3. 再探测一次,返回最终结论
|
||||
|
||||
这里**不抛 5001**:登录失败会以 `logged_in=false` + 各站 detail 正常返回,
|
||||
调用方据此决定是人工接管还是换账号。`relogin_enabled=false` 或 account.yaml
|
||||
缺失时同样返回 false(服务端日志里有具体原因)。
|
||||
|
||||
单次调用最长耗时由 `RAKUTEN_RELOGIN_TIMEOUT_SECONDS`(默认 300s)决定——
|
||||
撞验证码时流程要在有头浏览器上等人工,别拿短超时的客户端调它。
|
||||
"""
|
||||
sites = [payload.site.value] if payload.site else list(container.auth_session.sites)
|
||||
|
||||
attempted: dict[str, bool] = {}
|
||||
statuses: list[AuthSiteStatus] = []
|
||||
for site in sites:
|
||||
status = await container.auth_session.check(site)
|
||||
if status.logged_in:
|
||||
attempted[site] = False
|
||||
else:
|
||||
attempted[site] = True
|
||||
if await container.auth_session.try_relogin(site):
|
||||
status = await container.auth_session.check(site)
|
||||
statuses.append(_to_model(status))
|
||||
|
||||
return ApiResponse[AuthLoginData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=AuthLoginData(
|
||||
logged_in=all(status.logged_in for status in statuses),
|
||||
relogin_attempted=attempted,
|
||||
sites=statuses,
|
||||
),
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/reload",
|
||||
response_model=ApiResponse[AuthReloadData],
|
||||
|
||||
@@ -88,6 +88,7 @@ async def cart_status(
|
||||
"""查询购物车状态(轻量)
|
||||
|
||||
只调 cart count JSONP API,不渲染整页。返回登录态、商品件数与站点状态码。
|
||||
空车是正常结果:count=0 且 raw_status="101";获取失败走错误码 5002。
|
||||
"""
|
||||
site = _require_site(container)
|
||||
result = await site.cart_status() # type: ignore[union-attr]
|
||||
@@ -118,7 +119,11 @@ async def cart_clear(
|
||||
return ApiResponse[CartClearData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=CartClearData(**result),
|
||||
# result 还带清理后的页面 html(供 worker 落证据排查用),不进 HTTP 响应
|
||||
data=CartClearData(
|
||||
removed_count=result["removed_count"],
|
||||
cart_count=result["cart_count"],
|
||||
),
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@@ -41,3 +41,6 @@ class TradingContainer:
|
||||
worker_local_db: object | None = None # app.trading.worker.local_db.LocalDB
|
||||
worker_evidence: object | None = None # app.trading.worker.evidence.EvidenceStore
|
||||
worker_runner: object | None = None # app.trading.worker.runner.WorkerRunner
|
||||
# 账号只读查询的第二条循环(docs/order-gateway.md §11)。与下单 worker 同生共死,
|
||||
# 但跑在各自的 asyncio 任务里:下单一跑几分钟,查询不该排在它后面。
|
||||
query_runner: object | None = None # app.trading.worker.query_runner.QueryRunner
|
||||
|
||||
@@ -37,6 +37,22 @@ RAKUTEN_LOGIN_URL: Final = "https://www.rakuten.co.jp/myrakuten/"
|
||||
# 购物车页上表示「当前会话未登录」的文案。出现即判定登录态失效。
|
||||
RAKUTEN_LOGGED_OUT_MARKER: Final = "現在ログインしていません"
|
||||
|
||||
# 未登录时访问「需要登录才能看」的页面(订单列表/详情等)会被整页跳到 SSO 域。
|
||||
# 判据来源:login_runner 模块文档记录的实测——SSO 落地域名确实是这两个之一。
|
||||
# 但「订单页在登录态失效时一定跳到这里」这一步**没有**真实探测证据(要复现得
|
||||
# 先让一份真实登录态过期),是按站点通行行为的推断,用途也限定在
|
||||
# `looks_logged_out` 那种「判错只多花一次重登」的场景,不作为正向结论使用。
|
||||
RAKUTEN_SSO_URL_MARKERS: Final = (
|
||||
"login.account.rakuten.com",
|
||||
"grp01.id.rakuten.co.jp",
|
||||
)
|
||||
|
||||
# SSO 域下的「已登录但要求复核密码」路径。这**不是**登录态失效:结算流程里即使
|
||||
# 会话有效也会被站点风控要求重输密码(见 site_interact.py 模块文档与
|
||||
# _SESSION_UPGRADE_URL_MARKERS)。落在这些路径上时不能判未登录,否则会把风控
|
||||
# 拦截误当成 cookie 过期去重登。
|
||||
RAKUTEN_SSO_UPGRADE_PATH_MARKERS: Final = ("session/upgrade",)
|
||||
|
||||
# ---- 登录态请求指纹 ----
|
||||
# 这个 UA 必须与 scripts/login.py 起浏览器时用的一致:cookie 是在那个 UA 下拿到的,
|
||||
# 服务端再拿它发请求时换了 UA,可能触发站点的设备校验让登录态提前失效。
|
||||
@@ -101,3 +117,35 @@ def is_logged_in(site: str, *, final_url: str, body: str) -> bool:
|
||||
if site != "rakuten":
|
||||
raise ValueError(f"未知站点:{site}")
|
||||
return RAKUTEN_LOGGED_OUT_MARKER not in body
|
||||
|
||||
|
||||
def looks_logged_out(site: str, *, final_url: str, body: str) -> bool:
|
||||
"""在**非探针页**上判断「这次拿到的页面像是因为掉登录才长这样」
|
||||
|
||||
与 `is_logged_in` 的区别,别混用:
|
||||
|
||||
- `is_logged_in` 是探针页(购物车页)上的**正向**判据,用来回答「这套 cookie
|
||||
还有效吗」,是登录态的权威结论。
|
||||
- 本函数是订单列表页 / 订单详情页这类业务页上的**单边启发式**:返回 True 只
|
||||
表示「值得回探针页复核一次登录态」,返回 False **不代表登录着**(业务页
|
||||
正常渲染时本来就没有任何未登录信号)。所以它只该用在「判错了最多多花一次
|
||||
探测/重登」的重试决策上,不能拿来当登录态结论对外报告。
|
||||
|
||||
判据(前者是推断,后者是实测):
|
||||
1. 落地 URL 进了 SSO 域(`RAKUTEN_SSO_URL_MARKERS`),且不是「已登录但要求
|
||||
复核密码」的 session upgrade 路径——后者是站点风控,不是掉登录,
|
||||
误判会把风控拦截当 cookie 过期去重登。
|
||||
注意 SSO 域下的 `sign_in/password`(真正的密码输入页)**不**在豁免里:
|
||||
它既可能是全新登录的密码步、也可能是 upgrade 的后续步,含义有歧义,
|
||||
按「像掉登录」处理——代价只是多跑一次 `login_one`,而它自己会先探测
|
||||
登录态、已登录就直接跳过。
|
||||
2. 正文出现旧版未登录 marker(SSR 落地 HTML 上实测有效,新 SPA 渲染后不再
|
||||
输出,所以这条同样是单边的)。
|
||||
"""
|
||||
if site != "rakuten":
|
||||
raise ValueError(f"未知站点:{site}")
|
||||
lowered = (final_url or "").lower()
|
||||
if any(marker in lowered for marker in RAKUTEN_SSO_URL_MARKERS):
|
||||
if not any(path in lowered for path in RAKUTEN_SSO_UPGRADE_PATH_MARKERS):
|
||||
return True
|
||||
return RAKUTEN_LOGGED_OUT_MARKER in body
|
||||
|
||||
+72
-1
@@ -13,7 +13,8 @@
|
||||
SiteInteractor 始终启动:HTTP /api/auth/* 与 /api/cart/* 都要靠它持账号 cookie
|
||||
做站点交互(加购、校验、清空、删除)。worker 启动条件:RAKUTEN_ORDER_GATEWAY_URL
|
||||
非空。留空时不构造 worker 子组件(本地 DB / 证据目录 / 出站客户端都不创建),
|
||||
但 SiteInteractor 与 Playwright 仍会启动。
|
||||
但 SiteInteractor 与 Playwright 仍会启动。配了网关 URL 时会起**两条**常驻循环:
|
||||
下单 worker 与账号只读查询 worker(后者见 docs/order-gateway.md §11)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -56,11 +57,13 @@ def build_container() -> TradingContainer:
|
||||
from app.trading.worker.client import GatewayClient
|
||||
from app.trading.worker.evidence import EvidenceStore
|
||||
from app.trading.worker.local_db import LocalDB
|
||||
from app.trading.worker.query_runner import QueryRunner
|
||||
from app.trading.worker.runner import WorkerRunner
|
||||
|
||||
client = GatewayClient(
|
||||
settings.order_gateway_url,
|
||||
settings.bearer_token,
|
||||
settings=settings,
|
||||
timeout=max(60.0, settings.lease_max_wait_seconds + 10),
|
||||
)
|
||||
local_db = LocalDB(settings.trading_db_path_resolved)
|
||||
@@ -76,10 +79,43 @@ def build_container() -> TradingContainer:
|
||||
container.worker_local_db = local_db
|
||||
container.worker_evidence = evidence
|
||||
container.worker_runner = runner
|
||||
# 只读查询循环与下单 worker 共用同一个网关客户端与 SiteInteractor:
|
||||
# 前者是同一份连接池,后者那把锁正是账号级串行的保证。
|
||||
container.query_runner = QueryRunner(
|
||||
settings=settings, gateway_client=client, site=container.site
|
||||
)
|
||||
|
||||
return container
|
||||
|
||||
|
||||
async def auto_login_on_start(container: TradingContainer) -> None:
|
||||
"""启动后自动把登录态准备好(RAKUTEN_AUTO_LOGIN_ON_START=true 时)
|
||||
|
||||
为容器部署而存在:镜像里没有落盘的 storage_state,不希望每次新起容器都得先
|
||||
在宿主机跑一遍 scripts/login.py。逐站点探测,未登录就按 account.yaml 登一次。
|
||||
|
||||
刻意做成后台任务而不是启动阻塞:登录最长要等 relogin_timeout_seconds(默认
|
||||
300s,撞验证码时在等人工),阻塞会让 /health 在这段时间里连端口都不通。
|
||||
任何失败都只记日志——服务照常提供 /health 与 /api/auth/*,人工接管后调
|
||||
/api/auth/login 重试即可。
|
||||
"""
|
||||
for site in container.auth_session.sites:
|
||||
try:
|
||||
status = await container.auth_session.check(site)
|
||||
if status.logged_in:
|
||||
logger.info("启动自动登录跳过:site=%s 已是登录态", site)
|
||||
continue
|
||||
logger.info("启动自动登录:site=%s 当前未登录(%s)", site, status.detail)
|
||||
if await container.auth_session.try_relogin(site):
|
||||
status = await container.auth_session.check(site)
|
||||
logger.info(
|
||||
"启动自动登录结束:site=%s logged_in=%s detail=%s",
|
||||
site, status.logged_in, status.detail,
|
||||
)
|
||||
except Exception:
|
||||
logger.exception("启动自动登录异常:site=%s(服务继续运行)", site)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用生命周期管理:登录态会话 + 站点交互器(必起)+ 可选 worker"""
|
||||
@@ -101,7 +137,16 @@ async def lifespan(app: FastAPI):
|
||||
assert container.site is not None
|
||||
await container.site.start() # type: ignore[union-attr]
|
||||
|
||||
login_task: asyncio.Task | None = None
|
||||
if container.settings.auto_login_on_start:
|
||||
login_task = asyncio.create_task(
|
||||
auto_login_on_start(container), name="trading-auto-login"
|
||||
)
|
||||
else:
|
||||
logger.info("未开启 RAKUTEN_AUTO_LOGIN_ON_START,启动时不自动登录")
|
||||
|
||||
worker_task: asyncio.Task | None = None
|
||||
query_task: asyncio.Task | None = None
|
||||
if container.worker_runner is not None:
|
||||
# 顺序:本地 DB 在 worker 写证据前先就绪;SiteInteractor 已在外层启动
|
||||
assert container.worker_local_db is not None
|
||||
@@ -115,6 +160,13 @@ async def lifespan(app: FastAPI):
|
||||
container.settings.worker_id_effective,
|
||||
container.settings.order_gateway_url,
|
||||
)
|
||||
# 第二条循环:账号只读查询。独立于下单主循环,否则一笔下单跑几分钟,
|
||||
# 期间上游一句「账号里那笔订单什么状态」就得干等(docs/order-gateway.md §11)。
|
||||
assert container.query_runner is not None
|
||||
query_task = asyncio.create_task(
|
||||
container.query_runner.run(), name="trading-query-worker"
|
||||
)
|
||||
logger.info("账号只读查询 worker 已启动")
|
||||
else:
|
||||
logger.info(
|
||||
"未配置 RAKUTEN_ORDER_GATEWAY_URL,下单 worker 不启动(仅登录态与购物车接口)"
|
||||
@@ -123,6 +175,13 @@ async def lifespan(app: FastAPI):
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if login_task is not None and not login_task.done():
|
||||
# 关服务时正在登的那次不要了:浏览器由 login_one 自己的 async with 收尾
|
||||
login_task.cancel()
|
||||
try:
|
||||
await login_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if worker_task is not None:
|
||||
container.worker_runner.stop() # type: ignore[union-attr]
|
||||
worker_task.cancel()
|
||||
@@ -130,6 +189,18 @@ async def lifespan(app: FastAPI):
|
||||
await worker_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
# 付款后监控是独立于主循环的后台任务(见 runner.py::_spawn_monitor),
|
||||
# 取消 worker_task 不会连带取消它们,必须在关 SiteInteractor 前单独收尾。
|
||||
await container.worker_runner.cancel_monitors() # type: ignore[union-attr]
|
||||
if query_task is not None:
|
||||
# 与下单 worker 同样:先 stop 再 cancel,正在跑的那次读取直接放弃
|
||||
# (只读,放弃没有副作用;网关侧那张单会超时重投)。
|
||||
container.query_runner.stop() # type: ignore[union-attr]
|
||||
query_task.cancel()
|
||||
try:
|
||||
await query_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if container.site is not None:
|
||||
await container.site.close() # type: ignore[union-attr]
|
||||
if container.worker_local_db is not None:
|
||||
|
||||
+55
-29
@@ -26,37 +26,58 @@ class AuthSite(StrEnum):
|
||||
class AuthStatusRequest(BaseModel):
|
||||
"""登录态查询请求"""
|
||||
|
||||
site: AuthSite | None = None # 不传则返回所有已配置站点
|
||||
refresh: bool = True # 是否真实打一次请求探测;false 时只读缓存
|
||||
site: AuthSite | None = Field(default=None, description="站点标识;不传则返回所有已配置站点")
|
||||
refresh: bool = Field(default=True, description="是否真实打一次请求探测;false 时只读缓存")
|
||||
|
||||
|
||||
class AuthSiteStatus(BaseModel):
|
||||
"""单站登录态"""
|
||||
|
||||
site: str
|
||||
state_file_exists: bool # 是否已跑过 scripts/login.py
|
||||
logged_in: bool | None # None 表示尚未探测
|
||||
checked_age_seconds: float | None = None
|
||||
detail: str = ""
|
||||
site: str = Field(description="站点标识(rakuten)")
|
||||
state_file_exists: bool = Field(description="是否已跑过 scripts/login.py(本地登录态文件存在)")
|
||||
logged_in: bool | None = Field(description="是否已登录;null 表示尚未探测")
|
||||
checked_age_seconds: float | None = Field(default=None, description="上次探测距现在的秒数;未探测为 null")
|
||||
detail: str = Field(default="", description="补充说明(如未登录原因)")
|
||||
|
||||
|
||||
class AuthStatusData(BaseModel):
|
||||
"""登录态查询响应"""
|
||||
|
||||
sites: list[AuthSiteStatus] = Field(default_factory=list)
|
||||
sites: list[AuthSiteStatus] = Field(default_factory=list, description="各站点登录态列表")
|
||||
|
||||
|
||||
class AuthReloadRequest(BaseModel):
|
||||
"""重新加载登录态请求(人工登录完成后调用,免重启服务)"""
|
||||
|
||||
site: AuthSite | None = None # 不传则重载所有已配置站点
|
||||
site: AuthSite | None = Field(default=None, description="站点标识;不传则重载所有已配置站点")
|
||||
|
||||
|
||||
class AuthReloadData(BaseModel):
|
||||
"""重新加载登录态响应"""
|
||||
|
||||
reloaded: dict[str, int] = Field(default_factory=dict) # site -> cookie 条数
|
||||
sites: list[AuthSiteStatus] = Field(default_factory=list)
|
||||
reloaded: dict[str, int] = Field(default_factory=dict, description="重载结果:站点 -> cookie 条数")
|
||||
sites: list[AuthSiteStatus] = Field(default_factory=list, description="重载后的各站点登录态")
|
||||
|
||||
|
||||
class AuthLoginRequest(BaseModel):
|
||||
"""触发自动登录请求"""
|
||||
|
||||
site: AuthSite | None = Field(default=None, description="站点标识;不传则对所有已配置站点各跑一次")
|
||||
|
||||
|
||||
class AuthLoginData(BaseModel):
|
||||
"""触发自动登录响应
|
||||
|
||||
logged_in 是本次动作的最终结论(所有涉及站点都登录着才为 true);
|
||||
relogin_attempted 标出哪些站点真的跑了登录流程——已登录的站点直接跳过,
|
||||
不会白起一次浏览器。
|
||||
"""
|
||||
|
||||
logged_in: bool = Field(description="本次动作的最终结论:所有涉及站点都登录着才为 true")
|
||||
relogin_attempted: dict[str, bool] = Field(
|
||||
default_factory=dict, description="各站点是否真的跑了登录流程;已登录的站点直接跳过"
|
||||
)
|
||||
sites: list[AuthSiteStatus] = Field(default_factory=list, description="各站点登录态")
|
||||
|
||||
|
||||
class TradingHealthData(BaseModel):
|
||||
@@ -66,8 +87,8 @@ class TradingHealthData(BaseModel):
|
||||
POST /api/auth/status。
|
||||
"""
|
||||
|
||||
status: str
|
||||
auth: dict[str, Any] = Field(default_factory=dict)
|
||||
status: str = Field(description="服务状态,健康为 ok")
|
||||
auth: dict[str, Any] = Field(default_factory=dict, description="缓存的各站点登录态(不触发网络探测)")
|
||||
|
||||
|
||||
# ---- 购物车接口(/api/cart/*)----
|
||||
@@ -81,10 +102,15 @@ class CartAddRequest(BaseModel):
|
||||
choice 接受字符串("颜色:赤")或字符串列表(["颜色:赤", "サイズ:M"])。
|
||||
"""
|
||||
|
||||
item_url: str
|
||||
quantity: int = 1
|
||||
variant_id: str | None = None
|
||||
choice: str | list[str] | None = None
|
||||
item_url: str = Field(description="商品页 URL")
|
||||
quantity: int = Field(default=1, description="加购数量")
|
||||
variant_id: str | None = Field(
|
||||
default=None, description="多规格商品的规格 ID;不传时自动选第一个非售罄规格"
|
||||
)
|
||||
choice: str | list[str] | None = Field(
|
||||
default=None,
|
||||
description="商品选项,如 \"颜色:赤\";可传字符串或字符串列表,不传时必填选项自动选第一个候选值",
|
||||
)
|
||||
|
||||
|
||||
class CartStatusRequest(BaseModel):
|
||||
@@ -98,35 +124,35 @@ class CartClearRequest(BaseModel):
|
||||
class CartRemoveRequest(BaseModel):
|
||||
"""删除指定商品请求"""
|
||||
|
||||
item_id: str
|
||||
item_id: str = Field(description="要删除的商品 ID")
|
||||
|
||||
|
||||
class CartAddData(BaseModel):
|
||||
"""加购响应数据"""
|
||||
|
||||
added: bool = True
|
||||
item_id: str
|
||||
shop_bid: str
|
||||
cart_count: int # -1 表示加购成功但末尾 count 查询失败
|
||||
added: bool = Field(default=True, description="是否加购成功")
|
||||
item_id: str = Field(description="加购的商品 ID")
|
||||
shop_bid: str = Field(description="店铺 bid")
|
||||
cart_count: int = Field(description="加购后的购物车商品件数;-1 表示加购成功但末尾 count 查询失败")
|
||||
|
||||
|
||||
class CartStatusData(BaseModel):
|
||||
"""购物车状态响应数据"""
|
||||
|
||||
logged_in: bool
|
||||
count: int
|
||||
raw_status: str # 站点状态码字符串,"100" 表示正常
|
||||
logged_in: bool = Field(description="当前是否已登录")
|
||||
count: int = Field(description="购物车商品件数(空车为 0)")
|
||||
raw_status: str = Field(description="站点 cart count API 的状态码字符串,\"100\" 表示正常、\"101\" 表示空车")
|
||||
|
||||
|
||||
class CartClearData(BaseModel):
|
||||
"""清空购物车响应数据"""
|
||||
|
||||
removed_count: int # 实际点击「削除」按钮的次数
|
||||
cart_count: int # -1 表示末尾 count 查询失败
|
||||
removed_count: int = Field(description="实际点击「削除」按钮的次数")
|
||||
cart_count: int = Field(description="清空后的购物车商品件数;-1 表示末尾 count 查询失败")
|
||||
|
||||
|
||||
class CartRemoveData(BaseModel):
|
||||
"""删除指定商品响应数据"""
|
||||
|
||||
removed: bool # 末尾 cart HTML 已不含 item_id 时为 true
|
||||
item_id: str
|
||||
removed: bool = Field(description="末尾 cart HTML 已不含该 item_id 时为 true")
|
||||
item_id: str = Field(description="被删除的商品 ID")
|
||||
|
||||
@@ -37,6 +37,7 @@ from typing import Any
|
||||
import httpx
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.proxy import httpx_client_options
|
||||
from app.shared.errors import NotLoggedInError, UpstreamRequestError
|
||||
from app.trading.core import auth_site
|
||||
|
||||
@@ -89,6 +90,11 @@ class AuthSession:
|
||||
self._sites: dict[str, _SiteAuth] = {}
|
||||
# 自动重登的 site 级互斥锁:同账号同时只能一个登录流程(user_data_dir 被锁)
|
||||
self._relogin_locks: dict[str, asyncio.Lock] = {}
|
||||
# 每个 site 已完成的重登尝试计数。用途只有一个:让在锁上排队的调用方能分辨
|
||||
# 「我等的这段时间里已经有人替我登过了」。不能用 status().logged_in 代替——
|
||||
# 重登成功后的 reload() 会把 logged_in 重置成 None(未探测),排队者读到
|
||||
# None 会误判成「还没人登过」,于是 N 个并发调用串行触发 N 次真实登录。
|
||||
self._relogin_epochs: dict[str, int] = {}
|
||||
|
||||
# ---- 生命周期 ----
|
||||
|
||||
@@ -105,7 +111,7 @@ class AuthSession:
|
||||
headers=profile.headers(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
proxy=self._settings.httpx_proxy,
|
||||
**httpx_client_options(self._settings),
|
||||
http2=True,
|
||||
)
|
||||
self._sites[name] = _SiteAuth(name=name, client=client)
|
||||
@@ -211,8 +217,11 @@ class AuthSession:
|
||||
2. 重登成功 → 重新 check 一次,登录态转好即放行
|
||||
3. 重登失败 / 未启用 / account.yaml 缺失 → 抛 NotLoggedInError 让 worker 转 needs_human
|
||||
|
||||
重登只在这一层(任务前置检查)触发;任务执行过程中失效不重试,
|
||||
避免脏状态(cart 已提交但响应后 cookie 失效等场景)。
|
||||
**写**操作(加购 / 提交订单 / 付款)只在这一层(动作前置检查)触发重登;
|
||||
执行过程中失效不重试,避免脏状态(cart 已提交但响应后 cookie 失效等场景)。
|
||||
**只读**操作(订单列表 / 订单详情查询)另有一层执行中重试,见
|
||||
`site_interact.SiteInteractor._read_with_relogin_retry`——重跑一次查询没有
|
||||
副作用,这条边界是刻意区分的,不要把它推广到写操作上。
|
||||
"""
|
||||
status = await self.check(site)
|
||||
if status.logged_in:
|
||||
@@ -246,11 +255,34 @@ class AuthSession:
|
||||
|
||||
# site 级锁:同账号同 user_data_dir,并发重登会撞锁
|
||||
lock = self._relogin_locks.setdefault(site, asyncio.Lock())
|
||||
epoch_before_wait = self._relogin_epochs.get(site, 0)
|
||||
async with lock:
|
||||
# 拿锁后再 check 一次——可能别的协程刚重登过
|
||||
status = self.status(site)
|
||||
if status.logged_in:
|
||||
return True
|
||||
# 在锁上等过、且期间有人跑完了一轮重登 → 大概率不需要再登一次。
|
||||
# 这里必须**真探测**而不是读 self.status():重登成功后的 reload() 把
|
||||
# logged_in 重置成 None,读缓存会一律判「还没登上」,白跑一次 login_one
|
||||
# (最坏 N 个并发调用串行触发 N 次真实登录,各自最长 relogin_timeout)。
|
||||
if self._relogin_epochs.get(site, 0) != epoch_before_wait:
|
||||
# 探测失败(网络问题)时不在这里抛错——try_relogin 的契约是「降级
|
||||
# 返回 False / 继续走登录」,不是抛异常;这种情况直接落到下面正常
|
||||
# 跑一次 login_one(它自己也会先探测登录态,已登录就跳过填表)。
|
||||
try:
|
||||
status = await self.check(site)
|
||||
except UpstreamRequestError:
|
||||
logger.warning(
|
||||
"自动重登:site=%s 等锁后复核登录态失败,继续尝试登录", site, exc_info=True
|
||||
)
|
||||
else:
|
||||
if status.logged_in:
|
||||
logger.info("自动重登跳过:site=%s 等锁期间已由其他调用方登录完成", site)
|
||||
return True
|
||||
# 刚有人登过还是没登上,说明这轮站点侧就是登不上(验证码/密码错/
|
||||
# 风控)。同一波并发里再串一遍只是把每个调用方各卡一个
|
||||
# relogin_timeout,站点侧结果不会变——直接失败,交人工。
|
||||
logger.warning(
|
||||
"自动重登跳过:site=%s 等锁期间刚失败过一次,不在同一波并发里重复触发",
|
||||
site,
|
||||
)
|
||||
return False
|
||||
|
||||
# 读 account.yaml(失败时降级,不抛错)
|
||||
try:
|
||||
@@ -282,6 +314,11 @@ class AuthSession:
|
||||
except Exception:
|
||||
logger.exception("自动重登异常:site=%s account_id=%s", site, account.id)
|
||||
return False
|
||||
finally:
|
||||
# 成败都记一次「本站跑过一轮登录流程」。放 finally 是因为排队者要
|
||||
# 分辨的是「有没有人替我尝试过」,失败的尝试同样算——否则失败后
|
||||
# 排队的调用方会一个接一个重跑同一个注定失败的登录。
|
||||
self._relogin_epochs[site] = self._relogin_epochs.get(site, 0) + 1
|
||||
|
||||
if not ok:
|
||||
logger.warning("自动重登失败:site=%s account_id=%s", site, account.id)
|
||||
|
||||
@@ -25,6 +25,7 @@ from typing import Any
|
||||
import yaml
|
||||
|
||||
from app.shared.config import BASE_DIR, Settings
|
||||
from app.shared.proxy import playwright_launch_proxy
|
||||
from app.trading.core import auth_site
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -36,6 +37,16 @@ _POLL_INTERVAL_SECONDS = 5
|
||||
_ACCOUNTS_FILE = BASE_DIR / "account.yaml"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CreditCard:
|
||||
"""account.yaml 中 payment.credit-card 的结构化形式"""
|
||||
|
||||
number: str
|
||||
month: str
|
||||
year: str
|
||||
name: str
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Account:
|
||||
"""account.yaml 中的一条账号记录"""
|
||||
@@ -47,6 +58,13 @@ class Account:
|
||||
user_data_dir: Path
|
||||
state_filename: str
|
||||
default: bool
|
||||
# 电话号码:可选。仅用于 checkout 流程里「账号缺电话号码」时的补录步骤
|
||||
# (site_interact.py::_complete_phone_registration),登录本身不需要它。
|
||||
phone: str | None = None
|
||||
# 信用卡:可选。仅用于 checkout 流程里支付方式页「账号没有已保存卡」时的
|
||||
# 新卡代填步骤(site_interact.py::_fill_new_card_form)。该表单选择器从未
|
||||
# 见过真实 HTML,代填后不会自动提交,须人工核对再点提交,详见该方法文档字符串。
|
||||
credit_card: CreditCard | None = None
|
||||
|
||||
|
||||
# ---- YAML 读取与账号选择 ----
|
||||
@@ -103,6 +121,25 @@ def load_accounts() -> dict[str, list[Account]]:
|
||||
f"account.yaml: {site}[{idx}].{required} 必填"
|
||||
)
|
||||
account_id = str(rec["id"])
|
||||
payment = rec.get("payment") or {}
|
||||
cc_raw = payment.get("credit-card") if isinstance(payment, dict) else None
|
||||
credit_card = None
|
||||
if cc_raw:
|
||||
if not isinstance(cc_raw, dict):
|
||||
raise ValueError(
|
||||
f"account.yaml: {site}[{idx}].payment.credit-card 应为字典"
|
||||
)
|
||||
for required in ("card-no", "month", "year", "name"):
|
||||
if not cc_raw.get(required):
|
||||
raise ValueError(
|
||||
f"account.yaml: {site}[{idx}].payment.credit-card.{required} 必填"
|
||||
)
|
||||
credit_card = CreditCard(
|
||||
number=str(cc_raw["card-no"]),
|
||||
month=str(cc_raw["month"]),
|
||||
year=str(cc_raw["year"]),
|
||||
name=str(cc_raw["name"]),
|
||||
)
|
||||
accounts.append(
|
||||
Account(
|
||||
site=site,
|
||||
@@ -112,6 +149,8 @@ def load_accounts() -> dict[str, list[Account]]:
|
||||
user_data_dir=_resolve_user_data_dir(site, rec, account_id),
|
||||
state_filename=_resolve_state_filename(site, rec, is_first=(idx == 0)),
|
||||
default=bool(rec.get("default", False)),
|
||||
phone=str(rec["phone"]) if rec.get("phone") else None,
|
||||
credit_card=credit_card,
|
||||
)
|
||||
)
|
||||
result[site] = accounts
|
||||
@@ -174,18 +213,31 @@ def default_account_for(site: str, accounts_by_site: dict[str, list[Account]]) -
|
||||
|
||||
|
||||
async def _is_logged_in(page, site: str) -> bool:
|
||||
"""访问 profile.login_url 看落地是否被踢到 SSO
|
||||
"""访问 profile.probe_url(购物车页)读 `__INITIAL_STATE__.user.isLoggedIn`
|
||||
|
||||
`site` 目前恒为 `"rakuten"`:未登录时会被重定向到 login.account.rakuten.com
|
||||
或 /login 路径;登录后停在 my.rakuten.co.jp。保留 site 参数形态是为了与
|
||||
SiteAuthProfile 共用签名,方便未来加站点。
|
||||
`site` 目前恒为 `"rakuten"`。此前用「落地 URL 是否被踢到 SSO」判断
|
||||
profile.login_url(myrakuten):实测发现未登录时该页会落到 my.rakuten.co.jp
|
||||
而不重定向到 login.account.rakuten.com,导致误判为「已登录」,
|
||||
login_one() 因此跳过填表、保存一份只有 Akamai/负载均衡 cookie(无真实会话)
|
||||
的 storage_state——看似登录成功,实际购物车页仍判未登录。
|
||||
改用与 AuthSession._check_rakuten / site_interact.py 一致的判据(购物车页,
|
||||
Playwright 渲染后取 `__INITIAL_STATE__.user.isLoggedIn`),三处判据不再漂移。
|
||||
"""
|
||||
profile = auth_site.profile(site)
|
||||
await page.goto(profile.login_url, wait_until="domcontentloaded", timeout=60_000)
|
||||
final_url = page.url
|
||||
if site != "rakuten":
|
||||
raise ValueError(f"未知站点:{site}")
|
||||
return "login.account.rakuten.com" not in final_url and "/login" not in final_url
|
||||
profile = auth_site.profile(site)
|
||||
await page.goto(profile.probe_url, wait_until="domcontentloaded", timeout=60_000)
|
||||
try:
|
||||
await page.wait_for_function(
|
||||
"() => window.__INITIAL_STATE__ !== undefined", timeout=8_000
|
||||
)
|
||||
except Exception:
|
||||
pass # 未渲染出来时按 isLoggedIn=false 处理(下面 evaluate 拿到 None)
|
||||
state = await page.evaluate(
|
||||
"() => (window.__INITIAL_STATE__ && window.__INITIAL_STATE__.user) "
|
||||
"? window.__INITIAL_STATE__.user.isLoggedIn : null"
|
||||
)
|
||||
return bool(state)
|
||||
|
||||
|
||||
# ---- 自动填表(启发式选择器)----
|
||||
@@ -304,7 +356,7 @@ async def login_one(
|
||||
user_data_dir=str(account.user_data_dir),
|
||||
headless=False, # 自动登录仍需有头:撞验证码要人工接管
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
proxy=playwright_launch_proxy(settings),
|
||||
user_agent=profile.user_agent,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
@@ -319,12 +371,12 @@ async def login_one(
|
||||
)
|
||||
try:
|
||||
page = context.pages[0] if context.pages else await context.new_page()
|
||||
await page.goto(profile.login_url, wait_until="domcontentloaded", timeout=60_000)
|
||||
|
||||
# 已经登录则直接保存
|
||||
# 已经登录则直接保存(_is_logged_in 自己打购物车页判定,不依赖 login_url 落地)
|
||||
if await _is_logged_in(page, account.site):
|
||||
log(f"已处于登录态,跳过填表:{account.id}")
|
||||
else:
|
||||
await page.goto(profile.login_url, wait_until="domcontentloaded", timeout=60_000)
|
||||
filled = await _try_autofill(page, account)
|
||||
if not filled:
|
||||
log("未找到登录表单且未登录,请在浏览器里手动完成登录")
|
||||
|
||||
@@ -16,9 +16,11 @@ from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import AppError
|
||||
from app.shared.proxy import httpx_client_options
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
from app.trading.worker.models import LeaseTask
|
||||
from app.trading.worker.models import LeaseTask, QueryTask
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,13 +31,16 @@ class GatewayClient:
|
||||
一份 AsyncClient 实例贯穿 worker 整个生命周期,连接池由 httpx 管理。
|
||||
"""
|
||||
|
||||
def __init__(self, base_url: str, bearer_token: str, *, timeout: float = 60.0):
|
||||
def __init__(
|
||||
self, base_url: str, bearer_token: str, *, settings: Settings, timeout: float = 60.0
|
||||
):
|
||||
# 末尾去斜杠,避免 base + "/api/..." 拼出双斜杠
|
||||
self._base_url = base_url.rstrip("/")
|
||||
self._client = httpx.AsyncClient(
|
||||
base_url=self._base_url,
|
||||
headers={"Authorization": f"Bearer {bearer_token}"},
|
||||
timeout=timeout,
|
||||
**httpx_client_options(settings, target_url=self._base_url),
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
@@ -127,6 +132,16 @@ class GatewayClient:
|
||||
body = await self._request("POST", f"/api/orders/{task_id}/report", json=payload)
|
||||
return body["data"]
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any]:
|
||||
"""任务详情,返回网关响应里的 data 字段(含 created_at)
|
||||
|
||||
供 verify.verify_on_site 恢复核对用:LeaseTask(lease/reclaim 的响应)
|
||||
不带 created_at,规格 §5「按创建时间 ~ stale 之间」的窗口核对必须单独
|
||||
查一次这个接口才能拿到。
|
||||
"""
|
||||
body = await self._request("GET", f"/api/orders/{task_id}")
|
||||
return body["data"]
|
||||
|
||||
async def reclaim(self, task_id: str, worker_id: str) -> LeaseTask:
|
||||
"""恢复领取。返回的 LeaseTask 必然 lease_count > 1"""
|
||||
body = await self._request(
|
||||
@@ -143,3 +158,55 @@ class GatewayClient:
|
||||
lease_count=data.get("lease_count", 1),
|
||||
known_state=data.get("known_state"),
|
||||
)
|
||||
|
||||
# ---- 账号只读查询通道(docs/order-gateway.md §11)----
|
||||
|
||||
async def lease_query(self, worker_id: str, *, wait: int = 30) -> QueryTask | None:
|
||||
"""长轮询领取一张只读查询单。无单可领时返回 None
|
||||
|
||||
与 lease() 走的是两条互不干扰的通道:下单任务在跑的时候,查询照样能领到
|
||||
(账号级串行由 SiteInteractor 那把锁保证,不靠网关排队)。
|
||||
"""
|
||||
body = await self._request(
|
||||
"GET",
|
||||
"/api/account/queries/lease",
|
||||
params={"worker_id": worker_id, "wait": wait},
|
||||
)
|
||||
data = body.get("data")
|
||||
if not data:
|
||||
return None
|
||||
return QueryTask(
|
||||
query_id=data["query_id"],
|
||||
site=data["site"],
|
||||
kind=data["kind"],
|
||||
params=data.get("params") or {},
|
||||
lease_expires_at=data.get("lease_expires_at", ""),
|
||||
attempt=data.get("attempt", 1),
|
||||
)
|
||||
|
||||
async def report_query_result(
|
||||
self,
|
||||
query_id: str,
|
||||
worker_id: str,
|
||||
*,
|
||||
success: bool,
|
||||
result: dict[str, Any] | None = None,
|
||||
error_code: int | None = None,
|
||||
error_message: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""回报查询结果,返回网关响应里的 data 字段
|
||||
|
||||
网关可能回 6006(租约已被重投给下一轮)——那说明本次结果已经作废,
|
||||
调用方按「丢弃」处理即可,不要重发。
|
||||
"""
|
||||
payload: dict[str, Any] = {
|
||||
"worker_id": worker_id,
|
||||
"success": success,
|
||||
"result": result,
|
||||
"error_code": error_code,
|
||||
"error_message": error_message,
|
||||
}
|
||||
body = await self._request(
|
||||
"POST", f"/api/account/queries/{query_id}/result", json=payload
|
||||
)
|
||||
return body["data"]
|
||||
|
||||
@@ -26,3 +26,20 @@ class LeaseTask:
|
||||
lease_expires_at: str = ""
|
||||
lease_count: int = 1
|
||||
known_state: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class QueryTask:
|
||||
"""worker 从网关领到的账号只读查询单
|
||||
|
||||
对应网关 GET /api/account/queries/lease 的 QueryLeaseData。没有 known_state
|
||||
之类的字段——查询是一次性只读动作,`attempt > 1` 只说明上一轮超时被重投了,
|
||||
worker 不需要因此改变行为(重跑安全,这正是它与下单任务的根本差别)。
|
||||
"""
|
||||
|
||||
query_id: str
|
||||
site: str
|
||||
kind: str
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
lease_expires_at: str = ""
|
||||
attempt: int = 1
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
"""账号只读查询 worker:第二条常驻循环,与下单主循环并行
|
||||
|
||||
为什么单独一条循环而不是塞进 `runner.WorkerRunner`(docs/order-gateway.md §11):
|
||||
下单主循环执行一笔任务要几分钟(加购 → 确认页 → 提交 → 付款),期间它整个人
|
||||
都占在那笔任务上;查询要是排在同一条循环里,上游问一句「账号里那笔订单现在
|
||||
什么状态」就得等下单跑完。两条循环各自长轮询各自的通道,互不阻塞。
|
||||
|
||||
那「同一账号并发操作」的硬约束靠什么保证?靠 `SiteInteractor` 里那把
|
||||
`asyncio.Lock`——所有站点交互(下单写操作与这里的只读操作)都要先拿它,天然串行。
|
||||
网关侧因此**不需要**为查询再实现一遍「全局并发度 1」。
|
||||
|
||||
代价是:正在下单时,查询会卡在那把锁上等。所以每次执行都套
|
||||
`account_query_timeout_seconds` 超时,超时按失败回报,上游重发即可
|
||||
(只读,重发没有副作用)——总比让查询单一直挂到租约过期强。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from app.shared.errors import AppError, InvalidRequestError
|
||||
from app.shared.task_state import AccountQueryKind
|
||||
from app.trading.worker.client import GatewayClient
|
||||
from app.trading.worker.models import QueryTask
|
||||
from app.trading.worker.site_interact import SiteInteractor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.shared.config import Settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 没给 since 时的窗口下界。用 epoch 而不是 None,是为了让翻页终止条件
|
||||
# (`order_date < since`)与「不设下界」共用同一条代码路径,不多一个分支。
|
||||
_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
|
||||
# 超时时回报的错误码:沿用 2002(资源繁忙 / 等槽位超时),这里等的是账号锁。
|
||||
_TIMEOUT_ERR_CODE = 2002
|
||||
# 未预期异常回报的错误码,与 HTTP 层兜底处理器保持一致
|
||||
_UNEXPECTED_ERR_CODE = 1500
|
||||
|
||||
|
||||
def _parse_since(raw: Any) -> datetime:
|
||||
"""解析 params.since。缺省用 epoch(不设下界);给了但解析不出直接报错,不猜"""
|
||||
if raw in (None, ""):
|
||||
return _EPOCH
|
||||
if not isinstance(raw, str):
|
||||
raise InvalidRequestError(f"params.since 必须是 ISO8601 字符串,收到 {type(raw).__name__}")
|
||||
try:
|
||||
parsed = datetime.fromisoformat(raw.replace("Z", "+00:00"))
|
||||
except ValueError as exc:
|
||||
raise InvalidRequestError(f"params.since 不是合法的 ISO8601 时间:{raw}") from exc
|
||||
# 站点返回的下单时间带时区,比较两侧必须都 aware,否则直接 TypeError
|
||||
return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
|
||||
|
||||
|
||||
def _parse_max_pages(raw: Any, default: int) -> int:
|
||||
if raw in (None, ""):
|
||||
return default
|
||||
if not isinstance(raw, int) or isinstance(raw, bool):
|
||||
raise InvalidRequestError(f"params.max_pages 必须是整数,收到 {raw!r}")
|
||||
if raw < 1:
|
||||
raise InvalidRequestError(f"params.max_pages 必须 ≥ 1,收到 {raw}")
|
||||
return raw
|
||||
|
||||
|
||||
class QueryRunner:
|
||||
"""账号只读查询的常驻循环
|
||||
|
||||
只依赖网关客户端与 SiteInteractor:查询不落证据、不写本地 SQLite——它没有
|
||||
「执行事实」需要留痕,读到什么就回什么。
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
settings: "Settings",
|
||||
gateway_client: GatewayClient,
|
||||
site: SiteInteractor,
|
||||
):
|
||||
self._settings = settings
|
||||
self._gateway = gateway_client
|
||||
self._site = site
|
||||
self._running = False
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
|
||||
@property
|
||||
def worker_id(self) -> str:
|
||||
return self._settings.worker_id_effective
|
||||
|
||||
# ---- 主循环 ----
|
||||
|
||||
async def run(self) -> None:
|
||||
"""常驻循环:长轮询领查询单 → 执行 → 回结果"""
|
||||
self._running = True
|
||||
logger.info("账号查询 worker 启动:worker_id=%s", self.worker_id)
|
||||
while self._running:
|
||||
try:
|
||||
query = await self._gateway.lease_query(self.worker_id, wait=30)
|
||||
except AppError as exc:
|
||||
logger.warning("查询 lease 失败:%s (err=%s)", exc.message, exc.err_code)
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("查询 lease 异常")
|
||||
await asyncio.sleep(5)
|
||||
continue
|
||||
|
||||
if query is None:
|
||||
continue
|
||||
|
||||
try:
|
||||
await self.handle(query)
|
||||
except Exception: # noqa: BLE001
|
||||
# 任何未预期的异常都不应让这条循环退出
|
||||
logger.exception("处理查询单异常:query_id=%s", query.query_id)
|
||||
|
||||
# ---- 单张查询单 ----
|
||||
|
||||
async def handle(self, query: QueryTask) -> None:
|
||||
"""执行一张查询单并回报结果。任何失败都转成一次「失败回报」,不抛出去"""
|
||||
logger.info(
|
||||
"领到查询单:query_id=%s kind=%s attempt=%s",
|
||||
query.query_id, query.kind, query.attempt,
|
||||
)
|
||||
try:
|
||||
result = await asyncio.wait_for(
|
||||
self.execute(query),
|
||||
timeout=self._settings.account_query_timeout_seconds,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.warning(
|
||||
"查询超时(%s 秒,多半是下单任务正占着账号锁):query_id=%s",
|
||||
self._settings.account_query_timeout_seconds, query.query_id,
|
||||
)
|
||||
await self._report_safe(
|
||||
query,
|
||||
success=False,
|
||||
error_code=_TIMEOUT_ERR_CODE,
|
||||
error_message=(
|
||||
f"查询超过 {self._settings.account_query_timeout_seconds} 秒未完成"
|
||||
"(账号锁被下单任务占用或站点页面无响应),可稍后重发"
|
||||
),
|
||||
)
|
||||
return
|
||||
except AppError as exc:
|
||||
logger.warning(
|
||||
"查询失败:query_id=%s code=%s msg=%s",
|
||||
query.query_id, exc.err_code, exc.message,
|
||||
)
|
||||
await self._report_safe(
|
||||
query, success=False, error_code=exc.err_code, error_message=exc.message
|
||||
)
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
logger.exception("查询未预期异常:query_id=%s", query.query_id)
|
||||
await self._report_safe(
|
||||
query,
|
||||
success=False,
|
||||
error_code=_UNEXPECTED_ERR_CODE,
|
||||
error_message=f"{type(exc).__name__}: {exc}",
|
||||
)
|
||||
return
|
||||
|
||||
payload, oversized = self._enforce_result_size(result)
|
||||
if oversized is not None:
|
||||
await self._report_safe(
|
||||
query, success=False, error_code=1003, error_message=oversized
|
||||
)
|
||||
return
|
||||
await self._report_safe(query, success=True, result=payload)
|
||||
|
||||
async def execute(self, query: QueryTask) -> dict[str, Any]:
|
||||
"""按 kind 分发到 SiteInteractor 的只读方法,返回要回给上游的结果字典"""
|
||||
if query.site != "rakuten":
|
||||
raise InvalidRequestError(
|
||||
f"site={query.site} 不在交易服务范围内(账号查询仅支持 rakuten)"
|
||||
)
|
||||
|
||||
if query.kind == AccountQueryKind.ORDER_LIST.value:
|
||||
return await self._execute_order_list(query)
|
||||
if query.kind == AccountQueryKind.ORDER_DETAIL.value:
|
||||
return await self._execute_order_detail(query)
|
||||
raise InvalidRequestError(f"未知的查询种类:{query.kind}")
|
||||
|
||||
async def _execute_order_list(self, query: QueryTask) -> dict[str, Any]:
|
||||
"""订单列表:规范化条目 + 站点原始 orderListData(逐页)"""
|
||||
since = _parse_since(query.params.get("since"))
|
||||
max_pages = _parse_max_pages(
|
||||
query.params.get("max_pages"), self._settings.account_query_default_max_pages
|
||||
)
|
||||
|
||||
window = await self._site.list_recent_orders(since=since, max_pages=max_pages)
|
||||
return {
|
||||
"kind": AccountQueryKind.ORDER_LIST.value,
|
||||
"since": since.isoformat().replace("+00:00", "Z"),
|
||||
"max_pages": max_pages,
|
||||
# False=翻页没能覆盖完窗口(命中页数上限 / 页面结构不对 / 掉登录),
|
||||
# 此时「列表里没有某笔订单」不等于「账号里没有这笔订单」。
|
||||
"window_fully_covered": window.window_fully_covered,
|
||||
"orders": [
|
||||
{
|
||||
"order_number": entry.order_number,
|
||||
"order_date": entry.order_date,
|
||||
"shop_id": entry.shop_id,
|
||||
"shop_name": entry.shop_name,
|
||||
"items": [
|
||||
{
|
||||
"item_id": item.item_id,
|
||||
"item_name": item.item_name,
|
||||
"item_url": item.item_url,
|
||||
}
|
||||
for item in entry.items
|
||||
],
|
||||
}
|
||||
for entry in window.entries
|
||||
],
|
||||
# 站点 __INITIAL_STATE__.orderListData 原文,按翻页顺序。规范化字段
|
||||
# 只挑了下单/核对用得上的那几个,别的字段(金额、配送、状态文案)在这里。
|
||||
"raw_pages": window.raw_pages,
|
||||
}
|
||||
|
||||
async def _execute_order_detail(self, query: QueryTask) -> dict[str, Any]:
|
||||
"""单笔订单详情:已实测的配送阶段 + 未实测结构的页面原始状态"""
|
||||
order_number = query.params.get("order_number")
|
||||
if not order_number or not isinstance(order_number, str):
|
||||
raise InvalidRequestError("params.order_number 必填(站点注文番号)")
|
||||
|
||||
detail = await self._site.fetch_order_detail(order_number)
|
||||
status = detail.status
|
||||
return {
|
||||
"kind": AccountQueryKind.ORDER_DETAIL.value,
|
||||
"order_number": order_number,
|
||||
# found=False 不是错误:站点自己说下单后订单要 10 分钟左右才反映到
|
||||
# 订单页,刚下的单查不到属正常。
|
||||
"found": status.found,
|
||||
"stage_label": status.stage_label,
|
||||
"order_state": status.order_state.value if status.order_state else None,
|
||||
# 站点侧结构化配送状态码(如 "CHECKING_ORDER");列表页/stepper 路径
|
||||
# 拿不到时为 None。这是 raw 里 orderData 的规范化摘要,供上游快速对账。
|
||||
"delivery_status": status.delivery_status,
|
||||
# 详情页额外带出的站点原始 JSON 原样透传,由上游自担结构变动风险
|
||||
# (见 site_interact.OrderDetailSnapshot)。
|
||||
"raw": detail.raw,
|
||||
"raw_available": detail.raw is not None,
|
||||
}
|
||||
|
||||
# ---- 结果体积与回报 ----
|
||||
|
||||
def _enforce_result_size(self, result: dict[str, Any]) -> tuple[dict[str, Any], str | None]:
|
||||
"""体积闸门:超限先丢站点原始 JSON,仍超限则判失败
|
||||
|
||||
返回 (要回报的结果, 超限说明)。超限说明非 None 时表示这次要按失败回报——
|
||||
与其把几 MB 页面状态塞进网关 SQLite,不如让上游把窗口调小重来。
|
||||
"""
|
||||
limit = self._settings.query_result_max_bytes
|
||||
if self._size_of(result) <= limit:
|
||||
return result, None
|
||||
|
||||
trimmed = dict(result)
|
||||
trimmed["raw_pages"] = []
|
||||
trimmed["raw"] = None
|
||||
trimmed["raw_omitted"] = f"站点原始 JSON 超过 {limit} 字节上限,已丢弃,仅保留规范化字段"
|
||||
if self._size_of(trimmed) <= limit:
|
||||
logger.warning("查询结果超限,已丢弃站点原始 JSON:limit=%s", limit)
|
||||
return trimmed, None
|
||||
|
||||
return result, (
|
||||
f"查询结果超过 {limit} 字节上限,丢弃站点原始 JSON 后仍超限;"
|
||||
"请缩小查询窗口(减小 params.max_pages 或收紧 params.since)后重试"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _size_of(payload: dict[str, Any]) -> int:
|
||||
return len(json.dumps(payload, ensure_ascii=False).encode("utf-8"))
|
||||
|
||||
async def _report_safe(
|
||||
self,
|
||||
query: QueryTask,
|
||||
*,
|
||||
success: bool,
|
||||
result: dict[str, Any] | None = None,
|
||||
error_code: int | None = None,
|
||||
error_message: str = "",
|
||||
) -> None:
|
||||
"""回报结果,失败只记日志不抛——这条循环不能因为回报失败退出
|
||||
|
||||
网关回 6006 属于正常情况(本次租约已被重投给下一轮,结果作废),
|
||||
同样只记日志,不重发。
|
||||
"""
|
||||
try:
|
||||
await self._gateway.report_query_result(
|
||||
query.query_id,
|
||||
self.worker_id,
|
||||
success=success,
|
||||
result=result,
|
||||
error_code=error_code,
|
||||
error_message=error_message,
|
||||
)
|
||||
except AppError as exc:
|
||||
logger.warning(
|
||||
"回报查询结果被网关拒绝:query_id=%s code=%s msg=%s",
|
||||
query.query_id, exc.err_code, exc.message,
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("回报查询结果失败:query_id=%s", query.query_id)
|
||||
+193
-20
@@ -28,14 +28,14 @@ import contextlib
|
||||
import logging
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from app.shared.errors import AppError, OrderGuardError
|
||||
from app.shared.errors import AppError, CheckoutBlockedError, OrderGuardError
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
from app.trading.worker import verify
|
||||
from app.trading.worker.client import GatewayClient
|
||||
from app.trading.worker.evidence import EvidenceStore
|
||||
from app.trading.worker.local_db import LocalDB
|
||||
from app.trading.worker.models import LeaseTask
|
||||
from app.trading.worker.site_interact import SiteInteractor
|
||||
from app.trading.worker.site_interact import PageSnapshot, SiteInteractor
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.shared.config import Settings
|
||||
@@ -76,10 +76,28 @@ class WorkerRunner:
|
||||
self._evidence = evidence
|
||||
self._site = site
|
||||
self._running = False
|
||||
# 付款后监控后台任务:task_id → asyncio.Task。与主循环解耦(不阻塞领下一单),
|
||||
# 详见 _spawn_monitor / _monitor_order。用 set 而非 list 是因为只需要成员管理,
|
||||
# 不需要顺序。
|
||||
self._monitor_tasks: set[asyncio.Task] = set()
|
||||
|
||||
def stop(self) -> None:
|
||||
self._running = False
|
||||
|
||||
async def cancel_monitors(self) -> None:
|
||||
"""服务关闭时调用:取消所有还在跑的付款后监控后台任务并等它们收尾
|
||||
|
||||
必须在 SiteInteractor.close() 之前调用——监控任务还在用同一个
|
||||
Playwright context,先关 context 再取消会导致监控任务在 await 到一半时
|
||||
踩上已关闭的资源,报一堆无意义的异常。
|
||||
"""
|
||||
tasks = list(self._monitor_tasks)
|
||||
for t in tasks:
|
||||
t.cancel()
|
||||
for t in tasks:
|
||||
with contextlib.suppress(asyncio.CancelledError):
|
||||
await t
|
||||
|
||||
@property
|
||||
def worker_id(self) -> str:
|
||||
return self._settings.worker_id_effective
|
||||
@@ -144,7 +162,7 @@ class WorkerRunner:
|
||||
task.lease_count,
|
||||
task.task_id,
|
||||
)
|
||||
verdict = await verify.verify_on_site(task)
|
||||
verdict = await verify.verify_on_site(task, gateway=self._gateway, site=self._site)
|
||||
if verdict.verdict == verify.VerifyVerdict.ALREADY_ORDERED:
|
||||
await self._report_safe(
|
||||
task,
|
||||
@@ -158,12 +176,16 @@ class WorkerRunner:
|
||||
await self._db.mark_finished(task.task_id, OrderState.ORDERED.value)
|
||||
return
|
||||
|
||||
# NOT_ORDERED 也走 needs_human:当前 verify 桩不会返回这个值,但留接口给
|
||||
# 未来真正能可靠核对时——那时再决定 NOT_ORDERED 是否直接重新执行
|
||||
# NOT_ORDERED 也走 needs_human(2026-08-13 verify_on_site 已实现,会真的
|
||||
# 返回这个值,但刻意仍不自动重新执行):核对逻辑目前只有「账号 1 笔订单」
|
||||
# 的真实数据支撑 ALREADY_ORDERED 分支,NOT_ORDERED 分支只验证过逻辑本身,
|
||||
# 没有被真实多单场景跑过——在这条判断被更多真实数据验证之前,即使确认
|
||||
# 未下单也交人工决定是否重新提交,不自动触发新的下单动作。这是刻意的
|
||||
# 保守选择,不是遗漏;要不要放开需要显式决定,不在这里静默改。
|
||||
await self._report_safe(
|
||||
task,
|
||||
state=_coerce_state(task.known_state),
|
||||
detail=f"恢复核对无法定论:{verdict.detail}",
|
||||
detail=f"恢复核对结论({verdict.verdict.value}):{verdict.detail}",
|
||||
terminal=True,
|
||||
terminal_status=TaskStatus.NEEDS_HUMAN,
|
||||
)
|
||||
@@ -203,6 +225,21 @@ class WorkerRunner:
|
||||
terminal=True,
|
||||
terminal_status=TaskStatus.NEEDS_HUMAN,
|
||||
)
|
||||
except CheckoutBlockedError as exc:
|
||||
# 站点风控拦截(session upgrade / 3DS 等):转 needs_human 交人工接管,
|
||||
# 不当普通失败重试(规格 §10.1)
|
||||
logger.warning(
|
||||
"结算被站点风控拦截,转 needs_human:task_id=%s msg=%s",
|
||||
task.task_id,
|
||||
exc.message,
|
||||
)
|
||||
await self._report_safe(
|
||||
task,
|
||||
state=_coerce_state(task.known_state),
|
||||
detail=exc.message,
|
||||
terminal=True,
|
||||
terminal_status=TaskStatus.NEEDS_HUMAN,
|
||||
)
|
||||
except AppError as exc:
|
||||
logger.warning(
|
||||
"执行失败:task_id=%s code=%s msg=%s",
|
||||
@@ -219,11 +256,12 @@ class WorkerRunner:
|
||||
)
|
||||
|
||||
async def execute(self, task: LeaseTask) -> None:
|
||||
"""站点交互的实际调度:加购 → 校验 → 确认页 → 金额守卫 → 提交 → 付款
|
||||
"""站点交互的实际调度:清购物车 → 加购 → 校验 → 确认页 → 金额守卫 → 提交 → 付款
|
||||
|
||||
每一步的顺序:动作 → 落证据 → 写本地 SQLite → 回报 gateway。
|
||||
站点交互当前未实现(site_interact 抛 NotImplementedError),第一步就会
|
||||
转到 _execute_with_renewal 的 except 分支上报 needs_human。
|
||||
开单前的「清购物车」是本机侧卫生步骤(step 0,见下方代码注释):落本地
|
||||
步骤证据(页面 + 截图 + meta)但不上报 gateway、不记状态事件;站点交互
|
||||
失败会转 _execute_with_renewal 的 except 分支上报 needs_human / failed。
|
||||
"""
|
||||
await self._db.ensure_started(task.task_id, task.site, task.intent)
|
||||
|
||||
@@ -231,6 +269,40 @@ class WorkerRunner:
|
||||
if task.site != "rakuten":
|
||||
raise NotImplementedError(f"site={task.site} 暂不在交易服务范围内(仅 rakuten)")
|
||||
|
||||
# 步骤 0:开单前清理购物车。上一单若在提交(submit_order)之前失败——比如
|
||||
# 加购成功但 enter_checkout 被 session upgrade 拦截、或金额守卫拦下——残留的
|
||||
# 商品不会自己消失,会一直留在购物车里;下一次 add_to_cart 把新商品叠加在旧
|
||||
# 商品上,结算时会把上一单的一起买走(已实测踩到过)。所以每单开跑前先清空
|
||||
# 购物车,保证「一单 = 只买本单商品」的不变式——无论上一单是怎么失败的
|
||||
# (包括进程中途崩溃,残留都没人清),这里都从空车起步。
|
||||
#
|
||||
# 这条不在 gateway 上报、不记 order_events:它是本机侧的卫生操作,不是订单
|
||||
# 进度的状态迁移。但清理后的页面快照、整页截图与 meta 要落本地证据并登记
|
||||
# evidence_index——清理没跑干净被下面的闸门拦下转 needs_human 时,这份现场
|
||||
# 就是排查依据,所以证据必须先于闸门判断落盘。清理后 count 非 0(含
|
||||
# count API 拿不到结果返回 -1)说明清理没跑干净,**不能**带着残留往下
|
||||
# 加购——有把上一单买走的真实风险,按闸门语义拦截转 needs_human,
|
||||
# 宁可卡住等人核对,也不赌「残留不会被买走」。
|
||||
cleared = await self._site.clear_cart()
|
||||
evidence_ref = self._evidence.write_step(
|
||||
task.task_id, 0, "cart-clear",
|
||||
html=cleared.get("html") or None,
|
||||
png=cleared.get("screenshot") or None,
|
||||
meta={
|
||||
"step": "cart-clear",
|
||||
"removed_count": cleared.get("removed_count"),
|
||||
"cart_count": cleared.get("cart_count"),
|
||||
},
|
||||
)
|
||||
await self._db.index_evidence(task.task_id, 0, "cart-clear", evidence_ref)
|
||||
if cleared.get("cart_count", -1) != 0:
|
||||
raise OrderGuardError(
|
||||
"开单前清理购物车后仍未清空"
|
||||
f"(cart_count={cleared.get('cart_count')},removed_count="
|
||||
f"{cleared.get('removed_count')}):残留商品可能随本次下单一起被买走,"
|
||||
"中止转人工核对"
|
||||
)
|
||||
|
||||
# 步骤 1:加购
|
||||
await self._run_step(
|
||||
task, step_no=1, step_name="cart-add",
|
||||
@@ -248,14 +320,16 @@ class WorkerRunner:
|
||||
)
|
||||
|
||||
# 步骤 3:进入下单确认页 + 金额守卫
|
||||
checkout_html = await self._site.enter_checkout(task)
|
||||
summary = await self._site.parse_checkout(checkout_html)
|
||||
checkout = await self._site.enter_checkout(task)
|
||||
summary = await self._site.parse_checkout(checkout.html)
|
||||
self._enforce_amount_guard(task, summary.payable_yen)
|
||||
await self._run_step(
|
||||
task, step_no=3, step_name="order-confirm",
|
||||
action=self._noop(),
|
||||
state=OrderState.CREATED,
|
||||
detail=f"下单确认页已解析:应付 {summary.payable_yen} 円",
|
||||
html=checkout.html or None,
|
||||
png=checkout.screenshot or None,
|
||||
evidence_meta={
|
||||
"payable_yen": summary.payable_yen,
|
||||
"site_order_id": summary.site_order_id,
|
||||
@@ -264,13 +338,16 @@ class WorkerRunner:
|
||||
)
|
||||
|
||||
# 步骤 4:提交下单
|
||||
site_order_id = await self._site.submit_order(task)
|
||||
submit = await self._site.submit_order(task)
|
||||
site_order_id = submit.site_order_id
|
||||
await self._run_step(
|
||||
task, step_no=4, step_name="order-submit",
|
||||
action=self._noop(),
|
||||
state=OrderState.ORDERED,
|
||||
site_order_id=site_order_id,
|
||||
detail=f"已提交下单,站点订单号 {site_order_id}",
|
||||
html=submit.evidence.html or None,
|
||||
png=submit.evidence.screenshot or None,
|
||||
)
|
||||
|
||||
# 步骤 5:付款
|
||||
@@ -295,11 +372,91 @@ class WorkerRunner:
|
||||
)
|
||||
await self._db.mark_finished(task.task_id, OrderState.PAID.value)
|
||||
|
||||
# 步骤 6:付款后监控(非阻塞,常驻轮询;当前未实现)
|
||||
try:
|
||||
await self._site.monitor(task, site_order_id)
|
||||
except NotImplementedError:
|
||||
logger.info("付款后监控未实现,跳过:task_id=%s", task.task_id)
|
||||
# 步骤 6:付款后监控——后台常驻轮询,不阻塞主循环领下一单(规格 §6)
|
||||
self._spawn_monitor(task, site_order_id)
|
||||
|
||||
def _spawn_monitor(self, task: LeaseTask, site_order_id: str) -> None:
|
||||
"""把 _monitor_order 起成独立的后台任务,登记到 _monitor_tasks 便于关服时收尾"""
|
||||
monitor_task = asyncio.create_task(
|
||||
self._monitor_order(task, site_order_id), name=f"monitor-{task.task_id}"
|
||||
)
|
||||
self._monitor_tasks.add(monitor_task)
|
||||
monitor_task.add_done_callback(self._monitor_tasks.discard)
|
||||
|
||||
async def _monitor_order(self, task: LeaseTask, site_order_id: str) -> None:
|
||||
"""付款后台轮询:订单配送阶段变化时继续 report(规格 §6「付款后监控」)
|
||||
|
||||
任务本身已经在 execute() 里上报过 succeeded 终态——这里的 report 都是
|
||||
`terminal=False` 的追加上报(规格 §4.5:任务已 terminal 的仍可上报,
|
||||
gateway 追加到 task_reports),监控本身的成败不影响已经完成的下单结果。
|
||||
|
||||
轮询间隔/次数上限见 settings.order_monitor_poll_interval_seconds /
|
||||
order_monitor_max_checks;达到上限仍未看到「配達完了」不算失败,只是
|
||||
停止追踪(记一条 warning)。单次探测异常(登录态失效、页面打不开)只记
|
||||
日志、下一轮重试,不能把异常抛出去——这是后台任务,没有人等着 catch 它,
|
||||
未捕获异常会被 asyncio 直接吞掉且只在垃圾回收时打一条难查的警告。
|
||||
"""
|
||||
last_state: OrderState | None = None
|
||||
step_no = 6
|
||||
interval = self._settings.order_monitor_poll_interval_seconds
|
||||
max_checks = self._settings.order_monitor_max_checks
|
||||
for attempt in range(1, max_checks + 1):
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
snapshot = await self._site.check_order_status(site_order_id)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"订单监控探测失败(第 %s/%s 次),下一轮继续重试:"
|
||||
"task_id=%s site_order_id=%s",
|
||||
attempt, max_checks, task.task_id, site_order_id,
|
||||
exc_info=True,
|
||||
)
|
||||
continue
|
||||
|
||||
if not snapshot.found:
|
||||
logger.info(
|
||||
"订单监控:第 %s/%s 次仍未在详情页找到订单号,继续轮询:task_id=%s",
|
||||
attempt, max_checks, task.task_id,
|
||||
)
|
||||
continue
|
||||
|
||||
if snapshot.order_state is None or snapshot.order_state == last_state:
|
||||
continue
|
||||
|
||||
last_state = snapshot.order_state
|
||||
step_name = f"monitor-{snapshot.order_state.value}"
|
||||
detail = f"订单监控:进度「{snapshot.stage_label}」→ {snapshot.order_state.value}"
|
||||
evidence_ref = self._evidence.write_step(
|
||||
task.task_id, step_no, step_name,
|
||||
html=snapshot.html,
|
||||
png=snapshot.screenshot or None,
|
||||
meta={
|
||||
"step": step_name,
|
||||
"state": snapshot.order_state.value,
|
||||
"stage_label": snapshot.stage_label,
|
||||
},
|
||||
)
|
||||
await self._db.index_evidence(task.task_id, step_no, step_name, evidence_ref)
|
||||
await self._db.record_event(
|
||||
task.task_id, snapshot.order_state.value,
|
||||
detail=detail, evidence_ref=evidence_ref,
|
||||
)
|
||||
await self._report_safe(
|
||||
task,
|
||||
state=snapshot.order_state,
|
||||
detail=detail,
|
||||
site_order_id=site_order_id,
|
||||
)
|
||||
step_no += 1
|
||||
|
||||
if snapshot.order_state == OrderState.DELIVERED:
|
||||
logger.info("订单监控:已送达,停止轮询:task_id=%s", task.task_id)
|
||||
return
|
||||
|
||||
logger.warning(
|
||||
"订单监控达到最大轮询次数 %s 仍未看到「配達完了」,停止追踪:task_id=%s",
|
||||
max_checks, task.task_id,
|
||||
)
|
||||
|
||||
def _enforce_amount_guard(self, task: LeaseTask, payable_yen: int) -> None:
|
||||
"""金额守卫:实际应付超过 intent.max_total_yen 或 RAKUTEN_ORDER_MAX_TOTAL_YEN 时拦截"""
|
||||
@@ -333,16 +490,32 @@ class WorkerRunner:
|
||||
payable_yen: int | None = None,
|
||||
pay_deadline: str | None = None,
|
||||
evidence_meta: dict | None = None,
|
||||
html: str | None = None,
|
||||
png: bytes | None = None,
|
||||
) -> None:
|
||||
"""单步执行:动作 → 落证据 → 写本地 → 回报 gateway"""
|
||||
await action()
|
||||
"""单步执行:动作 → 落证据 → 写本地 → 回报 gateway
|
||||
|
||||
`action` 返回 PageSnapshot(add_to_cart / verify_cart / pay 等站点方法的
|
||||
证据载体)时,其 html / screenshot 即本步骤要落盘的页面与整页截图;
|
||||
显式传入的 `html` / `png` 优先级更高,供 step3/step4 等已经单独拿到
|
||||
页面的调用点使用。
|
||||
"""
|
||||
result = await action()
|
||||
if isinstance(result, PageSnapshot):
|
||||
if html is None:
|
||||
html = result.html or None
|
||||
if png is None:
|
||||
png = result.screenshot or None
|
||||
elif html is None and isinstance(result, str):
|
||||
# 旧契约兼容:站点方法返回字符串时视为页面 HTML
|
||||
html = result
|
||||
meta = {
|
||||
"step": step_name,
|
||||
"state": state.value,
|
||||
**(evidence_meta or {}),
|
||||
}
|
||||
evidence_ref = self._evidence.write_step(
|
||||
task.task_id, step_no, step_name, meta=meta
|
||||
task.task_id, step_no, step_name, html=html, png=png, meta=meta
|
||||
)
|
||||
await self._db.index_evidence(task.task_id, step_no, step_name, evidence_ref)
|
||||
await self._db.record_event(
|
||||
|
||||
+1683
-45
File diff suppressed because it is too large
Load Diff
@@ -4,16 +4,27 @@
|
||||
此时 worker 不能盲目重新提交(可能上次已经下单成功,只是回报断网),必须先查
|
||||
站点订单列表比对。
|
||||
|
||||
订单列表反查的实现需要实测(规格 §10 第 3 条:订单列表页能否按商品 + 时间窗口
|
||||
可靠地反查出「这单下没下」)。当前为桩:恒返回 `unknown`,让 runner 走
|
||||
needs_human 分支。**绝不**默认按「没下单」处理——那是猜,可能变成重复下单。
|
||||
2026-08-13 实现:核对逻辑用真实账号 + 真实订单验证过订单列表的数据结构(见
|
||||
app/trading/worker/site_interact.py::list_recent_orders 与 _parse_order_list 的
|
||||
模块文档),但当时账号只有 1 笔订单,ALREADY_ORDERED 分支有真实数据支撑,
|
||||
NOT_ORDERED / 多笔命中 / 多页翻页这几个分支目前只有逻辑,没有被真实多单数据
|
||||
跑过——**绝不**在没有把握时猜:任何一步信息不足都转 UNKNOWN,宁可卡住等人看
|
||||
一眼,也不赌一次重复下单。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from typing import TYPE_CHECKING
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.shared.errors import AppError
|
||||
from app.trading.worker.models import LeaseTask
|
||||
from app.trading.worker.site_interact import parse_order_datetime
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from app.trading.worker.client import GatewayClient
|
||||
from app.trading.worker.site_interact import SiteInteractor
|
||||
|
||||
|
||||
class VerifyVerdict(StrEnum):
|
||||
@@ -31,16 +42,85 @@ class VerifyResult:
|
||||
detail: str = ""
|
||||
|
||||
|
||||
async def verify_on_site(task: LeaseTask) -> VerifyResult:
|
||||
def _normalize_item_url(url: str | None) -> str | None:
|
||||
"""去掉查询串与末尾斜杠,只留 scheme+host+path 用于比对
|
||||
|
||||
订单列表返回的 itemUrl 实测带 `?variantId=...`(见 _parse_order_list 的真实
|
||||
样例),intent.item_url 通常不带——按查询串比对会产生假阴性,所以只比
|
||||
路径本身。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
parts = urlsplit(url)
|
||||
return f"{parts.scheme}://{parts.netloc}{parts.path.rstrip('/')}"
|
||||
|
||||
|
||||
async def verify_on_site(
|
||||
task: LeaseTask, *, gateway: "GatewayClient", site: "SiteInteractor"
|
||||
) -> VerifyResult:
|
||||
"""核对一笔任务是否已在站点上下过单
|
||||
|
||||
桩实现:永远返回 UNKNOWN。**绝不返回 NOT_ORDERED**——除非真实订单列表反查
|
||||
能可靠证明这一点,否则视为「无法判断」交人工,宁可卡住等人看一眼。
|
||||
|
||||
实现方需要补的实测:用 intent 里的商品 + 时间窗口(created_at 到 stale 之间)
|
||||
比对订单列表,能拿到 site_order_id 最好。
|
||||
核对链路:intent.item_url → 查任务创建时间(GET /api/orders/{task_id},
|
||||
LeaseTask 本身不带 created_at)→ 拉「创建时间之后」的订单列表 → 按商品 URL
|
||||
比对。任何一环拿不到足够信息都返回 UNKNOWN,不猜——尤其是 NOT_ORDERED,
|
||||
只有在确认翻完了窗口内的全部订单后才允许返回,否则「没找到」可能只是没翻
|
||||
到那一页。
|
||||
"""
|
||||
intent = task.intent or {}
|
||||
item_url = intent.get("item_url")
|
||||
if not item_url:
|
||||
return VerifyResult(
|
||||
VerifyVerdict.UNKNOWN, detail="intent 缺 item_url,无法比对商品"
|
||||
)
|
||||
target = _normalize_item_url(item_url)
|
||||
|
||||
try:
|
||||
task_detail = await gateway.get_task(task.task_id)
|
||||
except AppError as exc:
|
||||
return VerifyResult(
|
||||
VerifyVerdict.UNKNOWN, detail=f"查询任务创建时间失败:{exc.message}"
|
||||
)
|
||||
|
||||
created_at_raw = task_detail.get("created_at")
|
||||
created_at = parse_order_datetime(created_at_raw) if created_at_raw else None
|
||||
if created_at is None:
|
||||
return VerifyResult(
|
||||
VerifyVerdict.UNKNOWN, detail=f"任务创建时间不可解析:{created_at_raw!r}"
|
||||
)
|
||||
|
||||
try:
|
||||
window = await site.list_recent_orders(since=created_at)
|
||||
except Exception as exc: # noqa: BLE001 — 站点交互失败一律转 unknown,不重试
|
||||
return VerifyResult(
|
||||
VerifyVerdict.UNKNOWN,
|
||||
detail=f"订单列表查询失败:{type(exc).__name__}: {exc}",
|
||||
)
|
||||
|
||||
matches = [
|
||||
entry
|
||||
for entry in window.entries
|
||||
if any(_normalize_item_url(it.item_url) == target for it in entry.items)
|
||||
]
|
||||
if len(matches) == 1:
|
||||
return VerifyResult(
|
||||
VerifyVerdict.ALREADY_ORDERED,
|
||||
site_order_id=matches[0].order_number,
|
||||
detail=f"订单列表命中 1 笔匹配商品的订单({matches[0].order_number})",
|
||||
)
|
||||
if len(matches) > 1:
|
||||
return VerifyResult(
|
||||
VerifyVerdict.UNKNOWN,
|
||||
detail=(
|
||||
f"订单列表命中 {len(matches)} 笔匹配商品的订单,"
|
||||
f"无法唯一确定:{[m.order_number for m in matches]}"
|
||||
),
|
||||
)
|
||||
if window.window_fully_covered:
|
||||
return VerifyResult(
|
||||
VerifyVerdict.NOT_ORDERED,
|
||||
detail="已核对任务创建时间之后的全部订单,未找到匹配商品",
|
||||
)
|
||||
return VerifyResult(
|
||||
verdict=VerifyVerdict.UNKNOWN,
|
||||
detail="订单列表反查未实现:见 docs/order-gateway.md §10 第 3 条",
|
||||
VerifyVerdict.UNKNOWN,
|
||||
detail="订单列表未能确认翻完任务创建时间之后的全部订单",
|
||||
)
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# 抓取与有状态端部署
|
||||
#
|
||||
# 默认起抓取服务 scraping(:31107)与交易服务 trading(:31108):
|
||||
# docker compose up -d
|
||||
# 生产环境仍可按部署位置分别启动:服务器侧 `docker compose up -d scraping`,
|
||||
# 本地机侧 `docker compose up -d trading`。抓取服务匿名无状态,可独立多开。
|
||||
# 下单任务网关按设计部署在**服务器侧**(与抓取服务同机),是另一个部署单元,
|
||||
# 所以放在 gateway profile 里,不会被默认启动。真要在这台机上一起跑(自包含
|
||||
# 联调 / 单机部署)时:
|
||||
# docker compose --profile gateway up -d
|
||||
#
|
||||
# 两个服务都**只能单实例**:登录态 cookie 全局唯一、订单监控是常驻轮询、SQLite
|
||||
# 单连接 + 全局并发度 1。不要 `--scale`,不要在前面挂多副本。
|
||||
#
|
||||
# 部署前置(缺一样服务就跑不通,不是可选项):
|
||||
# 1. cp .env.example .env,至少填 RAKUTEN_BEARER_TOKEN / RAKUTEN_ORDER_GATEWAY_URL /
|
||||
# RAKUTEN_SCRAPER_BASE_URL(后两个留空则 worker 不启动,只剩购物车与登录态接口)。
|
||||
# 2. cp account.yaml.example account.yaml 并填真实凭据(session upgrade 复核密码、
|
||||
# 手机号补录、支付方式核对都要读它)。**必须先建文件再 up**,否则 Docker 会
|
||||
# 把这个挂载点当目录创建,容器里读到的是个空目录。
|
||||
# 3. 登录态 cookie(.auth/rakuten_state.json):**不必**先在宿主机准备——
|
||||
# RAKUTEN_AUTO_LOGIN_ON_START 默认开,容器起来就按 account.yaml 自己登一次。
|
||||
# 撞 reCAPTCHA / 设备验证会停在浏览器里等人工,这时把 RAKUTEN_VNC_ENABLED 设成
|
||||
# true(配 RAKUTEN_VNC_PASSWORD)连 127.0.0.1:5900 接管,完成后:
|
||||
# curl -X POST -H "Authorization: Bearer $TOKEN" http://127.0.0.1:31108/api/auth/login
|
||||
# 也可以照旧在宿主机跑 scripts/login.py 再挂载进来,两条路都行。
|
||||
#
|
||||
# 敏感面:.auth/(可直接冒充账号的 cookie)、account.yaml(明文密码+卡号)、
|
||||
# data/(真实姓名地址等 PII 与证据快照)全在宿主机目录里,注意宿主机权限与备份加密。
|
||||
|
||||
services:
|
||||
scraping:
|
||||
# 商品获取 API:乐天市场与ラクマ抓取,匿名无状态,可独立扩容。
|
||||
image: ${RAKUTEN_SCRAPING_IMAGE:-git.jerryyan.net/jp/rakuten-api}:${RAKUTEN_SCRAPING_TAG:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: rakuten-api
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
RAKUTEN_APP_HOST: 0.0.0.0
|
||||
RAKUTEN_APP_PORT: 31107
|
||||
RAKUTEN_APP_ENV: prod
|
||||
ports:
|
||||
# 默认仅供本机反向代理访问;需要直连时显式设 RAKUTEN_SCRAPING_BIND。
|
||||
- "${RAKUTEN_SCRAPING_BIND:-127.0.0.1}:31107:31107"
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "5"
|
||||
|
||||
trading:
|
||||
# 交易服务:加购 → 下单 → 付款 → 订单监控,以及 /api/auth/* /api/cart/*
|
||||
image: ${RAKUTEN_TRADING_IMAGE:-git.jerryyan.net/jp/rakuten-trading}:${RAKUTEN_TRADING_TAG:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.trading
|
||||
container_name: rakuten-trading
|
||||
restart: unless-stopped
|
||||
# PID 1 用 docker-init:Chromium 会派生一堆子进程,崩溃时需要有人收尸
|
||||
init: true
|
||||
# Chromium 默认 /dev/shm 只有 64MB,渲染稍重的结算页会直接 crash
|
||||
shm_size: 1gb
|
||||
# 停容器时给正在执行的下单任务留出收尾时间(uvicorn 关 lifespan → worker 停轮询
|
||||
# → 取消订单监控 → 关浏览器)。**停容器前最好先确认没有在途任务**:进程被硬杀
|
||||
# 时站点侧可能已经提交成功,本地没记录,恢复要走 verify_on_site 人工核对。
|
||||
stop_grace_period: 120s
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
# 容器内必须监听 0.0.0.0,对外只从 127.0.0.1 映射(见 ports)
|
||||
RAKUTEN_TRADING_HOST: 0.0.0.0
|
||||
RAKUTEN_TRADING_PORT: 31108
|
||||
RAKUTEN_HEALTH_PORT: 31108
|
||||
RAKUTEN_APP_ENV: prod
|
||||
# worker 标识:默认取主机名,容器主机名是随机 ID,重建就变——显式固定,
|
||||
# 便于在网关侧对上是哪台机在领任务
|
||||
RAKUTEN_WORKER_ID: ${RAKUTEN_WORKER_ID:-rakuten-trading-docker}
|
||||
# 有头 Chromium 需要虚拟显示(镜像内 Xvfb),别关
|
||||
RAKUTEN_XVFB_ENABLED: "true"
|
||||
# 启动即按 account.yaml 自动登录(后台跑,不阻塞端口)。撞 reCAPTCHA / 设备验证
|
||||
# 时会停在浏览器里等人工——那时开 VNC 接管,或事后调 POST /api/auth/login 重试。
|
||||
RAKUTEN_AUTO_LOGIN_ON_START: ${RAKUTEN_AUTO_LOGIN_ON_START:-true}
|
||||
RAKUTEN_RELOGIN_ENABLED: ${RAKUTEN_RELOGIN_ENABLED:-true}
|
||||
# 需要远程盯着浏览器(登录验证码、3DS、needs_human 人工接管)时改成 true,
|
||||
# 并设 RAKUTEN_VNC_PASSWORD;端口只映射到 127.0.0.1,走 SSH 隧道访问
|
||||
RAKUTEN_VNC_ENABLED: ${RAKUTEN_VNC_ENABLED:-false}
|
||||
RAKUTEN_VNC_PASSWORD: ${RAKUTEN_VNC_PASSWORD:-}
|
||||
ports:
|
||||
# 这些接口能操作真实账号,只对本机开放;要跨机访问请走内网地址或 SSH 隧道
|
||||
- "127.0.0.1:31108:31108"
|
||||
- "127.0.0.1:5900:5900"
|
||||
volumes:
|
||||
# 登录态 cookie:容器要能写(自动重登会重写 storage_state)
|
||||
- ./.auth:/app/.auth
|
||||
# 账号凭据:只读挂载。文件必须先存在,见文件头「部署前置」第 2 条
|
||||
- ./account.yaml:/app/account.yaml:ro
|
||||
# login.py 的持久化浏览器目录(保住设备指纹,减少重登触发风控核验)
|
||||
- ./.browser-data:/app/.browser-data
|
||||
# 订单 SQLite(执行事实的权威记录)+ 证据快照,丢了没法追溯下过什么单
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "5"
|
||||
# Chromium 的 namespace sandbox 在容器里可能被 seccomp/AppArmor 挡住(表现为
|
||||
# 浏览器起不来、Playwright 报 Target closed)。宿主机内核不允许非特权 user
|
||||
# namespace 时,按下面两条择一放行;两条都不想放,则需要在
|
||||
# site_interact.py 的 launch(args=...) 里加 --no-sandbox(改生产代码,需另行决定)。
|
||||
# security_opt:
|
||||
# - seccomp:unconfined
|
||||
# cap_add:
|
||||
# - SYS_ADMIN
|
||||
|
||||
gateway:
|
||||
# 下单任务网关:任务队列 + 状态镜像,上游业务系统在这里提交下单任务与查任务状态。
|
||||
# 正式拓扑里它在服务器侧、trading 在本地,两者靠出站长轮询联通(docs/order-gateway.md);
|
||||
# 这里只是为了单机自包含跑起来,默认不启动。
|
||||
profiles: ["gateway"]
|
||||
image: ${RAKUTEN_TRADING_IMAGE:-git.jerryyan.net/jp/rakuten-trading}:${RAKUTEN_TRADING_TAG:-latest}
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.trading
|
||||
container_name: rakuten-gateway
|
||||
restart: unless-stopped
|
||||
init: true
|
||||
command: ["python", "-m", "app.gateway.main"]
|
||||
env_file:
|
||||
- .env
|
||||
environment:
|
||||
RAKUTEN_GATEWAY_HOST: 0.0.0.0
|
||||
RAKUTEN_GATEWAY_PORT: 31109
|
||||
RAKUTEN_HEALTH_PORT: 31109
|
||||
RAKUTEN_APP_ENV: prod
|
||||
# 网关不碰浏览器,跳过 Xvfb
|
||||
RAKUTEN_XVFB_ENABLED: "false"
|
||||
ports:
|
||||
# 上游业务系统要能调到;对外暴露时请在反代上收紧来源,鉴权只有一个 Bearer token
|
||||
- "${RAKUTEN_GATEWAY_BIND:-127.0.0.1}:31109:31109"
|
||||
volumes:
|
||||
# 任务队列 SQLite(data/gateway.db)必须持久化,丢了等于丢一批下单任务
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
logging:
|
||||
driver: json-file
|
||||
options:
|
||||
max-size: "50m"
|
||||
max-file: "5"
|
||||
+290
-3
@@ -54,6 +54,11 @@ queued ──lease──> leased ──首次 report──> running ──termin
|
||||
└── 只有人工介入才能从 stale 回到 queued(见 §5)
|
||||
```
|
||||
|
||||
> 账号**只读**查询(上游问「账号里真实的订单长什么样」,
|
||||
> 见 [§11](#11-账号只读查询通道))有一套独立的查询单状态机与通道,不复用这张任务表——
|
||||
> 查询是可安全重投的,与「下单绝不重投」是两套相反的安全语义,分表就是要防止
|
||||
> 这两个 if 被改错。
|
||||
|
||||
## 4. order-gateway
|
||||
|
||||
技术栈与现有仓库保持一致:Python 3.13 + FastAPI + pydantic-settings + SQLite,
|
||||
@@ -69,6 +74,7 @@ CREATE TABLE tasks (
|
||||
task_id TEXT PRIMARY KEY, -- 幂等键,见 §4.2
|
||||
site TEXT NOT NULL, -- rakuten(交易服务仅覆盖乐天市场)
|
||||
intent_json TEXT NOT NULL, -- 下单意图原文,gateway 不解释内容
|
||||
callback_url TEXT, -- 终结类事件通知地址(可选),见 §4.8
|
||||
status TEXT NOT NULL, -- 见 §3
|
||||
lease_owner TEXT, -- worker_id
|
||||
lease_expires_at TEXT, -- ISO8601 UTC
|
||||
@@ -111,14 +117,16 @@ CREATE TABLE workers (
|
||||
"variant_id": "...", // 多规格商品必填
|
||||
"options": {},
|
||||
"max_total_yen": 30000 // 可选,覆盖本次的金额上限
|
||||
}
|
||||
},
|
||||
"callback_url": "https://upstream.example.com/hooks/rakuten-order" // 可选,终结类事件通知,见 §4.8
|
||||
}
|
||||
```
|
||||
|
||||
响应 `data`:`{"task_id": "...", "status": "queued", "created": true}`
|
||||
|
||||
**幂等**:同一个 `task_id` 重复提交不新建任务,返回既有任务且 `created=false`。
|
||||
上游重发不会变成两单。
|
||||
上游重发不会变成两单。注意 `callback_url` 只在首次创建时写入,幂等重发不更新
|
||||
既有任务的任何字段——要换通知地址必须用新 `task_id`。
|
||||
|
||||
### 4.3 GET /api/orders/lease — 本地长轮询领取
|
||||
|
||||
@@ -193,6 +201,48 @@ CREATE TABLE workers (
|
||||
付款期限监控**不在 gateway**:本地机 7×24 在线,那套逻辑放本地(§6),
|
||||
gateway 不重复实现一遍定时器。
|
||||
|
||||
### 4.8 回调通知(callback_url,2026-08-16)
|
||||
|
||||
上游提交任务时可带 `callback_url`(http/https 绝对 URL,其余提交时 422)。
|
||||
登记后网关在**终结类事件**发生时向该地址 POST 一条 JSON 通知,上游免去轮询。
|
||||
|
||||
触发时机只有两类,每类一次:
|
||||
|
||||
1. **任务到达终态**(`succeeded` / `failed` / `needs_human`):worker 的
|
||||
`terminal=true` report 真正把任务推入终态的那一次。中间态 report
|
||||
(in_cart / ordered / awaiting_payment 非 terminal)不通知;任务已终结后的
|
||||
后续 report(如付款后监控的 shipped/delivered)也不重复通知。
|
||||
2. **任务被置 stale**:租约过期被 sweep 标记(见 §5)。上游收到后应走人工
|
||||
排查 + reclaim 流程。
|
||||
|
||||
终态事件的 payload(无信封,直接是 JSON 对象):
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"task_id": "po-20260727-0001",
|
||||
"site": "rakuten",
|
||||
"event": "terminal",
|
||||
"status": "succeeded", // 终态:succeeded / failed / needs_human
|
||||
"state": "paid", // 本次 report 的订单状态
|
||||
"payable_yen": 12800,
|
||||
"pay_deadline": "2026-07-30T14:59:00Z",
|
||||
"site_order_id": "266123-20260727-0001234",
|
||||
"evidence_ref": "po-20260727-0001/05-payment",
|
||||
"detail": "付款完成",
|
||||
"reported_at": "2026-07-27T09:15:00Z"
|
||||
}
|
||||
```
|
||||
|
||||
stale 事件的 payload:`{"task_id", "site", "event": "stale", "status": "stale",
|
||||
"detail"}`。
|
||||
|
||||
投递语义是 **best-effort 单次尝试**:超时(`RAKUTEN_CALLBACK_TIMEOUT_SECONDS`,
|
||||
默认 10 秒)、非 2xx、连接错误都只记网关日志,不重试、不阻塞 report/sweep
|
||||
主流程。回调只是「省轮询」的提示,可能丢失——**权威状态仍以
|
||||
`GET /api/orders/{task_id}` 为准**,上游应保留对账轮询(可降低频率)。
|
||||
回调出站沿用统一代理策略(`app/shared/proxy.py`,按目标 host 判定 bypass);
|
||||
上游如需鉴权,可把 token 编进 callback_url 的 query 里自行校验。
|
||||
|
||||
## 5. 租约过期:绝不自动重投(本文最关键的一条)
|
||||
|
||||
常规任务队列在租约超时后会把任务放回队列重新分发。**这里必须禁止**:本地可能已经
|
||||
@@ -275,6 +325,7 @@ gateway 侧:
|
||||
- `RAKUTEN_LEASE_TTL_SECONDS`(默认 300)
|
||||
- `RAKUTEN_LEASE_MAX_WAIT_SECONDS`(默认 60)
|
||||
- `RAKUTEN_WORKER_OFFLINE_ALERT_SECONDS`(默认 300)
|
||||
- `RAKUTEN_CALLBACK_TIMEOUT_SECONDS`(默认 10,回调通知单次 HTTP 超时,见 §4.8)
|
||||
|
||||
trading 侧新增:
|
||||
|
||||
@@ -296,6 +347,8 @@ trading 侧新增:
|
||||
| 6002 | 租约无效:不是持有者、已过期或任务已终结 | 409 |
|
||||
| 6003 | 任务状态不允许该操作(如对已终结任务 reclaim) | 409 |
|
||||
| 6004 | 已有任务在执行中,本次不发放(正常返回空即可,仅诊断用) | 200 |
|
||||
| 6005 | 查询单不存在(账号只读查询通道,见 §11.4) | 404 |
|
||||
| 6006 | 查询单租约无效(账号只读查询通道,见 §11.4) | 409 |
|
||||
|
||||
## 9. 验收清单
|
||||
|
||||
@@ -324,7 +377,241 @@ trading 侧新增:
|
||||
1. 自动付款是否触发 3D Secure 或短信验证。若触发,这条路走不通,付款环节改为
|
||||
「下单到 `awaiting_payment` + 上报 `needs_human` 交人工」,其余环节不变。
|
||||
2. 下单确认页的实际应付金额、付款方式、付款期限、站点订单号各自在哪个字段。
|
||||
3. 订单列表页能否按商品 + 时间窗口可靠地反查出「这单下没下」(§5 的恢复核对依赖它)。
|
||||
3. 订单列表页能否按商品 + 时间窗口可靠地反查出「这单下没下」(§5 的恢复核对依赖它,
|
||||
`verify.verify_on_site` 仍是恒返回 unknown 的桩)。2026-08-13 已经拿到过
|
||||
`order.my.rakuten.co.jp` 的真实 HTML(用于实现付款后监控,见
|
||||
`site_interact.py::check_order_status` / `_parse_order_status`),页面按
|
||||
订单号能查到「注文番号」「配送阶段进度条」,但没有验证过按商品名/时间窗口
|
||||
反查、也没有验证过多页/分页场景——对实现 §5 恢复核对有参考价值,不能直接复用。
|
||||
|
||||
> **交易范围**:交易服务只覆盖乐天市场(rakuten)。ラクマ 的抓取仍在抓取服务里提供,
|
||||
> 但不进入交易链路——没有加购契约,也永不实现下单/付款/订单监控。
|
||||
|
||||
## 11. 账号只读查询通道
|
||||
|
||||
上游有时要的不是「网关记的这笔任务走到哪一步」,而是「**已登录账号在站点上真实的
|
||||
订单是什么样**」——两者会不一致:站点侧被商家取消、金额调整、或者根本是走别的
|
||||
渠道下的单,网关的状态镜像都看不见。
|
||||
|
||||
但账号只在本地机上(NAT 后无公网入口),上游只能打网关。所以这条通道与下单任务
|
||||
同构:上游把查询意图放进队列,本地 worker 出站长轮询领走、去站点上真读一次、回结果。
|
||||
|
||||
### 11.1 为什么不复用 `/api/orders` 那条队列
|
||||
|
||||
三条理由,任何一条单独成立就足够:
|
||||
|
||||
1. **全局并发度 1 是给写操作设的**。查询排在下单后面会被堵死,而只读本来可以并行。
|
||||
2. **「租约过期绝不自动重投」是写操作的安全约束**。只读操作重投是安全的,套上
|
||||
§5 那套只会制造一堆需要人工 reclaim 的 stale 噪音。
|
||||
3. **`task_reports` 的语义是订单状态镜像**(主键 `(task_id, state)`),塞查询结果
|
||||
会污染对账口径。
|
||||
|
||||
因此单开一张 `account_queries` 表、一套 `QueryStatus` 词汇表、一条 worker 循环。
|
||||
两条通道的语义对照写在 `app/gateway/query_queue.py` 顶部,改任一边前先看它。
|
||||
|
||||
```
|
||||
queued ──lease──> leased ──result──> succeeded
|
||||
▲ │ └─> failed
|
||||
│ │
|
||||
└─ 租约过期自动重投 ┘ (attempts 用尽 → failed;超过查询单 TTL → expired)
|
||||
```
|
||||
|
||||
**账号级串行靠什么保证**:本地 `SiteInteractor` 里那把 `asyncio.Lock`——下单的写
|
||||
操作与这里的只读操作都要先拿它。网关侧不重复实现一遍并发闸门。代价是下单跑着时
|
||||
查询要排队等锁,因此 worker 侧对每次执行套 `RAKUTEN_ACCOUNT_QUERY_TIMEOUT_SECONDS`,
|
||||
超时按失败回报,上游重发即可(只读,重发无副作用)。
|
||||
|
||||
### 11.2 接口
|
||||
|
||||
**纯异步**:提交只拿单号,结果去 GET 取。网关不提供「挂起等结果」的接口——下单
|
||||
可能占着账号锁跑几分钟,同步等待会把上游的连接一起卡住。
|
||||
|
||||
- `POST /api/account/queries` — 提交查询单(幂等,同 `query_id` 返回既有单)
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"query_id": "aq-20260816-0001", // 可选,不传服务端生成
|
||||
"site": "rakuten",
|
||||
"kind": "order_list", // order_list | order_detail
|
||||
"params": { // 网关原样透传,结构由 trading 侧定义
|
||||
"since": "2026-08-01T00:00:00Z", // order_list 可选,缺省不设下界
|
||||
"max_pages": 3 // order_list 可选,1..20
|
||||
// order_detail 必填:{"order_number": "306087-20260813-0863947697"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
响应 `data`:`{"query_id": "...", "status": "queued", "created": true}`
|
||||
|
||||
- `GET /api/account/queries/lease` — 本地长轮询领取(参数 `worker_id`、`wait`、`site`)。
|
||||
**不刷 worker 心跳**:心跳的语义是「下单 worker 还活着」,查询循环活着不代表它活着。
|
||||
- `POST /api/account/queries/{id}/result` — 本地回结果
|
||||
(`{worker_id, success, result?, error_code?, error_message?}`)
|
||||
- `GET /api/account/queries/{id}` — 取状态与结果
|
||||
- `GET /api/account/queries?status=&kind=&limit=&offset=` — 列表(运维排查)
|
||||
|
||||
### 11.3 结果结构:原样透传 + 规范化字段
|
||||
|
||||
两者都给。规范化字段是稳定契约,站点原始 JSON 是逃生舱——站点比我们的模型多给的
|
||||
字段(金额、配送、状态文案)不该在中间层被悄悄丢掉。
|
||||
|
||||
`kind=order_list`(复用 `list_recent_orders`,2026-08-13 实测过的路径):
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"kind": "order_list",
|
||||
"since": "2026-08-01T00:00:00Z",
|
||||
"max_pages": 3,
|
||||
"window_fully_covered": true, // ← 见下,false 时结论不可当「没有」用
|
||||
"orders": [{"order_number": "...", "order_date": "...", "shop_id": 306087,
|
||||
"shop_name": "...", "items": [{"item_id": 0, "item_name": "", "item_url": ""}]}],
|
||||
"raw_pages": [ /* 站点 __INITIAL_STATE__.orderListData 原文,按翻页顺序 */ ]
|
||||
}
|
||||
```
|
||||
|
||||
**`window_fully_covered=false` 时,「列表里没有某笔订单」不等于「账号里没有」**——
|
||||
它表示翻页在覆盖完窗口前就停了(命中 `max_pages`、页面改版、掉登录)。上游据此
|
||||
决定是缩小窗口重查还是交人工,不要把它当成确定性的否定结论。这与 §5 恢复核对里
|
||||
「核对不出结论就报 needs_human」是同一条原则。
|
||||
|
||||
`kind=order_detail`(复用 `fetch_order_detail`):
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"kind": "order_detail",
|
||||
"order_number": "306087-20260813-0863947697",
|
||||
"found": true, // false 不是错误:站点自己说下单后要 ~10 分钟才反映
|
||||
"stage_label": "出荷", // 站点原文(配送阶段)
|
||||
"order_state": "shipped", // 映射到 OrderState;映射不到为 null
|
||||
"delivery_status": "CHECKING_ORDER", // 站点结构化配送状态码;列表页/stepper 路径为 null
|
||||
"raw": { /* 页面 __INITIAL_STATE__ 原文 */ },
|
||||
"raw_available": true
|
||||
}
|
||||
```
|
||||
|
||||
> **实测边界(2026-08-16 更新)**:上次评估时「详情页 `__INITIAL_STATE__` 结构从未
|
||||
> 拿到真实样本」——**已用真账号真实爬过一次**(`scripts/probe_order_detail.py`,
|
||||
> 样本落盘 `.probe/order_detail/`,订单号 306087-20260813-0863947697)。确认:
|
||||
> - 详情页 `pageType="ph-detail"`,配送阶段有两处可解析信号:CSS 进度条
|
||||
> (`_parse_order_status`)**以及** `orderData.shippingList[].deliveryInfo.
|
||||
> deliveryStatus` 结构化枚举(`_parse_order_detail_status`,2026-08-16 新增)。
|
||||
> `stage_label` 优先取 `deliveryStatusTitle` 站点原文(如「ご注文確認中」)。
|
||||
> - 进度条解析修了一个此前未暴露的回归:不锚定 `<li class="item--3gWCU...">` 时,
|
||||
> 详情页的面包屑 `<li>` 会先于进度条第 0 项被 `findall` 抓走,导致带 `-active--`
|
||||
> 的当前阶段被错位跳过、永远判不出阶段(见 `_ORDER_STEPPER_ITEM_PATTERN` 注释)。
|
||||
> - `deliveryStatus` 的枚举→`OrderState` 映射**仍只实测过 `CHECKING_ORDER` 一个值**,
|
||||
> 未识别的枚举值一律不接管、交回 stepper 兜底(避免掐掉 monitor 的
|
||||
> SHIPPED/DELIVERED 上报)。`delivery_status` 字段把这个原始码原样带给上游对账。
|
||||
> - `raw` 仍原样透传,想要金额/收货地址/付款方式的调用方从 `raw.orderData` 自取
|
||||
> (2026-08-16 样本确认其结构:`orderSummary` / `itemObject.itemList[]` /
|
||||
> `shippingList[].deliveryInfo.deliveryAddress` / `paymentInfoModel` 等),
|
||||
> 本服务仍不自担字段抽取的去重/解释责任。
|
||||
|
||||
结果体积上限 `RAKUTEN_QUERY_RESULT_MAX_BYTES`:超限先丢 `raw_pages`/`raw` 并置
|
||||
`raw_omitted`;丢完仍超限则整单判失败,让上游缩小窗口重来——不把几 MB 页面状态
|
||||
塞进网关 SQLite。
|
||||
|
||||
### 11.4 错误码(新增 `600x`)
|
||||
|
||||
| code | 含义 | HTTP |
|
||||
| --- | --- | --- |
|
||||
| 6005 | 查询单不存在(或已过保留期被清理) | 404 |
|
||||
| 6006 | 查询单租约无效:不是持有者、已被重投给别人或已终结 | 409 |
|
||||
|
||||
worker 回结果撞上 6006 属于正常情况(上一轮超时、单子已重投),**丢弃本次结果即可,
|
||||
不要重发**——否则会覆盖新一轮的结果。
|
||||
|
||||
### 11.5 配置项
|
||||
|
||||
| 配置 | 默认 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `RAKUTEN_QUERY_LEASE_TTL_SECONDS` | 180 | 查询单租约 TTL,过期自动重投 |
|
||||
| `RAKUTEN_QUERY_TTL_SECONDS` | 900 | 查询单整体存活上限,超时置 expired |
|
||||
| `RAKUTEN_QUERY_MAX_ATTEMPTS` | 3 | 最多被领取几次,用尽置 failed |
|
||||
| `RAKUTEN_QUERY_RETENTION_SECONDS` | 604800 | 终态查询单保留时长,过期由 sweep 清理 |
|
||||
| `RAKUTEN_ACCOUNT_QUERY_TIMEOUT_SECONDS` | 120 | worker 单次站点读取上限(含等账号锁) |
|
||||
| `RAKUTEN_ACCOUNT_QUERY_DEFAULT_MAX_PAGES` | 3 | order_list 缺省翻页上限 |
|
||||
| `RAKUTEN_QUERY_RESULT_MAX_BYTES` | 1048576 | 结果 JSON 体积上限 |
|
||||
|
||||
### 11.6 验收清单
|
||||
|
||||
- [x] 同 `query_id` 提交两次只产生一张单
|
||||
- [x] `kind` 非法在提交时就 422,不等 worker 领走才失败
|
||||
- [x] 有查询在飞时,下单任务照样能被 lease(两条通道互不阻塞)
|
||||
- [x] 多张查询单可以同时在飞(没有并发度 1 闸门)
|
||||
- [x] 租约过期 → 回 `queued` 且能被再次领取,`attempt` 递增
|
||||
- [x] 重投次数用尽 → `failed`,不再无限重投
|
||||
- [x] 超过查询单 TTL → `expired`,且不会再被领取
|
||||
- [x] 重投后旧 worker 的迟到结果被拒(6006),不覆盖新结果
|
||||
- [x] 终态单过保留期被清理,进行中的单不受影响
|
||||
- [x] 站点原始 JSON 完整出现在结果里;翻页上限生效且如实标 `window_fully_covered`
|
||||
- [x] 站点报错(掉登录 5001 等)原样带着错误码回给上游
|
||||
- [x] 执行超时按失败回报,不把查询单挂到租约过期
|
||||
- [x] worker 任何异常都不逃出主循环,每张单都有一次回报
|
||||
|
||||
## 12. 定时下派通道:周期性发现账号订单并回写网关(2026-08-16)
|
||||
|
||||
### 12.1 解决什么
|
||||
|
||||
§11 的查询通道要拿订单详情,得**上游自己已经知道 order_number 再下派**。可账号在
|
||||
站点上可能走了别的渠道下单、被商家改状态,网关的状态镜像根本看不到。本通道让
|
||||
网关**自己定期盘点账号**:派一张 `order_list` 查询(走 §11 那条队列,本地 worker
|
||||
出站真读),把「网关编目里还没有的订单」**回写进新的 `account_orders` 表**,再逐笔
|
||||
派 `order_detail` 查询把配送阶段沉淀进去。之后上游直接查 `GET /api/account/orders`
|
||||
就能拿到账号里真实有哪些订单,不必自己记单号。
|
||||
|
||||
一句话:**gateway 从「等上游告诉它账号里有什么」变成「自己定时去账号里看,并把
|
||||
新订单写回自己」**。
|
||||
|
||||
### 12.2 实现与边界(最重要的两条)
|
||||
|
||||
**1. collector 只消费「规范化字段」,绝不解析站点原始 JSON。**「网关不解释
|
||||
query 的 params/result」这条原则照旧适用于上游自提的查询单;collector 自主派发的
|
||||
这批 `discover-*` 单是**网关自己的编排数据**,它们的结果里 worker 已经把
|
||||
`orders[].order_number/order_date/shop_id/shop_name` 与 `delivery_status/order_state`
|
||||
抽成了稳定契约字段(§11.3 明说这是「规范化字段是稳定契约」),collector 把它
|
||||
沉淀进编目不算解释站点内容。代码见 `app/gateway/collector.py::_parse_list_result`(
|
||||
只读这几个键)。
|
||||
|
||||
**2. 状态无痕(不新增任何编排跟踪表)。** collector 只在内存记「本进程派出去
|
||||
但还没处理结果」的 `_pending`;进程重启即清空,重启后下一轮 tick 派一张全新 list
|
||||
扫描,结果幂等 upsert 进编目,没有重复行。`order_detail` 的 query_id 用
|
||||
`discover-detail-<order_number>-<日期>` **按天分段**:同一天内重复 submit 命中幂等
|
||||
返回既有单(不会因每轮 tick 都满足「详情过期」而反复创建),跨天自然生成新单刷新
|
||||
详情;同一天内已派过详情的订单由 `_detail_dispatched_this_day` 在内存里挡掉,避免
|
||||
「found=False 订单一直算过期、每轮都重发」。已消费的 `discover-*` 查询单留在
|
||||
`account_queries`,由既有 sweep 按保留期(默认 7 天)清理。
|
||||
|
||||
### 12.3 接口
|
||||
|
||||
- `GET /api/account/orders?state=&limit=&offset=` — 列出编目订单,按最近出现倒序。
|
||||
编目行只含规范化字段(订单号/店铺/日期/配送状态),要看细节仍去拿对应订单的
|
||||
`order_detail` 查询单。
|
||||
- `GET /api/account/orders/{order_number}` — 单笔编目订单;编目里没有返回 6005。
|
||||
- `POST /api/account/discovery/trigger` — 手动立即派一轮 `order_list` 扫描
|
||||
(不等定时器到点),返回本轮下派的单号与 `created`。
|
||||
|
||||
collector 本身不回写「已经存在于下单任务表 `tasks` 的订单」——`account_orders` 是
|
||||
**账号真实订单的编目**,与 `tasks`(下单意图及其镜像)是两回事,两者通过
|
||||
`order_number` 天然对账(上游要确认「这单网上下了没」就是查 `tasks`,要「账号里
|
||||
实际有什么」就是查 `account_orders`)。
|
||||
|
||||
### 12.4 配置项
|
||||
|
||||
| 配置 | 默认 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `RAKUTEN_ACCOUNT_DISCOVERY_ENABLED` | true | 关闭则 collector 完全不启动 |
|
||||
| `RAKUTEN_ACCOUNT_DISCOVERY_INTERVAL_SECONDS` | 21600 | 两轮 `order_list` 扫描间隔(6h) |
|
||||
| `RAKUTEN_ACCOUNT_DISCOVERY_MAX_PAGES` | 3 | list 扫描翻页上限 |
|
||||
| `RAKUTEN_ACCOUNT_DETAIL_REFRESH_SECONDS` | 86400 | 单笔详情刷新间隔(24h) |
|
||||
|
||||
### 12.5 验收清单
|
||||
|
||||
- [x] 定时派 `order_list` / `order_detail`,结果落到 `account_orders` 编目
|
||||
- [x] 采集到编目中不存在的订单 → 回写新行;重复采集只刷新不重复
|
||||
- [x] 列表扫描不抹掉已取过的详情阶段(配送状态/详情时间保留)
|
||||
- [x] 同一天内同一订单的详情单只派一张;跨天自然刷新详情
|
||||
- [x] 手动 trigger 立即派一轮 list,返回的单在查询队列里可见
|
||||
- [x] `GET /api/account/orders` 列编目;单笔不存在返回 6005
|
||||
- [x] collector 只读规范化字段,不解析 raw/raw_pages 站点原始 JSON
|
||||
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
"""把三个服务的接口合并导出成一份 openapi.json(供 Apifox / Postman 快速测试)
|
||||
|
||||
仓库出三个进程、三个端口,但对外只想给一份文档,所以这里把三份 FastAPI 生成的
|
||||
spec 合成一份,并补上 FastAPI 自己表达不出来的两件事:
|
||||
|
||||
1. **每个接口属于哪个服务**:给每个 operation 单独写 `servers`(OpenAPI 允许
|
||||
operation 级覆盖),导入后每条请求的 base URL 就是它真正的端口,不用手动切换。
|
||||
2. **Bearer 鉴权**:`require_bearer_token` 是自己读 Authorization 头的普通依赖,
|
||||
不是 FastAPI 的 security scheme,生成的 spec 里完全看不见。这里遍历路由的依赖
|
||||
树识别出哪些接口真的要 token(而不是按路径猜),补上 securitySchemes + security。
|
||||
|
||||
`/health` 三个服务都有且路径相同 —— OpenAPI 的 paths 是以路径为键的,无法放三份。
|
||||
合并成一条:servers 列出三个服务,响应 schema 用 anyOf 罩住三种 HealthData,
|
||||
描述里写明按 server 切换。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/export_openapi.py # 写回 openapi.json
|
||||
.venv/Scripts/python.exe scripts/export_openapi.py --check # 只校验是否已是最新
|
||||
|
||||
tests/test_openapi_export.py 会跑 --check 的等价断言:加了新接口忘了重新导出,
|
||||
测试会直接失败,避免这份文档慢慢变成过期文档。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from fastapi import FastAPI # noqa: E402
|
||||
from fastapi.routing import APIRoute # noqa: E402
|
||||
|
||||
OUTPUT_PATH = Path(__file__).resolve().parent.parent / "openapi.json"
|
||||
|
||||
_SECURITY_SCHEME = "BearerAuth"
|
||||
|
||||
_INFO_DESCRIPTION = """乐天 / ラクマ 抓取 + 下单交易 HTTP API(三个服务合并成一份文档)。
|
||||
|
||||
- **抓取服务** `:31107` —— 匿名无状态,可多开实例。搜索/分类/商品/店铺,含 ラクマ。
|
||||
- **交易服务(有状态端)** `:31108` —— 持账号登录态,加购与登录态管理,**只能单实例**。
|
||||
- **下单任务网关** `:31109` —— 下单任务队列与状态查询,本地 worker 长轮询领任务,**只能单实例**。
|
||||
|
||||
每个接口的 `servers` 已按所属服务单独标注,导入后不需要手动切 base URL。
|
||||
除 `/health` 外全部需要请求头 `Authorization: Bearer <RAKUTEN_BEARER_TOKEN>`。
|
||||
响应统一是 `{success, msg, data, code}` 信封,字段与错误码说明见项目 README.md。
|
||||
|
||||
本文件由 scripts/export_openapi.py 生成,不要手改。"""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Service:
|
||||
"""一个部署单元在文档里的身份"""
|
||||
|
||||
key: str # 用于 operationId 前缀与 schema 重名时的前缀
|
||||
label: str # 展示名(tag 前缀 / Apifox 目录名)
|
||||
module: str # 入口模块,取其 create_app
|
||||
port: int
|
||||
|
||||
@property
|
||||
def url(self) -> str:
|
||||
return f"http://127.0.0.1:{self.port}"
|
||||
|
||||
|
||||
SERVICES = (
|
||||
Service(key="scraping", label="抓取服务", module="app.scraping.main", port=31107),
|
||||
Service(key="trading", label="交易服务", module="app.trading.main", port=31108),
|
||||
Service(key="gateway", label="下单网关", module="app.gateway.main", port=31109),
|
||||
)
|
||||
|
||||
_METHODS = ("get", "put", "post", "delete", "options", "head", "patch", "trace")
|
||||
|
||||
|
||||
def _create_app(service: Service) -> FastAPI:
|
||||
"""按模块名取 create_app 并建应用(不跑 lifespan,无副作用)"""
|
||||
module = __import__(service.module, fromlist=["create_app"])
|
||||
return module.create_app()
|
||||
|
||||
|
||||
def _api_routes(app: FastAPI) -> list[APIRoute]:
|
||||
"""摊平取出所有 APIRoute
|
||||
|
||||
新版 FastAPI(0.140)的 include_router 不再把子路由摊到 app.routes 上,而是包成
|
||||
_IncludedRouter,真正的路由挂在它的 original_router.routes 上。不递归下去的话
|
||||
一条 APIRoute 都拿不到,鉴权也就全都识别不出来。
|
||||
"""
|
||||
found: list[APIRoute] = []
|
||||
|
||||
def walk(routes: list[Any]) -> None:
|
||||
for route in routes:
|
||||
if isinstance(route, APIRoute):
|
||||
found.append(route)
|
||||
continue
|
||||
nested = getattr(route, "routes", None)
|
||||
if nested is None:
|
||||
inner = getattr(route, "original_router", None)
|
||||
nested = getattr(inner, "routes", None)
|
||||
if nested:
|
||||
walk(list(nested))
|
||||
|
||||
walk(list(app.routes))
|
||||
return found
|
||||
|
||||
|
||||
def _bearer_protected(app: FastAPI) -> set[tuple[str, str]]:
|
||||
"""遍历依赖树,找出真的挂了 require_bearer_token 的 (path, method)
|
||||
|
||||
不按路径规律猜:以后哪条接口加/去掉鉴权,文档要跟着自动变。
|
||||
"""
|
||||
from app.shared.api import require_bearer_token
|
||||
|
||||
def uses_token(dependant: Any, seen: set[int]) -> bool:
|
||||
if id(dependant) in seen:
|
||||
return False
|
||||
seen.add(id(dependant))
|
||||
if getattr(dependant, "call", None) is require_bearer_token:
|
||||
return True
|
||||
return any(uses_token(sub, seen) for sub in dependant.dependencies)
|
||||
|
||||
protected: set[tuple[str, str]] = set()
|
||||
for route in _api_routes(app):
|
||||
if uses_token(route.dependant, set()):
|
||||
protected.update((route.path, method.lower()) for method in route.methods)
|
||||
return protected
|
||||
|
||||
|
||||
def _rewrite_refs(node: Any, rename: dict[str, str]) -> Any:
|
||||
"""递归把 $ref 指向的 schema 名按 rename 表替换"""
|
||||
if isinstance(node, dict):
|
||||
result = {}
|
||||
for key, value in node.items():
|
||||
if key == "$ref" and isinstance(value, str):
|
||||
name = value.rsplit("/", 1)[-1]
|
||||
if value.startswith("#/components/schemas/") and name in rename:
|
||||
result[key] = f"#/components/schemas/{rename[name]}"
|
||||
continue
|
||||
result[key] = _rewrite_refs(value, rename)
|
||||
return result
|
||||
if isinstance(node, list):
|
||||
return [_rewrite_refs(item, rename) for item in node]
|
||||
return node
|
||||
|
||||
|
||||
def _merge_schemas(
|
||||
merged: dict[str, Any], incoming: dict[str, Any], service: Service
|
||||
) -> dict[str, str]:
|
||||
"""把一个服务的 schemas 并进总表,返回该服务需要的重命名表
|
||||
|
||||
同名同内容(如 ApiResponse 信封派生出的公共模型、HTTPValidationError)直接复用;
|
||||
同名不同内容才加服务前缀——三个服务的模型确实可能撞名,但不能让后来者悄悄覆盖前者。
|
||||
"""
|
||||
rename: dict[str, str] = {}
|
||||
for name, schema in incoming.items():
|
||||
if name not in merged:
|
||||
merged[name] = schema
|
||||
continue
|
||||
if merged[name] == schema:
|
||||
continue
|
||||
rename[name] = f"{service.key.capitalize()}{name}"
|
||||
for original, renamed in rename.items():
|
||||
merged[renamed] = incoming[original]
|
||||
return rename
|
||||
|
||||
|
||||
def _response_schemas(operation: dict[str, Any]) -> dict[str, Any]:
|
||||
"""取 200 响应的 JSON schema(没有则空 dict)"""
|
||||
content = operation.get("responses", {}).get("200", {}).get("content", {})
|
||||
return content.get("application/json", {}).get("schema", {}) or {}
|
||||
|
||||
|
||||
def _merge_same_path(existing: dict[str, Any], incoming: dict[str, Any]) -> None:
|
||||
"""同路径同方法(只有 /health):并 servers、并响应 schema、拼描述"""
|
||||
for server in incoming.get("servers", []):
|
||||
if server not in existing.setdefault("servers", []):
|
||||
existing["servers"].append(server)
|
||||
|
||||
existing_schema = _response_schemas(existing)
|
||||
incoming_schema = _response_schemas(incoming)
|
||||
if existing_schema and incoming_schema and existing_schema != incoming_schema:
|
||||
options = existing_schema.get("anyOf", [existing_schema])
|
||||
if incoming_schema not in options:
|
||||
options = [*options, incoming_schema]
|
||||
existing["responses"]["200"]["content"]["application/json"]["schema"] = {
|
||||
"anyOf": options,
|
||||
"title": "各服务的健康检查响应",
|
||||
}
|
||||
|
||||
incoming_description = incoming.get("description", "").strip()
|
||||
if incoming_description and incoming_description not in existing.get("description", ""):
|
||||
existing["description"] = (
|
||||
f"{existing.get('description', '').rstrip()}\n\n---\n\n{incoming_description}"
|
||||
)
|
||||
|
||||
|
||||
def build_spec() -> dict[str, Any]:
|
||||
"""合并三个服务的 OpenAPI 文档"""
|
||||
paths: dict[str, Any] = {}
|
||||
schemas: dict[str, Any] = {}
|
||||
tags: list[dict[str, str]] = []
|
||||
|
||||
for service in SERVICES:
|
||||
app = _create_app(service)
|
||||
spec = copy.deepcopy(app.openapi())
|
||||
protected = _bearer_protected(app)
|
||||
|
||||
rename = _merge_schemas(
|
||||
schemas, spec.get("components", {}).get("schemas", {}), service
|
||||
)
|
||||
service_paths = _rewrite_refs(spec.get("paths", {}), rename)
|
||||
|
||||
# 鉴权是从路由依赖树里认出来的,路径对不上就等于漏标——宁可构建失败,
|
||||
# 也不要导出一份「看起来不需要 token」的文档
|
||||
unmatched = sorted(
|
||||
f"{method.upper()} {path}"
|
||||
for path, method in protected
|
||||
if method not in service_paths.get(path, {})
|
||||
)
|
||||
if unmatched:
|
||||
raise RuntimeError(
|
||||
f"{service.key}: 这些需要鉴权的路由在 OpenAPI 里找不到对应 operation:"
|
||||
f"{unmatched}(include_router 加了 prefix?)"
|
||||
)
|
||||
|
||||
for path, path_item in service_paths.items():
|
||||
for method, operation in path_item.items():
|
||||
if method not in _METHODS:
|
||||
continue
|
||||
operation["servers"] = [{"url": service.url, "description": service.label}]
|
||||
# operationId 必须全局唯一:三个服务的 /health 生成的都是 health_health_get
|
||||
operation["operationId"] = f"{service.key}_{operation.get('operationId', method)}"
|
||||
raw_tags = operation.get("tags") or ["default"]
|
||||
operation["tags"] = [f"{service.label}/{tag}" for tag in raw_tags]
|
||||
for tag in operation["tags"]:
|
||||
if all(item["name"] != tag for item in tags):
|
||||
tags.append({"name": tag, "description": f"{service.label}({service.url})"})
|
||||
# Apifox 目录:与 tag 一致,导入后直接按服务分组
|
||||
operation["x-apifox-folder"] = operation["tags"][0]
|
||||
if (path, method) in protected:
|
||||
operation["security"] = [{_SECURITY_SCHEME: []}]
|
||||
|
||||
if path not in paths:
|
||||
paths[path] = path_item
|
||||
continue
|
||||
for method, operation in path_item.items():
|
||||
if method in paths[path]:
|
||||
_merge_same_path(paths[path][method], operation)
|
||||
else:
|
||||
paths[path][method] = operation
|
||||
|
||||
return {
|
||||
"openapi": "3.1.0",
|
||||
"info": {
|
||||
"title": "Rakuten API(抓取 / 交易 / 下单网关)",
|
||||
"version": "0.1.0",
|
||||
"description": _INFO_DESCRIPTION,
|
||||
"x-apifox-folder": "Rakuten",
|
||||
},
|
||||
"servers": [
|
||||
{"url": service.url, "description": f"{service.label} :{service.port}"}
|
||||
for service in SERVICES
|
||||
],
|
||||
"tags": tags,
|
||||
"paths": paths,
|
||||
"components": {
|
||||
"schemas": schemas,
|
||||
"securitySchemes": {
|
||||
_SECURITY_SCHEME: {
|
||||
"type": "http",
|
||||
"scheme": "bearer",
|
||||
"description": "值取配置项 RAKUTEN_BEARER_TOKEN;三个服务共用同一个 token",
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def dump(spec: dict[str, Any]) -> str:
|
||||
"""固定序列化形式,便于 --check 直接比字符串"""
|
||||
return json.dumps(spec, ensure_ascii=False, indent=2, sort_keys=False) + "\n"
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser(description="导出合并后的 openapi.json")
|
||||
parser.add_argument(
|
||||
"--check",
|
||||
action="store_true",
|
||||
help="只校验 openapi.json 是否与当前代码一致,不写文件;不一致时退出码 1",
|
||||
)
|
||||
parser.add_argument("--output", default=str(OUTPUT_PATH))
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
content = dump(build_spec())
|
||||
output = Path(args.output)
|
||||
|
||||
if args.check:
|
||||
current = output.read_text(encoding="utf-8") if output.exists() else ""
|
||||
if current == content:
|
||||
print(f"openapi.json 已是最新:{output}")
|
||||
return 0
|
||||
print(
|
||||
f"openapi.json 与当前代码不一致:{output}\n"
|
||||
"请重新导出:.venv/Scripts/python.exe scripts/export_openapi.py",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
output.write_text(content, encoding="utf-8")
|
||||
spec = json.loads(content)
|
||||
print(f"已写入 {output}")
|
||||
print(f"接口数:{sum(1 for item in spec['paths'].values() for _ in item)}(路径 {len(spec['paths'])} 条)")
|
||||
for path in spec["paths"]:
|
||||
methods = [m.upper() for m in spec["paths"][path] if m in _METHODS]
|
||||
print(f" {','.join(methods):6} {path}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,157 @@
|
||||
"""收货地址确认页真实 HTML 探针
|
||||
|
||||
走 SiteInteractor 的真实生产代码路径(add_to_cart → verify_cart → enter_checkout),
|
||||
在 _confirm_default_address 被调用前把该页 HTML 落盘,验证/纠正选择器是否与真实页面
|
||||
一致。**不调用 submit_order / pay**——enter_checkout 落地到确认页就返回,这里直接
|
||||
丢弃该页,不产生下单动作。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_address_confirm.py
|
||||
.venv/Scripts/python.exe scripts/probe_address_confirm.py --item-url https://item.rakuten.co.jp/...
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.services.auth_session import AuthSession # noqa: E402
|
||||
from app.trading.worker.models import LeaseTask # noqa: E402
|
||||
from app.trading.worker.site_interact import SiteInteractor # noqa: E402
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "address_confirm"
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# fafachai/10000033 已下架/不可购买;改用 2026-08-13 真实下单时用过的商品
|
||||
# (data/evidence/checkout-live-20260813/i00-cart-initial.html 里的 itemUrl)
|
||||
DEFAULT_ITEM_URL = "https://item.rakuten.co.jp/moccasin/ds001iwrgesaaa2/"
|
||||
|
||||
|
||||
async def _start_visible(site: SiteInteractor, settings) -> None:
|
||||
"""复制 SiteInteractor.start() 但 headless=False,供本地诊断肉眼看浏览器
|
||||
|
||||
不改动生产代码(生产 start() 硬编码 headless=True),直接绕过它、把结果
|
||||
塞进 site 的私有属性——只在这个一次性诊断脚本里这么干。
|
||||
"""
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from app.trading.core import auth_site
|
||||
|
||||
state_path = settings.auth_state_path / auth_site.profile("rakuten").state_filename
|
||||
storage_state = str(state_path) if state_path.exists() else None
|
||||
|
||||
site._playwright = await async_playwright().start()
|
||||
site._browser = await site._playwright.chromium.launch(
|
||||
headless=False,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
site._context = await site._browser.new_context(
|
||||
storage_state=storage_state,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
print(f"[visible] 浏览器已启动(headless=False),storage_state={storage_state or '(none)'}")
|
||||
|
||||
|
||||
async def run(item_url: str) -> int:
|
||||
settings = get_settings()
|
||||
auth = AuthSession(settings)
|
||||
await auth.start()
|
||||
site = SiteInteractor(auth_session=auth, settings=settings)
|
||||
await _start_visible(site, settings)
|
||||
|
||||
captured: dict[str, str] = {}
|
||||
orig_confirm = site._confirm_default_address
|
||||
|
||||
async def spy_confirm(page, *, task_id: str) -> None:
|
||||
html = await page.content()
|
||||
captured["html"] = html
|
||||
captured["url"] = page.url
|
||||
path = PROBE_DIR / "address-confirm-page.html"
|
||||
path.write_text(html, encoding="utf-8")
|
||||
print(f"[捕获] 收货地址确认页 -> {path} ({len(html)} bytes), url={page.url}")
|
||||
await orig_confirm(page, task_id=task_id)
|
||||
|
||||
site._confirm_default_address = spy_confirm
|
||||
|
||||
orig_session_upgrade = site._complete_session_upgrade
|
||||
|
||||
async def spy_session_upgrade(page, *, task_id: str) -> None:
|
||||
html = await page.content()
|
||||
path = PROBE_DIR / "session-upgrade-before.html"
|
||||
path.write_text(html, encoding="utf-8")
|
||||
print(f"[捕获] session upgrade 页(点提交前)-> {path} ({len(html)} bytes), url={page.url}")
|
||||
try:
|
||||
await orig_session_upgrade(page, task_id=task_id)
|
||||
finally:
|
||||
html_after = await page.content()
|
||||
(PROBE_DIR / "session-upgrade-after.html").write_text(html_after, encoding="utf-8")
|
||||
|
||||
site._complete_session_upgrade = spy_session_upgrade
|
||||
|
||||
task = LeaseTask(
|
||||
task_id="probe-addr-1",
|
||||
site="rakuten",
|
||||
intent={"item_url": item_url, "quantity": 1},
|
||||
)
|
||||
|
||||
try:
|
||||
print("=== add_to_cart ===")
|
||||
cart_add_html = await site.add_to_cart(task)
|
||||
if cart_add_html:
|
||||
(PROBE_DIR / "01-cart-add.html").write_text(cart_add_html, encoding="utf-8")
|
||||
print(f" cart-add 页已存盘 ({len(cart_add_html)} bytes)")
|
||||
print("=== verify_cart ===")
|
||||
verify_html = await site.verify_cart(task)
|
||||
if verify_html:
|
||||
(PROBE_DIR / "02-cart-verify.html").write_text(verify_html, encoding="utf-8")
|
||||
print(f" cart-verify 页已存盘 ({len(verify_html)} bytes)")
|
||||
print("=== enter_checkout(落地到确认页或撞上未捕获的中间步骤为止)===")
|
||||
try:
|
||||
html = await site.enter_checkout(task)
|
||||
print(f"enter_checkout 返回,长度={len(html)}")
|
||||
(PROBE_DIR / "03-enter-checkout-final.html").write_text(html, encoding="utf-8")
|
||||
except Exception as exc:
|
||||
print(f"enter_checkout 抛出:{type(exc).__name__}: {exc}")
|
||||
finally:
|
||||
# 不调用 submit_order/pay;enter_checkout 若成功会把 page 留存在
|
||||
# _checkout_pages,这里直接丢弃,不产生下单动作
|
||||
page = site._checkout_pages.pop(task.task_id, None)
|
||||
if page is not None:
|
||||
print(f" 最终落地 url={page.url}")
|
||||
await page.close()
|
||||
|
||||
if "html" not in captured:
|
||||
print("\n没有触发 _confirm_default_address——账号可能没走到这一步"
|
||||
"(比如没有电话补录/地址确认步骤,或选择器判断条件本身有问题)")
|
||||
finally:
|
||||
try:
|
||||
await site.clear_cart()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"清理购物车失败(忽略):{type(exc).__name__}: {exc}")
|
||||
await site.close()
|
||||
await auth.close()
|
||||
|
||||
print(f"\n探针输出目录:{PROBE_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--item-url", default=DEFAULT_ITEM_URL)
|
||||
args = parser.parse_args()
|
||||
return await run(args.item_url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,126 @@
|
||||
"""cart count API 探针:实测「空购物车」与「获取失败」两种响应形态
|
||||
|
||||
背景:_query_cart_count 目前只认 status=="100",其余一律 CartOperationError。
|
||||
需要真账号确认:购物车为空时 cart-api.step.rakuten.co.jp 到底返回什么——
|
||||
如果空车响应与「请求被拒」长的不一样却被同一错误吞掉,就无法区分
|
||||
「车是空的」与「获取失败」。
|
||||
|
||||
只读探测,不做任何加购/删除操作。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_cart_count.py
|
||||
.venv/Scripts/python.exe scripts/probe_cart_count.py --headful
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.core import auth_site # noqa: E402
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "cart_count"
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
CART_COUNT_API = "https://cart-api.step.rakuten.co.jp/rms/mall/cart/count/all/jsonp/"
|
||||
CART_PAGE = auth_site.RAKUTEN_CART_URL
|
||||
|
||||
|
||||
def save(name: str, content: str) -> Path:
|
||||
path = PROBE_DIR / name
|
||||
path.write_text(content, encoding="utf-8")
|
||||
print(f" saved -> {path} ({len(content)} bytes)")
|
||||
return path
|
||||
|
||||
|
||||
async def call_count_api(context, *, referer: str | None, tag: str) -> None:
|
||||
headers = {"Referer": referer} if referer else {}
|
||||
resp = await context.request.get(CART_COUNT_API + "?sid=1010", headers=headers)
|
||||
body = await resp.text()
|
||||
print(f" [{tag}] http_status={resp.status} body={body!r}")
|
||||
save(f"{tag}.txt", f"http_status={resp.status}\n{body}")
|
||||
|
||||
|
||||
async def run(*, headful: bool) -> int:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
settings = get_settings()
|
||||
state_path = settings.auth_state_path / "rakuten_state.json"
|
||||
if not state_path.exists():
|
||||
print(f"找不到登录态文件:{state_path}")
|
||||
return 1
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=not headful,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
storage_state=state_path,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
|
||||
print("=== 1. 带正确 Referer 调 cart count API(生产路径)===")
|
||||
await call_count_api(context, referer=CART_PAGE, tag="10-with-referer")
|
||||
|
||||
print("=== 2. 不带 Referer 调 cart count API(观测「被拒」形态)===")
|
||||
await call_count_api(context, referer=None, tag="11-no-referer")
|
||||
|
||||
print("=== 3. 错误 sid 调 cart count API(另一种异常入参)===")
|
||||
resp = await context.request.get(
|
||||
CART_COUNT_API + "?sid=999999", headers={"Referer": CART_PAGE}
|
||||
)
|
||||
body = await resp.text()
|
||||
print(f" [bad-sid] http_status={resp.status} body={body!r}")
|
||||
save("12-bad-sid.txt", f"http_status={resp.status}\n{body}")
|
||||
|
||||
print("=== 4. 打开购物车页,看当前车状态(只读)===")
|
||||
page = await context.new_page()
|
||||
await page.goto(CART_PAGE, wait_until="domcontentloaded", timeout=30_000)
|
||||
await page.wait_for_timeout(5000)
|
||||
state = await page.evaluate(
|
||||
"""() => {
|
||||
const s = window.__INITIAL_STATE__;
|
||||
if (!s || !s.cart) return {has_state: !!s, keys: s ? Object.keys(s) : []};
|
||||
return {
|
||||
has_state: true,
|
||||
cart_keys: Object.keys(s.cart),
|
||||
shopUrlList_len: Array.isArray(s.cart.shopUrlList) ? s.cart.shopUrlList.length : null,
|
||||
};
|
||||
}"""
|
||||
)
|
||||
print(f" __INITIAL_STATE__.cart: {state}")
|
||||
html = await page.content()
|
||||
save("20-cart-page.html", html)
|
||||
delete_btns = await page.locator('button[aria-label^="削除"]').count()
|
||||
print(f" 削除按钮数={delete_btns} final_url={page.url}")
|
||||
|
||||
print("=== 5. 车页打开后再调一次 count API(贴近 clear_cart 末尾场景)===")
|
||||
await call_count_api(context, referer=CART_PAGE, tag="21-after-cart-page")
|
||||
|
||||
await browser.close()
|
||||
|
||||
print(f"\n探针输出目录:{PROBE_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--headful", action="store_true")
|
||||
args = parser.parse_args()
|
||||
return await run(headful=args.headful)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -24,7 +24,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
import httpx
|
||||
|
||||
from app.shared.config import get_settings
|
||||
from app.shared.config import Settings, get_settings
|
||||
from app.shared.proxy import httpx_client_options
|
||||
from app.trading.core import auth_site
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "checkout"
|
||||
@@ -42,12 +43,13 @@ def load_cookies(state_path: Path) -> list[dict]:
|
||||
return state.get("cookies", [])
|
||||
|
||||
|
||||
def build_client(cookies: list[dict]) -> httpx.AsyncClient:
|
||||
def build_client(cookies: list[dict], settings: Settings) -> httpx.AsyncClient:
|
||||
client = httpx.AsyncClient(
|
||||
headers=auth_site.PROFILES["rakuten"].headers(),
|
||||
timeout=30.0,
|
||||
follow_redirects=True,
|
||||
http2=True,
|
||||
**httpx_client_options(settings),
|
||||
)
|
||||
for cookie in cookies:
|
||||
name = cookie.get("name")
|
||||
@@ -210,7 +212,7 @@ async def main() -> int:
|
||||
cookies = load_cookies(state_path)
|
||||
print(f"加载 {len(cookies)} 条 cookie 自 {state_path}")
|
||||
|
||||
async with build_client(cookies) as client:
|
||||
async with build_client(cookies, settings) as client:
|
||||
await probe_readonly(client)
|
||||
if args.mutate:
|
||||
if not args.item_url:
|
||||
|
||||
@@ -56,6 +56,7 @@ async def run(item_url: str, *, headful: bool) -> int:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=not headful,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
|
||||
@@ -0,0 +1,331 @@
|
||||
"""新卡代填 + 自动提交 真实站点探针
|
||||
|
||||
_fill_new_card_form / _submit_new_card_form 目前只有 2026-08-13 人工手填过一次
|
||||
(OS 级 SendKeys,非 Playwright .fill()),2026-08-14 新加的自动提交代码
|
||||
(_submit_new_card_form)从未跑过真实站点。当前账号已有一张匹配 account.yaml
|
||||
配置的已保存卡,_select_payment_method 走的是「直接点次へ」分支,不会自然触发
|
||||
新卡代填——本探针绕开这个判断,强制走「新しいカードを追加する」表单,直接调用
|
||||
生产代码 _fill_new_card_form + _submit_new_card_form,验证三个 iframe 字段选择器
|
||||
与自动提交回显判定在真实站点上是否成立。
|
||||
|
||||
**用的是 account.yaml 里配置的同一张卡号**,不是别的卡;预期效果是在账号已保存卡
|
||||
列表里多一条记录(不产生额外扣款/开卡费)。**不调用 submit_order / pay**——
|
||||
验证完成后直接丢弃 page,不产生下单动作。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_new_card.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.services import login_runner # noqa: E402
|
||||
from app.trading.services.auth_session import AuthSession # noqa: E402
|
||||
from app.trading.worker.models import LeaseTask # noqa: E402
|
||||
from app.trading.worker.site_interact import SiteInteractor # noqa: E402
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "new_card"
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
DEFAULT_ITEM_URL = "https://item.rakuten.co.jp/moccasin/ds001iwrgesaaa2/"
|
||||
|
||||
|
||||
async def _start_visible(site: SiteInteractor, settings) -> None:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from app.trading.core import auth_site
|
||||
|
||||
state_path = settings.auth_state_path / auth_site.profile("rakuten").state_filename
|
||||
storage_state = str(state_path) if state_path.exists() else None
|
||||
|
||||
site._playwright = await async_playwright().start()
|
||||
site._browser = await site._playwright.chromium.launch(
|
||||
headless=False,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
site._context = await site._browser.new_context(
|
||||
storage_state=storage_state,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
print(f"[visible] 浏览器已启动(headless=False),storage_state={storage_state or '(none)'}")
|
||||
|
||||
|
||||
async def run(item_url: str) -> int:
|
||||
settings = get_settings()
|
||||
accounts_by_site = login_runner.load_accounts()
|
||||
account = login_runner.default_account_for("rakuten", accounts_by_site)
|
||||
if account is None or account.credit_card is None:
|
||||
print("account.yaml 未配置 rakuten 默认账号的 credit_card,无法代填,中止")
|
||||
return 1
|
||||
card = account.credit_card
|
||||
print(f"将代填卡:尾号 {card.number[-4:]},有效期 {card.month}/{card.year}")
|
||||
|
||||
auth = AuthSession(settings)
|
||||
await auth.start()
|
||||
site = SiteInteractor(auth_session=auth, settings=settings)
|
||||
await _start_visible(site, settings)
|
||||
|
||||
task = LeaseTask(
|
||||
task_id="probe-new-card-1",
|
||||
site="rakuten",
|
||||
intent={"item_url": item_url, "quantity": 1},
|
||||
)
|
||||
|
||||
try:
|
||||
print("=== add_to_cart ===")
|
||||
await site.add_to_cart(task)
|
||||
print("=== verify_cart ===")
|
||||
await site.verify_cart(task)
|
||||
print("=== enter_checkout(预期直落 order-confirmation)===")
|
||||
try:
|
||||
await site.enter_checkout(task)
|
||||
except Exception as exc:
|
||||
print(f"enter_checkout 抛出:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
|
||||
page = site._checkout_pages.get(task.task_id)
|
||||
if page is None:
|
||||
print("enter_checkout 没有留存 page,无法继续")
|
||||
return 1
|
||||
print(f" 落地 url={page.url}")
|
||||
|
||||
print('=== 点击「支払い方法」区块「変更」,跳回 /payment ===')
|
||||
try:
|
||||
change_btn = page.locator('text="支払い方法"').first.locator(
|
||||
'xpath=following::button[@aria-label="変更"][1]'
|
||||
)
|
||||
await change_btn.click(timeout=10_000)
|
||||
await page.wait_for_timeout(2_000)
|
||||
except Exception as exc:
|
||||
print(f"点击「変更」失败:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
print(f" 跳转后 url={page.url}")
|
||||
|
||||
label = settings.order_payment_method
|
||||
print(f'=== 选中支付方式「{label}」===')
|
||||
try:
|
||||
option = page.locator(f'text="{label}"').first
|
||||
await option.click(timeout=10_000)
|
||||
await page.wait_for_timeout(1_000)
|
||||
except Exception as exc:
|
||||
print(f"选中「{label}」失败:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
|
||||
(PROBE_DIR / "01-payment-page-before-add-card.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
await page.screenshot(
|
||||
path=str(PROBE_DIR / "01-after-select-label.png"), full_page=True
|
||||
)
|
||||
|
||||
print(f'=== 截图显示行内有折叠箭头,再点一次「{label}」尝试展开该行 ===')
|
||||
try:
|
||||
await option.click(timeout=10_000)
|
||||
await page.wait_for_timeout(1_000)
|
||||
except Exception as exc:
|
||||
print(f"第二次点击「{label}」失败:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
await page.screenshot(
|
||||
path=str(PROBE_DIR / "01b-after-second-click.png"), full_page=True
|
||||
)
|
||||
is_visible_now = await page.locator(
|
||||
'text="新しいカードを追加する"'
|
||||
).first.is_visible()
|
||||
print(f" 第二次点击后「新しいカードを追加する」可见={is_visible_now}")
|
||||
|
||||
print('=== 诊断「新しいカードを追加する」有几个匹配、哪个可见 ===')
|
||||
from app.trading.worker.site_interact import _ADD_CARD_LINK_TEXT
|
||||
|
||||
candidates = page.locator(f'text="{_ADD_CARD_LINK_TEXT}"')
|
||||
count = await candidates.count()
|
||||
print(f" 匹配数量={count}")
|
||||
visible_index = None
|
||||
for i in range(count):
|
||||
el = candidates.nth(i)
|
||||
is_visible = await el.is_visible()
|
||||
print(f" [{i}] visible={is_visible}")
|
||||
if is_visible and visible_index is None:
|
||||
visible_index = i
|
||||
|
||||
if visible_index is None:
|
||||
print(" 没有任何一个匹配是可见的,诊断祖先节点的可见性状态")
|
||||
(PROBE_DIR / "02-no-visible-add-card-link.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
diag = await candidates.first.evaluate(
|
||||
"""el => {
|
||||
const out = [];
|
||||
let node = el;
|
||||
for (let depth = 0; depth < 12 && node; depth++) {
|
||||
const cs = getComputedStyle(node);
|
||||
const rect = node.getBoundingClientRect();
|
||||
out.push({
|
||||
depth,
|
||||
tag: node.tagName,
|
||||
cls: (node.className || '').toString().slice(0, 80),
|
||||
display: cs.display,
|
||||
visibility: cs.visibility,
|
||||
opacity: cs.opacity,
|
||||
height: rect.height,
|
||||
width: rect.width,
|
||||
});
|
||||
node = node.parentElement;
|
||||
}
|
||||
return out;
|
||||
}"""
|
||||
)
|
||||
for row in diag:
|
||||
print(f" depth={row['depth']} tag={row['tag']} cls={row['cls']!r} "
|
||||
f"display={row['display']} visibility={row['visibility']} "
|
||||
f"opacity={row['opacity']} size={row['width']}x{row['height']}")
|
||||
await page.screenshot(
|
||||
path=str(PROBE_DIR / "02-no-visible-add-card-link.png"), full_page=True
|
||||
)
|
||||
print(f" 截图已存盘:{PROBE_DIR / '02-no-visible-add-card-link.png'}")
|
||||
return 1
|
||||
|
||||
print(f" 将点击可见的第 [{visible_index}] 个匹配")
|
||||
try:
|
||||
await candidates.nth(visible_index).click(timeout=10_000)
|
||||
await page.wait_for_timeout(1_000)
|
||||
except Exception as exc:
|
||||
print(f"点击「{_ADD_CARD_LINK_TEXT}」失败:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
|
||||
print("=== 强制走新卡代填:直接调用三个 iframe 字段 + 名義人代填(跳过 _fill_new_card_form 内部的点击)===")
|
||||
try:
|
||||
month = str(card.month).zfill(2)
|
||||
year = str(card.year)
|
||||
from app.trading.worker.site_interact import (
|
||||
_CARD_MONTH_MOUNT_SELECTOR,
|
||||
_CARD_NAME_LABEL_TEXT,
|
||||
_CARD_NUMBER_MOUNT_SELECTOR,
|
||||
_CARD_YEAR_MOUNT_SELECTOR,
|
||||
)
|
||||
|
||||
async def _fill_frame(mount_selector: str, value: str, label_: str) -> None:
|
||||
frame = page.frame_locator(mount_selector)
|
||||
field = frame.locator("input, select").first
|
||||
await field.wait_for(state="visible", timeout=8_000)
|
||||
tag = await field.evaluate("el => el.tagName.toLowerCase()")
|
||||
if tag == "select":
|
||||
await field.select_option(value)
|
||||
else:
|
||||
await field.fill(value)
|
||||
print(f" 已填 {label_}")
|
||||
|
||||
await _fill_frame(_CARD_NUMBER_MOUNT_SELECTOR, card.number, "卡号")
|
||||
await _fill_frame(_CARD_MONTH_MOUNT_SELECTOR, month, "有効期限(月)")
|
||||
await _fill_frame(_CARD_YEAR_MOUNT_SELECTOR, year, "有効期限(年)")
|
||||
name_locator = page.locator(f'text="{_CARD_NAME_LABEL_TEXT}"').locator(
|
||||
"xpath=following::input[1]"
|
||||
)
|
||||
await name_locator.wait_for(state="visible", timeout=8_000)
|
||||
await name_locator.fill(card.name)
|
||||
print(" 已填 名義人")
|
||||
# 上一轮实测:名義人 input 填完后焦点仍在它上面,它所在的
|
||||
# aria-modal="true" 弹层拦截了后续对「追加する」提交按钮的点击
|
||||
# (pointer events intercepted)。blur 掉这个 input 再继续。
|
||||
await name_locator.evaluate("el => el.blur()")
|
||||
await page.wait_for_timeout(300)
|
||||
print(" 已 blur 名義人 input")
|
||||
print(" _fill_new_card_form 等效逻辑成功返回")
|
||||
except Exception as exc:
|
||||
print(f"新卡代填抛出:{type(exc).__name__}: {exc}")
|
||||
(PROBE_DIR / "02-fill-new-card-error.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
return 1
|
||||
|
||||
(PROBE_DIR / "02-after-fill-new-card.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
|
||||
print('=== 诊断「追加する」按钮有几个匹配、哪个真正可点 ===')
|
||||
from app.trading.worker.site_interact import (
|
||||
_ADD_CARD_SUBMIT_TEXT,
|
||||
_SAVED_CARD_SIGNAL_PATTERN,
|
||||
)
|
||||
|
||||
submit_candidates = page.locator(f'button:has-text("{_ADD_CARD_SUBMIT_TEXT}")')
|
||||
submit_count = await submit_candidates.count()
|
||||
print(f" 匹配数量={submit_count}")
|
||||
clicked = False
|
||||
for i in range(submit_count):
|
||||
el = submit_candidates.nth(i)
|
||||
is_visible = await el.is_visible()
|
||||
print(f" [{i}] visible={is_visible}")
|
||||
if not is_visible:
|
||||
continue
|
||||
try:
|
||||
await el.click(timeout=5_000)
|
||||
print(f" 成功点击第 [{i}] 个「{_ADD_CARD_SUBMIT_TEXT}」")
|
||||
clicked = True
|
||||
break
|
||||
except Exception as exc:
|
||||
print(f" 第 [{i}] 个点击失败(继续试下一个):{type(exc).__name__}: {exc}")
|
||||
|
||||
if not clicked:
|
||||
print(" 所有候选都点不中,中止")
|
||||
(PROBE_DIR / "03-submit-new-card-error.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
return 1
|
||||
|
||||
confirmed = False
|
||||
for _ in range(10):
|
||||
await page.wait_for_timeout(1_500)
|
||||
html_now = await page.content()
|
||||
if _SAVED_CARD_SIGNAL_PATTERN.search(html_now):
|
||||
confirmed = True
|
||||
break
|
||||
print(f" 提交后已保存卡信号确认={confirmed},当前 url={page.url}")
|
||||
if not confirmed:
|
||||
(PROBE_DIR / "03-submit-new-card-error.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
return 1
|
||||
|
||||
(PROBE_DIR / "03-after-submit-new-card.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
print("新卡代填 + 自动提交全链路验证完成,不继续点「次へ」/不调用 submit_order/pay")
|
||||
|
||||
finally:
|
||||
page = site._checkout_pages.pop(task.task_id, None)
|
||||
if page is not None:
|
||||
print(f" 丢弃 checkout page(不调用 submit_order/pay),最终 url={page.url}")
|
||||
await page.close()
|
||||
try:
|
||||
await site.clear_cart()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"清理购物车失败(忽略):{type(exc).__name__}: {exc}")
|
||||
await site.close()
|
||||
await auth.close()
|
||||
|
||||
print(f"\n探针输出目录:{PROBE_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--item-url", default=DEFAULT_ITEM_URL)
|
||||
args = parser.parse_args()
|
||||
return await run(args.item_url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,212 @@
|
||||
"""订单详情页探针:为账号只读查询通道(docs/order-gateway.md §11)取详情页真实 DOM 结构
|
||||
|
||||
目标:`fetch_order_detail` 目前只把详情页 `window.__INITIAL_STATE__` 原样透传
|
||||
(从未有过真实样本,只做「解析得动就带走」,见 site_interact.py::_parse_initial_state
|
||||
上方与 OrderDetailSnapshot 文档)。本探针用真账号登录态打开一条真实订单的详情页
|
||||
(`_ORDER_DETAIL_URL_TEMPLATE` 那条 `act=detail_page_view` 路径),把整页 HTML、
|
||||
以及解析出的 `__INITIAL_STATE__` JSON 都落盘,作为后续补规范化字段抽取的第一份
|
||||
真实样本。
|
||||
|
||||
只读导航,不做任何加购/下单操作。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_order_detail.py
|
||||
.venv/Scripts/python.exe scripts/probe_order_detail.py --headful
|
||||
.venv/Scripts/python.exe scripts/probe_order_detail.py --order 306087-20260813-0863947697
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
# Windows 控制台默认 GBK,打印含日文/波浪符(如 U+301C 〜)的中文会抛
|
||||
# UnicodeEncodeError——统一改 stdout 为 UTF-8,跟 save() 落盘的编码保持一致
|
||||
# (记忆 project://jp-rakuten/index 里「Git Bash 打中文 body 乱码」是同一问题的另一面)
|
||||
if hasattr(sys.stdout, "reconfigure"):
|
||||
try:
|
||||
sys.stdout.reconfigure(encoding="utf-8")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.core import auth_site # noqa: E402
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "order_detail"
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 与 site_interact.py::_ORDER_DETAIL_URL_TEMPLATE 保持一致
|
||||
ORDER_DETAIL_URL_TEMPLATE = (
|
||||
"https://order.my.rakuten.co.jp/purchase-history/"
|
||||
"?order_number={order_number}&shop_id={shop_id}&act=detail_page_view"
|
||||
)
|
||||
DEFAULT_ORDER_ID = "306087-20260813-0863947697"
|
||||
|
||||
|
||||
def save(name: str, content: str | bytes) -> Path:
|
||||
path = PROBE_DIR / name
|
||||
if isinstance(content, bytes):
|
||||
path.write_bytes(content)
|
||||
else:
|
||||
path.write_text(content, encoding="utf-8")
|
||||
print(f" saved -> {path} ({len(content)} bytes)")
|
||||
return path
|
||||
|
||||
|
||||
def parse_initial_state(html: str) -> dict | None:
|
||||
m = re.search(
|
||||
r"window\.__INITIAL_STATE__\s*=\s*(.+?);\s*window\.",
|
||||
html,
|
||||
re.DOTALL,
|
||||
)
|
||||
if not m:
|
||||
return None
|
||||
try:
|
||||
return json.loads(m.group(1))
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
|
||||
|
||||
async def run(*, headful: bool, order_id: str) -> int:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
settings = get_settings()
|
||||
state_path = settings.auth_state_path / "rakuten_state.json"
|
||||
if not state_path.exists():
|
||||
print(f"找不到登录态文件:{state_path}")
|
||||
return 1
|
||||
|
||||
shop_id = order_id.split("-", 1)[0]
|
||||
url = ORDER_DETAIL_URL_TEMPLATE.format(order_number=order_id, shop_id=shop_id)
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=not headful,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
storage_state=state_path,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
api_calls: list[tuple[int, str]] = []
|
||||
|
||||
def on_response(r):
|
||||
api_calls.append((r.status, str(r.url)))
|
||||
|
||||
page.on("response", on_response)
|
||||
|
||||
print(f"=== 访问订单详情页(order_id={order_id})===")
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30_000)
|
||||
await page.wait_for_timeout(3000)
|
||||
except Exception as exc:
|
||||
print(f"导航失败:{type(exc).__name__}: {exc}")
|
||||
|
||||
html = await page.content()
|
||||
final_url = page.url
|
||||
print(f" final_url={final_url} body_len={len(html)}")
|
||||
print(f" 含订单号 {order_id}: {order_id in html}")
|
||||
logged_out = auth_site.looks_logged_out(
|
||||
"rakuten", final_url=final_url, body=html
|
||||
)
|
||||
print(f" looks_logged_out={logged_out}")
|
||||
|
||||
save("00-detail-default.html", html)
|
||||
|
||||
state = parse_initial_state(html)
|
||||
if state is None:
|
||||
print(" !! 解析不到 __INITIAL_STATE__(可能 PC 模板 / 掉登录 / 反爬)")
|
||||
else:
|
||||
print(f" __INITIAL_STATE__ 顶层键:{sorted(state.keys())}")
|
||||
print(f" pageType={state.get('pageType')!r}")
|
||||
|
||||
# 打印与订单/阶段相关的子结构概览,方便人眼核对
|
||||
print("\n=== 与订单字段相关的线索 ===")
|
||||
flat_search = {}
|
||||
def walk(node, prefix=""):
|
||||
if isinstance(node, dict):
|
||||
for k, v in node.items():
|
||||
p = f"{prefix}.{k}" if prefix else k
|
||||
if isinstance(v, (dict, list)):
|
||||
walk(v, p)
|
||||
elif isinstance(v, (str, int, float)) and 0 < len(str(v)) <= 80:
|
||||
flat_search[p] = v
|
||||
elif isinstance(node, list):
|
||||
for i, v in enumerate(node[:5]):
|
||||
walk(v, f"{prefix}[{i}]")
|
||||
walk(state)
|
||||
for pat in ["order", "stage", "配送", "出荷", "配達", "金額", "amount",
|
||||
"address", "payment", "shop", "item", "status"]:
|
||||
hits = {k: v for k, v in flat_search.items() if pat in k.lower()}
|
||||
if hits:
|
||||
print(f" 键含 {pat!r}(前 15 条):")
|
||||
for k, v in list(hits.items())[:15]:
|
||||
print(f" {k} = {v!r}")
|
||||
|
||||
save("01-detail-initial-state.json", json.dumps(
|
||||
state, ensure_ascii=False, indent=2,
|
||||
))
|
||||
# 顶层平铺一份,方便快速看
|
||||
save("02-detail-initial-state-top.json", json.dumps(
|
||||
{k: (v if not isinstance(v, (dict, list)) else "…") for k, v in state.items()},
|
||||
ensure_ascii=False, indent=2,
|
||||
))
|
||||
|
||||
# 配送阶段进度条(_ORDER_STEPPER_ITEM_PATTERN 那套),确认详情页同样适用
|
||||
steps = re.findall(
|
||||
r'<li class="([^"]*)">.*?<div class="title--2uGVi">([^<]*)</div></li>',
|
||||
html,
|
||||
re.DOTALL,
|
||||
)
|
||||
print(f"\n=== 配送阶段进度条(_ORDER_STEPPER_ITEM_PATTERN)===")
|
||||
if steps:
|
||||
for cls, label in steps:
|
||||
print(f" stage={label!r} active={'-active--' in cls}")
|
||||
else:
|
||||
print(" (未匹配到进度条 li 结构)")
|
||||
|
||||
print("\n=== 捕获的 XHR(末 20 条)===")
|
||||
for status, u in api_calls[-20:]:
|
||||
print(f" {status} {u[:160]}")
|
||||
|
||||
# 截图留底(无头模式下可能是白屏/异常渲染,正好暴露反爬)
|
||||
try:
|
||||
await page.screenshot(path=str(PROBE_DIR / "03-detail-screenshot.png"), full_page=True)
|
||||
print(" 截图 ->", PROBE_DIR / "03-detail-screenshot.png")
|
||||
except Exception as exc:
|
||||
print(f" 截图失败:{type(exc).__name__}: {exc}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
print(f"\n探针输出目录:{PROBE_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--headful", action="store_true")
|
||||
parser.add_argument(
|
||||
"--order",
|
||||
default=DEFAULT_ORDER_ID,
|
||||
help="要探测的订单号(默认已知订单 2026-08-13 实测那单)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
return await run(headful=args.headful, order_id=args.order)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,131 @@
|
||||
"""订单列表页探针:为实现 verify_on_site(规格 §5 恢复核对)取真实 DOM 结构
|
||||
|
||||
只读导航,不做任何加购/下单操作。目标:确认 order.my.rakuten.co.jp 的订单列表页
|
||||
(不带 order_number/shop_id 参数)能否按「商品名 + 时间窗口」反查出某个任务是否已
|
||||
下单——这是 app/trading/worker/verify.py::verify_on_site 目前的桩要补的实测。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_order_list.py
|
||||
.venv/Scripts/python.exe scripts/probe_order_list.py --headful
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.core import auth_site # noqa: E402
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "order_list"
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ORDER_LIST_URL = "https://order.my.rakuten.co.jp/purchase-history/"
|
||||
KNOWN_ORDER_ID = "306087-20260813-0863947697"
|
||||
|
||||
|
||||
def save(name: str, content: str) -> Path:
|
||||
path = PROBE_DIR / name
|
||||
path.write_text(content, encoding="utf-8")
|
||||
print(f" saved -> {path} ({len(content)} bytes)")
|
||||
return path
|
||||
|
||||
|
||||
async def run(*, headful: bool) -> int:
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
settings = get_settings()
|
||||
state_path = settings.auth_state_path / "rakuten_state.json"
|
||||
if not state_path.exists():
|
||||
print(f"找不到登录态文件:{state_path}")
|
||||
return 1
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=not headful,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
storage_state=state_path,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
api_calls: list[tuple[int, str]] = []
|
||||
|
||||
def on_response(r):
|
||||
url = str(r.url)
|
||||
if any(kw in url for kw in ["purchase-history", "order", "history"]):
|
||||
api_calls.append((r.status, url))
|
||||
|
||||
page.on("response", on_response)
|
||||
|
||||
print("=== 访问订单列表页(不带参数)===")
|
||||
await page.goto(ORDER_LIST_URL, wait_until="domcontentloaded", timeout=30_000)
|
||||
await page.wait_for_timeout(3000)
|
||||
html = await page.content()
|
||||
save("00-list-default.html", html)
|
||||
print(f" final_url={page.url} body_len={len(html)}")
|
||||
print(f" 含已知订单号 {KNOWN_ORDER_ID}: {KNOWN_ORDER_ID in html}")
|
||||
|
||||
# 尝试常见的日期范围/分页查询参数,看服务端是否支持按时间窗口过滤
|
||||
candidate_qs = [
|
||||
"?period=3months",
|
||||
"?range=3",
|
||||
f"?order_number={KNOWN_ORDER_ID.split('-')[0]}",
|
||||
]
|
||||
for qs in candidate_qs:
|
||||
url = ORDER_LIST_URL + qs
|
||||
print(f"\n=== 尝试 {url} ===")
|
||||
try:
|
||||
resp = await page.goto(url, wait_until="domcontentloaded", timeout=20_000)
|
||||
await page.wait_for_timeout(2000)
|
||||
html2 = await page.content()
|
||||
status = resp.status if resp else None
|
||||
print(f" status={status} final_url={page.url} body_len={len(html2)}")
|
||||
safe_name = re.sub(r"[^\w]+", "_", qs) or "root"
|
||||
save(f"01{safe_name}.html", html2)
|
||||
except Exception as exc:
|
||||
print(f" 失败:{type(exc).__name__}: {exc}")
|
||||
|
||||
print("\n=== 捕获的 XHR ===")
|
||||
for status, url in api_calls[-30:]:
|
||||
print(f" {status} {url[:160]}")
|
||||
|
||||
# 在默认列表页上找商品名/日期相关的文本线索
|
||||
print("\n=== 默认列表页文本线索(商品名/日期候选片段)===")
|
||||
for pat in [
|
||||
r'"itemName"\s*:\s*"([^"]{2,60})"',
|
||||
r'"orderDate"\s*:\s*"([^"]{2,40})"',
|
||||
r'"orderNumber"\s*:\s*"([^"]{2,40})"',
|
||||
r"\d{4}[年/]\d{1,2}[月/]\d{1,2}日?",
|
||||
]:
|
||||
matches = re.findall(pat, html)
|
||||
print(f" pattern={pat!r} -> {matches[:5]}")
|
||||
|
||||
await browser.close()
|
||||
|
||||
print(f"\n探针输出目录:{PROBE_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--headful", action="store_true")
|
||||
args = parser.parse_args()
|
||||
return await run(headful=args.headful)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,161 @@
|
||||
"""支付方式选择页真实 HTML 探针
|
||||
|
||||
走真实 add_to_cart → verify_cart → enter_checkout 落地到 order-confirmation 后,
|
||||
该账号是「默认地址+默认卡」直落确认页,不会自然经过 /pay 选择页(与 /ship 地址
|
||||
确认页被跳过是同一原因)。本探针从确认页上真实存在的「支払い方法」区块「変更」
|
||||
按钮手动跳回选择页,在这一页上真实调用生产代码 SiteInteractor._select_payment_method,
|
||||
验证「已有匹配已保存卡 → 点『次へ』继续」这条分支在真实站点上是否成立
|
||||
(重点验证 _PAYMENT_NEXT_BUTTON_SELECTOR = 'button:has-text("次へ")' ——
|
||||
这跟 session upgrade 页那个已证实猜错的选择器是同一种写法,未经真实验证)。
|
||||
|
||||
**不调用 submit_order / pay**——validate 完成后直接丢弃 page,不产生下单动作。
|
||||
不主动触发「新しいカードを追加する」代填/提交分支(会在真实账号上注册一张新卡,
|
||||
需要单独决定是否执行)。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_payment_method.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.services.auth_session import AuthSession # noqa: E402
|
||||
from app.trading.worker.models import LeaseTask # noqa: E402
|
||||
from app.trading.worker.site_interact import SiteInteractor # noqa: E402
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "payment_method"
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
DEFAULT_ITEM_URL = "https://item.rakuten.co.jp/moccasin/ds001iwrgesaaa2/"
|
||||
|
||||
|
||||
async def _start_visible(site: SiteInteractor, settings) -> None:
|
||||
"""复制 SiteInteractor.start() 但 headless=False,供本地诊断肉眼看浏览器
|
||||
|
||||
生产 start() 本身已经改成 headless=False 了(2026-08-14 实测确认无头模式
|
||||
会让结算 SPA 表现异常),这里保留独立实现只是为了不依赖生产 start() 的
|
||||
内部签名,跟 probe_address_confirm.py 保持一致的写法。
|
||||
"""
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from app.trading.core import auth_site
|
||||
|
||||
state_path = settings.auth_state_path / auth_site.profile("rakuten").state_filename
|
||||
storage_state = str(state_path) if state_path.exists() else None
|
||||
|
||||
site._playwright = await async_playwright().start()
|
||||
site._browser = await site._playwright.chromium.launch(
|
||||
headless=False,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
site._context = await site._browser.new_context(
|
||||
storage_state=storage_state,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
print(f"[visible] 浏览器已启动(headless=False),storage_state={storage_state or '(none)'}")
|
||||
|
||||
|
||||
async def run(item_url: str) -> int:
|
||||
settings = get_settings()
|
||||
auth = AuthSession(settings)
|
||||
await auth.start()
|
||||
site = SiteInteractor(auth_session=auth, settings=settings)
|
||||
await _start_visible(site, settings)
|
||||
|
||||
task = LeaseTask(
|
||||
task_id="probe-payment-1",
|
||||
site="rakuten",
|
||||
intent={"item_url": item_url, "quantity": 1},
|
||||
)
|
||||
|
||||
try:
|
||||
print("=== add_to_cart ===")
|
||||
await site.add_to_cart(task)
|
||||
print("=== verify_cart ===")
|
||||
await site.verify_cart(task)
|
||||
print("=== enter_checkout(预期直落 order-confirmation)===")
|
||||
try:
|
||||
html = await site.enter_checkout(task)
|
||||
print(f"enter_checkout 返回,长度={len(html)}")
|
||||
except Exception as exc:
|
||||
print(f"enter_checkout 抛出:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
|
||||
page = site._checkout_pages.get(task.task_id)
|
||||
if page is None:
|
||||
print("enter_checkout 没有留存 page,无法继续")
|
||||
return 1
|
||||
print(f" 落地 url={page.url}")
|
||||
(PROBE_DIR / "01-order-confirmation.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
|
||||
print('=== 点击「支払い方法」区块的「変更」按钮,跳回支付方式选择页 ===')
|
||||
try:
|
||||
change_btn = page.locator('text="支払い方法"').first.locator(
|
||||
'xpath=following::button[@aria-label="変更"][1]'
|
||||
)
|
||||
await change_btn.click(timeout=10_000)
|
||||
await page.wait_for_timeout(2_000)
|
||||
except Exception as exc:
|
||||
print(f"点击「変更」失败:{type(exc).__name__}: {exc}")
|
||||
return 1
|
||||
print(f" 跳转后 url={page.url}")
|
||||
html_pay_page = await page.content()
|
||||
(PROBE_DIR / "02-payment-method-page.html").write_text(
|
||||
html_pay_page, encoding="utf-8"
|
||||
)
|
||||
print(f" 支付方式选择页已存盘 ({len(html_pay_page)} bytes)")
|
||||
|
||||
print("=== 真实调用 SiteInteractor._select_payment_method ===")
|
||||
try:
|
||||
await site._select_payment_method(page, task_id=task.task_id)
|
||||
print(f"_select_payment_method 成功返回,当前 url={page.url}")
|
||||
(PROBE_DIR / "03-after-select-payment-method.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
except Exception as exc:
|
||||
print(f"_select_payment_method 抛出:{type(exc).__name__}: {exc}")
|
||||
(PROBE_DIR / "03-select-payment-method-error.html").write_text(
|
||||
await page.content(), encoding="utf-8"
|
||||
)
|
||||
print(" 失败时页面 HTML 已存盘,供排查真实「次へ」按钮结构")
|
||||
|
||||
finally:
|
||||
page = site._checkout_pages.pop(task.task_id, None)
|
||||
if page is not None:
|
||||
print(f" 丢弃 checkout page(不调用 submit_order/pay),最终 url={page.url}")
|
||||
await page.close()
|
||||
try:
|
||||
await site.clear_cart()
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"清理购物车失败(忽略):{type(exc).__name__}: {exc}")
|
||||
await site.close()
|
||||
await auth.close()
|
||||
|
||||
print(f"\n探针输出目录:{PROBE_DIR}")
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--item-url", default=DEFAULT_ITEM_URL)
|
||||
args = parser.parse_args()
|
||||
return await run(args.item_url)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -164,7 +164,11 @@ async def main() -> int:
|
||||
log: list[str] = []
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(headless=True, args=["--no-first-run"])
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
storage_state=state_path,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
|
||||
@@ -179,7 +179,11 @@ async def main() -> int:
|
||||
log: list[str] = []
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(headless=True, args=["--no-first-run"])
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
storage_state=state_path,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
|
||||
@@ -121,6 +121,7 @@ async def main() -> int:
|
||||
browser = await pw.chromium.launch(
|
||||
headless=True,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
context = await browser.new_context(
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""搜索页抓取延迟探针:复现线上 ~5s 的慢请求,定位耗时阶段
|
||||
|
||||
测量:
|
||||
1. 首次抓取的 预热耗时 / 目标页耗时
|
||||
2. 同会话连续抓取的稳态耗时(预热复用是否生效)
|
||||
3. 对比「无预热直连」的耗时(验证 Akamai 限速行为是否变化)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.scraping.services.browser_fallback import BrowserFallback # noqa: E402
|
||||
from app.scraping.services.site_session import SiteSession # noqa: E402
|
||||
from app.scraping.core import site # noqa: E402
|
||||
|
||||
URL = "https://search.rakuten.co.jp/search/mall/kitty/?p=2"
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
settings = get_settings()
|
||||
print(f"proxy={bool(settings.proxy_server)} timeout={settings.request_timeout_seconds} "
|
||||
f"ttl={settings.session_ttl_seconds} attempts={settings.http_max_attempts}")
|
||||
|
||||
session = SiteSession(settings, BrowserFallback(settings))
|
||||
await session.start()
|
||||
profile = session._profiles["pc"]
|
||||
|
||||
# 1) 首次抓取(含预热)
|
||||
t0 = time.monotonic()
|
||||
await session._ensure_warm(profile)
|
||||
t_warm = time.monotonic() - t0
|
||||
print(f"[1] 预热: {t_warm:.2f}s cookies={sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES))}")
|
||||
|
||||
t0 = time.monotonic()
|
||||
page = await session.fetch_html(URL, mobile=False)
|
||||
t_first = time.monotonic() - t0
|
||||
print(f"[1] 首次抓取: {t_first:.2f}s html={len(page)}")
|
||||
|
||||
# 2) 稳态:同会话连抓 3 次
|
||||
for i in range(2, 5):
|
||||
t0 = time.monotonic()
|
||||
page = await session.fetch_html(URL, mobile=False)
|
||||
print(f"[{i}] 稳态抓取: {time.monotonic() - t0:.2f}s html={len(page)}")
|
||||
|
||||
await session.close()
|
||||
|
||||
# 3) 对照:全新客户端、无预热、直接打目标页(Akamai 限速基线)
|
||||
import httpx
|
||||
from app.shared.proxy import httpx_client_options
|
||||
|
||||
async with httpx.AsyncClient(
|
||||
headers=site.default_headers(mobile=False),
|
||||
timeout=settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
http2=True,
|
||||
**httpx_client_options(settings),
|
||||
) as cold:
|
||||
t0 = time.monotonic()
|
||||
resp = await cold.get(URL)
|
||||
print(f"[对照] 无预热直连: {time.monotonic() - t0:.2f}s status={resp.status_code} "
|
||||
f"html={len(resp.text)} cookies={sorted(c for c in resp.cookies.keys())}")
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,850 @@
|
||||
"""官方子站(books / brandavenue / biccamera)加购→下单链路探针
|
||||
|
||||
背景:主站 ichiba 的「加购→确认页→提交」已经用真账号跑通(见
|
||||
project://jp-rakuten/trading-split),但 [[cart_contract]] 记录的三个官方子站
|
||||
(楽天ブックス / Rakuten Fashion / ビックカメラ楽天市場店)只有**加购契约**的
|
||||
静态记录,checkout 这一段的选择器与流程从没在子站上真实走过。本探针把这段空白
|
||||
补上:每个子站自动挑一件便宜的在售商品,用真账号走到**下单确认页为止**。
|
||||
|
||||
**绝不提交订单**:全流程有两道闸——
|
||||
1. `_FORBIDDEN_BUTTON_TEXTS` 黑名单,任何候选按钮文案命中就不点、直接停;
|
||||
2. 一旦页面出现确认页特征(「注文を確定する」等),立即停手并落证据。
|
||||
所以本脚本不会产生订单,也不会扣款。它**会**产生的真实副作用:
|
||||
- 往真实账号的购物车里加商品(结束时尝试清空,清不掉的会在报告里点名);
|
||||
- 可能触发 session upgrade(用 account.yaml 里的密码自动复核,与生产同一条路径);
|
||||
- 在站点上留下订单草稿(未提交,站点侧一般随会话失效,不影响账号)。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/probe_subsite_checkout.py # 三个子站依次跑
|
||||
.venv/Scripts/python.exe scripts/probe_subsite_checkout.py --site books
|
||||
.venv/Scripts/python.exe scripts/probe_subsite_checkout.py --site biccamera \
|
||||
--item-url https://item.rakuten.co.jp/biccamera/4904530012150/
|
||||
|
||||
输出:.probe/subsite/<site>/NN-*.html|png 逐步快照 + report-<site>.json 结构化结论。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
import traceback
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.services.auth_session import AuthSession # noqa: E402
|
||||
from app.trading.worker.site_interact import ( # noqa: E402
|
||||
SiteInteractor,
|
||||
_parse_checkout_summary,
|
||||
)
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "subsite"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SubsiteSpec:
|
||||
"""一个官方子站的探测配置"""
|
||||
|
||||
name: str
|
||||
host: str # 商品页最终落地的域名,用来确认确实跳出了 item.rakuten.co.jp
|
||||
sid: str # 市场侧 shop_id,用于用搜索页自动挑商品
|
||||
max_price: int # 自动挑商品的价格上限(挑不到就抬价重试)
|
||||
# item.rakuten.co.jp 的 302 有时只把人送到子站首页而丢掉商品(brandavenue 实测),
|
||||
# 这时用商品编号直接拼子站 URL 兜底。upper_code=True 表示子站 URL 用大写编号。
|
||||
direct_url: str = ""
|
||||
upper_code: bool = False
|
||||
|
||||
|
||||
SUBSITES: dict[str, SubsiteSpec] = {
|
||||
"books": SubsiteSpec(
|
||||
name="books",
|
||||
host="books.rakuten.co.jp",
|
||||
sid="213310",
|
||||
max_price=1200,
|
||||
direct_url="https://books.rakuten.co.jp/rb/{code}/",
|
||||
),
|
||||
"brandavenue": SubsiteSpec(
|
||||
name="brandavenue",
|
||||
host="brandavenue.rakuten.co.jp",
|
||||
sid="279405",
|
||||
max_price=6000,
|
||||
direct_url="https://brandavenue.rakuten.co.jp/item/{code}/",
|
||||
upper_code=True,
|
||||
),
|
||||
"biccamera": SubsiteSpec(
|
||||
name="biccamera",
|
||||
host="biccamera.rakuten.co.jp",
|
||||
sid="269553",
|
||||
max_price=1200,
|
||||
direct_url="https://biccamera.rakuten.co.jp/item/{code}/",
|
||||
),
|
||||
}
|
||||
|
||||
_SEARCH_URL = "https://search.rakuten.co.jp/search/mall/-/?sid={sid}&max={max_price}&min=100"
|
||||
|
||||
# ---- 加购按钮候选(子站各自的文案未知,列一串按顺序试,命中哪条记在报告里)----
|
||||
_ADD_CART_TEXTS = (
|
||||
"カートに入れる",
|
||||
"カートに追加",
|
||||
"かごに追加",
|
||||
"買い物かごに入れる",
|
||||
"カートへ入れる",
|
||||
"レジに進む",
|
||||
)
|
||||
|
||||
# ---- 「继续下一步」候选按钮文案:从加购落地页一路点到确认页 ----
|
||||
_NEXT_STEP_TEXTS = (
|
||||
"購入手続き",
|
||||
"ご購入手続き",
|
||||
"購入手続きへ",
|
||||
"レジに進む",
|
||||
"注文手続きへ",
|
||||
"ご注文手続き",
|
||||
"次へ",
|
||||
"進む",
|
||||
)
|
||||
|
||||
# ---- 绝对不点的按钮:命中即停手。宁可少走一步,也不能提交订单 ----
|
||||
_FORBIDDEN_BUTTON_TEXTS = (
|
||||
"注文を確定",
|
||||
"ご注文を確定",
|
||||
"この内容で注文",
|
||||
"購入を確定",
|
||||
"注文する",
|
||||
"決済する",
|
||||
"支払う",
|
||||
"購入する",
|
||||
)
|
||||
|
||||
# ---- 已经走到下单确认页的判据 ----
|
||||
# **不能按 HTML 全文匹配文案**:楽天ブックス 的购物车页正文里就写着「注文画面の
|
||||
# 「注文を確定する」ボタンを押した時点で…」这样的注意事项,全文匹配会把购物车页
|
||||
# 误判成确认页(2026-08-14 books 探测实测踩到)。改成「页面上真实存在一个可见的
|
||||
# 确认按钮」或 URL 命中确认页路径两个判据。
|
||||
_CONFIRM_PAGE_URL_MARKERS = ("order-confirmation", "/step/confirm", "ConfirmOrder")
|
||||
|
||||
# 确认页判定专用的按钮文案:比 _FORBIDDEN_BUTTON_TEXTS 窄。黑名单里的「購入する」
|
||||
# 会命中商品页上的「Rakuten Fashionアプリで購入する」(装 App 引导),拿它判定
|
||||
# 确认页会把商品页误判成确认页(2026-08-14 brandavenue 实测)。判定要窄、
|
||||
# 点击闸门要宽,两者不能共用一份列表。
|
||||
_CONFIRM_BUTTON_TEXTS = ("注文を確定", "ご注文を確定", "この内容で注文", "購入を確定")
|
||||
|
||||
# session upgrade / 中间步骤判据:与生产 site_interact 保持一致的口径
|
||||
_SESSION_UPGRADE_URL_MARKERS = ("session/upgrade", "sign_in/password")
|
||||
|
||||
# 2026-08-14 biccamera 探测新发现的一步:密码复核通过后,SSO 会再要求同意
|
||||
# **该子站自己的**利用規約/プライバシーポリシー(URL 还是 session/upgrade,
|
||||
# 只是 hash 变成 #/agree/service)。这一步是账号级的协议同意,不是下单动作,
|
||||
# 默认**不自动点**,要显式加 --accept-subsite-terms 才代用户同意。
|
||||
_AGREE_STEP_URL_MARKER = "#/agree/"
|
||||
_AGREE_STEP_TEXT_MARKER = "ご利用いただくサービスに関するご確認"
|
||||
_PHONE_REGISTRATION_MARKER = "会員情報の追加登録"
|
||||
# 同一个「会員情報の追加登録」标题下还有另一种变体(2026-08-14 biccamera 实测,
|
||||
# hash #/profiling/1):必填项是**誕生日 + 性別**,页面明写「登録した情報は、
|
||||
# 変更できません」。这类不可撤销的个人信息不能由脚本代填,探针识别到就停手。
|
||||
_PROFILE_REGISTRATION_MARKERS = ("誕生日", "性別")
|
||||
_PROFILE_STEP_URL_MARKER = "#/profiling"
|
||||
_ADDRESS_STEP_URL_MARKER = "/ship"
|
||||
_PAYMENT_STEP_URL_MARKER = "/pay"
|
||||
|
||||
_MAX_HOPS = 8
|
||||
# 一个子站最多试几个候选商品(多规格商品可能整件都缺货,见 discover_candidates)
|
||||
_MAX_CANDIDATES = 4
|
||||
|
||||
|
||||
@dataclass
|
||||
class SiteReport:
|
||||
"""单个子站的探测结论(最终写成 report-<site>.json)"""
|
||||
|
||||
site: str
|
||||
item_url: str = ""
|
||||
item_price: int | None = None
|
||||
item_name: str = ""
|
||||
landing_host: str = ""
|
||||
add_cart_contract: dict[str, Any] = field(default_factory=dict)
|
||||
add_cart_button: str = ""
|
||||
add_cart_landing_url: str = ""
|
||||
add_cart_ok: bool = False
|
||||
cart_signals: dict[str, Any] = field(default_factory=dict)
|
||||
ichiba_cart_count: int | None = None
|
||||
hops: list[dict[str, str]] = field(default_factory=list)
|
||||
reached_confirm: bool = False
|
||||
confirm_buttons: list[str] = field(default_factory=list)
|
||||
payable_yen: int | None = None
|
||||
stopped_reason: str = ""
|
||||
errors: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
class Probe:
|
||||
"""把一个子站从挑商品走到确认页;每一步都落快照,任何异常只记录不中断整轮"""
|
||||
|
||||
def __init__(self, site: SiteInteractor, spec: SubsiteSpec, *, accept_terms: bool):
|
||||
self._site = site
|
||||
self._spec = spec
|
||||
self._accept_terms = accept_terms
|
||||
self._dir = PROBE_DIR / spec.name
|
||||
self._dir.mkdir(parents=True, exist_ok=True)
|
||||
self._step = 0
|
||||
self.report = SiteReport(site=spec.name)
|
||||
|
||||
# ---- 通用工具 ----
|
||||
|
||||
def log(self, message: str) -> None:
|
||||
print(f"[{self._spec.name}] {message}", flush=True)
|
||||
|
||||
async def snapshot(self, page, label: str) -> str:
|
||||
"""落一份 HTML + 截图,返回 HTML;截图失败不影响主流程"""
|
||||
self._step += 1
|
||||
stem = f"{self._step:02d}-{label}"
|
||||
html = await page.content()
|
||||
(self._dir / f"{stem}.html").write_text(html, encoding="utf-8")
|
||||
try:
|
||||
await page.screenshot(path=str(self._dir / f"{stem}.png"), full_page=False)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.log(f"截图失败(忽略):{type(exc).__name__}: {exc}")
|
||||
self.log(f"快照 {stem} url={page.url}")
|
||||
return html
|
||||
|
||||
async def _visible_texts(self, page) -> list[str]:
|
||||
"""页面上所有可点控件的文案,供报告里留证据(子站文案基本靠这个发现)"""
|
||||
return await page.evaluate(
|
||||
"""() => Array.from(
|
||||
document.querySelectorAll('button, a[role="button"], input[type="submit"], div[role="button"]')
|
||||
).map(el => (el.innerText || el.value || '').trim().replace(/\\s+/g, ' '))
|
||||
.filter(t => t && t.length <= 30).slice(0, 60)"""
|
||||
)
|
||||
|
||||
# ---- 步骤 1:挑一件便宜的在售商品 ----
|
||||
|
||||
async def discover_candidates(self, page) -> list[dict[str, Any]]:
|
||||
"""用市场搜索页按 shop_id 过滤,按价格升序给出候选商品
|
||||
|
||||
走浏览器而不是 httpx:search.rakuten.co.jp 对 curl/httpx 直接 503,
|
||||
浏览器带完整指纹能正常拿到 __INITIAL_STATE__。
|
||||
返回列表而不是单个:搜索页的 isSoldOut 是**商品级**的,多规格商品可能
|
||||
「商品在售但每个颜色尺码都缺货」(brandavenue 实测第一个候选就是这样),
|
||||
调用方要能换下一个候选试。
|
||||
"""
|
||||
url = _SEARCH_URL.format(sid=self._spec.sid, max_price=self._spec.max_price)
|
||||
await self._goto_with_retry(page, url)
|
||||
items = await page.evaluate(
|
||||
"""() => {
|
||||
const s = window.__INITIAL_STATE__;
|
||||
const list = s && s.state && s.state.data && s.state.data.ichibaSearch
|
||||
? s.state.data.ichibaSearch.items : null;
|
||||
if (!list) return null;
|
||||
return list.filter(i => !i.isSoldOut)
|
||||
.map(i => ({price: i.price, url: i.url, name: i.name}));
|
||||
}"""
|
||||
)
|
||||
if not items:
|
||||
raise RuntimeError(f"搜索页没拿到商品列表(sid={self._spec.sid})")
|
||||
items.sort(key=lambda i: i["price"])
|
||||
self.log(f"候选 {len(items)} 件,最低 {items[0]['price']}円")
|
||||
return items
|
||||
|
||||
def record_item(self, candidate: dict[str, Any]) -> None:
|
||||
self.report.item_url = candidate["url"]
|
||||
self.report.item_price = candidate["price"]
|
||||
self.report.item_name = candidate["name"]
|
||||
self.log(f"试 {candidate['price']}円 {candidate['url']}")
|
||||
|
||||
# ---- 步骤 2:打开商品页,抽这个子站的加购契约 ----
|
||||
|
||||
async def open_item(self, page, item_url: str) -> None:
|
||||
await self._goto_with_retry(page, item_url)
|
||||
await page.wait_for_timeout(3_000)
|
||||
# 302 落到子站首页(路径只剩 /)时,用商品编号直接拼子站商品页兜底
|
||||
if page.url.rstrip("/").endswith(self._spec.host) and self._spec.direct_url:
|
||||
code = [part for part in item_url.split("?")[0].split("/") if part][-1]
|
||||
direct = self._spec.direct_url.format(
|
||||
code=code.upper() if self._spec.upper_code else code
|
||||
)
|
||||
self.log(f"302 只落到子站首页,改用直连商品页:{direct}")
|
||||
self.report.errors.append(
|
||||
f"item.rakuten.co.jp 的 302 丢掉了商品(落到 {page.url}),改用 {direct}"
|
||||
)
|
||||
await self._goto_with_retry(page, direct)
|
||||
await page.wait_for_timeout(3_000)
|
||||
self.report.landing_host = page.url.split("/")[2] if "//" in page.url else ""
|
||||
await self.snapshot(page, "item-page")
|
||||
if self._spec.host not in page.url:
|
||||
self.report.errors.append(
|
||||
f"商品页没有落到 {self._spec.host},实际 url={page.url}"
|
||||
)
|
||||
self.report.add_cart_contract = await self._extract_contract(page)
|
||||
self.log(f"加购契约:{json.dumps(self.report.add_cart_contract, ensure_ascii=False)[:400]}")
|
||||
|
||||
async def _goto_with_retry(self, page, url: str, attempts: int = 3) -> None:
|
||||
"""item.rakuten.co.jp → 子站的 302 跳转偶发 ERR_HTTP2_PROTOCOL_ERROR
|
||||
|
||||
2026-08-14 brandavenue 探测实测:同一个 URL 第一次 goto 直接 HTTP2 协议
|
||||
报错,重试就过。这是站点/网络侧的偶发问题,不是选择器或登录态问题,所以
|
||||
重试而不是当失败——但重试次数用尽仍失败就如实抛出,不静默跳过这个子站。
|
||||
"""
|
||||
last: Exception | None = None
|
||||
for attempt in range(attempts):
|
||||
try:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=45_000)
|
||||
return
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last = exc
|
||||
self.log(f"goto 第 {attempt + 1} 次失败:{type(exc).__name__},2s 后重试")
|
||||
await page.wait_for_timeout(2_000)
|
||||
raise RuntimeError(f"打开 {url} 连续 {attempts} 次失败:{last}")
|
||||
|
||||
async def _extract_contract(self, page) -> dict[str, Any]:
|
||||
"""按子站各自的数据源抽加购要素,核对 [[cart_contract]] 的记录是否还成立"""
|
||||
if self._spec.name == "biccamera":
|
||||
return await page.evaluate(
|
||||
"""() => {
|
||||
const it = window.__NUXT__ && window.__NUXT__.state && window.__NUXT__.state.item;
|
||||
if (!it) return {error: 'no __NUXT__.state.item'};
|
||||
return {
|
||||
source: '__NUXT__.state.item',
|
||||
item_id: it.item_id, item_number: it.item_number, shop_id: it.shop_id,
|
||||
add_cart_api_url: it.add_cart_api_url,
|
||||
buying_procedure_url: it.buying_procedure_url,
|
||||
cart_addable: it.cart_addable, price_with_tax: it.price_with_tax,
|
||||
};
|
||||
}"""
|
||||
)
|
||||
if self._spec.name == "books":
|
||||
return await page.evaluate(
|
||||
"""() => {
|
||||
const forms = Array.from(document.querySelectorAll('form'))
|
||||
.filter(f => (f.action || '').includes('Cart'));
|
||||
if (!forms.length) return {error: 'no cart form'};
|
||||
const f = forms[0];
|
||||
const fields = {};
|
||||
f.querySelectorAll('input, select').forEach(i => {
|
||||
if (i.name) fields[i.name] = i.value;
|
||||
});
|
||||
return {source: 'form[action*=Cart]', action: f.action, method: f.method, fields};
|
||||
}"""
|
||||
)
|
||||
return await page.evaluate(
|
||||
"""() => {
|
||||
const s = window.__INITIAL_STATE__;
|
||||
const p = s && s.itemDetail && s.itemDetail.data && s.itemDetail.data.product;
|
||||
if (!p) return {error: 'no itemDetail.data.product'};
|
||||
const rms = p.rms_info || {};
|
||||
return {
|
||||
source: '__INITIAL_STATE__.itemDetail.data.product',
|
||||
rms_item_id: rms.rms_item_id, shop_id: (s.env || {}).shop_id,
|
||||
cart_url_type: rms.cart_url_type ?? p.cart_url_type ?? null,
|
||||
sku_count: (p.product_sku || []).length,
|
||||
inventory_count: (rms.inventory_list || []).length,
|
||||
first_variant: (rms.inventory_list || [])[0] || null,
|
||||
};
|
||||
}"""
|
||||
)
|
||||
|
||||
# ---- 步骤 3:点站点自己的加购按钮(不重放表单,看真实交互长什么样)----
|
||||
|
||||
async def add_to_cart(self, page) -> None:
|
||||
if self._spec.name == "brandavenue":
|
||||
await self._add_to_cart_brandavenue(page)
|
||||
await self._assert_in_cart(page, "かごに追加(SKU モーダル)")
|
||||
self.report.add_cart_ok = True
|
||||
return
|
||||
await self._select_required_options(page)
|
||||
button = await self._find_clickable(page, _ADD_CART_TEXTS)
|
||||
if button is None:
|
||||
texts = await self._visible_texts(page)
|
||||
self.report.errors.append(f"商品页没找到加购按钮,页面控件文案:{texts}")
|
||||
await self.snapshot(page, "add-cart-notfound")
|
||||
raise RuntimeError("找不到加购按钮")
|
||||
|
||||
locator, text = button
|
||||
self.report.add_cart_button = text
|
||||
self.log(f"点击加购按钮「{text}」")
|
||||
await locator.click(timeout=15_000)
|
||||
await page.wait_for_timeout(4_000)
|
||||
self.report.add_cart_landing_url = page.url
|
||||
await self.snapshot(page, "after-add-cart")
|
||||
|
||||
await self._assert_in_cart(page, text)
|
||||
self.report.add_cart_ok = True
|
||||
|
||||
async def _assert_in_cart(self, page, button_text: str) -> None:
|
||||
"""点了不等于进车:两个独立信号,任一成立就算进车
|
||||
|
||||
1. 落地 URL 带 `added_item=`——市场侧加购成功的标准落地(brandavenue 走这条)
|
||||
2. cart count JSONP API 的 count > 0(books / biccamera 走这条)
|
||||
两个信号都要记进报告:**count API 并不总可靠**——brandavenue 明明已经
|
||||
进车并跳到 sp.cart.step 的 cart 页,count API 仍返回 status=101
|
||||
「value not found」(2026-08-14 实测,末尾 clear_cart 真删掉了 4 件,
|
||||
证明东西确实在车里)。只信 count API 会把成功判成失败。
|
||||
"""
|
||||
added_signal = "added_item=" in page.url
|
||||
count: int | None = None
|
||||
try:
|
||||
_, count = await self._site._query_cart_count() # noqa: SLF001
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.log(f"cart count 查询失败:{type(exc).__name__}: {exc}")
|
||||
self.report.cart_signals = {
|
||||
"landing_url_has_added_item": added_signal,
|
||||
"cart_count_api": count,
|
||||
}
|
||||
if added_signal or (count or 0) > 0:
|
||||
return
|
||||
texts = await self._visible_texts(page)
|
||||
self.report.errors.append(
|
||||
f"点了「{button_text}」但两个进车信号都不成立,可能还有规格/数量选择层。"
|
||||
f"当时控件文案:{texts}"
|
||||
)
|
||||
raise RuntimeError("加购未生效(URL 无 added_item 且 cart count 为 0)")
|
||||
|
||||
async def _add_to_cart_brandavenue(self, page) -> None:
|
||||
"""Rakuten Fashion 的加购是三步,不是一步
|
||||
|
||||
实测(2026-08-14):页面上的「かごに追加」只负责**弹出**「カラー・サイズを選ぶ」
|
||||
模态层;真正入车的按钮在模态层里(同样叫「かごに追加」,但带
|
||||
data-tracking="sku-modal-size-cart")。中间必须先选颜色再选尺码。
|
||||
另外搜索页的 isSoldOut 只表示「整件商品还挂着」,颜色/尺码级缺货要看
|
||||
product_sku[].inventory_exist_flg——最便宜的候选商品全尺码「なし」,
|
||||
点加购只会弹「再入荷リクエスト」。
|
||||
"""
|
||||
variants = await page.evaluate(
|
||||
"""() => {
|
||||
const s = window.__INITIAL_STATE__;
|
||||
const p = s && s.itemDetail && s.itemDetail.data && s.itemDetail.data.product;
|
||||
if (!p) return [];
|
||||
return (p.product_sku || [])
|
||||
.filter(e => String(e.inventory_exist_flg) === '1')
|
||||
.map(e => ({color: e.product_color_name, size: e.product_size_name}));
|
||||
}"""
|
||||
)
|
||||
if not variants:
|
||||
raise RuntimeError("该商品所有颜色/尺码都缺货(product_sku 无 inventory_exist_flg=1)")
|
||||
target = variants[0]
|
||||
self.log(f"选中规格:颜色={target['color']} 尺码={target['size']}(共 {len(variants)} 个有货组合)")
|
||||
self.report.add_cart_contract["picked_variant"] = target
|
||||
|
||||
opener = await self._find_clickable(page, ("かごに追加",))
|
||||
if opener is None:
|
||||
raise RuntimeError("商品页没找到「かごに追加」(打开规格模态层的按钮)")
|
||||
self.report.add_cart_button = "かごに追加 → SKU モーダル → かごに追加"
|
||||
await opener[0].click(timeout=15_000)
|
||||
await page.wait_for_timeout(2_500)
|
||||
await self.snapshot(page, "sku-modal")
|
||||
|
||||
for label, value in (("颜色", target["color"]), ("尺码", target["size"])):
|
||||
if not value:
|
||||
continue
|
||||
if await self._click_variant_chip(page, value):
|
||||
self.log(f"{label}「{value}」已点选")
|
||||
await page.wait_for_timeout(1_500)
|
||||
else:
|
||||
self.report.errors.append(f"{label}「{value}」在模态层里没找到可点的选项")
|
||||
await self.snapshot(page, "variant-selected")
|
||||
|
||||
# 点尺码这一下**本身就入车**并整页跳到市场侧 cart(实测落地
|
||||
# sp.cart.step.rakuten.co.jp/cart?shop_bid=279405&added_item=<rms_item_id>),
|
||||
# 模态层里那个带 data-tracking="sku-modal-size-cart" 的「かごに追加」根本
|
||||
# 等不到——页面已经走了。所以只在没跳走时才去点它兜底。
|
||||
if "added_item=" not in page.url:
|
||||
modal_add = page.locator('button:has([data-tracking="sku-modal-size-cart"])').first
|
||||
try:
|
||||
await modal_add.click(timeout=10_000)
|
||||
except Exception as exc:
|
||||
raise RuntimeError(
|
||||
f"选完尺码没自动入车,模态层里的「かごに追加」也点不到:"
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
) from exc
|
||||
await page.wait_for_timeout(4_000)
|
||||
self.report.add_cart_landing_url = page.url
|
||||
await self.snapshot(page, "after-add-cart")
|
||||
|
||||
async def _click_variant_chip(self, page, value: str) -> bool:
|
||||
"""点一个规格选项:颜色是带 img[alt] 的按钮,尺码是文案就等于尺码的按钮"""
|
||||
for selector in (
|
||||
f'button:has(img[alt="{value}"])',
|
||||
f'button[aria-label="{value}"]',
|
||||
f'li:has-text("{value}") button',
|
||||
):
|
||||
candidates = page.locator(selector)
|
||||
try:
|
||||
total = await candidates.count()
|
||||
except Exception:
|
||||
continue
|
||||
for index in range(min(total, 20)):
|
||||
locator = candidates.nth(index)
|
||||
try:
|
||||
if not await locator.is_visible():
|
||||
continue
|
||||
await locator.click(timeout=5_000)
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
# 尺码这类纯文本 chip:按「可见控件文案完全等于目标值」精确点,避免撞到别的数字
|
||||
clicked = await page.evaluate(
|
||||
"""(value) => {
|
||||
const els = Array.from(document.querySelectorAll('button, div[role="button"], label, li'));
|
||||
for (const el of els) {
|
||||
const r = el.getBoundingClientRect();
|
||||
if (r.width <= 0 || r.height <= 0) continue;
|
||||
if ((el.innerText || '').trim() === value) { el.click(); return true; }
|
||||
}
|
||||
return false;
|
||||
}""",
|
||||
value,
|
||||
)
|
||||
return bool(clicked)
|
||||
|
||||
async def _select_required_options(self, page) -> None:
|
||||
"""加购前尽力选一个可售规格(Rakuten Fashion 的尺码/颜色是必选)
|
||||
|
||||
只做「有明显的下拉框就选第一个非空项」这一档:真实规格控件长什么样正是
|
||||
本探针要探的东西,猜太多反而会掩盖真实结构,选不出来就继续往下走,
|
||||
让站点自己报错、把错误文案记进报告。
|
||||
"""
|
||||
selects = page.locator("select")
|
||||
for index in range(min(await selects.count(), 3)):
|
||||
select = selects.nth(index)
|
||||
try:
|
||||
values = await select.evaluate(
|
||||
"el => Array.from(el.options).filter(o => o.value && !o.disabled).map(o => o.value)"
|
||||
)
|
||||
if values:
|
||||
await select.select_option(values[0])
|
||||
self.log(f"下拉框 #{index} 选中 {values[0]}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.log(f"下拉框 #{index} 选择失败(忽略):{type(exc).__name__}")
|
||||
|
||||
# ---- 步骤 4:购物车归属确认 ----
|
||||
|
||||
async def check_cart(self, page) -> None:
|
||||
"""加购后查市场侧 cart count:子站是否共用 ichiba 购物车,这一步能直接看出来"""
|
||||
try:
|
||||
_, count = await self._site._query_cart_count() # noqa: SLF001
|
||||
self.report.ichiba_cart_count = count
|
||||
self.log(f"ichiba cart count = {count}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.report.errors.append(f"cart count 查询失败:{type(exc).__name__}: {exc}")
|
||||
|
||||
# ---- 步骤 5:一路点到确认页为止 ----
|
||||
|
||||
async def drive_checkout(self, page) -> None:
|
||||
for hop in range(_MAX_HOPS):
|
||||
await page.wait_for_timeout(2_000)
|
||||
url = page.url
|
||||
html = await self.snapshot(page, f"hop{hop}")
|
||||
hop_record = {"hop": str(hop), "url": url}
|
||||
|
||||
confirm_buttons = await self._confirm_buttons(page)
|
||||
if confirm_buttons or any(m in url for m in _CONFIRM_PAGE_URL_MARKERS):
|
||||
hop_record["action"] = f"确认页命中(确认按钮={confirm_buttons}),停手(不提交)"
|
||||
self.report.hops.append(hop_record)
|
||||
self.report.reached_confirm = True
|
||||
self.report.confirm_buttons = confirm_buttons
|
||||
self.report.stopped_reason = "已到下单确认页,按要求不提交"
|
||||
await self._parse_amount(html)
|
||||
return
|
||||
|
||||
# 同样必须排在 session upgrade 判据前面(URL 仍带 session/upgrade)。
|
||||
# 生年月日/性別 这种「注册后不可变更」的个人信息,脚本一律不代填。
|
||||
if _PROFILE_STEP_URL_MARKER in url or (
|
||||
_PHONE_REGISTRATION_MARKER in html
|
||||
and all(m in html for m in _PROFILE_REGISTRATION_MARKERS)
|
||||
):
|
||||
hop_record["action"] = "会員情報の追加登録(誕生日+性別)页,脚本不代填,停手"
|
||||
self.report.hops.append(hop_record)
|
||||
self.report.stopped_reason = (
|
||||
"卡在「会員情報の追加登録」的誕生日+性別 变体(hash #/profiling/1):"
|
||||
"两项都是必填且页面明写「登録した情報は、変更できません」,"
|
||||
"属于不可撤销的个人信息,脚本不代填,必须人工填一次"
|
||||
)
|
||||
return
|
||||
|
||||
# 必须排在 session upgrade 判据前面:同意页的 URL 里仍然带
|
||||
# session/upgrade,只有 hash 不同,顺序反了会拿密码框选择器去撞同意页
|
||||
if _AGREE_STEP_URL_MARKER in url or _AGREE_STEP_TEXT_MARKER in html:
|
||||
if not self._accept_terms:
|
||||
hop_record["action"] = "子站利用規約同意页,未授权自动同意,停手"
|
||||
self.report.hops.append(hop_record)
|
||||
self.report.stopped_reason = (
|
||||
"卡在子站利用規約/プライバシーポリシー同意页("
|
||||
"#/agree/service):这是账号级协议同意,需显式授权,"
|
||||
"加 --accept-subsite-terms 才会自动点「次へ」"
|
||||
)
|
||||
return
|
||||
hop_record["action"] = "子站利用規約同意页:点「次へ」(已显式授权)"
|
||||
self.report.hops.append(hop_record)
|
||||
agree = await self._find_clickable(page, ("次へ",))
|
||||
if agree is None:
|
||||
self.report.stopped_reason = "同意页没找到「次へ」"
|
||||
return
|
||||
await agree[0].click(timeout=15_000)
|
||||
continue
|
||||
|
||||
if any(marker in url for marker in _SESSION_UPGRADE_URL_MARKERS):
|
||||
hop_record["action"] = "session upgrade:调生产 _complete_session_upgrade"
|
||||
self.report.hops.append(hop_record)
|
||||
await self._site._complete_session_upgrade( # noqa: SLF001
|
||||
page, task_id=f"probe-{self._spec.name}"
|
||||
)
|
||||
continue
|
||||
|
||||
if _PHONE_REGISTRATION_MARKER in html:
|
||||
hop_record["action"] = "电话补录:调生产 _complete_phone_registration"
|
||||
self.report.hops.append(hop_record)
|
||||
await self._site._complete_phone_registration( # noqa: SLF001
|
||||
page, task_id=f"probe-{self._spec.name}"
|
||||
)
|
||||
continue
|
||||
|
||||
if _ADDRESS_STEP_URL_MARKER in url:
|
||||
hop_record["action"] = "地址确认:调生产 _confirm_default_address"
|
||||
self.report.hops.append(hop_record)
|
||||
await self._site._confirm_default_address( # noqa: SLF001
|
||||
page, task_id=f"probe-{self._spec.name}"
|
||||
)
|
||||
continue
|
||||
|
||||
if _PAYMENT_STEP_URL_MARKER in url:
|
||||
hop_record["action"] = "支付方式:调生产 _select_payment_method"
|
||||
self.report.hops.append(hop_record)
|
||||
await self._site._select_payment_method( # noqa: SLF001
|
||||
page, task_id=f"probe-{self._spec.name}"
|
||||
)
|
||||
continue
|
||||
|
||||
found = await self._find_clickable(page, _NEXT_STEP_TEXTS)
|
||||
if found is None:
|
||||
texts = await self._visible_texts(page)
|
||||
hop_record["action"] = f"没有可继续的按钮,停。页面控件文案:{texts}"
|
||||
self.report.hops.append(hop_record)
|
||||
self.report.stopped_reason = "找不到下一步按钮"
|
||||
return
|
||||
|
||||
locator, text = found
|
||||
hop_record["action"] = f"点击「{text}」"
|
||||
self.report.hops.append(hop_record)
|
||||
self.log(f"hop{hop} 点击「{text}」")
|
||||
await locator.click(timeout=15_000)
|
||||
|
||||
self.report.stopped_reason = f"跳转超过 {_MAX_HOPS} 跳仍未到确认页"
|
||||
|
||||
async def _parse_amount(self, html: str) -> None:
|
||||
"""确认页金额解析:直接用生产的 _parse_checkout_summary,验证它在子站是否也成立"""
|
||||
try:
|
||||
summary = _parse_checkout_summary(html)
|
||||
self.report.payable_yen = summary.payable_yen
|
||||
self.log(f"确认页金额解析成功:{summary.payable_yen} 円")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
self.report.errors.append(
|
||||
f"确认页金额解析失败(生产 _parse_checkout_summary):{type(exc).__name__}: {exc}"
|
||||
)
|
||||
|
||||
async def _confirm_buttons(self, page) -> list[str]:
|
||||
"""页面上**可见的**下单确认按钮文案;空列表表示还没到确认页
|
||||
|
||||
只看真实控件(button / role=button / submit)且尺寸非零,不看正文文案——
|
||||
正文里出现「注文を確定する」这五个字的说明性文字并不代表这是确认页。
|
||||
"""
|
||||
return await page.evaluate(
|
||||
"""(texts) => Array.from(document.querySelectorAll(
|
||||
'button, div[role="button"], a[role="button"], input[type="submit"]'
|
||||
))
|
||||
.filter(el => {
|
||||
const r = el.getBoundingClientRect();
|
||||
return r.width > 0 && r.height > 0;
|
||||
})
|
||||
.map(el => (el.innerText || el.value || '').trim().replace(/\\s+/g, ' '))
|
||||
.filter(t => t && texts.some(x => t.includes(x)))""",
|
||||
list(_CONFIRM_BUTTON_TEXTS),
|
||||
)
|
||||
|
||||
# ---- 按钮定位:文案匹配 + 黑名单闸门 ----
|
||||
|
||||
async def _find_clickable(self, page, texts: tuple[str, ...]):
|
||||
"""按候选文案找第一个可见可点的控件;命中黑名单立即抛错,绝不返回
|
||||
|
||||
`button:has-text()` 是子串匹配,「購入手続き」会同时命中促销按钮
|
||||
「カード入会&購入手続き」——主站踩过这个坑(见 site_interact
|
||||
_CHECKOUT_BUTTON_SELECTOR 注释),所以这里按 aria-label 精确匹配优先,
|
||||
再退到文案子串匹配,并且每个候选都要过一遍黑名单。
|
||||
"""
|
||||
for text in texts:
|
||||
for selector in (
|
||||
f'button[aria-label="{text}"]',
|
||||
f'button:has-text("{text}")',
|
||||
f'a:has-text("{text}")',
|
||||
f'div[role="button"]:has-text("{text}")',
|
||||
f'input[type="submit"][value*="{text}"]',
|
||||
):
|
||||
candidates = page.locator(selector)
|
||||
# 子站页面同一个按钮常有 PC/SP 两套 DOM,只有一套可见;.first 会选中
|
||||
# 隐藏的那个然后死等 visible 超时(biccamera 实测踩过),必须逐个筛可见的
|
||||
try:
|
||||
total = await candidates.count()
|
||||
except Exception:
|
||||
continue
|
||||
for index in range(min(total, 10)):
|
||||
locator = candidates.nth(index)
|
||||
try:
|
||||
if not await locator.is_visible():
|
||||
continue
|
||||
except Exception:
|
||||
continue
|
||||
actual = text
|
||||
if "input" not in selector:
|
||||
try:
|
||||
actual = ((await locator.inner_text()) or "").strip() or text
|
||||
except Exception:
|
||||
actual = text
|
||||
if any(bad in actual for bad in _FORBIDDEN_BUTTON_TEXTS):
|
||||
raise RuntimeError(
|
||||
f"候选按钮「{actual}」命中提交订单黑名单,停手(selector={selector})"
|
||||
)
|
||||
return locator, actual
|
||||
return None
|
||||
|
||||
|
||||
async def _start_visible(site: SiteInteractor, settings) -> None:
|
||||
"""headless=False 启动(与 probe_payment_method.py 同一写法)
|
||||
|
||||
无头模式下结算 SPA 表现异常,见 site_interact.start() 的注释。
|
||||
"""
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from app.trading.core import auth_site
|
||||
|
||||
state_path = settings.auth_state_path / auth_site.profile("rakuten").state_filename
|
||||
storage_state = str(state_path) if state_path.exists() else None
|
||||
|
||||
site._playwright = await async_playwright().start() # noqa: SLF001
|
||||
site._browser = await site._playwright.chromium.launch( # noqa: SLF001
|
||||
headless=False,
|
||||
channel=settings.browser_channel or None,
|
||||
proxy=settings.playwright_proxy,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
site._context = await site._browser.new_context( # noqa: SLF001
|
||||
storage_state=storage_state,
|
||||
user_agent=auth_site.RAKUTEN_USER_AGENT,
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844},
|
||||
is_mobile=True,
|
||||
has_touch=True,
|
||||
)
|
||||
print(f"[visible] 浏览器已启动,storage_state={storage_state or '(none)'}", flush=True)
|
||||
|
||||
|
||||
async def probe_one(
|
||||
site: SiteInteractor, spec: SubsiteSpec, item_url: str | None, *, accept_terms: bool
|
||||
) -> SiteReport:
|
||||
probe = Probe(site, spec, accept_terms=accept_terms)
|
||||
page = await site._context.new_page() # noqa: SLF001
|
||||
try:
|
||||
if item_url:
|
||||
candidates = [{"url": item_url, "price": None, "name": "(指定)"}]
|
||||
else:
|
||||
candidates = (await probe.discover_candidates(page))[:_MAX_CANDIDATES]
|
||||
for index, candidate in enumerate(candidates):
|
||||
probe.record_item(candidate)
|
||||
try:
|
||||
await probe.open_item(page, candidate["url"])
|
||||
await probe.add_to_cart(page)
|
||||
break
|
||||
except Exception as exc: # noqa: BLE001
|
||||
probe.report.errors.append(
|
||||
f"候选 {index + 1}/{len(candidates)}({candidate['url']})加购失败:"
|
||||
f"{type(exc).__name__}: {exc}"
|
||||
)
|
||||
probe.log(f"候选 {index + 1} 加购失败,换下一个:{type(exc).__name__}: {exc}")
|
||||
if not probe.report.add_cart_ok:
|
||||
raise RuntimeError(f"{len(candidates)} 个候选商品都没能加购成功")
|
||||
await probe.check_cart(page)
|
||||
await probe.drive_checkout(page)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
probe.report.errors.append(f"{type(exc).__name__}: {exc}")
|
||||
probe.report.stopped_reason = probe.report.stopped_reason or f"异常中断:{type(exc).__name__}"
|
||||
probe.log(f"异常:{type(exc).__name__}: {exc}")
|
||||
traceback.print_exc()
|
||||
try:
|
||||
await probe.snapshot(page, "error")
|
||||
except Exception: # noqa: BLE001
|
||||
pass
|
||||
finally:
|
||||
await page.close()
|
||||
# 每个子站跑完都清一次购物车:下一个子站要从空车开始,否则确认页会混单
|
||||
try:
|
||||
result = await site.clear_cart()
|
||||
probe.log(f"清空购物车:{result}")
|
||||
if result.get("cart_count") not in (0, -1):
|
||||
probe.report.errors.append(f"购物车未清干净:{result}")
|
||||
except Exception as exc: # noqa: BLE001
|
||||
probe.report.errors.append(f"清空购物车失败:{type(exc).__name__}: {exc}")
|
||||
|
||||
(PROBE_DIR / f"report-{spec.name}.json").write_text(
|
||||
json.dumps(probe.report.__dict__, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
return probe.report
|
||||
|
||||
|
||||
async def run(sites: list[str], item_url: str | None, *, accept_terms: bool) -> int:
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
settings = get_settings()
|
||||
auth = AuthSession(settings)
|
||||
await auth.start()
|
||||
site = SiteInteractor(auth_session=auth, settings=settings)
|
||||
await _start_visible(site, settings)
|
||||
|
||||
reports: list[SiteReport] = []
|
||||
try:
|
||||
try:
|
||||
first = await site.clear_cart()
|
||||
print(f"[setup] 起始清空购物车:{first}", flush=True)
|
||||
except Exception as exc: # noqa: BLE001
|
||||
print(f"[setup] 起始清空失败(继续):{type(exc).__name__}: {exc}", flush=True)
|
||||
|
||||
for name in sites:
|
||||
print(f"\n{'=' * 20} {name} {'=' * 20}", flush=True)
|
||||
reports.append(
|
||||
await probe_one(site, SUBSITES[name], item_url, accept_terms=accept_terms)
|
||||
)
|
||||
finally:
|
||||
await site.close()
|
||||
await auth.close()
|
||||
|
||||
print("\n==== 汇总 ====", flush=True)
|
||||
for report in reports:
|
||||
print(
|
||||
f"{report.site}: 加购={'OK' if report.add_cart_ok else 'NG'} "
|
||||
f"落地域名={report.landing_host} cart_count={report.ichiba_cart_count} "
|
||||
f"到确认页={'是' if report.reached_confirm else '否'} "
|
||||
f"金额={report.payable_yen} 停在={report.stopped_reason}",
|
||||
flush=True,
|
||||
)
|
||||
for err in report.errors:
|
||||
print(f" ! {err}", flush=True)
|
||||
print(f"\n证据目录:{PROBE_DIR}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
async def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--site", choices=[*SUBSITES, "all"], default="all")
|
||||
parser.add_argument("--item-url", default=None, help="跳过自动选品,直接用这个商品 URL")
|
||||
parser.add_argument(
|
||||
"--accept-subsite-terms",
|
||||
action="store_true",
|
||||
help="遇到子站利用規約同意页时代用户点「次へ」(账号级协议同意,默认不做)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
sites = list(SUBSITES) if args.site == "all" else [args.site]
|
||||
if args.item_url and len(sites) > 1:
|
||||
parser.error("--item-url 只能配合单个 --site 使用")
|
||||
return await run(sites, args.item_url, accept_terms=args.accept_subsite_terms)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(main()))
|
||||
@@ -0,0 +1,100 @@
|
||||
"""真账号验证:clear_cart 两个场景——空车返回成功 / 有商品正确清除
|
||||
|
||||
场景 1(空车):clear_cart 应 removed_count=0、cart_count=0(status=101 是合法
|
||||
空车,不是获取失败),并带回清理后的页面 HTML 与整页截图(runner step 0 落
|
||||
证据用)。
|
||||
场景 2(有商品):先加购 1 件商品,clear_cart 应 removed_count>=1、cart_count=0,
|
||||
末尾再用 cart_status 独立复核确实是空车。
|
||||
|
||||
两个场景的截图会写到 .probe/cart_clear/ 供人工查看。
|
||||
|
||||
前置:登录态有效(.auth/ 下有 storage_state);浏览器必须非无头(站点风控要求,
|
||||
会弹出真实浏览器窗口)。商品 URL 与其他探针(probe_new_card 等)一致。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/verify_cart_clear.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.services.auth_session import AuthSession # noqa: E402
|
||||
from app.trading.worker.site_interact import SiteInteractor # noqa: E402
|
||||
|
||||
ITEM_URL = "https://item.rakuten.co.jp/moccasin/ds001iwrgesaaa2/"
|
||||
|
||||
PROBE_DIR = Path(__file__).resolve().parent.parent / ".probe" / "cart_clear"
|
||||
_PNG_MAGIC = b"\x89PNG\r\n\x1a\n"
|
||||
|
||||
|
||||
def _check_png(data: bytes, label: str) -> None:
|
||||
assert data[:8] == _PNG_MAGIC, f"{label} 截图不是 PNG({len(data)}B)"
|
||||
|
||||
|
||||
async def run() -> int:
|
||||
settings = get_settings()
|
||||
auth = AuthSession(settings)
|
||||
await auth.start()
|
||||
site = SiteInteractor(auth_session=auth, settings=settings)
|
||||
await site.start()
|
||||
PROBE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
# 前置:先清一次,保证从空车起步(顺便清掉上次验证可能残留的商品)
|
||||
pre = await site.clear_cart()
|
||||
print(f"前置 clear -> removed={pre['removed_count']} cart_count={pre['cart_count']}")
|
||||
assert pre["cart_count"] == 0, f"前置清理后应为空车,实际 {pre['cart_count']}"
|
||||
|
||||
# 场景 1:空车 —— 应直接返回清理成功,不做任何点击
|
||||
empty = await site.clear_cart()
|
||||
print(
|
||||
f"场景1 空车 -> removed={empty['removed_count']} "
|
||||
f"cart_count={empty['cart_count']} html={len(empty['html'])}B "
|
||||
f"png={len(empty['screenshot'])}B"
|
||||
)
|
||||
assert empty["removed_count"] == 0, f"空车不应有点击,实际 {empty['removed_count']}"
|
||||
assert empty["cart_count"] == 0, f"空车 cart_count 应为 0,实际 {empty['cart_count']}"
|
||||
assert empty["html"], "空车也应带回页面 HTML(runner 落证据用)"
|
||||
_check_png(empty["screenshot"], "场景1 空车")
|
||||
(PROBE_DIR / "scenario1-empty.png").write_bytes(empty["screenshot"])
|
||||
|
||||
# 场景 2:有商品 —— 先加购 1 件,再清,应真删掉
|
||||
added = await site.add_to_cart_payload(item_url=ITEM_URL, quantity=1)
|
||||
print(f"加购 -> item_id={added['item_id']} cart_count={added['cart_count']}")
|
||||
assert added["cart_count"] >= 1, "加购后购物车应有商品"
|
||||
_check_png(added["screenshot"], "加购后商品页")
|
||||
|
||||
cleared = await site.clear_cart()
|
||||
print(
|
||||
f"场景2 有商品 -> removed={cleared['removed_count']} "
|
||||
f"cart_count={cleared['cart_count']} html={len(cleared['html'])}B "
|
||||
f"png={len(cleared['screenshot'])}B"
|
||||
)
|
||||
assert cleared["removed_count"] >= 1, (
|
||||
f"有商品时至少点 1 次「削除」,实际 {cleared['removed_count']}"
|
||||
)
|
||||
assert cleared["cart_count"] == 0, f"清理后应为空车,实际 {cleared['cart_count']}"
|
||||
assert cleared["html"], "清理后应带回页面 HTML"
|
||||
_check_png(cleared["screenshot"], "场景2 清理后")
|
||||
(PROBE_DIR / "scenario2-cleared.png").write_bytes(cleared["screenshot"])
|
||||
|
||||
# 复核:独立的 status 查询确认确实是空车(不只信 clear 末尾的一次校验)
|
||||
status = await site.cart_status()
|
||||
print(f"复核 status -> {status}")
|
||||
assert status["count"] == 0 and status["raw_status"] == "101", (
|
||||
f"复核应为空车(count=0, raw_status=101),实际 {status}"
|
||||
)
|
||||
|
||||
print(f"\n验证通过:空车正确返回成功;有商品时正确清除并复核为空车。截图见 {PROBE_DIR}")
|
||||
return 0
|
||||
finally:
|
||||
await site.close()
|
||||
await auth.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(run()))
|
||||
@@ -0,0 +1,51 @@
|
||||
"""真账号验证:空车场景下 cart_status / clear_cart 不再被误判为获取失败
|
||||
|
||||
修复前行为(2026-08-16 探针实测):空车时 cart count API 返回 status=101,
|
||||
_query_cart_count 一律当失败抛 CartOperationError → cart_status 报错 5002、
|
||||
clear_cart 末尾校验拿 cart_count=-1(runner 据此误判「开单前清理未清空」)。
|
||||
修复后:status=101 识别为合法空车,count=0。
|
||||
|
||||
用法:
|
||||
.venv/Scripts/python.exe scripts/verify_cart_empty.py
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||
|
||||
from app.shared.config import get_settings # noqa: E402
|
||||
from app.trading.services.auth_session import AuthSession # noqa: E402
|
||||
from app.trading.worker.site_interact import SiteInteractor # noqa: E402
|
||||
|
||||
|
||||
async def run() -> int:
|
||||
settings = get_settings()
|
||||
auth = AuthSession(settings)
|
||||
await auth.start()
|
||||
site = SiteInteractor(auth_session=auth, settings=settings)
|
||||
await site.start()
|
||||
try:
|
||||
status = await site.cart_status()
|
||||
print(f"cart_status -> {status}")
|
||||
assert status["logged_in"] is True
|
||||
assert status["count"] == 0, f"预期空车 count=0,实际 {status['count']}"
|
||||
assert status["raw_status"] == "101", f"预期 raw_status=101,实际 {status['raw_status']}"
|
||||
|
||||
cleared = await site.clear_cart()
|
||||
print(f"clear_cart -> {cleared}")
|
||||
assert cleared["cart_count"] == 0, (
|
||||
f"空车 clear 后 cart_count 应为 0(修复前是 -1),实际 {cleared['cart_count']}"
|
||||
)
|
||||
|
||||
print("\n验证通过:空车按正常结果返回(count=0 / raw_status=101),未误判为获取失败")
|
||||
return 0
|
||||
finally:
|
||||
await site.close()
|
||||
await auth.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(asyncio.run(run()))
|
||||
@@ -330,10 +330,16 @@ async def test_try_relogin_is_disabled_when_flag_off(tmp_path):
|
||||
|
||||
|
||||
async def test_try_relogin_serializes_concurrent_calls_same_site(tmp_path, monkeypatch):
|
||||
"""同 site 并发触发只跑一次 login_one:靠 site 级锁串行化"""
|
||||
"""同 site 并发触发**只跑一次** login_one:锁串行化 + 排队者复核登录态后跳过
|
||||
|
||||
不能只断言「没并发执行」:重登成功后的 reload() 会把 logged_in 重置成 None,
|
||||
排队者如果读缓存去重就会一律判「还没人登上」,于是 N 个并发调用串行触发 N 次
|
||||
真实登录(每次最长 relogin_timeout)。这里断言 count == 1 就是钉住这一点。
|
||||
"""
|
||||
_patch_accounts_file(monkeypatch, tmp_path)
|
||||
|
||||
invocations = {"count": 0, "in_flight_max": 0, "current": 0}
|
||||
state = {"logged_in": False}
|
||||
from app.trading.services import login_runner
|
||||
|
||||
async def counting_login_one(account, s, *, timeout_seconds=300, progress=None):
|
||||
@@ -348,12 +354,19 @@ async def test_try_relogin_serializes_concurrent_calls_same_site(tmp_path, monke
|
||||
"rakuten",
|
||||
[{"name": "X", "value": "1", "domain": ".rakuten.co.jp", "path": "/"}],
|
||||
)
|
||||
# 登录成功后探针页也必须随之翻转,否则桩自相矛盾(login_one 说成功、站点说
|
||||
# 没登上),测不出真实行为
|
||||
state["logged_in"] = True
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(login_runner, "login_one", counting_login_one)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
text = CART_LOGGED_IN if state["logged_in"] else CART_LOGGED_OUT
|
||||
return httpx.Response(200, text=text)
|
||||
|
||||
settings = make_settings(tmp_path, relogin_enabled=True, relogin_timeout_seconds=10)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT))
|
||||
session = await build_session(settings, handler)
|
||||
|
||||
try:
|
||||
# 第一次状态为 logged_in=None,三个并发都进入 try_relogin
|
||||
@@ -362,12 +375,43 @@ async def test_try_relogin_serializes_concurrent_calls_same_site(tmp_path, monke
|
||||
session.try_relogin("rakuten"),
|
||||
session.try_relogin("rakuten"),
|
||||
)
|
||||
# login_one 至少被调一次(串行下后续可能因 status 已 logged_in 跳过)
|
||||
assert invocations["count"] >= 1
|
||||
# 关键:login_one 永远没并发执行
|
||||
assert invocations["count"] == 1
|
||||
# login_one 永远没并发执行(同账号同 user_data_dir,撞锁会失败)
|
||||
assert invocations["in_flight_max"] == 1
|
||||
# 结果都成功(要么真重登,要么拿到锁后发现已 logged_in)
|
||||
assert all(results)
|
||||
# 三个调用方都拿到「已登录」结论:一个真登录,两个复核后跳过
|
||||
assert results == [True, True, True]
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_try_relogin_does_not_retry_after_queued_failure(tmp_path, monkeypatch):
|
||||
"""一次重登失败后,排在锁上的并发调用不再重复触发同一个注定失败的登录
|
||||
|
||||
站点侧登不上(验证码/密码错/风控)时,把每个调用方各卡一个 relogin_timeout
|
||||
不会改变结果,只会让整批任务慢几倍。
|
||||
"""
|
||||
_patch_accounts_file(monkeypatch, tmp_path)
|
||||
|
||||
calls = {"count": 0}
|
||||
from app.trading.services import login_runner
|
||||
|
||||
async def failing_login_one(account, s, *, timeout_seconds=300, progress=None):
|
||||
calls["count"] += 1
|
||||
await asyncio.sleep(0.05) # 让其余并发调用叠到锁上
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(login_runner, "login_one", failing_login_one)
|
||||
|
||||
settings = make_settings(tmp_path, relogin_enabled=True, relogin_timeout_seconds=10)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT))
|
||||
try:
|
||||
results = await asyncio.gather(
|
||||
session.try_relogin("rakuten"),
|
||||
session.try_relogin("rakuten"),
|
||||
session.try_relogin("rakuten"),
|
||||
)
|
||||
assert results == [False, False, False]
|
||||
assert calls["count"] == 1
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
@@ -29,6 +29,9 @@ def gateway_client(tmp_path: Path, monkeypatch):
|
||||
"""
|
||||
db_path = tmp_path / "gw.db"
|
||||
monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(db_path))
|
||||
# 定时下派通道(§12)会在一启动就派 order_list 扫描,污染查询队列语义测试,
|
||||
# 这类「队列/租赁/鉴权」用例统一关掉它(它有自己的专门测试文件)。
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "false")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
from app.gateway.main import create_app
|
||||
|
||||
@@ -0,0 +1,356 @@
|
||||
"""终结类事件回调(callback_url)测试,对应 docs/order-gateway.md §4.8
|
||||
|
||||
三层覆盖:
|
||||
|
||||
- API 层:提交校验(非法地址 422)、详情透出 callback_url、幂等重发不改地址、
|
||||
HTTP 全流程下 terminal report 恰好通知一次(注入记录型假 notifier)。
|
||||
- 队列层:中间态不通知、终结通知一次、终结后的后续 report 不重复通知、
|
||||
租约过期 sweep 触发 stale 通知、未登记地址不通知。
|
||||
- 通知器层:真实发送路径(httpx.MockTransport)——成功送达、上游 5xx、
|
||||
连接异常都不抛出、只记日志(best-effort)。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import aiosqlite
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.gateway.callback import CallbackNotifier
|
||||
from app.gateway.db import GatewayDB
|
||||
from app.gateway.task_queue import TaskQueue
|
||||
from app.shared.config import Settings, get_settings
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
|
||||
TOKEN = get_settings().bearer_token
|
||||
AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
CALLBACK_URL = "https://upstream.example.com/hooks/rakuten-order"
|
||||
|
||||
|
||||
class FakeNotifier:
|
||||
"""记录型假通知器:与 CallbackNotifier 同接口,同步记录便于断言"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.sent: list[tuple[str, dict]] = []
|
||||
|
||||
def notify(self, callback_url: str, payload: dict) -> None:
|
||||
self.sent.append((callback_url, payload))
|
||||
|
||||
|
||||
# ---- API 层 ----
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gateway_client(tmp_path: Path, monkeypatch):
|
||||
db_path = tmp_path / "gw.db"
|
||||
monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "false")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
from app.gateway.main import create_app
|
||||
|
||||
app = create_app()
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_notifier(gateway_client):
|
||||
"""把容器里 TaskQueue 的通知器换成记录型假实现,用完还原"""
|
||||
container = gateway_client.app.state.container
|
||||
original = container.task_queue.notifier
|
||||
fake = FakeNotifier()
|
||||
container.task_queue.notifier = fake
|
||||
yield fake
|
||||
container.task_queue.notifier = original
|
||||
|
||||
|
||||
def _submit(client, *, task_id: str, callback_url: str | None = CALLBACK_URL):
|
||||
payload: dict = {"task_id": task_id, "site": "rakuten", "intent": {}}
|
||||
if callback_url is not None:
|
||||
payload["callback_url"] = callback_url
|
||||
return client.post("/api/orders", json=payload, headers=AUTH)
|
||||
|
||||
|
||||
def test_submit_with_callback_url_roundtrip(gateway_client):
|
||||
response = _submit(gateway_client, task_id="t1")
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"]["created"] is True
|
||||
|
||||
detail = gateway_client.get("/api/orders/t1", headers=AUTH).json()["data"]
|
||||
assert detail["callback_url"] == CALLBACK_URL
|
||||
|
||||
|
||||
def test_submit_without_callback_url_defaults_to_null(gateway_client):
|
||||
_submit(gateway_client, task_id="t1", callback_url=None)
|
||||
detail = gateway_client.get("/api/orders/t1", headers=AUTH).json()["data"]
|
||||
assert detail["callback_url"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_url",
|
||||
[
|
||||
"ftp://upstream.example.com/hook", # 非 http/https scheme
|
||||
"not-a-url", # 没有 scheme 与 host
|
||||
"https://", # 有 scheme 没 host
|
||||
],
|
||||
)
|
||||
def test_submit_rejects_invalid_callback_url(gateway_client, bad_url):
|
||||
response = _submit(gateway_client, task_id="t1", callback_url=bad_url)
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_idempotent_resubmit_keeps_original_callback_url(gateway_client):
|
||||
"""同 task_id 重发带不同 callback_url:不新建、不更新地址"""
|
||||
_submit(gateway_client, task_id="t1", callback_url=CALLBACK_URL)
|
||||
r2 = _submit(gateway_client, task_id="t1", callback_url="https://other.example.com/hook")
|
||||
assert r2.json()["data"]["created"] is False
|
||||
|
||||
detail = gateway_client.get("/api/orders/t1", headers=AUTH).json()["data"]
|
||||
assert detail["callback_url"] == CALLBACK_URL
|
||||
|
||||
|
||||
def test_terminal_report_sends_exactly_one_callback(gateway_client, fake_notifier):
|
||||
_submit(gateway_client, task_id="t1")
|
||||
gateway_client.get("/api/orders/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
|
||||
response = gateway_client.post(
|
||||
"/api/orders/t1/report",
|
||||
json={
|
||||
"worker_id": "w1",
|
||||
"state": OrderState.PAID.value,
|
||||
"payable_yen": 9800,
|
||||
"site_order_id": "ord-1",
|
||||
"terminal": True,
|
||||
"terminal_status": TaskStatus.SUCCEEDED.value,
|
||||
"detail": "付款完成",
|
||||
},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
|
||||
assert len(fake_notifier.sent) == 1
|
||||
url, payload = fake_notifier.sent[0]
|
||||
assert url == CALLBACK_URL
|
||||
assert payload["event"] == "terminal"
|
||||
assert payload["task_id"] == "t1"
|
||||
assert payload["site"] == "rakuten"
|
||||
assert payload["status"] == TaskStatus.SUCCEEDED.value
|
||||
assert payload["state"] == OrderState.PAID.value
|
||||
assert payload["payable_yen"] == 9800
|
||||
assert payload["site_order_id"] == "ord-1"
|
||||
assert payload["detail"] == "付款完成"
|
||||
assert payload["reported_at"]
|
||||
|
||||
|
||||
def test_non_terminal_report_sends_no_callback(gateway_client, fake_notifier):
|
||||
_submit(gateway_client, task_id="t1")
|
||||
gateway_client.get("/api/orders/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
gateway_client.post(
|
||||
"/api/orders/t1/report",
|
||||
json={"worker_id": "w1", "state": OrderState.IN_CART.value},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert fake_notifier.sent == []
|
||||
|
||||
|
||||
def test_report_after_terminal_sends_no_more_callbacks(gateway_client, fake_notifier):
|
||||
"""任务终结后的后续 report(付款后监控)与幂等重报都不再通知"""
|
||||
_submit(gateway_client, task_id="t1")
|
||||
gateway_client.get("/api/orders/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
terminal_report = {
|
||||
"worker_id": "w1",
|
||||
"state": OrderState.PAID.value,
|
||||
"terminal": True,
|
||||
"terminal_status": TaskStatus.SUCCEEDED.value,
|
||||
}
|
||||
gateway_client.post("/api/orders/t1/report", json=terminal_report, headers=AUTH)
|
||||
assert len(fake_notifier.sent) == 1
|
||||
|
||||
# 同一 (task_id, state) 幂等重报 + 终结后的监控上报,都不应再触发
|
||||
gateway_client.post("/api/orders/t1/report", json=terminal_report, headers=AUTH)
|
||||
gateway_client.post(
|
||||
"/api/orders/t1/report",
|
||||
json={"worker_id": "w1", "state": OrderState.SHIPPED.value},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert len(fake_notifier.sent) == 1
|
||||
|
||||
|
||||
def test_needs_human_inferred_terminal_sends_callback(gateway_client, fake_notifier):
|
||||
"""terminal=true 不显式给终态时按 state 推断(非 paid/cancelled → needs_human)"""
|
||||
_submit(gateway_client, task_id="t1")
|
||||
gateway_client.get("/api/orders/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
gateway_client.post(
|
||||
"/api/orders/t1/report",
|
||||
json={
|
||||
"worker_id": "w1",
|
||||
"state": OrderState.AWAITING_PAYMENT.value,
|
||||
"terminal": True,
|
||||
"detail": "弹了 3DS",
|
||||
},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert len(fake_notifier.sent) == 1
|
||||
_, payload = fake_notifier.sent[0]
|
||||
assert payload["status"] == TaskStatus.NEEDS_HUMAN.value
|
||||
|
||||
|
||||
# ---- 队列层 ----
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def queue(tmp_path: Path):
|
||||
db = GatewayDB(tmp_path / "gw.db")
|
||||
await db.start()
|
||||
fake = FakeNotifier()
|
||||
q = TaskQueue(
|
||||
db, lease_ttl_seconds=0, worker_offline_alert_seconds=300, notifier=fake
|
||||
)
|
||||
yield q, fake
|
||||
await db.close()
|
||||
|
||||
|
||||
async def test_sweep_stale_sends_callback(queue):
|
||||
"""租约过期被 sweep 置 stale 时触发 stale 通知(上游需要人工 reclaim)"""
|
||||
q, fake = queue
|
||||
await q.submit(
|
||||
task_id="t1", site="rakuten", intent={}, callback_url=CALLBACK_URL
|
||||
)
|
||||
# lease_ttl=0:领走即过期,下一次 sweep 置 stale
|
||||
await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
|
||||
swept = await q.sweep()
|
||||
assert swept == 1
|
||||
assert len(fake.sent) == 1
|
||||
url, payload = fake.sent[0]
|
||||
assert url == CALLBACK_URL
|
||||
assert payload["event"] == "stale"
|
||||
assert payload["status"] == TaskStatus.STALE.value
|
||||
assert payload["task_id"] == "t1"
|
||||
|
||||
|
||||
async def test_sweep_without_callback_url_sends_nothing(queue):
|
||||
q, fake = queue
|
||||
await q.submit(task_id="t1", site="rakuten", intent={})
|
||||
await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
|
||||
swept = await q.sweep()
|
||||
assert swept == 1
|
||||
assert fake.sent == []
|
||||
|
||||
|
||||
async def test_terminal_without_callback_url_sends_nothing(queue):
|
||||
q, fake = queue
|
||||
await q.submit(task_id="t1", site="rakuten", intent={})
|
||||
await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
await q.report(
|
||||
"t1",
|
||||
worker_id="w1",
|
||||
state=OrderState.PAID.value,
|
||||
payable_yen=None,
|
||||
pay_deadline=None,
|
||||
site_order_id=None,
|
||||
evidence_ref=None,
|
||||
detail="",
|
||||
terminal=True,
|
||||
terminal_status=None,
|
||||
)
|
||||
assert fake.sent == []
|
||||
|
||||
|
||||
# ---- DB 迁移 ----
|
||||
|
||||
|
||||
async def test_start_migrates_tasks_table_without_callback_url(tmp_path: Path):
|
||||
"""既有库(无 callback_url 列的老表)启动时补列,原数据不丢"""
|
||||
db_path = tmp_path / "old.db"
|
||||
conn = await aiosqlite.connect(str(db_path))
|
||||
await conn.execute(
|
||||
"CREATE TABLE tasks ("
|
||||
"task_id TEXT PRIMARY KEY, site TEXT NOT NULL, intent_json TEXT NOT NULL, "
|
||||
"status TEXT NOT NULL, lease_owner TEXT, lease_expires_at TEXT, "
|
||||
"lease_count INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL, "
|
||||
"updated_at TEXT NOT NULL)"
|
||||
)
|
||||
await conn.execute(
|
||||
"INSERT INTO tasks VALUES ('t1', 'rakuten', '{}', 'queued', "
|
||||
"NULL, NULL, 0, '2026-08-16T00:00:00Z', '2026-08-16T00:00:00Z')"
|
||||
)
|
||||
await conn.commit()
|
||||
await conn.close()
|
||||
|
||||
db = GatewayDB(db_path)
|
||||
await db.start()
|
||||
try:
|
||||
task = await db.get_task("t1")
|
||||
assert task is not None
|
||||
assert task.callback_url is None
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
# ---- 通知器层(真实发送路径,MockTransport)----
|
||||
|
||||
|
||||
def _bare_settings() -> Settings:
|
||||
"""不读 .env 的最小配置,通知器只要代理策略字段"""
|
||||
return Settings(_env_file=None)
|
||||
|
||||
|
||||
async def test_notifier_delivers_payload():
|
||||
received: list[httpx.Request] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
received.append(request)
|
||||
return httpx.Response(200)
|
||||
|
||||
notifier = CallbackNotifier(
|
||||
settings=_bare_settings(),
|
||||
timeout_seconds=5,
|
||||
client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
payload = {"task_id": "t1", "event": "terminal", "status": "succeeded"}
|
||||
notifier.notify(CALLBACK_URL, payload)
|
||||
await notifier.aclose() # 等在途通知发完
|
||||
|
||||
assert len(received) == 1
|
||||
assert str(received[0].url) == CALLBACK_URL
|
||||
assert json.loads(received[0].content) == payload
|
||||
|
||||
|
||||
async def test_notifier_swallows_upstream_5xx(caplog):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(500)
|
||||
|
||||
notifier = CallbackNotifier(
|
||||
settings=_bare_settings(),
|
||||
timeout_seconds=5,
|
||||
client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
with caplog.at_level("WARNING", logger="app.gateway.callback"):
|
||||
notifier.notify(CALLBACK_URL, {"task_id": "t1"})
|
||||
await notifier.aclose() # 不抛出
|
||||
|
||||
assert any("非成功状态" in rec.message for rec in caplog.records)
|
||||
|
||||
|
||||
async def test_notifier_swallows_connection_error(caplog):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("connection refused", request=request)
|
||||
|
||||
notifier = CallbackNotifier(
|
||||
settings=_bare_settings(),
|
||||
timeout_seconds=5,
|
||||
client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
||||
)
|
||||
with caplog.at_level("WARNING", logger="app.gateway.callback"):
|
||||
notifier.notify(CALLBACK_URL, {"task_id": "t1"})
|
||||
await notifier.aclose() # 不抛出
|
||||
|
||||
assert any("发送失败" in rec.message for rec in caplog.records)
|
||||
@@ -0,0 +1,290 @@
|
||||
"""定时下派通道测试(docs/order-gateway.md §12)
|
||||
|
||||
collector 靠 QueryQueue + GatewayDB 真实装配驱动(与 test_gateway_queries.py 同一套
|
||||
思路),用「真实队列 + 假 worker 回结果」来模拟完整闭环:
|
||||
|
||||
派 order_list → lease → worker 回带订单的 result → collector 收割 → 写进
|
||||
account_orders 编目(回写网关)→ 派 order_detail → 回 detail → 更新配送阶段
|
||||
|
||||
跑测试时关掉 collector 的常驻循环,只驱动单次 tick()/各方法,避免引入 sleep。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.gateway.collector import OrderDiscoveryCollector, _parse_list_result
|
||||
from app.gateway.db import GatewayDB
|
||||
from app.gateway.query_queue import QueryQueue
|
||||
from app.shared.config import get_settings
|
||||
|
||||
TOKEN = get_settings().bearer_token
|
||||
AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
|
||||
ORDER = {
|
||||
"order_number": "306087-20260813-0863947697",
|
||||
"order_date": "2026-08-13T09:47:28.000Z",
|
||||
"shop_id": 306087,
|
||||
"shop_name": "テスト店舗",
|
||||
"items": [],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def collector(tmp_path: Path, monkeypatch):
|
||||
"""真实装配的 (collector, db, queue) 三元组,collector 的常驻循环不启动"""
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "true")
|
||||
get_settings.cache_clear()
|
||||
db = GatewayDB(tmp_path / "c.db")
|
||||
await db.start()
|
||||
queue = QueryQueue(
|
||||
db, lease_ttl_seconds=60, query_ttl_seconds=600,
|
||||
max_attempts=3, retention_seconds=7 * 24 * 3600,
|
||||
)
|
||||
c = OrderDiscoveryCollector(
|
||||
settings=get_settings(), db=db, query_queue=queue,
|
||||
)
|
||||
try:
|
||||
yield c, db, queue
|
||||
finally:
|
||||
await db.close()
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
async def _complete_query(queue, query_id: str, result: dict, *, success: bool = True):
|
||||
"""模拟 worker:lease 这张单并回一个结果"""
|
||||
leased = await queue.lease(worker_id="w-fake", wait=0, site=None, max_wait=60)
|
||||
assert leased.query_id == query_id
|
||||
await queue.submit_result(
|
||||
query_id, worker_id="w-fake", success=success, result=result,
|
||||
error_code=None if success else 1234,
|
||||
error_message="" if success else "boom",
|
||||
)
|
||||
|
||||
|
||||
# ---- 纯函数:从列表结果里抽规范化订单 ----
|
||||
|
||||
def test_parse_list_result_extracts_valid_rows():
|
||||
result = {
|
||||
"orders": [
|
||||
{**ORDER},
|
||||
{"order_number": "x", "order_date": "t"},
|
||||
{}, # 缺 order_number,应被丢弃
|
||||
"not-a-dict", # 非 dict,应被丢弃
|
||||
]
|
||||
}
|
||||
rows = _parse_list_result(result)
|
||||
assert len(rows) == 2
|
||||
assert rows[0]["order_number"] == ORDER["order_number"]
|
||||
|
||||
|
||||
def test_parse_list_result_empty_on_no_orders():
|
||||
assert _parse_list_result(None) == []
|
||||
assert _parse_list_result({}) == []
|
||||
assert _parse_list_result({"orders": "oops"}) == []
|
||||
|
||||
|
||||
# ---- 回写:采集到的订单写进编目 ----
|
||||
|
||||
async def test_ingest_list_result_writes_back_new_order(collector):
|
||||
c, db, queue = collector
|
||||
await c._queue.submit(query_id="q1", site="rakuten", kind="order_list", params={})
|
||||
# 模拟 collector 已派这张单并在等结果(_collect_results 只处理 _pending 里的单)
|
||||
c._pending["q1"] = "order_list"
|
||||
await _complete_query(queue, "q1", {"orders": [ORDER], "window_fully_covered": True})
|
||||
|
||||
await c._collect_results() # 收割 q1 的结果,回写编目
|
||||
|
||||
row = await db.get_account_order(ORDER["order_number"])
|
||||
assert row is not None
|
||||
assert row.shop_name == "テスト店舗"
|
||||
assert row.delivery_status is None # 列表结果不带配送阶段,回到编目后仍待详情
|
||||
assert row.detail_fetched_at is None
|
||||
|
||||
|
||||
async def test_reingest_same_order_no_duplicate(collector):
|
||||
"""同一订单反复被采集,编目里只有一行(upsert 以 order_number 为主键)"""
|
||||
c, db, _ = collector
|
||||
for _ in range(2):
|
||||
await c._ingest_list_result({"orders": [ORDER]}, query_id="q")
|
||||
rows, total = await db.list_account_orders()
|
||||
assert total == 1
|
||||
assert rows[0].order_number == ORDER["order_number"]
|
||||
|
||||
|
||||
async def test_list_reingest_preserves_detail_data(collector):
|
||||
"""订单已取过详情后,再来一轮 list 扫描不得把配送阶段/详情时间抹掉"""
|
||||
c, db, _ = collector
|
||||
# 先经 detail 沉淀配送阶段
|
||||
await c._ingest_list_result({"orders": [ORDER]}, query_id="q")
|
||||
await c._ingest_detail_result(
|
||||
{
|
||||
"order_number": ORDER["order_number"], "found": True,
|
||||
"delivery_status": "CHECKING_ORDER", "order_state": "created",
|
||||
}
|
||||
)
|
||||
before = await db.get_account_order(ORDER["order_number"])
|
||||
assert before.detail_fetched_at is not None
|
||||
|
||||
# 再来一轮列表扫描
|
||||
await c._ingest_list_result({"orders": [ORDER]}, query_id="q2")
|
||||
|
||||
after = await db.get_account_order(ORDER["order_number"])
|
||||
assert after.delivery_status == "CHECKING_ORDER"
|
||||
assert after.detail_fetched_at == before.detail_fetched_at
|
||||
|
||||
|
||||
async def test_found_false_detail_does_not_wipe_status(collector):
|
||||
"""详情 found=False(订单还没反映出来)时,不覆盖已沉淀的配送阶段"""
|
||||
c, db, _ = collector
|
||||
await c._ingest_list_result({"orders": [ORDER]}, query_id="q")
|
||||
await c._ingest_detail_result(
|
||||
{
|
||||
"order_number": ORDER["order_number"], "found": True,
|
||||
"delivery_status": "CHECKING_ORDER", "order_state": "created",
|
||||
}
|
||||
)
|
||||
|
||||
# 一瞬抖动:这单暂时查不到(found=False,结果里配送字段为 null)
|
||||
await c._ingest_detail_result(
|
||||
{"order_number": ORDER["order_number"], "found": False,
|
||||
"delivery_status": None, "order_state": None}
|
||||
)
|
||||
|
||||
after = await db.get_account_order(ORDER["order_number"])
|
||||
assert after.delivery_status == "CHECKING_ORDER" # 没被抹掉
|
||||
assert after.detail_fetched_at is not None
|
||||
|
||||
|
||||
# ---- 完整闭环:派 list → 消化 → 派 detail → 更新 ----
|
||||
|
||||
async def test_full_discovery_loop_populates_catalog(collector):
|
||||
"""一次 tick 派 list,回来后被写进编目,且触发 detail 下派与更新"""
|
||||
c, db, queue = collector
|
||||
|
||||
# 第一次 tick:派出一张 order_list 扫描
|
||||
await c._dispatch_list_if_due()
|
||||
assert c._pending # 有一张在飞
|
||||
|
||||
# writer worker 领走并回一个有订单的列表
|
||||
(qid,) = c._pending
|
||||
await _complete_query(queue, qid, {"orders": [ORDER], "window_fully_covered": True})
|
||||
|
||||
# 收割 + 派 stale detail
|
||||
await c.tick()
|
||||
|
||||
# 编目里有了订单
|
||||
row = await db.get_account_order(ORDER["order_number"])
|
||||
assert row is not None and row.detail_fetched_at is None
|
||||
|
||||
# detail 下派已产生一张 order_detail 单
|
||||
pending_detail = [q for q, k in c._pending.items() if k == "order_detail"]
|
||||
assert pending_detail
|
||||
|
||||
# worker 回 detail 结果
|
||||
dqid = pending_detail[0]
|
||||
await _complete_query(
|
||||
queue, dqid,
|
||||
{
|
||||
"order_number": ORDER["order_number"],
|
||||
"found": True,
|
||||
"delivery_status": "CHECKING_ORDER",
|
||||
"order_state": "created",
|
||||
},
|
||||
)
|
||||
await c._collect_results()
|
||||
|
||||
after = await db.get_account_order(ORDER["order_number"])
|
||||
assert after.delivery_status == "CHECKING_ORDER"
|
||||
assert after.detail_fetched_at is not None
|
||||
|
||||
|
||||
async def test_detail_dispatch_is_same_day_idempotent(collector):
|
||||
"""同一天内轮询到同一订单,detail 单只派一张(幂等 submit 命中既有单)"""
|
||||
c, db, _ = collector
|
||||
await c._ingest_list_result({"orders": [ORDER]}, query_id="q")
|
||||
|
||||
await c._dispatch_one_detail(ORDER["order_number"])
|
||||
first = set(c._pending)
|
||||
await c._dispatch_one_detail(ORDER["order_number"])
|
||||
second = set(c._pending)
|
||||
|
||||
assert first == second
|
||||
assert len(first) == 1
|
||||
|
||||
|
||||
async def test_stale_detail_not_redispatched_within_day(collector):
|
||||
"""详情 found=False(订单还没反映)时,当天不再重复派同一订单的详情"""
|
||||
c, db, _ = collector
|
||||
await c._ingest_list_result({"orders": [ORDER]}, query_id="q")
|
||||
|
||||
# 第一轮:派详情 → 回 found=False → 订单仍算「未取过详情」
|
||||
await c._dispatch_stale_details()
|
||||
(dqid,) = [q for q, k in c._pending.items() if k == "order_detail"]
|
||||
await _complete_query(
|
||||
c._queue, dqid,
|
||||
{"order_number": ORDER["order_number"], "found": False},
|
||||
)
|
||||
await c._collect_results()
|
||||
assert (await db.get_account_order(ORDER["order_number"])).detail_fetched_at is None
|
||||
|
||||
# 第二轮:不应再为同一订单派详情(当天已派过)
|
||||
before = set(c._pending)
|
||||
await c._dispatch_stale_details()
|
||||
assert set(c._pending) == before
|
||||
|
||||
|
||||
async def test_no_list_redispatch_while_pending(collector):
|
||||
"""有在飞的 list 单时,即使间隔到了也不重复派"""
|
||||
c, _, _ = collector
|
||||
await c._dispatch_list_if_due()
|
||||
assert len(c._pending) == 1
|
||||
# 模拟时间流逝:把 last_list_submitted 拨回到很久以前
|
||||
c._last_list_submitted = None
|
||||
await c._dispatch_list_if_due()
|
||||
# 幂等?list 用的是自动生成的 q- 前缀 id,重新 submit 会再新建——
|
||||
# 但「有在飞 list」会挡住。此时 pending 里那张 q- 单还没到终态,不应再派。
|
||||
assert len([k for k in c._pending.values() if k == "order_list"]) == 1
|
||||
|
||||
|
||||
# ---- HTTP 层:编目读取与手动触发 ----
|
||||
|
||||
@pytest.fixture
|
||||
def gateway_client(tmp_path: Path, monkeypatch):
|
||||
monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(tmp_path / "gw.db"))
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "false")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
from app.gateway.main import create_app
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
app = create_app()
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
def test_get_cataloged_order_not_found_6005(gateway_client):
|
||||
response = gateway_client.get("/api/account/orders/nope", headers=AUTH)
|
||||
assert response.status_code == 404
|
||||
assert response.json()["code"] == 6005
|
||||
|
||||
|
||||
def test_trigger_discovery_dispatch_list_sweep(gateway_client):
|
||||
"""手动触发应派一张 order_list 单(走真实队列)"""
|
||||
# 触发里 collector 真实存在;手动派单不应因 discovery disabled 而失效——
|
||||
# 但这个夹具关了 discovery,trigger 只是 submit 一张单,仍应正常。
|
||||
body = gateway_client.post("/api/account/discovery/trigger", headers=AUTH).json()
|
||||
assert body["success"] is True
|
||||
assert body["data"]["query_id"].startswith("q-")
|
||||
# 这张单真的进了查询队列
|
||||
detail = gateway_client.get(f"/api/account/queries/{body['data']['query_id']}", headers=AUTH).json()
|
||||
assert detail["data"]["kind"] == "order_list"
|
||||
|
||||
|
||||
def test_catalog_endpoints_reject_missing_token(gateway_client):
|
||||
assert gateway_client.get("/api/account/orders").status_code == 401
|
||||
assert gateway_client.get("/api/account/orders/x").status_code == 401
|
||||
assert gateway_client.post("/api/account/discovery/trigger").status_code == 401
|
||||
@@ -26,6 +26,7 @@ AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
def gateway_client(tmp_path: Path, monkeypatch):
|
||||
db_path = tmp_path / "gw.db"
|
||||
monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "false")
|
||||
# worker_offline_alert_seconds 设小一点,方便测 /health 失联判定
|
||||
monkeypatch.setenv("RAKUTEN_WORKER_OFFLINE_ALERT_SECONDS", "1")
|
||||
get_settings.cache_clear()
|
||||
|
||||
@@ -0,0 +1,391 @@
|
||||
"""账号只读查询通道测试(docs/order-gateway.md §11)
|
||||
|
||||
分两层:
|
||||
- HTTP 层:鉴权、幂等、6005/6006、lease → result 全链路、列表与 /health
|
||||
- 队列语义层:直接驱动 QueryQueue,测那些靠 HTTP 不好造的时序——租约超时重投、
|
||||
重投次数用尽、整体 TTL 过期、迟到结果被拒、过保留期清理
|
||||
|
||||
这里最关键的一条是「只读可重投」:它与下单任务的「绝不自动重投」正好相反,
|
||||
两条通道的语义分歧全在这几个用例里,改任何一边前先看它们。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.gateway.db import GatewayDB, QueryRow
|
||||
from app.gateway.query_queue import QueryQueue, to_iso, utcnow
|
||||
from app.shared.config import get_settings
|
||||
from app.shared.errors import QueryLeaseInvalidError
|
||||
from app.shared.task_state import AccountQueryKind, QueryStatus
|
||||
|
||||
TOKEN = get_settings().bearer_token
|
||||
AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
|
||||
QUERIES = "/api/account/queries"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gateway_client(tmp_path: Path, monkeypatch):
|
||||
"""起一个独立 DB 的网关应用(与 test_gateway_api.py 同一套装配方式)"""
|
||||
monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(tmp_path / "gw.db"))
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "false")
|
||||
get_settings.cache_clear()
|
||||
try:
|
||||
from app.gateway.main import create_app
|
||||
|
||||
app = create_app()
|
||||
with TestClient(app) as client:
|
||||
yield client
|
||||
finally:
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def queue(tmp_path: Path):
|
||||
"""直接可用的 (QueryQueue, GatewayDB) 对
|
||||
|
||||
队列语义用例要模拟「租约已过期」「结果早就写完了」这类时间流逝,直接改库里的
|
||||
时间戳比 sleep 快得多,所以把 db 一并交出去。
|
||||
"""
|
||||
db = GatewayDB(tmp_path / "q.db")
|
||||
await db.start()
|
||||
try:
|
||||
yield (
|
||||
QueryQueue(
|
||||
db,
|
||||
lease_ttl_seconds=60,
|
||||
query_ttl_seconds=600,
|
||||
max_attempts=3,
|
||||
retention_seconds=7 * 24 * 3600,
|
||||
),
|
||||
db,
|
||||
)
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
def _submit(client: TestClient, **overrides) -> dict:
|
||||
payload = {
|
||||
"query_id": "q1",
|
||||
"site": "rakuten",
|
||||
"kind": AccountQueryKind.ORDER_LIST.value,
|
||||
"params": {"max_pages": 1},
|
||||
}
|
||||
payload.update(overrides)
|
||||
return client.post(QUERIES, json=payload, headers=AUTH).json()
|
||||
|
||||
|
||||
# ---- HTTP:鉴权与参数校验 ----
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path,method",
|
||||
[
|
||||
(QUERIES, "POST"),
|
||||
(f"{QUERIES}/lease", "GET"),
|
||||
(f"{QUERIES}/q1", "GET"),
|
||||
],
|
||||
)
|
||||
def test_query_endpoints_reject_missing_token(gateway_client, path, method):
|
||||
response = gateway_client.request(method, path)
|
||||
assert response.status_code == 401
|
||||
assert response.json()["code"] == 1001
|
||||
|
||||
|
||||
def test_unknown_kind_is_rejected_at_submit(gateway_client):
|
||||
"""kind 拼错当场 422,而不是等 worker 领走、执行、回报失败才知道"""
|
||||
response = gateway_client.post(
|
||||
QUERIES,
|
||||
json={"site": "rakuten", "kind": "order_lst", "params": {}},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == 1002
|
||||
|
||||
|
||||
# ---- HTTP:提交、幂等、领取、回结果 ----
|
||||
|
||||
|
||||
def test_submit_returns_query_id_and_queued(gateway_client):
|
||||
body = _submit(gateway_client)
|
||||
assert body["success"] is True
|
||||
assert body["data"]["query_id"] == "q1"
|
||||
assert body["data"]["status"] == QueryStatus.QUEUED.value
|
||||
assert body["data"]["created"] is True
|
||||
|
||||
|
||||
def test_submit_is_idempotent_on_same_query_id(gateway_client):
|
||||
assert _submit(gateway_client)["data"]["created"] is True
|
||||
assert _submit(gateway_client)["data"]["created"] is False
|
||||
|
||||
|
||||
def test_submit_generates_query_id_when_absent(gateway_client):
|
||||
body = gateway_client.post(
|
||||
QUERIES,
|
||||
json={"site": "rakuten", "kind": AccountQueryKind.ORDER_DETAIL.value, "params": {}},
|
||||
headers=AUTH,
|
||||
).json()
|
||||
assert body["data"]["query_id"].startswith("q-")
|
||||
|
||||
|
||||
def test_lease_returns_null_when_no_query(gateway_client):
|
||||
response = gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
assert response.json()["data"] is None
|
||||
|
||||
|
||||
def test_lease_then_result_completes_the_query(gateway_client):
|
||||
_submit(gateway_client)
|
||||
|
||||
leased = gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH).json()
|
||||
assert leased["data"]["query_id"] == "q1"
|
||||
assert leased["data"]["kind"] == AccountQueryKind.ORDER_LIST.value
|
||||
assert leased["data"]["params"] == {"max_pages": 1}
|
||||
assert leased["data"]["attempt"] == 1
|
||||
|
||||
result = {"kind": "order_list", "orders": [{"order_number": "306087-20260813-0863947697"}]}
|
||||
reported = gateway_client.post(
|
||||
f"{QUERIES}/q1/result",
|
||||
json={"worker_id": "w1", "success": True, "result": result},
|
||||
headers=AUTH,
|
||||
).json()
|
||||
assert reported["data"]["status"] == QueryStatus.SUCCEEDED.value
|
||||
|
||||
detail = gateway_client.get(f"{QUERIES}/q1", headers=AUTH).json()["data"]
|
||||
assert detail["status"] == QueryStatus.SUCCEEDED.value
|
||||
assert detail["result"] == result
|
||||
assert detail["error"] is None
|
||||
assert detail["lease_owner"] is None
|
||||
|
||||
|
||||
def test_failed_result_carries_error_code_and_message(gateway_client):
|
||||
_submit(gateway_client)
|
||||
gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
gateway_client.post(
|
||||
f"{QUERIES}/q1/result",
|
||||
json={
|
||||
"worker_id": "w1",
|
||||
"success": False,
|
||||
"error_code": 5001,
|
||||
"error_message": "账号未登录",
|
||||
},
|
||||
headers=AUTH,
|
||||
)
|
||||
detail = gateway_client.get(f"{QUERIES}/q1", headers=AUTH).json()["data"]
|
||||
assert detail["status"] == QueryStatus.FAILED.value
|
||||
assert detail["error"] == {"code": 5001, "message": "账号未登录"}
|
||||
|
||||
|
||||
def test_two_queries_can_be_in_flight_at_once(gateway_client):
|
||||
"""与下单任务不同:只读查询没有「全局并发度 1」的闸门"""
|
||||
_submit(gateway_client, query_id="q1")
|
||||
_submit(gateway_client, query_id="q2")
|
||||
|
||||
first = gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH).json()
|
||||
second = gateway_client.get(f"{QUERIES}/lease?worker_id=w2&wait=0", headers=AUTH).json()
|
||||
|
||||
assert first["data"]["query_id"] == "q1"
|
||||
assert second["data"]["query_id"] == "q2"
|
||||
|
||||
|
||||
def test_order_lease_is_not_disturbed_by_pending_query(gateway_client):
|
||||
"""两条通道互不干扰:有查询在飞,下单任务照领"""
|
||||
_submit(gateway_client)
|
||||
gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
|
||||
gateway_client.post(
|
||||
"/api/orders",
|
||||
json={"task_id": "t1", "site": "rakuten", "intent": {}},
|
||||
headers=AUTH,
|
||||
)
|
||||
leased = gateway_client.get("/api/orders/lease?worker_id=w1&wait=0", headers=AUTH).json()
|
||||
assert leased["data"]["task_id"] == "t1"
|
||||
|
||||
|
||||
# ---- HTTP:错误码 ----
|
||||
|
||||
|
||||
def test_get_unknown_query_returns_6005(gateway_client):
|
||||
response = gateway_client.get(f"{QUERIES}/no-such", headers=AUTH)
|
||||
assert response.status_code == 404
|
||||
assert response.json()["code"] == 6005
|
||||
|
||||
|
||||
def test_result_from_non_owner_returns_6006(gateway_client):
|
||||
_submit(gateway_client)
|
||||
gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
|
||||
response = gateway_client.post(
|
||||
f"{QUERIES}/q1/result",
|
||||
json={"worker_id": "w2", "success": True, "result": {}},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert response.status_code == 409
|
||||
assert response.json()["code"] == 6006
|
||||
|
||||
|
||||
def test_result_on_terminal_query_returns_6006(gateway_client):
|
||||
_submit(gateway_client)
|
||||
gateway_client.get(f"{QUERIES}/lease?worker_id=w1&wait=0", headers=AUTH)
|
||||
gateway_client.post(
|
||||
f"{QUERIES}/q1/result",
|
||||
json={"worker_id": "w1", "success": True, "result": {"a": 1}},
|
||||
headers=AUTH,
|
||||
)
|
||||
|
||||
response = gateway_client.post(
|
||||
f"{QUERIES}/q1/result",
|
||||
json={"worker_id": "w1", "success": True, "result": {"a": 2}},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert response.status_code == 409
|
||||
assert response.json()["code"] == 6006
|
||||
# 结果没有被第二次回报覆盖
|
||||
detail = gateway_client.get(f"{QUERIES}/q1", headers=AUTH).json()["data"]
|
||||
assert detail["result"] == {"a": 1}
|
||||
|
||||
|
||||
# ---- HTTP:列表与健康检查 ----
|
||||
|
||||
|
||||
def test_list_queries_filters_by_kind(gateway_client):
|
||||
_submit(gateway_client, query_id="q1", kind=AccountQueryKind.ORDER_LIST.value)
|
||||
_submit(gateway_client, query_id="q2", kind=AccountQueryKind.ORDER_DETAIL.value)
|
||||
|
||||
body = gateway_client.get(
|
||||
f"{QUERIES}?kind={AccountQueryKind.ORDER_DETAIL.value}", headers=AUTH
|
||||
).json()
|
||||
assert body["data"]["total"] == 1
|
||||
assert body["data"]["items"][0]["query_id"] == "q2"
|
||||
|
||||
|
||||
def test_health_reports_queued_query_count(gateway_client):
|
||||
_submit(gateway_client)
|
||||
body = gateway_client.get("/health").json()
|
||||
assert body["data"]["queued_query_count"] == 1
|
||||
|
||||
|
||||
# ---- 队列语义:只读可重投(与下单任务的核心分歧)----
|
||||
|
||||
|
||||
async def test_expired_lease_goes_back_to_queued(queue):
|
||||
"""租约过期→回 queued 自动重投。下单任务在这里是 stale 且绝不重投"""
|
||||
q, db = queue
|
||||
await q.submit(query_id="q1", site="rakuten", kind="order_list", params={})
|
||||
lease = await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
assert lease is not None
|
||||
|
||||
await db.update_query("q1", lease_expires_at=to_iso(utcnow() - timedelta(seconds=1)))
|
||||
assert await q.sweep() == 1
|
||||
|
||||
detail = await q.get_detail("q1")
|
||||
assert detail.status == QueryStatus.QUEUED
|
||||
assert detail.lease_owner is None
|
||||
|
||||
again = await q.lease(worker_id="w2", wait=0, site=None, max_wait=60)
|
||||
assert again is not None
|
||||
assert again.attempt == 2
|
||||
|
||||
|
||||
async def test_late_result_after_reinvest_is_rejected(queue):
|
||||
"""超时重投后,上一轮 worker 迟到的结果必须被拒,不能覆盖新一轮"""
|
||||
q, db = queue
|
||||
await q.submit(query_id="q1", site="rakuten", kind="order_list", params={})
|
||||
await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
await db.update_query("q1", lease_expires_at=to_iso(utcnow() - timedelta(seconds=1)))
|
||||
await q.sweep()
|
||||
await q.lease(worker_id="w2", wait=0, site=None, max_wait=60)
|
||||
|
||||
with pytest.raises(QueryLeaseInvalidError):
|
||||
await q.submit_result(
|
||||
"q1", worker_id="w1", success=True, result={"stale": True},
|
||||
error_code=None, error_message="",
|
||||
)
|
||||
|
||||
|
||||
async def test_reinvest_stops_at_max_attempts(queue):
|
||||
"""重投次数用尽 → failed,不再无限重投同一张卡死的单"""
|
||||
q, db = queue
|
||||
await q.submit(query_id="q1", site="rakuten", kind="order_list", params={})
|
||||
for _ in range(3):
|
||||
await q.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
await db.update_query("q1", lease_expires_at=to_iso(utcnow() - timedelta(seconds=1)))
|
||||
await q.sweep()
|
||||
|
||||
detail = await q.get_detail("q1")
|
||||
assert detail.status == QueryStatus.FAILED
|
||||
assert detail.attempts == 3
|
||||
assert "不再重投" in detail.error.message
|
||||
|
||||
|
||||
async def test_query_expires_after_ttl(tmp_path):
|
||||
"""整体 TTL 到点 → expired(本地 worker 不在线的典型表现)"""
|
||||
db = GatewayDB(tmp_path / "q.db")
|
||||
await db.start()
|
||||
try:
|
||||
queue = QueryQueue(
|
||||
db, lease_ttl_seconds=60, query_ttl_seconds=60, max_attempts=3,
|
||||
retention_seconds=3600,
|
||||
)
|
||||
# 直接插一张「10 分钟前创建」的单,比 sleep 现实
|
||||
old = to_iso(utcnow() - timedelta(minutes=10))
|
||||
await db.insert_query(
|
||||
QueryRow(
|
||||
query_id="q1", site="rakuten", kind="order_list", params_json="{}",
|
||||
status=QueryStatus.QUEUED.value, lease_owner=None, lease_expires_at=None,
|
||||
attempts=0, result_json=None, error_code=None, error_message=None,
|
||||
created_at=old, updated_at=old, completed_at=None,
|
||||
)
|
||||
)
|
||||
|
||||
assert await queue.sweep() == 1
|
||||
detail = await queue.get_detail("q1")
|
||||
assert detail.status == QueryStatus.EXPIRED
|
||||
assert "worker 不在线" in detail.error.message
|
||||
# 过期的单不会再被领走
|
||||
assert await queue.lease(worker_id="w1", wait=0, site=None, max_wait=60) is None
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def test_sweep_purges_queries_past_retention(tmp_path):
|
||||
"""终态查询单过保留期被清掉——结果里带站点原始 JSON,不清会一直涨"""
|
||||
db = GatewayDB(tmp_path / "q.db")
|
||||
await db.start()
|
||||
try:
|
||||
queue = QueryQueue(
|
||||
db, lease_ttl_seconds=60, query_ttl_seconds=600, max_attempts=3,
|
||||
retention_seconds=1,
|
||||
)
|
||||
await queue.submit(query_id="q1", site="rakuten", kind="order_list", params={})
|
||||
await queue.lease(worker_id="w1", wait=0, site=None, max_wait=60)
|
||||
await queue.submit_result(
|
||||
"q1", worker_id="w1", success=True, result={"a": 1},
|
||||
error_code=None, error_message="",
|
||||
)
|
||||
await db.update_query("q1", completed_at=to_iso(utcnow() - timedelta(seconds=60)))
|
||||
|
||||
await queue.sweep()
|
||||
assert await db.get_query("q1") is None
|
||||
finally:
|
||||
await db.close()
|
||||
|
||||
|
||||
async def test_sweep_keeps_unfinished_queries(tmp_path):
|
||||
"""保留期清理只动终态单,进行中的一律不碰"""
|
||||
db = GatewayDB(tmp_path / "q.db")
|
||||
await db.start()
|
||||
try:
|
||||
queue = QueryQueue(
|
||||
db, lease_ttl_seconds=60, query_ttl_seconds=600, max_attempts=3,
|
||||
retention_seconds=1,
|
||||
)
|
||||
await queue.submit(query_id="q1", site="rakuten", kind="order_list", params={})
|
||||
await queue.sweep()
|
||||
assert await db.get_query("q1") is not None
|
||||
finally:
|
||||
await db.close()
|
||||
@@ -21,6 +21,7 @@ AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
def gateway_client(tmp_path: Path, monkeypatch):
|
||||
db_path = tmp_path / "gw.db"
|
||||
monkeypatch.setenv("RAKUTEN_GATEWAY_DB_PATH", str(db_path))
|
||||
monkeypatch.setenv("RAKUTEN_ACCOUNT_DISCOVERY_ENABLED", "false")
|
||||
# TTL 设小一点,让 lease 一过期就能被 sweep 转 stale
|
||||
monkeypatch.setenv("RAKUTEN_LEASE_TTL_SECONDS", "1")
|
||||
get_settings.cache_clear()
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
"""openapi.json 导出测试:守住「一份文档覆盖三个服务」的合并结果
|
||||
|
||||
这份文档是外部(Apifox / Postman / 上游联调)唯一的接口来源,合并逻辑错了就会
|
||||
把人引到不存在的接口、或漏标鉴权。下面的用例全部在内存里 build_spec() 检查合并
|
||||
结果,不读仓库里的 openapi.json——那个文件是 gitignore 的生成物,CI 的全新 clone
|
||||
里根本不存在,断言「文件内容 == 当前导出结果」在 CI 上必然失败。
|
||||
|
||||
因此「改了接口忘了重新导出」没有自动兜底,得手动跑:
|
||||
.venv/Scripts/python.exe scripts/export_openapi.py --check
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = ROOT / "scripts" / "export_openapi.py"
|
||||
|
||||
_METHODS = ("get", "put", "post", "delete", "options", "head", "patch", "trace")
|
||||
|
||||
|
||||
def _load_exporter():
|
||||
"""按路径加载 scripts/export_openapi.py(scripts 不是包)
|
||||
|
||||
必须先塞进 sys.modules 再 exec:dataclass 解析类型注解时会去
|
||||
sys.modules[cls.__module__] 找命名空间,没登记就直接 AttributeError。
|
||||
"""
|
||||
spec = importlib.util.spec_from_file_location("export_openapi", SCRIPT)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def exporter():
|
||||
return _load_exporter()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def spec(exporter):
|
||||
return exporter.build_spec()
|
||||
|
||||
|
||||
def _operations(document: dict) -> list[tuple[str, str, dict]]:
|
||||
return [
|
||||
(path, method, operation)
|
||||
for path, item in document["paths"].items()
|
||||
for method, operation in item.items()
|
||||
if method in _METHODS
|
||||
]
|
||||
|
||||
|
||||
def test_covers_all_three_services(spec):
|
||||
paths = spec["paths"]
|
||||
# 抓取
|
||||
assert "/api/search" in paths
|
||||
assert "/api/rakuma/search" in paths
|
||||
# 交易(有状态端)
|
||||
assert "/api/auth/login" in paths
|
||||
assert "/api/cart/add" in paths
|
||||
# 网关
|
||||
assert "/api/orders" in paths
|
||||
assert "/api/orders/lease" in paths
|
||||
|
||||
|
||||
def test_each_operation_points_at_its_own_service(spec):
|
||||
"""operation 级 servers:导入后不必手动切 base URL"""
|
||||
by_path = {path: operation["servers"][0]["url"] for path, _, operation in _operations(spec)}
|
||||
assert by_path["/api/search"].endswith(":31107")
|
||||
assert by_path["/api/cart/add"].endswith(":31108")
|
||||
assert by_path["/api/orders/lease"].endswith(":31109")
|
||||
# /health 三个服务都有,合成一条并列出三个 server
|
||||
health_servers = {s["url"] for s in spec["paths"]["/health"]["get"]["servers"]}
|
||||
assert len(health_servers) == 3
|
||||
|
||||
|
||||
def test_only_health_is_public(spec):
|
||||
"""除 /health 外都必须标 Bearer 鉴权——漏标会让调用方以为能匿名调"""
|
||||
unprotected = [
|
||||
f"{method.upper()} {path}"
|
||||
for path, method, operation in _operations(spec)
|
||||
if "security" not in operation
|
||||
]
|
||||
assert unprotected == ["GET /health"]
|
||||
assert "BearerAuth" in spec["components"]["securitySchemes"]
|
||||
|
||||
|
||||
def test_operation_ids_are_unique(spec):
|
||||
"""三个服务的 /health 原本都叫 health_health_get,撞了会让导入工具丢接口"""
|
||||
ids = [operation["operationId"] for _, _, operation in _operations(spec)]
|
||||
assert len(ids) == len(set(ids))
|
||||
|
||||
|
||||
def test_all_refs_resolve(spec):
|
||||
"""合并 schema 时如果重名处理错了,$ref 会指向不存在的定义"""
|
||||
names = set(spec["components"]["schemas"])
|
||||
refs = set(re.findall(r"#/components/schemas/([^\"]+)", json.dumps(spec)))
|
||||
assert not refs - names
|
||||
@@ -0,0 +1,115 @@
|
||||
"""统一出站代理策略的单元测试。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.proxy import httpx_client_options, playwright_launch_proxy
|
||||
from app.trading.worker import client as worker_client
|
||||
from app.trading.worker.client import GatewayClient
|
||||
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def _settings(**overrides) -> Settings:
|
||||
return Settings(_env_file=None, **overrides)
|
||||
|
||||
|
||||
def test_disabled_proxy_does_not_inherit_host_environment() -> None:
|
||||
settings = _settings()
|
||||
|
||||
assert httpx_client_options(settings) == {"trust_env": False}
|
||||
assert playwright_launch_proxy(settings) is None
|
||||
|
||||
|
||||
def test_authenticated_proxy_encodes_httpx_credentials() -> None:
|
||||
settings = _settings(
|
||||
proxy_server="http://proxy.example:8080",
|
||||
proxy_username="user@example.com",
|
||||
proxy_password="pa:ss /#",
|
||||
proxy_bypass="localhost,.internal.example",
|
||||
)
|
||||
|
||||
assert httpx_client_options(settings) == {
|
||||
"trust_env": False,
|
||||
"proxy": "http://user%40example.com:pa%3Ass%20%2F%23@proxy.example:8080",
|
||||
}
|
||||
assert playwright_launch_proxy(settings) == {
|
||||
"server": "http://proxy.example:8080",
|
||||
"username": "user@example.com",
|
||||
"password": "pa:ss /#",
|
||||
"bypass": "localhost,.internal.example",
|
||||
}
|
||||
|
||||
|
||||
def test_internal_hosts_bypass_httpx_proxy() -> None:
|
||||
settings = _settings(
|
||||
proxy_server="http://proxy.example:8080",
|
||||
proxy_bypass="localhost,.internal.example,*.svc",
|
||||
)
|
||||
|
||||
assert settings.proxy_bypasses("http://localhost:31109")
|
||||
assert settings.proxy_bypasses("https://api.internal.example/path")
|
||||
assert settings.proxy_bypasses("https://orders.svc")
|
||||
assert not settings.proxy_bypasses("https://www.rakuten.co.jp/")
|
||||
assert httpx_client_options(settings, target_url="http://localhost:31109") == {
|
||||
"trust_env": False
|
||||
}
|
||||
assert httpx_client_options(settings, target_url="https://www.rakuten.co.jp/") == {
|
||||
"trust_env": False,
|
||||
"proxy": "http://proxy.example:8080",
|
||||
}
|
||||
|
||||
|
||||
def test_gateway_client_uses_its_base_url_for_proxy_policy(monkeypatch) -> None:
|
||||
settings = _settings(proxy_server="http://proxy.example:8080")
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
def fake_httpx_options(actual_settings: Settings, *, target_url: str | None = None) -> dict[str, object]:
|
||||
captured["settings"] = actual_settings
|
||||
captured["target_url"] = target_url
|
||||
return {"trust_env": False}
|
||||
|
||||
monkeypatch.setattr(worker_client, "httpx_client_options", fake_httpx_options)
|
||||
client = GatewayClient("https://gateway.example", "token", settings=settings)
|
||||
try:
|
||||
assert captured == {"settings": settings, "target_url": "https://gateway.example"}
|
||||
finally:
|
||||
asyncio.run(client.aclose())
|
||||
|
||||
|
||||
def test_every_project_http_client_and_browser_launch_has_proxy_policy() -> None:
|
||||
"""新增出站入口时,强制它显式接入统一代理策略。"""
|
||||
missing_httpx_policy: list[Path] = []
|
||||
missing_browser_proxy: list[Path] = []
|
||||
|
||||
for root_name in ("app", "scripts"):
|
||||
for path in (PROJECT_ROOT / root_name).rglob("*.py"):
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"))
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, ast.Call) or not isinstance(node.func, ast.Attribute):
|
||||
continue
|
||||
if (
|
||||
node.func.attr == "AsyncClient"
|
||||
and isinstance(node.func.value, ast.Name)
|
||||
and node.func.value.id == "httpx"
|
||||
):
|
||||
uses_policy = any(
|
||||
keyword.arg is None
|
||||
and isinstance(keyword.value, ast.Call)
|
||||
and isinstance(keyword.value.func, ast.Name)
|
||||
and keyword.value.func.id == "httpx_client_options"
|
||||
for keyword in node.keywords
|
||||
)
|
||||
if not uses_policy:
|
||||
missing_httpx_policy.append(path)
|
||||
if node.func.attr in {"launch", "launch_persistent_context"} and not any(
|
||||
keyword.arg == "proxy" for keyword in node.keywords
|
||||
):
|
||||
missing_browser_proxy.append(path)
|
||||
|
||||
assert not missing_httpx_policy
|
||||
assert not missing_browser_proxy
|
||||
@@ -0,0 +1,417 @@
|
||||
"""账号只读查询 worker 测试(app/trading/worker/query_runner.py)
|
||||
|
||||
站点交互与网关都用桩替换,测的是 QueryRunner 自己的编排:
|
||||
- 两种 kind 各自把站点结果整理成什么样的 result(原始 JSON 有没有原样带出来)
|
||||
- 失败怎么转成一次「失败回报」——超时、站点 AppError、参数不合法、未知 kind
|
||||
- 结果体积闸门:先丢站点原始 JSON,仍超限就判失败让上游缩小窗口
|
||||
|
||||
一条贯穿始终的约定:**worker 永远不把异常抛回主循环**,每张查询单都要有一次
|
||||
回报(成功或失败),否则网关侧那张单只能干等到租约超时。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import AppError, NotLoggedInError
|
||||
from app.shared.task_state import AccountQueryKind, OrderState
|
||||
from app.trading.worker.models import QueryTask
|
||||
from app.trading.worker.query_runner import QueryRunner
|
||||
from app.trading.worker.site_interact import (
|
||||
OrderDetailSnapshot,
|
||||
OrderListEntry,
|
||||
OrderListItem,
|
||||
OrderListWindow,
|
||||
OrderStatusSnapshot,
|
||||
)
|
||||
|
||||
|
||||
class _FakeGateway:
|
||||
"""记录回报内容的网关客户端替身
|
||||
|
||||
`lease_query` 里那句 `sleep(0)` 是必需的:真实实现是 HTTP 长轮询,一定会把
|
||||
控制权交回事件循环;纯内存替身不 await 任何东西的话,run() 会一直霸占循环,
|
||||
同一个 event loop 里的停机任务永远排不上,测试直接挂死。
|
||||
"""
|
||||
|
||||
def __init__(self, queries: list[QueryTask | None] | None = None):
|
||||
self._queries = list(queries or [])
|
||||
self.reports: list[dict[str, Any]] = []
|
||||
self.lease_calls = 0
|
||||
|
||||
async def lease_query(self, worker_id: str, *, wait: int = 30) -> QueryTask | None:
|
||||
self.lease_calls += 1
|
||||
await asyncio.sleep(0)
|
||||
if not self._queries:
|
||||
return None
|
||||
return self._queries.pop(0)
|
||||
|
||||
async def report_query_result(
|
||||
self,
|
||||
query_id: str,
|
||||
worker_id: str,
|
||||
*,
|
||||
success: bool,
|
||||
result: dict[str, Any] | None = None,
|
||||
error_code: int | None = None,
|
||||
error_message: str = "",
|
||||
) -> dict[str, Any]:
|
||||
self.reports.append(
|
||||
{
|
||||
"query_id": query_id,
|
||||
"worker_id": worker_id,
|
||||
"success": success,
|
||||
"result": result,
|
||||
"error_code": error_code,
|
||||
"error_message": error_message,
|
||||
}
|
||||
)
|
||||
return {"query_id": query_id, "status": "succeeded" if success else "failed"}
|
||||
|
||||
|
||||
class _FakeSite:
|
||||
"""SiteInteractor 替身:按脚本返回窗口/详情,或抛出指定异常"""
|
||||
|
||||
def __init__(self, *, window=None, detail=None, error: Exception | None = None, delay: float = 0):
|
||||
self._window = window
|
||||
self._detail = detail
|
||||
self._error = error
|
||||
self._delay = delay
|
||||
self.list_calls: list[dict[str, Any]] = []
|
||||
self.detail_calls: list[str] = []
|
||||
|
||||
async def list_recent_orders(self, *, since, max_pages=None):
|
||||
self.list_calls.append({"since": since, "max_pages": max_pages})
|
||||
await asyncio.sleep(self._delay)
|
||||
if self._error:
|
||||
raise self._error
|
||||
return self._window
|
||||
|
||||
async def fetch_order_detail(self, site_order_id: str):
|
||||
self.detail_calls.append(site_order_id)
|
||||
await asyncio.sleep(self._delay)
|
||||
if self._error:
|
||||
raise self._error
|
||||
return self._detail
|
||||
|
||||
|
||||
def _runner(site: _FakeSite, gateway: _FakeGateway, **settings_kwargs) -> QueryRunner:
|
||||
settings = Settings(worker_id="w1", **settings_kwargs)
|
||||
return QueryRunner(settings=settings, gateway_client=gateway, site=site) # type: ignore[arg-type]
|
||||
|
||||
|
||||
def _query(kind: str, params: dict[str, Any] | None = None, **kwargs) -> QueryTask:
|
||||
return QueryTask(
|
||||
query_id=kwargs.get("query_id", "q1"),
|
||||
site=kwargs.get("site", "rakuten"),
|
||||
kind=kind,
|
||||
params=params or {},
|
||||
attempt=kwargs.get("attempt", 1),
|
||||
)
|
||||
|
||||
|
||||
def _window(*, covered: bool = True) -> OrderListWindow:
|
||||
return OrderListWindow(
|
||||
entries=[
|
||||
OrderListEntry(
|
||||
order_number="306087-20260813-0863947697",
|
||||
order_date="2026-08-13T09:47:28.000Z",
|
||||
shop_id=306087,
|
||||
shop_name="BACKYARD FAMILY インテリアタウン",
|
||||
items=[
|
||||
OrderListItem(
|
||||
item_url="https://item.rakuten.co.jp/backyard/x/",
|
||||
item_name="壁掛けフック",
|
||||
item_id=10012345,
|
||||
)
|
||||
],
|
||||
)
|
||||
],
|
||||
window_fully_covered=covered,
|
||||
raw_pages=[{"ordersFound": 1, "orderList": [{"orderNumber": "306087-20260813-0863947697"}]}],
|
||||
)
|
||||
|
||||
|
||||
# ---- order_list ----
|
||||
|
||||
|
||||
async def test_order_list_returns_normalized_and_raw():
|
||||
site = _FakeSite(window=_window())
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value, {"max_pages": 2}))
|
||||
|
||||
report = gateway.reports[0]
|
||||
assert report["success"] is True
|
||||
result = report["result"]
|
||||
assert result["kind"] == "order_list"
|
||||
assert result["window_fully_covered"] is True
|
||||
assert result["orders"][0]["order_number"] == "306087-20260813-0863947697"
|
||||
assert result["orders"][0]["items"][0]["item_id"] == 10012345
|
||||
# 站点原文原样带出,规范化模型没覆盖的字段上游还能自己取
|
||||
assert result["raw_pages"][0]["ordersFound"] == 1
|
||||
assert site.list_calls[0]["max_pages"] == 2
|
||||
|
||||
|
||||
async def test_order_list_since_is_passed_through():
|
||||
site = _FakeSite(window=_window())
|
||||
runner = _runner(site, _FakeGateway())
|
||||
|
||||
await runner.handle(
|
||||
_query(AccountQueryKind.ORDER_LIST.value, {"since": "2026-08-01T00:00:00Z"})
|
||||
)
|
||||
|
||||
assert site.list_calls[0]["since"] == datetime(2026, 8, 1, tzinfo=timezone.utc)
|
||||
|
||||
|
||||
async def test_order_list_without_since_uses_epoch_and_default_max_pages():
|
||||
site = _FakeSite(window=_window())
|
||||
runner = _runner(site, _FakeGateway(), account_query_default_max_pages=5)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
assert site.list_calls[0]["since"] == datetime(1970, 1, 1, tzinfo=timezone.utc)
|
||||
assert site.list_calls[0]["max_pages"] == 5
|
||||
|
||||
|
||||
async def test_order_list_reports_partial_coverage_honestly():
|
||||
"""翻页没覆盖完窗口时必须如实标出来——上游据此知道「没查到」不等于「没有」"""
|
||||
site = _FakeSite(window=_window(covered=False))
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
assert gateway.reports[0]["result"]["window_fully_covered"] is False
|
||||
|
||||
|
||||
async def test_invalid_since_is_reported_as_failure():
|
||||
site = _FakeSite(window=_window())
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value, {"since": "上周"}))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert gateway.reports[0]["error_code"] == 1003
|
||||
assert site.list_calls == [] # 参数不合法就没去碰站点
|
||||
|
||||
|
||||
async def test_invalid_max_pages_is_reported_as_failure():
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(_FakeSite(window=_window()), gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value, {"max_pages": 0}))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert gateway.reports[0]["error_code"] == 1003
|
||||
|
||||
|
||||
# ---- order_detail ----
|
||||
|
||||
|
||||
async def test_order_detail_returns_stage_and_raw():
|
||||
detail = OrderDetailSnapshot(
|
||||
status=OrderStatusSnapshot(
|
||||
found=True, stage_label="出荷", order_state=OrderState.SHIPPED, html="<html/>"
|
||||
),
|
||||
raw={"pageType": "ph-detail"},
|
||||
)
|
||||
site = _FakeSite(detail=detail)
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(
|
||||
_query(AccountQueryKind.ORDER_DETAIL.value, {"order_number": "306087-20260813-0863947697"})
|
||||
)
|
||||
|
||||
result = gateway.reports[0]["result"]
|
||||
assert result["found"] is True
|
||||
assert result["stage_label"] == "出荷"
|
||||
assert result["order_state"] == OrderState.SHIPPED.value
|
||||
assert result["raw"] == {"pageType": "ph-detail"}
|
||||
assert result["raw_available"] is True
|
||||
assert site.detail_calls == ["306087-20260813-0863947697"]
|
||||
|
||||
|
||||
async def test_order_detail_not_found_is_success_not_failure():
|
||||
"""站点自己说订单要 10 分钟才反映出来:查不到是正常结果,不是查询失败"""
|
||||
site = _FakeSite(detail=OrderDetailSnapshot(status=OrderStatusSnapshot(found=False), raw=None))
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_DETAIL.value, {"order_number": "x-1-2"}))
|
||||
|
||||
assert gateway.reports[0]["success"] is True
|
||||
assert gateway.reports[0]["result"]["found"] is False
|
||||
assert gateway.reports[0]["result"]["raw_available"] is False
|
||||
|
||||
|
||||
async def test_order_detail_requires_order_number():
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(_FakeSite(), gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_DETAIL.value, {}))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert gateway.reports[0]["error_code"] == 1003
|
||||
|
||||
|
||||
# ---- 失败路径 ----
|
||||
|
||||
|
||||
async def test_unknown_kind_is_reported_as_failure():
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(_FakeSite(), gateway)
|
||||
|
||||
await runner.handle(_query("order_everything"))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert "未知的查询种类" in gateway.reports[0]["error_message"]
|
||||
|
||||
|
||||
async def test_non_rakuten_site_is_rejected():
|
||||
"""交易服务只覆盖乐天市场,ラクマ 永不进入账号链路"""
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(_FakeSite(), gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value, site="rakuma"))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert "rakuma" in gateway.reports[0]["error_message"]
|
||||
|
||||
|
||||
async def test_site_app_error_keeps_its_error_code():
|
||||
"""站点侧错误码原样回给上游(掉登录=5001),不被翻译成一个笼统的失败"""
|
||||
site = _FakeSite(error=NotLoggedInError(site="rakuten"))
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert gateway.reports[0]["error_code"] == 5001
|
||||
|
||||
|
||||
async def test_unexpected_exception_is_reported_not_raised():
|
||||
site = _FakeSite(error=RuntimeError("boom"))
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert gateway.reports[0]["error_code"] == 1500
|
||||
assert "RuntimeError" in gateway.reports[0]["error_message"]
|
||||
|
||||
|
||||
async def test_timeout_is_reported_as_retryable_failure():
|
||||
"""账号锁被下单占着 → 超时按失败回报(上游重发即可),不挂到租约过期"""
|
||||
site = _FakeSite(window=_window(), delay=0.2)
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(site, gateway, account_query_timeout_seconds=0)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert gateway.reports[0]["error_code"] == 2002
|
||||
assert "可稍后重发" in gateway.reports[0]["error_message"]
|
||||
|
||||
|
||||
async def test_report_failure_does_not_escape():
|
||||
"""回报本身失败(比如租约已被重投)只记日志,不能把循环带崩"""
|
||||
|
||||
class _RejectingGateway(_FakeGateway):
|
||||
async def report_query_result(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
|
||||
raise AppError(message="lease invalid", code="X", err_code=6006, status_code=409)
|
||||
|
||||
runner = _runner(_FakeSite(window=_window()), _RejectingGateway())
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value)) # 不抛异常即通过
|
||||
|
||||
|
||||
# ---- 结果体积闸门 ----
|
||||
|
||||
|
||||
async def test_oversized_result_drops_raw_first():
|
||||
big_window = _window()
|
||||
big_window.raw_pages = [{"padding": "x" * 5000}]
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(_FakeSite(window=big_window), gateway, query_result_max_bytes=2000)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
report = gateway.reports[0]
|
||||
assert report["success"] is True
|
||||
assert report["result"]["raw_pages"] == []
|
||||
assert "raw_omitted" in report["result"]
|
||||
# 规范化字段还在——上游至少拿得到订单号
|
||||
assert report["result"]["orders"][0]["order_number"] == "306087-20260813-0863947697"
|
||||
|
||||
|
||||
async def test_result_still_oversized_after_dropping_raw_fails():
|
||||
"""丢掉原始 JSON 仍超限:判失败并告诉上游缩小窗口,不硬塞进网关库"""
|
||||
window = _window()
|
||||
window.entries = [
|
||||
OrderListEntry(order_number=f"o{i}", order_date="2026-08-13T09:47:28.000Z")
|
||||
for i in range(200)
|
||||
]
|
||||
gateway = _FakeGateway()
|
||||
runner = _runner(_FakeSite(window=window), gateway, query_result_max_bytes=500)
|
||||
|
||||
await runner.handle(_query(AccountQueryKind.ORDER_LIST.value))
|
||||
|
||||
assert gateway.reports[0]["success"] is False
|
||||
assert "缩小查询窗口" in gateway.reports[0]["error_message"]
|
||||
|
||||
|
||||
# ---- 主循环 ----
|
||||
|
||||
|
||||
async def test_run_loop_handles_then_stops():
|
||||
"""领到单就处理;stop() 后循环退出,不吃掉后续任务"""
|
||||
gateway = _FakeGateway([_query(AccountQueryKind.ORDER_LIST.value), None])
|
||||
runner = _runner(_FakeSite(window=_window()), gateway)
|
||||
|
||||
async def stop_soon():
|
||||
# 让 run() 先跑几轮(领单 → 处理 → 再领一次拿到 None),再停
|
||||
for _ in range(20):
|
||||
await asyncio.sleep(0)
|
||||
runner.stop()
|
||||
|
||||
await asyncio.wait_for(asyncio.gather(runner.run(), stop_soon()), timeout=5)
|
||||
|
||||
assert [r["query_id"] for r in gateway.reports] == ["q1"]
|
||||
|
||||
|
||||
async def test_run_loop_survives_lease_error(monkeypatch):
|
||||
"""lease 报错不能让循环退出(网关重启、网络抖动都算正常)"""
|
||||
real_sleep = asyncio.sleep
|
||||
|
||||
class _FlakyGateway(_FakeGateway):
|
||||
"""前两次 lease 都报错,第二次顺便把循环停掉,避免测试无限转"""
|
||||
|
||||
def __init__(self, stopper):
|
||||
super().__init__()
|
||||
self._stopper = stopper
|
||||
|
||||
async def lease_query(self, worker_id: str, *, wait: int = 30):
|
||||
self.lease_calls += 1
|
||||
if self.lease_calls >= 2:
|
||||
self._stopper()
|
||||
raise AppError(message="gateway down", code="X", err_code=3001)
|
||||
|
||||
holder: dict[str, QueryRunner] = {}
|
||||
gateway = _FlakyGateway(lambda: holder["runner"].stop())
|
||||
runner = _runner(_FakeSite(), gateway)
|
||||
holder["runner"] = runner
|
||||
# 出错后那句 sleep(5) 在测试里没必要真等;捕获原函数再替换,否则递归自调
|
||||
monkeypatch.setattr(asyncio, "sleep", lambda *_: real_sleep(0))
|
||||
|
||||
await runner.run()
|
||||
|
||||
assert gateway.lease_calls == 2
|
||||
+1121
-18
File diff suppressed because it is too large
Load Diff
@@ -89,6 +89,25 @@ async def test_warmup_happens_before_first_fetch_and_is_reused():
|
||||
assert seen.count(TARGET) == 2
|
||||
|
||||
|
||||
async def test_successful_warmup_is_reused_when_upstream_sets_no_cookie():
|
||||
"""首页可能返回 200 但不下发 Akamai cookie,仍不得重复预热。"""
|
||||
seen: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(str(request.url))
|
||||
return httpx.Response(200, text=GOOD_PAGE)
|
||||
|
||||
session = await build_session(handler)
|
||||
try:
|
||||
await session.fetch_html(TARGET, mobile=False)
|
||||
await session.fetch_html(TARGET, mobile=False)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
assert seen.count("https://www.rakuten.co.jp/") == 1
|
||||
assert seen.count(TARGET) == 2
|
||||
|
||||
|
||||
async def test_missing_state_marker_is_treated_as_blocked():
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, text="<html>no state here</html>")
|
||||
|
||||
+100
-1
@@ -24,7 +24,10 @@ class StubAuthSession:
|
||||
def __init__(self) -> None:
|
||||
self.checked: list[str] = []
|
||||
self.reloaded: list[str] = []
|
||||
self.relogin_calls: list[str] = []
|
||||
self.logged_in = True
|
||||
# try_relogin 的结果;None 表示「登录成功并转为登录态」
|
||||
self.relogin_result: bool | None = None
|
||||
|
||||
@property
|
||||
def sites(self) -> tuple[str, ...]:
|
||||
@@ -53,6 +56,14 @@ class StubAuthSession:
|
||||
self.reloaded.append(site)
|
||||
return 3
|
||||
|
||||
async def try_relogin(self, site: str) -> bool:
|
||||
"""默认「登录成功」:翻成登录态并返回 True;relogin_result 可注入失败"""
|
||||
self.relogin_calls.append(site)
|
||||
if self.relogin_result is None:
|
||||
self.logged_in = True
|
||||
return True
|
||||
return self.relogin_result
|
||||
|
||||
async def require_logged_in(self, site: str) -> None:
|
||||
if not self.logged_in:
|
||||
raise NotLoggedInError(site=site, detail="stub")
|
||||
@@ -168,7 +179,9 @@ def test_health_does_not_probe_the_site(client, stub):
|
||||
# ---- 鉴权 ----
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/api/auth/status", "/api/auth/reload"])
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["/api/auth/status", "/api/auth/login", "/api/auth/reload"]
|
||||
)
|
||||
def test_auth_endpoints_reject_missing_token(client, path):
|
||||
response = client.post(path, json={})
|
||||
assert response.status_code == 401
|
||||
@@ -213,6 +226,92 @@ def test_status_rejects_unknown_site(client):
|
||||
assert response.json()["code"] == 1002
|
||||
|
||||
|
||||
# ---- 自动登录 ----
|
||||
|
||||
|
||||
def test_login_skips_when_already_logged_in(client, stub):
|
||||
"""已登录时不该白起一次登录流程(起浏览器 + 打站点,代价不小)"""
|
||||
response = client.post("/api/auth/login", json={}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["data"]["logged_in"] is True
|
||||
assert body["data"]["relogin_attempted"] == {"rakuten": False}
|
||||
assert stub.relogin_calls == []
|
||||
# 结论必须来自真实探测,不能只看缓存
|
||||
assert stub.checked == ["rakuten"]
|
||||
|
||||
|
||||
def test_login_triggers_relogin_when_logged_out(client, stub):
|
||||
stub.logged_in = False
|
||||
response = client.post("/api/auth/login", json={"site": "rakuten"}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["data"]["logged_in"] is True
|
||||
assert body["data"]["relogin_attempted"] == {"rakuten": True}
|
||||
assert stub.relogin_calls == ["rakuten"]
|
||||
# 登录后必须再探测一次确认,不能拿登录流程的自述当结论
|
||||
assert stub.checked == ["rakuten", "rakuten"]
|
||||
|
||||
|
||||
def test_login_reports_failure_without_raising(client, stub):
|
||||
"""登录失败按 logged_in=false 正常返回,不是 5001
|
||||
|
||||
调用方要据此决定人工接管还是换账号;抛错会把「为什么失败」压成一个错误码。
|
||||
"""
|
||||
stub.logged_in = False
|
||||
stub.relogin_result = False
|
||||
response = client.post("/api/auth/login", json={}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["success"] is True
|
||||
assert body["data"]["logged_in"] is False
|
||||
assert body["data"]["relogin_attempted"] == {"rakuten": True}
|
||||
|
||||
|
||||
def test_login_rejects_unknown_site(client):
|
||||
response = client.post("/api/auth/login", json={"site": "mercari"}, headers=AUTH)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == 1002
|
||||
|
||||
|
||||
# ---- 启动时自动登录(RAKUTEN_AUTO_LOGIN_ON_START)----
|
||||
|
||||
|
||||
class _FakeContainer:
|
||||
def __init__(self, auth_session) -> None:
|
||||
self.auth_session = auth_session
|
||||
|
||||
|
||||
async def test_auto_login_on_start_skips_when_logged_in():
|
||||
from app.trading.main import auto_login_on_start
|
||||
|
||||
stub = StubAuthSession()
|
||||
await auto_login_on_start(_FakeContainer(stub))
|
||||
assert stub.relogin_calls == []
|
||||
|
||||
|
||||
async def test_auto_login_on_start_logs_in_when_logged_out():
|
||||
from app.trading.main import auto_login_on_start
|
||||
|
||||
stub = StubAuthSession()
|
||||
stub.logged_in = False
|
||||
await auto_login_on_start(_FakeContainer(stub))
|
||||
assert stub.relogin_calls == ["rakuten"]
|
||||
|
||||
|
||||
async def test_auto_login_on_start_swallows_errors():
|
||||
"""探测抛错也不能把启动流程带崩——服务要能起来报「未登录」"""
|
||||
from app.trading.main import auto_login_on_start
|
||||
|
||||
class Boom(StubAuthSession):
|
||||
async def check(self, site: str):
|
||||
raise RuntimeError("站点不可达")
|
||||
|
||||
stub = Boom()
|
||||
await auto_login_on_start(_FakeContainer(stub))
|
||||
assert stub.relogin_calls == []
|
||||
|
||||
|
||||
# ---- 登录态重载 ----
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""verify.verify_on_site 单元测试
|
||||
|
||||
用桩 gateway(只需要 get_task)和桩 site(只需要 list_recent_orders)跑通全部
|
||||
分支——不涉及 Playwright/httpx,纯逻辑测试。核对链路本身(intent.item_url →
|
||||
查任务创建时间 → 拉订单列表 → 按商品 URL 比对)2026-08-13 用真实账号验证过
|
||||
数据结构(见 test_site_interact.py 里 _parse_order_list 那组测试),但当时账号
|
||||
只有 1 笔订单,这里 NOT_ORDERED / 多笔命中 / 查询失败这几个分支只覆盖逻辑本身,
|
||||
没有被真实多单数据跑过。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.shared.errors import AppError
|
||||
from app.trading.worker import verify
|
||||
from app.trading.worker.models import LeaseTask
|
||||
from app.trading.worker.site_interact import OrderListEntry, OrderListItem, OrderListWindow
|
||||
|
||||
|
||||
def _make_task(*, item_url: str | None = "https://item.rakuten.co.jp/shop/x/") -> LeaseTask:
|
||||
intent: dict[str, Any] = {}
|
||||
if item_url is not None:
|
||||
intent["item_url"] = item_url
|
||||
return LeaseTask(task_id="t1", site="rakuten", intent=intent, lease_count=2)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeGateway:
|
||||
created_at: str | None = "2026-08-01T00:00:00Z"
|
||||
fail_with: Exception | None = None
|
||||
calls: list[str] = field(default_factory=list)
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any]:
|
||||
self.calls.append(task_id)
|
||||
if self.fail_with is not None:
|
||||
raise self.fail_with
|
||||
return {"task_id": task_id, "created_at": self.created_at}
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakeSite:
|
||||
window: OrderListWindow | None = None
|
||||
fail_with: Exception | None = None
|
||||
received_since: datetime | None = None
|
||||
|
||||
async def list_recent_orders(self, *, since: datetime) -> OrderListWindow:
|
||||
self.received_since = since
|
||||
if self.fail_with is not None:
|
||||
raise self.fail_with
|
||||
assert self.window is not None
|
||||
return self.window
|
||||
|
||||
|
||||
def _entry(order_number: str, item_url: str) -> OrderListEntry:
|
||||
return OrderListEntry(
|
||||
order_number=order_number,
|
||||
order_date="2026-08-10T00:00:00Z",
|
||||
items=[OrderListItem(item_url=item_url)],
|
||||
)
|
||||
|
||||
|
||||
# ---- intent 缺 item_url:不查任何东西,直接 unknown ----
|
||||
|
||||
|
||||
async def test_missing_item_url_returns_unknown_without_calling_gateway():
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite()
|
||||
result = await verify.verify_on_site(_make_task(item_url=None), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.UNKNOWN
|
||||
assert gateway.calls == []
|
||||
|
||||
|
||||
# ---- 查任务创建时间失败/不可解析 ----
|
||||
|
||||
|
||||
async def test_get_task_failure_returns_unknown():
|
||||
gateway = FakeGateway(fail_with=AppError(message="网关挂了", code="X", err_code=9999))
|
||||
site = FakeSite()
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.UNKNOWN
|
||||
assert "网关挂了" in result.detail
|
||||
|
||||
|
||||
async def test_unparseable_created_at_returns_unknown():
|
||||
gateway = FakeGateway(created_at="不是日期")
|
||||
site = FakeSite()
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.UNKNOWN
|
||||
|
||||
|
||||
# ---- 站点订单列表查询失败 ----
|
||||
|
||||
|
||||
async def test_site_list_failure_returns_unknown():
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite(fail_with=RuntimeError("Playwright 挂了"))
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.UNKNOWN
|
||||
assert "Playwright 挂了" in result.detail
|
||||
|
||||
|
||||
# ---- 命中 1 笔:已下单(2026-08-13 真实账号验证过的分支)----
|
||||
|
||||
|
||||
async def test_exact_one_match_returns_already_ordered():
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite(
|
||||
window=OrderListWindow(
|
||||
entries=[_entry("306087-20260813-0863947697", "https://item.rakuten.co.jp/shop/x/")],
|
||||
window_fully_covered=True,
|
||||
)
|
||||
)
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.ALREADY_ORDERED
|
||||
assert result.site_order_id == "306087-20260813-0863947697"
|
||||
|
||||
|
||||
async def test_match_ignores_query_string_difference():
|
||||
"""订单列表的 itemUrl 带 ?variantId=...,intent.item_url 通常不带——只比路径"""
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite(
|
||||
window=OrderListWindow(
|
||||
entries=[_entry("o1", "https://item.rakuten.co.jp/shop/x/?variantId=abc-1")],
|
||||
window_fully_covered=True,
|
||||
)
|
||||
)
|
||||
result = await verify.verify_on_site(
|
||||
_make_task(item_url="https://item.rakuten.co.jp/shop/x/"), gateway=gateway, site=site
|
||||
)
|
||||
assert result.verdict == verify.VerifyVerdict.ALREADY_ORDERED
|
||||
|
||||
|
||||
# ---- 命中 0 笔且窗口确认覆盖完:未下单 ----
|
||||
|
||||
|
||||
async def test_no_match_and_fully_covered_returns_not_ordered():
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite(
|
||||
window=OrderListWindow(
|
||||
entries=[_entry("o1", "https://item.rakuten.co.jp/other-shop/y/")],
|
||||
window_fully_covered=True,
|
||||
)
|
||||
)
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.NOT_ORDERED
|
||||
|
||||
|
||||
# ---- 命中 0 笔但窗口没确认覆盖完:不能猜没下单 ----
|
||||
|
||||
|
||||
async def test_no_match_and_not_fully_covered_returns_unknown():
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite(window=OrderListWindow(entries=[], window_fully_covered=False))
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.UNKNOWN
|
||||
|
||||
|
||||
# ---- 命中多笔:无法唯一确定 ----
|
||||
|
||||
|
||||
async def test_multiple_matches_returns_unknown():
|
||||
gateway = FakeGateway()
|
||||
site = FakeSite(
|
||||
window=OrderListWindow(
|
||||
entries=[
|
||||
_entry("o1", "https://item.rakuten.co.jp/shop/x/"),
|
||||
_entry("o2", "https://item.rakuten.co.jp/shop/x/"),
|
||||
],
|
||||
window_fully_covered=True,
|
||||
)
|
||||
)
|
||||
result = await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert result.verdict == verify.VerifyVerdict.UNKNOWN
|
||||
assert "o1" in result.detail and "o2" in result.detail
|
||||
|
||||
|
||||
# ---- since 正确传递给 site.list_recent_orders(用任务创建时间,不是别的)----
|
||||
|
||||
|
||||
async def test_since_passed_to_site_matches_task_created_at():
|
||||
gateway = FakeGateway(created_at="2026-08-05T12:00:00Z")
|
||||
site = FakeSite(window=OrderListWindow(entries=[], window_fully_covered=True))
|
||||
await verify.verify_on_site(_make_task(), gateway=gateway, site=site)
|
||||
assert site.received_since == datetime(2026, 8, 5, 12, 0, 0, tzinfo=timezone.utc)
|
||||
+519
-17
@@ -12,20 +12,28 @@ evidence store / site,覆盖 §6 主循环的分支:
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from app.shared.errors import OrderGuardError
|
||||
from app.shared.errors import CheckoutBlockedError, OrderGuardError
|
||||
from app.shared.task_state import OrderState, TaskStatus
|
||||
from app.trading.worker import verify
|
||||
from app.trading.worker.evidence import EvidenceStore
|
||||
from app.trading.worker.local_db import LocalDB
|
||||
from app.trading.worker.models import LeaseTask
|
||||
from app.trading.worker.runner import WorkerRunner
|
||||
from app.trading.worker.site_interact import CheckoutSummary, SiteInteractor
|
||||
from app.trading.worker.site_interact import (
|
||||
CheckoutSummary,
|
||||
OrderStatusSnapshot,
|
||||
PageSnapshot,
|
||||
SiteInteractor,
|
||||
SubmitOutcome,
|
||||
)
|
||||
|
||||
|
||||
# ---- 桩:网关客户端 ----
|
||||
@@ -39,11 +47,18 @@ class FakeGateway:
|
||||
renews: int = 0
|
||||
fail_report_with: Exception | None = None
|
||||
|
||||
# verify_on_site 恢复核对会调 get_task 拿 created_at;测试默认给一个可解析
|
||||
# 的时间戳,具体核对分支由 monkeypatch 替换 verify.verify_on_site 控制
|
||||
task_created_at: str = "2026-01-01T00:00:00Z"
|
||||
|
||||
async def lease(
|
||||
self, worker_id: str, *, wait: int = 30, site: str | None = None
|
||||
) -> LeaseTask | None:
|
||||
return None # 主循环测试不通过 lease 喂数据,直接调 handle
|
||||
|
||||
async def get_task(self, task_id: str) -> dict[str, Any]:
|
||||
return {"task_id": task_id, "created_at": self.task_created_at}
|
||||
|
||||
async def renew(self, task_id: str, worker_id: str) -> dict[str, Any]:
|
||||
self.renews += 1
|
||||
return {"task_id": task_id, "lease_expires_at": "2099-01-01T00:00:00Z"}
|
||||
@@ -123,6 +138,10 @@ def runner(tmp_path, local_db, evidence, site) -> WorkerRunner:
|
||||
class _FakeSettings:
|
||||
worker_id = "test"
|
||||
order_max_total_yen = 30000
|
||||
# 测试环境不真的等 3 小时:间隔设 0,次数设小,_monitor_order 的循环
|
||||
# 靠 asyncio.sleep(0) 立刻推进,不拖慢测试
|
||||
order_monitor_poll_interval_seconds = 0
|
||||
order_monitor_max_checks = 5
|
||||
|
||||
@property
|
||||
def worker_id_effective(self) -> str:
|
||||
@@ -159,6 +178,18 @@ def _make_task(
|
||||
)
|
||||
|
||||
|
||||
def _set_site_clear(site, result: dict) -> None:
|
||||
"""把 site.clear_cart 替换成返回指定结果的桩
|
||||
|
||||
runner.execute() 现在开单一律先 clear_cart(step 0,见 runner 里注释),
|
||||
测试若不复用它,site 桩会真的去调未启动的 SiteInteractor 而崩。
|
||||
"""
|
||||
async def _f() -> dict:
|
||||
return result
|
||||
|
||||
site.clear_cart = _f # type: ignore[assignment]
|
||||
|
||||
|
||||
# ---- 已完成的任务:补报而不重新执行(§9 第 9 条)----
|
||||
|
||||
|
||||
@@ -188,15 +219,17 @@ async def test_already_finished_task_is_reported_not_re_executed(
|
||||
async def test_recovery_task_with_unknown_verdict_reports_needs_human(
|
||||
runner: WorkerRunner, local_db: LocalDB, monkeypatch
|
||||
):
|
||||
"""lease_count > 1 + verify 桩返回 unknown → 转 needs_human,不执行"""
|
||||
"""lease_count > 1 + verify 返回 unknown(intent 缺 item_url)→ 转 needs_human,不执行"""
|
||||
|
||||
# verify 桩默认返回 unknown,不用 monkeypatch
|
||||
# verify.verify_on_site 已是真实实现(2026-08-13),但 _make_task 默认 intent 为空,
|
||||
# 缺 item_url 时函数第一步就短路返回 unknown,不用 monkeypatch 也能测这条分支
|
||||
await runner.handle(_make_task(task_id="t1", lease_count=2))
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
assert "无法定论" in terminal["detail"]
|
||||
assert "unknown" in terminal["detail"]
|
||||
assert "item_url" in terminal["detail"]
|
||||
|
||||
|
||||
async def test_recovery_task_with_already_ordered_verdict_reports_succeeded(
|
||||
@@ -204,7 +237,7 @@ async def test_recovery_task_with_already_ordered_verdict_reports_succeeded(
|
||||
):
|
||||
"""lease_count > 1 + verify 返回 already_ordered → 补报 succeeded,不重新下单"""
|
||||
|
||||
async def _already_ordered(task): # noqa: ANN001
|
||||
async def _already_ordered(task, **kwargs): # noqa: ANN001
|
||||
return verify.VerifyResult(
|
||||
verdict=verify.VerifyVerdict.ALREADY_ORDERED,
|
||||
site_order_id="ord-1",
|
||||
@@ -221,8 +254,34 @@ async def test_recovery_task_with_already_ordered_verdict_reports_succeeded(
|
||||
assert terminal["site_order_id"] == "ord-1"
|
||||
assert terminal["terminal_status"] == TaskStatus.SUCCEEDED
|
||||
|
||||
# 本地 DB 也应当被标记为已完成
|
||||
assert await local_db.has_finished("t1") is True
|
||||
|
||||
async def test_recovery_task_with_not_ordered_verdict_still_reports_needs_human(
|
||||
runner: WorkerRunner, local_db: LocalDB, monkeypatch
|
||||
):
|
||||
"""lease_count > 1 + verify 确认 not_ordered → 仍转 needs_human,不自动重新下单
|
||||
|
||||
刻意的保守选择(见 runner._handle_recovery 里的注释):NOT_ORDERED 分支目前
|
||||
只有逻辑正确性,没有被真实多单数据验证过,放开自动重新执行前必须显式决定。
|
||||
"""
|
||||
|
||||
async def _not_ordered(task, **kwargs): # noqa: ANN001
|
||||
return verify.VerifyResult(
|
||||
verdict=verify.VerifyVerdict.NOT_ORDERED,
|
||||
detail="已核对任务创建时间之后的全部订单,未找到匹配商品",
|
||||
)
|
||||
|
||||
monkeypatch.setattr(verify, "verify_on_site", _not_ordered)
|
||||
|
||||
await runner.handle(_make_task(task_id="t1", lease_count=2))
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
assert "not_ordered" in terminal["detail"]
|
||||
|
||||
# needs_human 分支不标记本地完成(与 UNKNOWN 分支一致):只有 ALREADY_ORDERED
|
||||
# 才 mark_finished,避免把「转人工」误当成本地已完成的终态
|
||||
assert await local_db.has_finished("t1") is False
|
||||
|
||||
|
||||
# ---- 站点交互未实现 → needs_human(§10 接口缝)----
|
||||
@@ -231,18 +290,28 @@ async def test_recovery_task_with_already_ordered_verdict_reports_succeeded(
|
||||
async def test_unimplemented_site_interaction_becomes_needs_human(
|
||||
runner: WorkerRunner, local_db: LocalDB
|
||||
):
|
||||
"""SiteInteractor 默认 enter_checkout 抛 NotImplementedError → runner 转 needs_human
|
||||
"""站点交互抛 NotImplementedError → runner 转 needs_human
|
||||
|
||||
add_to_cart 与 verify_cart 已实现(但站点未启动时会从 Playwright 调用失败,
|
||||
测试里先替换为 noop 让流程跑到 enter_checkout)。
|
||||
add_to_cart / verify_cart / enter_checkout / parse_checkout / submit_order /
|
||||
pay / check_order_status 现在均已实现,这里用桩显式模拟「某一步没实现」,
|
||||
覆盖 runner._execute_with_renewal 里 NotImplementedError → needs_human 的分支,
|
||||
与具体哪个方法真的未实现解耦。
|
||||
"""
|
||||
|
||||
# 让已实现的两步 noop,触发未实现的 enter_checkout
|
||||
async def _noop(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _checkout_html(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"fake-checkout-png")
|
||||
|
||||
async def _unimplemented(html): # noqa: ANN001
|
||||
raise NotImplementedError("模拟:假装 parse_checkout 还没实现")
|
||||
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _noop # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _unimplemented # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
@@ -252,6 +321,37 @@ async def test_unimplemented_site_interaction_becomes_needs_human(
|
||||
assert "未实现" in terminal["detail"]
|
||||
|
||||
|
||||
# ---- 结算被站点风控拦截 → needs_human(规格 §10.1)----
|
||||
|
||||
|
||||
async def test_checkout_blocked_becomes_needs_human(
|
||||
runner: WorkerRunner, local_db: LocalDB
|
||||
):
|
||||
"""enter_checkout 抛 CheckoutBlockedError(session upgrade 被拦截)→ 转 needs_human
|
||||
|
||||
与普通 AppError(转 FAILED)区分开:这类阻断是站点主动要求人工验证,
|
||||
不是本服务的失败,需要人工用有头浏览器接管当前登录态完成验证。
|
||||
"""
|
||||
|
||||
async def _noop(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _blocked(task): # noqa: ANN001
|
||||
raise CheckoutBlockedError("session upgrade 提交密码后 30s 内 URL 未变化,判定被站点风控拦截")
|
||||
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _noop # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _blocked # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
assert "风控" in terminal["detail"]
|
||||
|
||||
|
||||
# ---- ラクマ 不在交易范围 → needs_human ----
|
||||
|
||||
|
||||
@@ -281,7 +381,7 @@ async def test_amount_guard_blocks_when_payable_exceeds_limit(
|
||||
call_log.append(task.task_id + ":step")
|
||||
|
||||
async def _checkout_html(task): # noqa: ANN001
|
||||
return "<html>checkout</html>"
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"fake-checkout-png")
|
||||
|
||||
async def _parse(html: str):
|
||||
return CheckoutSummary(payable_yen=50000)
|
||||
@@ -291,13 +391,17 @@ async def test_amount_guard_blocks_when_payable_exceeds_limit(
|
||||
async def _submit(task): # noqa: ANN001
|
||||
nonlocal submit_called
|
||||
submit_called = True
|
||||
return "ord-1"
|
||||
return SubmitOutcome(
|
||||
site_order_id="ord-1",
|
||||
evidence=PageSnapshot(html="<html>complete</html>", screenshot=b"fake-submit-png"),
|
||||
)
|
||||
|
||||
runner._site.add_to_cart = _ok # type: ignore[assignment]
|
||||
runner._site.verify_cart = _ok # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _parse # type: ignore[assignment]
|
||||
runner._site.submit_order = _submit # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
# intent.max_total_yen 缺省,回落到 settings.order_max_total_yen=30000,
|
||||
# 实际 50000 > 30000 → 拦截
|
||||
@@ -320,7 +424,7 @@ async def test_amount_guard_honors_intent_max_total_yen(
|
||||
return None
|
||||
|
||||
async def _checkout_html(task): # noqa: ANN001
|
||||
return "<html>checkout</html>"
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"fake-checkout-png")
|
||||
|
||||
async def _parse(html: str):
|
||||
return CheckoutSummary(payable_yen=8000)
|
||||
@@ -337,6 +441,7 @@ async def test_amount_guard_honors_intent_max_total_yen(
|
||||
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _parse # type: ignore[assignment]
|
||||
runner._site.submit_order = _submit # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
# 全局上限 30000,但 intent 给 5000 → 实际 8000 > 5000 → 拦截
|
||||
await runner.handle(_make_task(task_id="t1", intent={"max_total_yen": 5000}))
|
||||
@@ -347,6 +452,260 @@ async def test_amount_guard_honors_intent_max_total_yen(
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
|
||||
|
||||
# ---- 开单前清理购物车(step 0):上一单失败残留不被新单买走 ----
|
||||
|
||||
|
||||
async def test_execute_clears_cart_before_adding(
|
||||
runner: WorkerRunner, local_db: LocalDB
|
||||
):
|
||||
"""开单一律先 clear_cart,且必须先于 add_to_cart 发生
|
||||
|
||||
上一单若在下单前失败(加购后、提交前),残留不会自动消失;不清掉就直接
|
||||
加购会把新商品叠在旧商品上,结算时把上一单的一起买走。这里断言顺序:
|
||||
clear 在 add_to_cart 之前,且 clear 返回 count=0 才继续往下加购。
|
||||
"""
|
||||
call_order: list[str] = []
|
||||
|
||||
async def _clear(): # noqa: ANN001
|
||||
call_order.append("clear")
|
||||
return {"removed_count": 0, "cart_count": 0}
|
||||
|
||||
async def _noop(task): # noqa: ANN001
|
||||
call_order.append("add")
|
||||
return None
|
||||
|
||||
async def _verify(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _checkout_html(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"fake-checkout-png")
|
||||
|
||||
async def _parse(html: str):
|
||||
return CheckoutSummary(payable_yen=297)
|
||||
|
||||
async def _submit(task): # noqa: ANN001
|
||||
return SubmitOutcome(
|
||||
site_order_id="ord-1",
|
||||
evidence=PageSnapshot(html="<html>complete</html>", screenshot=b"fake-submit-png"),
|
||||
)
|
||||
|
||||
async def _pay(task, site_order_id): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _unchanged(order_id: str): # noqa: ANN001
|
||||
# 不要让后台监控真的落到未启动的 SiteInteractor 上:found=True 但状态无
|
||||
# 变化(order_state=None),监控循环会一直轮询到上限退出,不污染断言
|
||||
return OrderStatusSnapshot(found=True, order_state=None)
|
||||
|
||||
runner._site.clear_cart = _clear # type: ignore[assignment]
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _verify # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _parse # type: ignore[assignment]
|
||||
runner._site.submit_order = _submit # type: ignore[assignment]
|
||||
runner._site.pay = _pay # type: ignore[assignment]
|
||||
runner._site.check_order_status = _unchanged # type: ignore[assignment]
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
assert call_order == ["clear", "add"], f"clear 必须在 add 之前:{call_order}"
|
||||
|
||||
|
||||
async def test_execute_blocks_when_cart_not_empty_after_clear(
|
||||
runner: WorkerRunner, local_db: LocalDB
|
||||
):
|
||||
"""清理后 count 仍非 0 → 视为清理没跑干净,转 needs_human,绝不往下加购
|
||||
|
||||
残留可能属于上一单失败留下的商品,带着它加购/结算会把上一单的一起买走。
|
||||
按闸门语义宁可卡住等人核对,也不赌「残留不会被买走」;关键断言是
|
||||
add_to_cart/submit_order 都未被调用。
|
||||
"""
|
||||
any_add_or_submit = {"called": False}
|
||||
|
||||
async def _clear(): # noqa: ANN001
|
||||
return {"removed_count": 2, "cart_count": 2} # 清理后仍剩 2 件
|
||||
|
||||
async def _add(task): # noqa: ANN001
|
||||
any_add_or_submit["called"] = True
|
||||
return None
|
||||
|
||||
runner._site.clear_cart = _clear # type: ignore[assignment]
|
||||
runner._site.add_to_cart = _add # type: ignore[assignment]
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
assert any_add_or_submit["called"] is False # 关键:没有加购、更没有提交
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
assert "未清空" in terminal["detail"]
|
||||
|
||||
|
||||
async def test_execute_blocks_when_clear_count_unknown(
|
||||
runner: WorkerRunner, local_db: LocalDB
|
||||
):
|
||||
"""清理后 count API 拿不到结果(cart_count=-1)→ 同样按未清空处理
|
||||
|
||||
无法确认购物车已空,就不能带着残留往下加购。clear_cart 在末尾 count API
|
||||
失败时返回 cart_count=-1——此时不能默认它是 0(那等于把「确认已空」当成
|
||||
「可能非空」),转 needs_human 让现场可查。
|
||||
"""
|
||||
add_called = False
|
||||
|
||||
async def _clear(): # noqa: ANN001
|
||||
return {"removed_count": 0, "cart_count": -1}
|
||||
|
||||
async def _add(task): # noqa: ANN001
|
||||
nonlocal add_called
|
||||
add_called = True
|
||||
return None
|
||||
|
||||
runner._site.clear_cart = _clear # type: ignore[assignment]
|
||||
runner._site.add_to_cart = _add # type: ignore[assignment]
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
assert add_called is False
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["terminal_status"] == TaskStatus.NEEDS_HUMAN
|
||||
assert "未清空" in terminal["detail"]
|
||||
|
||||
|
||||
async def test_clear_cart_step_writes_evidence_but_no_gateway_report(
|
||||
runner: WorkerRunner, evidence: EvidenceStore
|
||||
):
|
||||
"""step 0 落本地证据:清理后的页面 HTML 与 meta.json 在盘上,但不回报 gateway
|
||||
|
||||
clear_cart 返回的 html 是清理结束后的 cart 页现场,meta 记 removed_count /
|
||||
cart_count;它仍是本机卫生步骤——evidence_index 可翻查,gateway 收不到
|
||||
这一步的 evidence_ref。
|
||||
"""
|
||||
async def _clear(): # noqa: ANN001
|
||||
return {
|
||||
"removed_count": 1,
|
||||
"cart_count": 0,
|
||||
"html": "<html>cart</html>",
|
||||
"screenshot": b"fake-clear-png",
|
||||
}
|
||||
|
||||
async def _noop(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _unimplemented(task): # noqa: ANN001
|
||||
# 流程走到 enter_checkout 为止:step 0 的证据此时已经落盘
|
||||
raise NotImplementedError("模拟:enter_checkout 未实现,到此为止")
|
||||
|
||||
runner._site.clear_cart = _clear # type: ignore[assignment]
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _noop # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _unimplemented # type: ignore[assignment]
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
step_dir = evidence.step_dir("t1")
|
||||
html_text = (step_dir / "00-cart-clear.html").read_text(encoding="utf-8")
|
||||
assert html_text == "<html>cart</html>"
|
||||
assert (step_dir / "00-cart-clear.png").read_bytes() == b"fake-clear-png"
|
||||
meta = json.loads((step_dir / "00-cart-clear.meta.json").read_text(encoding="utf-8"))
|
||||
assert meta == {"step": "cart-clear", "removed_count": 1, "cart_count": 0}
|
||||
# 本机卫生步骤:gateway 的任何 report 都不携带 step-0 的 evidence_ref
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
assert all(r["evidence_ref"] != "t1/00-cart-clear" for r in gateway.reports)
|
||||
|
||||
|
||||
async def test_clear_cart_evidence_written_when_guard_blocks(
|
||||
runner: WorkerRunner, evidence: EvidenceStore
|
||||
):
|
||||
"""闸门拦截(清理后 cart_count 非 0)时 step-0 证据同样已落盘
|
||||
|
||||
被拦转 needs_human 时这份现场就是排查依据——证据先于闸门判断落盘。
|
||||
"""
|
||||
async def _clear(): # noqa: ANN001
|
||||
return {
|
||||
"removed_count": 2,
|
||||
"cart_count": 2,
|
||||
"html": "<html>leftover</html>",
|
||||
"screenshot": b"fake-leftover-png",
|
||||
}
|
||||
|
||||
runner._site.clear_cart = _clear # type: ignore[assignment]
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
step_dir = evidence.step_dir("t1")
|
||||
assert (step_dir / "00-cart-clear.html").exists()
|
||||
assert (step_dir / "00-cart-clear.png").read_bytes() == b"fake-leftover-png"
|
||||
meta = json.loads((step_dir / "00-cart-clear.meta.json").read_text(encoding="utf-8"))
|
||||
assert meta["cart_count"] == 2
|
||||
|
||||
|
||||
async def test_every_order_step_writes_html_png_and_meta(
|
||||
runner: WorkerRunner, evidence: EvidenceStore
|
||||
):
|
||||
"""下单流程每一步(step 0 清车 + step 1-5)都落 html + png + meta 三件套
|
||||
|
||||
排查问题时每一步都要有现场可看:页面 HTML、整页截图、步骤 meta 缺一不可。
|
||||
"""
|
||||
async def _clear(): # noqa: ANN001
|
||||
return {
|
||||
"removed_count": 0,
|
||||
"cart_count": 0,
|
||||
"html": "<html>cart</html>",
|
||||
"screenshot": b"png-0",
|
||||
}
|
||||
|
||||
async def _add(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>added</html>", screenshot=b"png-1")
|
||||
|
||||
async def _verify(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>verified</html>", screenshot=b"png-2")
|
||||
|
||||
async def _checkout(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"png-3")
|
||||
|
||||
async def _parse(html: str):
|
||||
return CheckoutSummary(payable_yen=297)
|
||||
|
||||
async def _submit(task): # noqa: ANN001
|
||||
return SubmitOutcome(
|
||||
site_order_id="ord-1",
|
||||
evidence=PageSnapshot(html="<html>done</html>", screenshot=b"png-4"),
|
||||
)
|
||||
|
||||
async def _pay(task, site_order_id): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>paid</html>", screenshot=b"png-5")
|
||||
|
||||
async def _unchanged(order_id: str): # noqa: ANN001
|
||||
# 监控不污染断言:found=True 但无状态变化,轮询到上限自行退出
|
||||
return OrderStatusSnapshot(found=True, order_state=None)
|
||||
|
||||
runner._site.clear_cart = _clear # type: ignore[assignment]
|
||||
runner._site.add_to_cart = _add # type: ignore[assignment]
|
||||
runner._site.verify_cart = _verify # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _checkout # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _parse # type: ignore[assignment]
|
||||
runner._site.submit_order = _submit # type: ignore[assignment]
|
||||
runner._site.pay = _pay # type: ignore[assignment]
|
||||
runner._site.check_order_status = _unchanged # type: ignore[assignment]
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
step_dir = evidence.step_dir("t1")
|
||||
expected = {
|
||||
"00-cart-clear": b"png-0",
|
||||
"01-cart-add": b"png-1",
|
||||
"02-cart-check": b"png-2",
|
||||
"03-order-confirm": b"png-3",
|
||||
"04-order-submit": b"png-4",
|
||||
"05-payment": b"png-5",
|
||||
}
|
||||
for stem, png in expected.items():
|
||||
assert (step_dir / f"{stem}.png").read_bytes() == png, f"{stem} 缺整页截图"
|
||||
assert (step_dir / f"{stem}.html").exists(), f"{stem} 缺页面 HTML"
|
||||
assert (step_dir / f"{stem}.meta.json").exists(), f"{stem} 缺 meta"
|
||||
|
||||
|
||||
# ---- 证据在 report 之前落盘(§9 第 11 条)----
|
||||
|
||||
|
||||
@@ -372,16 +731,159 @@ async def test_evidence_files_exist_before_each_report(
|
||||
|
||||
runner._gateway_for_test.report = _spy_report # type: ignore[attr-defined]
|
||||
|
||||
# 让 add_to_cart / verify_cart 正常,enter_checkout 抛 NotImplemented,
|
||||
# 触发前两步写证据 + report,到第三步转 needs_human
|
||||
# 让 add_to_cart / verify_cart / enter_checkout 正常,parse_checkout 抛
|
||||
# NotImplemented(模拟某一步没实现),触发前几步写证据 + report,最终转 needs_human
|
||||
async def _noop(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _checkout_html(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"fake-checkout-png")
|
||||
|
||||
async def _unimplemented(html): # noqa: ANN001
|
||||
raise NotImplementedError("模拟:假装 parse_checkout 还没实现")
|
||||
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _noop # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _unimplemented # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
# 至少有一个 step 调了 report,且每次 report 之前证据都在
|
||||
assert seen_evidence_at_report, "应当至少有一次带 evidence_ref 的 report"
|
||||
assert all(seen_evidence_at_report), "某次 report 之前证据文件未落盘"
|
||||
|
||||
|
||||
# ---- 付款后监控(规格 §6「付款后监控」)----
|
||||
|
||||
|
||||
async def test_successful_order_spawns_monitor_task_and_cancel_monitors_cleans_up(
|
||||
runner: WorkerRunner, local_db: LocalDB
|
||||
):
|
||||
"""付款成功后 execute() 应起一个后台监控任务、不阻塞返回;cancel_monitors 能干净收尾
|
||||
|
||||
check_order_status 桩故意永不返回(挂在一个手动控制的 Event 上),模拟「监控任务
|
||||
还在跑」这个可观察状态——用来确认 execute() 没有同步 await 监控循环
|
||||
(不然 handle() 早该被这个永不返回的调用卡死,测试会超时而不是正常结束)。
|
||||
"""
|
||||
|
||||
async def _noop(task): # noqa: ANN001
|
||||
return None
|
||||
|
||||
async def _checkout_html(task): # noqa: ANN001
|
||||
return PageSnapshot(html="<html>checkout</html>", screenshot=b"fake-checkout-png")
|
||||
|
||||
async def _parse(html: str):
|
||||
return CheckoutSummary(payable_yen=297)
|
||||
|
||||
async def _submit(task): # noqa: ANN001
|
||||
return SubmitOutcome(
|
||||
site_order_id="306087-20260813-0863947697",
|
||||
evidence=PageSnapshot(html="<html>complete</html>", screenshot=b"fake-submit-png"),
|
||||
)
|
||||
|
||||
async def _pay(task, site_order_id): # noqa: ANN001
|
||||
return None
|
||||
|
||||
never_resolves = asyncio.Event()
|
||||
|
||||
async def _hang_check(order_id: str): # noqa: ANN001
|
||||
await never_resolves.wait()
|
||||
|
||||
runner._site.add_to_cart = _noop # type: ignore[assignment]
|
||||
runner._site.verify_cart = _noop # type: ignore[assignment]
|
||||
runner._site.enter_checkout = _checkout_html # type: ignore[assignment]
|
||||
runner._site.parse_checkout = _parse # type: ignore[assignment]
|
||||
runner._site.submit_order = _submit # type: ignore[assignment]
|
||||
runner._site.pay = _pay # type: ignore[assignment]
|
||||
runner._site.check_order_status = _hang_check # type: ignore[assignment]
|
||||
_set_site_clear(runner._site, {"removed_count": 0, "cart_count": 0})
|
||||
|
||||
await runner.handle(_make_task(task_id="t1"))
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
terminal = gateway.last_terminal_report()
|
||||
assert terminal["state"] == OrderState.PAID
|
||||
assert terminal["terminal_status"] == TaskStatus.SUCCEEDED
|
||||
|
||||
assert len(runner._monitor_tasks) == 1
|
||||
|
||||
await runner.cancel_monitors()
|
||||
assert len(runner._monitor_tasks) == 0
|
||||
|
||||
|
||||
async def test_monitor_order_reports_state_changes_and_stops_at_delivered(
|
||||
runner: WorkerRunner, evidence: EvidenceStore
|
||||
):
|
||||
"""状态没变化不重复 report;到「配達完了」立刻停止轮询,不再多探测一次"""
|
||||
snapshots = [
|
||||
OrderStatusSnapshot(found=False),
|
||||
OrderStatusSnapshot(found=True, stage_label="ショップ", order_state=None, html="<html>shop</html>"),
|
||||
OrderStatusSnapshot(
|
||||
found=True, stage_label="出荷", order_state=OrderState.SHIPPED, html="<html>shipped</html>"
|
||||
),
|
||||
OrderStatusSnapshot(
|
||||
found=True, stage_label="配達完了", order_state=OrderState.DELIVERED, html="<html>delivered</html>"
|
||||
),
|
||||
]
|
||||
calls: list[OrderStatusSnapshot] = []
|
||||
|
||||
async def _check(order_id: str): # noqa: ANN001
|
||||
calls.append(snapshots[len(calls)])
|
||||
return calls[-1]
|
||||
|
||||
runner._site.check_order_status = _check # type: ignore[assignment]
|
||||
|
||||
await runner._monitor_order(_make_task(task_id="t1"), "306087-20260813-0863947697")
|
||||
|
||||
assert len(calls) == 4 # 配達完了那次之后立刻返回,不会再多轮询一次
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
reported_states = [r["state"] for r in gateway.reports]
|
||||
assert reported_states == [OrderState.SHIPPED, OrderState.DELIVERED]
|
||||
assert all(r["terminal"] is False for r in gateway.reports) # 追加上报,不重新终结任务
|
||||
|
||||
base = evidence.step_dir("t1")
|
||||
assert (base / "06-monitor-shipped.html").exists()
|
||||
assert (base / "07-monitor-delivered.html").exists()
|
||||
|
||||
|
||||
async def test_monitor_order_stops_after_max_checks_without_finding_order(
|
||||
runner: WorkerRunner,
|
||||
):
|
||||
"""一直查不到订单(found=False):轮询到上限后正常退出,不报错、不 report"""
|
||||
|
||||
async def _check(order_id: str): # noqa: ANN001
|
||||
return OrderStatusSnapshot(found=False)
|
||||
|
||||
runner._site.check_order_status = _check # type: ignore[assignment]
|
||||
|
||||
await runner._monitor_order(_make_task(task_id="t1"), "ord-1")
|
||||
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
assert gateway.reports == []
|
||||
|
||||
|
||||
async def test_monitor_order_swallows_check_errors_and_keeps_polling(
|
||||
runner: WorkerRunner,
|
||||
):
|
||||
"""单次探测异常(如登录态失效)只记日志继续重试,不会让后台任务崩掉"""
|
||||
call_count = 0
|
||||
|
||||
async def _check(order_id: str): # noqa: ANN001
|
||||
nonlocal call_count
|
||||
call_count += 1
|
||||
if call_count <= 2:
|
||||
raise RuntimeError("模拟登录态失效")
|
||||
return OrderStatusSnapshot(
|
||||
found=True, stage_label="配達完了", order_state=OrderState.DELIVERED, html="<html>ok</html>"
|
||||
)
|
||||
|
||||
runner._site.check_order_status = _check # type: ignore[assignment]
|
||||
|
||||
await runner._monitor_order(_make_task(task_id="t1"), "ord-1")
|
||||
|
||||
assert call_count == 3 # 前两次异常被吞掉继续重试,第三次成功拿到 DELIVERED 后停止
|
||||
gateway: FakeGateway = runner._gateway_for_test # type: ignore[attr-defined]
|
||||
assert [r["state"] for r in gateway.reports] == [OrderState.DELIVERED]
|
||||
|
||||
Reference in New Issue
Block a user