You've already forked pruchase_jhw
Compare commits
9 Commits
cc47e1dd57
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
| d1679cdfff | |||
| 0bac10c016 | |||
| fe708377e7 | |||
| d6ebf68728 | |||
| 729e5a0f35 | |||
| 29175270c8 | |||
| 8b54706ae7 | |||
| ddcc7262b6 | |||
| 687c28e8fc |
@@ -0,0 +1,20 @@
|
|||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
.venv
|
||||||
|
.claude
|
||||||
|
.idea
|
||||||
|
.vscode
|
||||||
|
**/__pycache__
|
||||||
|
*.pyc
|
||||||
|
chrome-profile
|
||||||
|
archives
|
||||||
|
logs
|
||||||
|
tmp
|
||||||
|
deploy/chrome-profile
|
||||||
|
deploy/archives
|
||||||
|
deploy/logs
|
||||||
|
.env.dev
|
||||||
|
.env.example
|
||||||
|
accounts.yaml
|
||||||
|
README.md
|
||||||
|
docker-compose.yml
|
||||||
+9
-2
@@ -19,17 +19,24 @@ REDIS_HOST=127.0.0.1
|
|||||||
REDIS_PORT=6379
|
REDIS_PORT=6379
|
||||||
REDIS_PASSWORD=
|
REDIS_PASSWORD=
|
||||||
REDIS_DB=0
|
REDIS_DB=0
|
||||||
REDIS_STREAM_KEY=jp_purchase_task_dev
|
REDIS_STREAM_KEY=jp_surugaya_purchase_task_dev
|
||||||
REDIS_CONSUMER_GROUP=purchase_group
|
REDIS_CONSUMER_GROUP=purchase_group
|
||||||
REDIS_CONSUMER_NAME=purchase_worker_dev
|
REDIS_CONSUMER_NAME=purchase_worker_dev
|
||||||
REDIS_DELAY_ZSET_KEY=jp_purchase_delay_task_dev
|
REDIS_DELAY_ZSET_KEY=jp_surugaya_trade_delay_dev
|
||||||
REDIS_DELAY_POLL_INTERVAL=3600
|
REDIS_DELAY_POLL_INTERVAL=3600
|
||||||
REDIS_DELAY_BATCH_SIZE=10
|
REDIS_DELAY_BATCH_SIZE=10
|
||||||
|
REDIS_NEWS_HASH_KEY=jp_surugaya_news_dev
|
||||||
|
|
||||||
|
# 通知定时抓取
|
||||||
|
NEWS_FETCH_ENABLED=true
|
||||||
|
NEWS_FETCH_INTERVAL_SECONDS=300
|
||||||
|
|
||||||
# 浏览器配置
|
# 浏览器配置
|
||||||
BROWSER_HEADLESS=false
|
BROWSER_HEADLESS=false
|
||||||
BROWSER_EXECUTABLE_PATH=
|
BROWSER_EXECUTABLE_PATH=
|
||||||
BROWSER_KEEP_OPEN=false
|
BROWSER_KEEP_OPEN=false
|
||||||
|
# 启动时登录预检:预热浏览器后逐账号确认登录态,未登录则自动登录(默认开启)
|
||||||
|
LOGIN_PRECHECK_ENABLED=true
|
||||||
|
|
||||||
# 账号配置
|
# 账号配置
|
||||||
ACCOUNTS_CONFIG_PATH=./accounts.yaml
|
ACCOUNTS_CONFIG_PATH=./accounts.yaml
|
||||||
|
|||||||
+68
@@ -0,0 +1,68 @@
|
|||||||
|
# syntax=docker/dockerfile:1
|
||||||
|
# 基础镜像版本必须与 uv.lock 中的 playwright 版本一致(当前 1.60.0),
|
||||||
|
# 浏览器二进制与系统依赖已预装在 /ms-playwright。
|
||||||
|
FROM mcr.microsoft.com/playwright/python:v1.60.0-noble
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
# X 虚拟桌面 + VNC/noVNC + 录屏/截图/图像匹配运行依赖 + 日文字体 + 进程编排
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
xvfb \
|
||||||
|
fluxbox \
|
||||||
|
x11vnc \
|
||||||
|
novnc \
|
||||||
|
websockify \
|
||||||
|
ffmpeg \
|
||||||
|
scrot \
|
||||||
|
libgl1 \
|
||||||
|
fonts-noto-cjk \
|
||||||
|
supervisor \
|
||||||
|
curl \
|
||||||
|
tzdata \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /usr/local/bin/
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# 私有 PyPI 索引(git.jerryyan.net)若需要认证,构建时传入:
|
||||||
|
# docker build --build-arg UV_INDEX_JERRYYAN_USERNAME=xxx --build-arg UV_INDEX_JERRYYAN_PASSWORD=xxx .
|
||||||
|
ARG UV_INDEX_JERRYYAN_USERNAME
|
||||||
|
ARG UV_INDEX_JERRYYAN_PASSWORD
|
||||||
|
|
||||||
|
# 依赖层缓存:仅复制依赖清单后同步,源码变更不触发依赖重装
|
||||||
|
COPY pyproject.toml uv.lock ./
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/uv uv sync --frozen --no-dev
|
||||||
|
|
||||||
|
COPY src ./src
|
||||||
|
COPY resources ./resources
|
||||||
|
COPY .env.prod ./.env.prod
|
||||||
|
|
||||||
|
# 容器配置模板:运行时若挂载的 /app/config 中无 accounts.yaml,entrypoint 会用它初始化
|
||||||
|
COPY deploy/config/accounts.yaml /app/deploy-defaults/accounts.yaml
|
||||||
|
|
||||||
|
COPY docker/supervisord.conf /etc/supervisor/supervisord.conf
|
||||||
|
COPY docker/entrypoint.sh /entrypoint.sh
|
||||||
|
# 防御 Windows 仓库 CRLF 换行导致脚本无法执行
|
||||||
|
RUN sed -i 's/\r$//' /entrypoint.sh && chmod +x /entrypoint.sh
|
||||||
|
|
||||||
|
ENV APP_ENV=prod \
|
||||||
|
TZ=Asia/Shanghai \
|
||||||
|
DISPLAY=:99 \
|
||||||
|
SCREEN_RES=1920x1080x24 \
|
||||||
|
ACCOUNTS_CONFIG_PATH=/app/config/accounts.yaml \
|
||||||
|
PYTHONUNBUFFERED=1
|
||||||
|
|
||||||
|
# 关键数据声明为卷(兜底):即使 docker run 时未显式挂载,
|
||||||
|
# 浏览器登录态(accounts 的 user_data_dir 父目录)与账号配置也会落入匿名卷持久化。
|
||||||
|
# accounts.yaml 中每个账号的 user_data_dir 都应指向 /app/chrome-profile/ 下的子目录。
|
||||||
|
VOLUME ["/app/chrome-profile", "/app/config"]
|
||||||
|
|
||||||
|
# 6080: noVNC web | 5900: 原生 VNC | 8080: 截图/账号配置 API
|
||||||
|
EXPOSE 5900 6080 8080
|
||||||
|
|
||||||
|
# 启动需等待 Xvfb + 浏览器预热 + API 拉起
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
|
||||||
|
CMD curl -fs "http://localhost:8080/api/screenshots?days=1" > /dev/null || exit 1
|
||||||
|
|
||||||
|
ENTRYPOINT ["/entrypoint.sh"]
|
||||||
@@ -203,3 +203,50 @@ XADD purchase:tasks * task_id "task_test_1001" topic "surugaya_tasks" purchase_a
|
|||||||
5. 若 `record_video="1"`,任务结束后会输出视频文件名与绝对路径,并归档到:`archives/YYYYMMDD/surugaya_tasks/video_task_test_1001_{timestamp}.mp4`。
|
5. 若 `record_video="1"`,任务结束后会输出视频文件名与绝对路径,并归档到:`archives/YYYYMMDD/surugaya_tasks/video_task_test_1001_{timestamp}.mp4`。
|
||||||
6. 任务结束后自动进行 `XACK` 确认并安全关舱。
|
6. 任务结束后自动进行 `XACK` 确认并安全关舱。
|
||||||
7. 若任务执行失败且通知模块已启用,系统会向 `MAIL_TO_ADDRESSES` 中配置的收件人发送失败邮件,邮件正文包含任务 ID、账号、错误原因、截图路径和录屏路径。
|
7. 若任务执行失败且通知模块已启用,系统会向 `MAIL_TO_ADDRESSES` 中配置的收件人发送失败邮件,邮件正文包含任务 ID、账号、错误原因、截图路径和录屏路径。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 🖥 Windows 裸机部署
|
||||||
|
|
||||||
|
如需在 Windows 物理机或 Windows 云服务器上以裸机方式部署本项目与 `../jp_surugaya`,请参考 [`docs/DEPLOYMENT_WINDOWS.md`](docs/DEPLOYMENT_WINDOWS.md)。
|
||||||
|
|
||||||
|
## 🐳 Docker 部署(Linux 服务器 + noVNC 远程桌面)
|
||||||
|
|
||||||
|
容器内通过 supervisord 编排 5 个进程:`Xvfb`(虚拟显示器 :99)→ `fluxbox`(窗口管理器)→ `x11vnc`(VNC server)→ `noVNC/websockify`(浏览器远程桌面)→ 业务进程。浏览器以有头模式跑在虚拟桌面上,Google Pay 模板匹配、pyautogui 鼠标操作与 FFmpeg 录屏(Linux 下自动切换为 `x11grab`)均在该桌面执行。
|
||||||
|
|
||||||
|
### 构建与启动
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 构建(构建机需能访问私有 PyPI 索引 git.jerryyan.net 拉取 surugaya-common;
|
||||||
|
# 若索引需认证,追加 --build-arg UV_INDEX_JERRYYAN_USERNAME=xxx --build-arg UV_INDEX_JERRYYAN_PASSWORD=xxx)
|
||||||
|
docker compose build
|
||||||
|
|
||||||
|
# 启动(VNC_PASSWORD 必填,是 noVNC/VNC 的访问密码)
|
||||||
|
export VNC_PASSWORD=your_password
|
||||||
|
docker compose up -d
|
||||||
|
```
|
||||||
|
|
||||||
|
### 访问入口
|
||||||
|
|
||||||
|
| 入口 | 地址 | 用途 |
|
||||||
|
| ---- | ---- | ---- |
|
||||||
|
| noVNC 远程桌面 | `http://<host>:6080/vnc.html` | 查看/操作容器内浏览器(鼠标键盘全可用,Google Pay 最终确认等人工环节在此操作) |
|
||||||
|
| 截图查看器 | `http://<host>:8080/` | 浏览归档截图 |
|
||||||
|
| 账号配置 | `http://<host>:8080/accounts` | 在线编辑 accounts.yaml,支持"保存并重启"(优雅停机后由 supervisord 自动拉起,按新配置重新预热浏览器) |
|
||||||
|
|
||||||
|
> ⚠️ 8080 与 6080 端口均无应用层鉴权(accounts.yaml 含账号密码),切勿直接暴露公网,应置于内网或经反向代理加鉴权后访问。
|
||||||
|
|
||||||
|
### 持久化目录(compose 已挂载)
|
||||||
|
|
||||||
|
- `deploy/chrome-profile/` → `/app/chrome-profile`:浏览器登录态,**必须持久化**;
|
||||||
|
- `deploy/config/accounts.yaml` → `/app/config/accounts.yaml`:账号配置(容器内 `ACCOUNTS_CONFIG_PATH` 已指向该路径),首次启动时若不存在会用镜像内模板初始化;
|
||||||
|
- `deploy/archives/` → `/archives`、`deploy/logs/` → `/logs`:截图/录屏归档与日志。
|
||||||
|
|
||||||
|
> 说明:`/app/chrome-profile` 与 `/app/config` 已在 Dockerfile 中声明 `VOLUME`——即使绕过 compose 直接 `docker run` 且忘记挂载,登录态与账号配置也会落入匿名卷而不会随容器删除丢失。新增账号无需改挂载:在 accounts.yaml 中把 `user_data_dir` 指向 `/app/chrome-profile/` 下新的子目录即可。
|
||||||
|
|
||||||
|
### 部署注意事项
|
||||||
|
|
||||||
|
1. **代理地址**:容器内 `127.0.0.1` 指向容器自身。代理跑在宿主机时,accounts.yaml 中应填 `http://host.docker.internal:<port>`(compose 已配置 `host-gateway`)。
|
||||||
|
2. **登录态迁移**:可将 Windows 的 `chrome-profile/<账号>` 目录拷入 `deploy/chrome-profile/`,但建议首次部署后通过 noVNC 人工核验登录态,失效则在远程桌面内重新登录一次。
|
||||||
|
3. **模板图重采集(重要)**:`resources/images/` 下的模板图截自 Windows 屏幕,Linux 下字体/渲染差异可能导致 OpenCV 匹配(阈值 0.85)不达标。识别失败时系统会把整屏截图存到 `archives/<日期>/google_pay_debug/`(可经 8080 截图查看器浏览),从中裁出新模板替换后重新构建镜像即可。
|
||||||
|
4. **分辨率**:虚拟桌面默认 `1920x1080x24`,可通过环境变量 `SCREEN_RES` 调整;调整后模板图需按新分辨率重新采集。
|
||||||
|
|||||||
+2
-2
@@ -17,8 +17,8 @@ sites:
|
|||||||
- account_id: "trdian023@tokugen.jp"
|
- account_id: "trdian023@tokugen.jp"
|
||||||
username: "trdian023@tokugen.jp"
|
username: "trdian023@tokugen.jp"
|
||||||
password: "trdian023.."
|
password: "trdian023.."
|
||||||
user_data_dir: "C:\\Users\\yan1h\\Code\\jp_pruchase_zdh_5\\chrome-profile\\787235076"
|
user_data_dir: "C:\\Users\\Administrator\\Code\\pruchase_jhw\\chrome-profile\\787235076"
|
||||||
proxy: "" # Optional residential proxy (e.g. "http://127.0.0.1:10808")
|
proxy: "http://127.0.0.1:7890" # Optional residential proxy (e.g. "http://127.0.0.1:10808")
|
||||||
|
|
||||||
# Future sites can be appended here
|
# Future sites can be appended here
|
||||||
mercari: []
|
mercari: []
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# ==========================================
|
||||||
|
# Site Accounts Configuration (容器部署版)
|
||||||
|
# ==========================================
|
||||||
|
# 容器内路径约定:
|
||||||
|
# - user_data_dir 必须位于 /app/chrome-profile/ 下(compose 已挂载持久化卷)
|
||||||
|
# - 本文件由 compose 挂载到 /app/config/accounts.yaml,可在 http://<host>:8080/accounts 在线编辑
|
||||||
|
#
|
||||||
|
# 代理注意:容器内 127.0.0.1 指向容器自身!
|
||||||
|
# - 代理跑在宿主机时,使用 http://host.docker.internal:<port>(compose 已配置 host-gateway)
|
||||||
|
# - 或直接填写代理服务器的内网/公网地址
|
||||||
|
|
||||||
|
sites:
|
||||||
|
surugaya:
|
||||||
|
- account_id: "trdian023@tokugen.jp"
|
||||||
|
username: "trdian023@tokugen.jp"
|
||||||
|
password: "trdian023.."
|
||||||
|
user_data_dir: "/app/chrome-profile/787235076"
|
||||||
|
proxy: ""
|
||||||
|
|
||||||
|
# Future sites can be appended here
|
||||||
|
mercari: []
|
||||||
|
yahoo: []
|
||||||
@@ -0,0 +1,28 @@
|
|||||||
|
services:
|
||||||
|
purchase-worker:
|
||||||
|
build: .
|
||||||
|
image: jp-purchase-zdh-5:latest
|
||||||
|
container_name: jp-purchase-worker
|
||||||
|
restart: unless-stopped
|
||||||
|
# Chromium 共享内存需求大,避免 /dev/shm 不足导致渲染崩溃
|
||||||
|
shm_size: "2g"
|
||||||
|
environment:
|
||||||
|
# noVNC/VNC 访问密码,部署机必须设置:export VNC_PASSWORD=xxx
|
||||||
|
VNC_PASSWORD: ${VNC_PASSWORD:?请在宿主机设置环境变量 VNC_PASSWORD}
|
||||||
|
# 虚拟桌面分辨率(重采集模板图后如需调整在此修改)
|
||||||
|
# SCREEN_RES: "1920x1080x24"
|
||||||
|
# 容器内通过 host.docker.internal 访问宿主机服务(如本机代理)
|
||||||
|
extra_hosts:
|
||||||
|
- "host.docker.internal:host-gateway"
|
||||||
|
ports:
|
||||||
|
- "6080:6080" # noVNC:http://<host>:6080/vnc.html
|
||||||
|
- "8080:8080" # 截图查看 / 账号配置 API
|
||||||
|
# - "5900:5900" # 原生 VNC 客户端,按需开放
|
||||||
|
volumes:
|
||||||
|
# 浏览器登录态(user_data_dir),必须持久化
|
||||||
|
- ./deploy/chrome-profile:/app/chrome-profile
|
||||||
|
# accounts.yaml 所在目录(容器内 ACCOUNTS_CONFIG_PATH=/app/config/accounts.yaml)
|
||||||
|
- ./deploy/config:/app/config
|
||||||
|
# 截图/录屏归档与日志
|
||||||
|
- ./deploy/archives:/archives
|
||||||
|
- ./deploy/logs:/logs
|
||||||
@@ -0,0 +1,18 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# VNC 密码必填,避免远程桌面裸奔
|
||||||
|
: "${VNC_PASSWORD:?必须通过环境变量 VNC_PASSWORD 设置 noVNC/VNC 访问密码}"
|
||||||
|
|
||||||
|
mkdir -p /archives /logs /app/config /app/chrome-profile
|
||||||
|
|
||||||
|
# 生成 x11vnc 密码文件
|
||||||
|
x11vnc -storepasswd "$VNC_PASSWORD" /root/.vncpass > /dev/null 2>&1
|
||||||
|
|
||||||
|
# 挂载的配置目录中没有 accounts.yaml 时,用镜像内模板初始化
|
||||||
|
if [ ! -f /app/config/accounts.yaml ] && [ -f /app/deploy-defaults/accounts.yaml ]; then
|
||||||
|
echo "[entrypoint] /app/config/accounts.yaml 不存在,使用镜像内模板初始化。"
|
||||||
|
cp /app/deploy-defaults/accounts.yaml /app/config/accounts.yaml
|
||||||
|
fi
|
||||||
|
|
||||||
|
exec /usr/bin/supervisord -c /etc/supervisor/supervisord.conf
|
||||||
@@ -0,0 +1,66 @@
|
|||||||
|
[supervisord]
|
||||||
|
nodaemon=true
|
||||||
|
user=root
|
||||||
|
logfile=/dev/null
|
||||||
|
logfile_maxbytes=0
|
||||||
|
pidfile=/tmp/supervisord.pid
|
||||||
|
|
||||||
|
[unix_http_server]
|
||||||
|
file=/tmp/supervisor.sock
|
||||||
|
|
||||||
|
[rpcinterface:supervisor]
|
||||||
|
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||||
|
|
||||||
|
[supervisorctl]
|
||||||
|
serverurl=unix:///tmp/supervisor.sock
|
||||||
|
|
||||||
|
; ---------- 虚拟桌面 ----------
|
||||||
|
[program:xvfb]
|
||||||
|
command=/usr/bin/Xvfb %(ENV_DISPLAY)s -screen 0 %(ENV_SCREEN_RES)s -ac -nolisten tcp
|
||||||
|
priority=10
|
||||||
|
autorestart=true
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
redirect_stderr=true
|
||||||
|
|
||||||
|
[program:fluxbox]
|
||||||
|
command=/usr/bin/fluxbox
|
||||||
|
priority=20
|
||||||
|
autorestart=true
|
||||||
|
startretries=10
|
||||||
|
stdout_logfile=/dev/null
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
redirect_stderr=true
|
||||||
|
|
||||||
|
; ---------- VNC / noVNC ----------
|
||||||
|
[program:x11vnc]
|
||||||
|
command=/usr/bin/x11vnc -display %(ENV_DISPLAY)s -rfbauth /root/.vncpass -rfbport 5900 -forever -shared -noxdamage -quiet
|
||||||
|
priority=30
|
||||||
|
autorestart=true
|
||||||
|
startretries=10
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
redirect_stderr=true
|
||||||
|
|
||||||
|
[program:novnc]
|
||||||
|
command=/usr/bin/websockify --web /usr/share/novnc 6080 localhost:5900
|
||||||
|
priority=40
|
||||||
|
autorestart=true
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
redirect_stderr=true
|
||||||
|
|
||||||
|
; ---------- 业务进程 ----------
|
||||||
|
; autorestart=true:进程任何形式退出都会被拉起,
|
||||||
|
; 配合 API 的 POST /api/restart(SIGTERM 优雅停机)实现"保存配置后重启"。
|
||||||
|
[program:app]
|
||||||
|
command=/app/.venv/bin/python -m src.main
|
||||||
|
directory=/app
|
||||||
|
priority=50
|
||||||
|
autorestart=true
|
||||||
|
startretries=3
|
||||||
|
stopsignal=TERM
|
||||||
|
stopwaitsecs=120
|
||||||
|
stdout_logfile=/dev/stdout
|
||||||
|
stdout_logfile_maxbytes=0
|
||||||
|
redirect_stderr=true
|
||||||
@@ -0,0 +1,467 @@
|
|||||||
|
# Windows 裸机部署指南
|
||||||
|
|
||||||
|
> 本文档说明如何在 Windows 裸机环境下部署以下两个互相配合的服务,不依赖 Docker 或 WSL。
|
||||||
|
> 部署对象:
|
||||||
|
>
|
||||||
|
> - `jp_pruchase_zdh_5`:采购自动化 worker(浏览器自动化下单、定时抓取通知/订单、截图归档)。
|
||||||
|
> - `../jp_surugaya`:骏河屋抓取 API 服务(搜索/详情/通知/订单查询,并可向 Redis 投递采购任务)。
|
||||||
|
>
|
||||||
|
> 两个服务通过 **Redis** 共享任务队列与状态数据;Redis 由 `../jp_surugaya` 侧统一部署,本服务(采购 worker)只需配置连接信息。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. 部署架构
|
||||||
|
|
||||||
|
```text
|
||||||
|
┌─────────────────────────────────────────────────────────────┐
|
||||||
|
│ Windows 服务器 │
|
||||||
|
│ ┌──────────────────┐ ┌──────────────────────────┐ │
|
||||||
|
│ │ jp_surugaya │◄───────►│ Redis (Windows) │ │
|
||||||
|
│ │ :31106 │ │ :6379 │ │
|
||||||
|
│ └──────────────────┘ └───────────┬──────────────┘ │
|
||||||
|
│ ▲ │ │
|
||||||
|
│ │ HTTP API │ Redis Stream │
|
||||||
|
│ ▼ ▼ │
|
||||||
|
│ ┌──────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ jp_pruchase_zdh_5 │ │
|
||||||
|
│ │ - 消费采购任务 │ │
|
||||||
|
│ │ - 浏览器自动化(Playwright) │ │
|
||||||
|
│ │ - 截图/录屏归档到 archives/ │ │
|
||||||
|
│ │ - 截图查看 API :8080 │ │
|
||||||
|
│ └──────────────────────────────────────────────────────┘ │
|
||||||
|
└─────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
- **Redis**:由 `../jp_surugaya` 侧统一部署和维护,采购 worker 通过其连接信息接入。
|
||||||
|
- **jp_surugaya**:对外提供 HTTP API(默认 `0.0.0.0:31106`),内部通过 Redis 读写任务与抓取结果。
|
||||||
|
- **jp_pruchase_zdh_5**:不直接对外暴露业务端口,但内置截图查看 API(默认 `0.0.0.0:8080`),建议仅本地或内网访问。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. 环境准备
|
||||||
|
|
||||||
|
### 2.1 操作系统
|
||||||
|
|
||||||
|
- Windows 10/11 64 位,或 Windows Server 2019/2022。
|
||||||
|
- 建议固定显示分辨率(例如 `1920x1080`),并保证登录会话处于交互式桌面环境(本地显示器或持续 RDP 会话)。
|
||||||
|
|
||||||
|
### 2.2 Python
|
||||||
|
|
||||||
|
- 安装 **Python 3.12+**(推荐从 [python.org](https://www.python.org/downloads/windows/) 下载)。
|
||||||
|
- 安装时勾选 **Add python.exe to PATH**。
|
||||||
|
- 验证:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
python --version
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.3 uv(推荐)
|
||||||
|
|
||||||
|
两个项目均使用 `uv` 作为包管理器。安装方式:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex"
|
||||||
|
```
|
||||||
|
|
||||||
|
安装完成后重启 PowerShell,并验证:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv --version
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2.4 Redis 接入信息
|
||||||
|
|
||||||
|
Redis 由 `../jp_surugaya` 侧统一部署,本服务不需要单独安装 Redis。
|
||||||
|
|
||||||
|
部署采购 worker 前,只需确认以下信息:
|
||||||
|
|
||||||
|
- Redis 主机地址、端口、密码、DB 编号。
|
||||||
|
- 采购 worker 与 `../jp_surugaya` 必须访问同一个 Redis 实例。
|
||||||
|
|
||||||
|
若本地开发测试时需要临时 Redis,可另行准备 Windows 版 Redis 或 Memurai;生产环境建议统一使用 `../jp_surugaya` 提供的实例。
|
||||||
|
|
||||||
|
### 2.5 浏览器
|
||||||
|
|
||||||
|
两个项目都基于 Playwright。首次部署时安装 Chromium:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv run playwright install chromium
|
||||||
|
```
|
||||||
|
|
||||||
|
> 默认使用 Playwright 自带的 Chromium。若 `accounts.yaml` / `.env` 中指定了 `browser_channel=chrome`,请额外安装 Google Chrome 并确保其在系统 PATH 中可被找到。
|
||||||
|
|
||||||
|
### 2.6 FFmpeg(可选)
|
||||||
|
|
||||||
|
仅在需要开启任务级桌面录屏时安装:
|
||||||
|
|
||||||
|
1. 从 [ffmpeg.org](https://ffmpeg.org/download.html#build-windows) 下载 Windows build。
|
||||||
|
2. 解压后将 `bin` 目录加入系统 PATH。
|
||||||
|
3. 验证:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
ffmpeg -version
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. 目录规划示例
|
||||||
|
|
||||||
|
建议将两个仓库放在同一父目录下,便于统一管理:
|
||||||
|
|
||||||
|
```text
|
||||||
|
C:\jp-deploy\
|
||||||
|
├── jp_pruchase_zdh_5\ # 采购 worker
|
||||||
|
├── jp_surugaya\ # 抓取 API(含 Redis)
|
||||||
|
├── chrome-profile\ # 浏览器登录态持久化目录
|
||||||
|
│ ├── acc_01\
|
||||||
|
│ └── acc_02\
|
||||||
|
├── archives\ # 截图/录屏归档(可选,也可放在项目内)
|
||||||
|
└── logs\ # 日志汇总(可选)
|
||||||
|
```
|
||||||
|
|
||||||
|
后续示例均基于以上目录。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. 部署 jp_surugaya(抓取服务)
|
||||||
|
|
||||||
|
### 4.1 进入项目目录
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cd C:\jp-deploy\jp_surugaya
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.2 配置环境变量
|
||||||
|
|
||||||
|
复制示例文件:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
copy .env.example .env
|
||||||
|
```
|
||||||
|
|
||||||
|
编辑 `.env`,生产环境最小可用示例如下:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
# 服务监听
|
||||||
|
SURUGAYA_APP_HOST=0.0.0.0
|
||||||
|
SURUGAYA_APP_PORT=31106
|
||||||
|
SURUGAYA_APP_ENV=prod
|
||||||
|
|
||||||
|
# 日志
|
||||||
|
SURUGAYA_LOG_LEVEL=INFO
|
||||||
|
SURUGAYA_LOG_TO_FILE=true
|
||||||
|
SURUGAYA_LOG_DIR=logs
|
||||||
|
|
||||||
|
# API 鉴权(必须替换为强随机字符串)
|
||||||
|
SURUGAYA_BEARER_TOKEN=REPLACE_WITH_STRONG_RANDOM_TOKEN
|
||||||
|
|
||||||
|
# 浏览器
|
||||||
|
SURUGAYA_BROWSER_ENABLED=true
|
||||||
|
SURUGAYA_BROWSER_HEADLESS=false
|
||||||
|
SURUGAYA_BROWSER_CHANNEL=
|
||||||
|
SURUGAYA_BROWSER_KEEP_OPEN_ON_ERROR=false
|
||||||
|
|
||||||
|
# 代理(建议日本 IP)
|
||||||
|
SURUGAYA_PROXY_SERVER=http://127.0.0.1:7890
|
||||||
|
SURUGAYA_PROXY_USERNAME=
|
||||||
|
SURUGAYA_PROXY_PASSWORD=
|
||||||
|
|
||||||
|
# Redis(与采购 worker 使用同一实例)
|
||||||
|
SURUGAYA_REDIS_HOST=127.0.0.1
|
||||||
|
SURUGAYA_REDIS_PORT=6379
|
||||||
|
SURUGAYA_REDIS_PASSWORD=
|
||||||
|
SURUGAYA_REDIS_DB=0
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注意:
|
||||||
|
> - `SURUGAYA_APP_ENV` 要与采购 worker 的 `APP_ENV` 保持一致(推荐均为 `prod`),否则 Redis key 后缀不同,会导致通知/订单数据互相看不到。
|
||||||
|
> - 若不需要无头调试,Windows 裸机建议保持 `SURUGAYA_BROWSER_HEADLESS=false`,便于观察 Cloudflare 页面。
|
||||||
|
|
||||||
|
### 4.3 安装依赖与浏览器
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv sync
|
||||||
|
uv run playwright install chromium
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.4 启动服务
|
||||||
|
|
||||||
|
项目已提供 PowerShell 启动脚本,推荐直接使用:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
.\start-server.ps1
|
||||||
|
```
|
||||||
|
|
||||||
|
可选参数:
|
||||||
|
|
||||||
|
- `-Port 31106`:指定端口。
|
||||||
|
- `-HostName 0.0.0.0`:指定监听地址。
|
||||||
|
- `-NoKillExisting`:端口占用时不自动结束旧进程。
|
||||||
|
- `-DryRun`:仅做检查,不实际启动。
|
||||||
|
|
||||||
|
如果不希望使用脚本,也可以手动启动:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv run python -m app.main
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4.5 验证
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
curl http://127.0.0.1:31106/health
|
||||||
|
```
|
||||||
|
|
||||||
|
测试带鉴权的搜索接口:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
curl -H "Authorization: Bearer REPLACE_WITH_STRONG_RANDOM_TOKEN" `
|
||||||
|
-H "Content-Type: application/json" `
|
||||||
|
-X POST "http://127.0.0.1:31106/api/search" `
|
||||||
|
-d '{"search_word":"cat","category":"","page":1}'
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. 部署 jp_pruchase_zdh_5(采购 worker)
|
||||||
|
|
||||||
|
### 5.1 进入项目目录
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cd C:\jp-deploy\jp_pruchase_zdh_5
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5.2 配置环境变量
|
||||||
|
|
||||||
|
复制示例文件为生产配置:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
copy .env.example .env.prod
|
||||||
|
```
|
||||||
|
|
||||||
|
编辑 `.env.prod`,生产环境最小可用示例如下:
|
||||||
|
|
||||||
|
```ini
|
||||||
|
APP_ENV=prod
|
||||||
|
LOG_LEVEL=INFO
|
||||||
|
LOG_DIR=./logs
|
||||||
|
|
||||||
|
# Redis(必须与 jp_surugaya 同一实例)
|
||||||
|
REDIS_HOST=127.0.0.1
|
||||||
|
REDIS_PORT=6379
|
||||||
|
REDIS_PASSWORD=
|
||||||
|
REDIS_DB=0
|
||||||
|
|
||||||
|
# 消费者标识(多 worker 部署时建议为每个实例设置唯一名称)
|
||||||
|
REDIS_CONSUMER_GROUP=purchase_group
|
||||||
|
REDIS_CONSUMER_NAME=purchase_worker_01
|
||||||
|
|
||||||
|
# Redis key 名称(与 jp_surugaya 共用,prod 环境不带 _dev 后缀)
|
||||||
|
# 这些 key 的默认值已由 surugaya-common 的 redis_keys 定义一致,通常无需显式覆盖
|
||||||
|
REDIS_STREAM_KEY=jp_surugaya_purchase_task
|
||||||
|
REDIS_DELAY_ZSET_KEY=jp_surugaya_trade_delay
|
||||||
|
REDIS_NEWS_HASH_KEY=jp_surugaya_news
|
||||||
|
REDIS_TRADE_FETCH_STREAM_KEY=jp_surugaya_trade_fetch
|
||||||
|
REDIS_TRADE_FETCH_CONSUMER_GROUP=surugaya_trade_fetch_group
|
||||||
|
REDIS_TRADE_FETCH_CONSUMER_NAME=surugaya_trade_fetch_worker
|
||||||
|
|
||||||
|
# 节点标识(用于 trade delay zset,最终 key 为 REDIS_DELAY_ZSET_KEY:NODE_ID)
|
||||||
|
NODE_ID=node_1
|
||||||
|
|
||||||
|
# 通知定时抓取
|
||||||
|
NEWS_FETCH_ENABLED=true
|
||||||
|
NEWS_FETCH_INTERVAL_SECONDS=300
|
||||||
|
|
||||||
|
# 浏览器
|
||||||
|
BROWSER_HEADLESS=false
|
||||||
|
BROWSER_EXECUTABLE_PATH=
|
||||||
|
BROWSER_KEEP_OPEN=false
|
||||||
|
LOGIN_PRECHECK_ENABLED=true
|
||||||
|
|
||||||
|
# 配置与资源路径
|
||||||
|
ACCOUNTS_CONFIG_PATH=./accounts.yaml
|
||||||
|
SCREENSHOT_DIR=./archives
|
||||||
|
|
||||||
|
# 任务上报(按需配置回调地址与签名密钥)
|
||||||
|
CALLBACK_URL=
|
||||||
|
APPID=
|
||||||
|
APP_SECRET=
|
||||||
|
|
||||||
|
# 通知配置(失败时发邮件)
|
||||||
|
NOTIFY_ENABLED=false
|
||||||
|
NOTIFY_CHANNELS=email
|
||||||
|
NOTIFY_ON_TASK_FAILED=true
|
||||||
|
|
||||||
|
MAIL_ENABLED=true
|
||||||
|
MAIL_SMTP_HOST=
|
||||||
|
MAIL_SMTP_PORT=465
|
||||||
|
MAIL_SMTP_USE_SSL=true
|
||||||
|
MAIL_SMTP_STARTTLS=false
|
||||||
|
MAIL_USERNAME=
|
||||||
|
MAIL_PASSWORD=
|
||||||
|
MAIL_FROM_ADDRESS=
|
||||||
|
MAIL_FROM_NAME=JP Purchase Bot
|
||||||
|
MAIL_TO_ADDRESSES=ops@example.com
|
||||||
|
MAIL_SUBJECT_PREFIX=[JP-Purchase]
|
||||||
|
MAIL_TIMEOUT_SECONDS=10
|
||||||
|
```
|
||||||
|
|
||||||
|
> 注意:
|
||||||
|
> - Redis 实例由 `../jp_surugaya` 侧统一部署,本配置只需填写正确的连接信息。
|
||||||
|
> - `APP_ENV=prod` 时,Stream / Hash / ZSet 的 key 默认不带 `_dev` 后缀;若 `jp_surugaya` 与采购 worker 的 prod/dev 环境不一致,会出现通知/订单数据互相看不到、任务投递后无法消费等问题。
|
||||||
|
> - `REDIS_DELAY_ZSET_KEY` 最终使用的 key 为 `{REDIS_DELAY_ZSET_KEY}:{NODE_ID}`,因此 `NODE_ID` 必须与 `jp_surugaya` 调用 `/api/order/add_trade_monitor` 时传入的 `node_id` 保持一致。
|
||||||
|
|
||||||
|
### 5.3 配置账号
|
||||||
|
|
||||||
|
编辑项目根目录的 `accounts.yaml`。Windows 裸机部署示例:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
sites:
|
||||||
|
surugaya:
|
||||||
|
- account_id: "acc_01"
|
||||||
|
username: "your_username@example.com"
|
||||||
|
password: "your_password"
|
||||||
|
# 使用绝对路径,建议使用正斜杠
|
||||||
|
user_data_dir: "C:/jp-deploy/chrome-profile/acc_01"
|
||||||
|
proxy: "http://127.0.0.1:7890"
|
||||||
|
|
||||||
|
- account_id: "acc_02"
|
||||||
|
username: "your_username2@example.com"
|
||||||
|
password: "your_password2"
|
||||||
|
user_data_dir: "C:/jp-deploy/chrome-profile/acc_02"
|
||||||
|
proxy: "http://127.0.0.1:7890"
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
> 提示:
|
||||||
|
> - `user_data_dir` 用于持久化浏览器登录态;首次启动前确保目录已存在。
|
||||||
|
> - 如需清除登录态重新登录,直接删除对应目录即可。
|
||||||
|
> - 密码若为空字符串 `""`,系统会尝试使用已有 Cookie 登录。
|
||||||
|
|
||||||
|
### 5.4 安装依赖与浏览器
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv sync
|
||||||
|
uv run playwright install chromium
|
||||||
|
```
|
||||||
|
|
||||||
|
> 说明:`jp_pruchase_zdh_5` 依赖的 `surugaya-common` 来自公开 PyPI 索引,其源码与发布源头位于 `../jp_surugaya/packages/surugaya_common`。如需本地调试或修改共享包,请在 `../jp_surugaya` 仓库中开发并重新安装/发布。
|
||||||
|
|
||||||
|
### 5.5 启动 worker
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
$env:APP_ENV="prod"
|
||||||
|
uv run python -m src.main
|
||||||
|
```
|
||||||
|
|
||||||
|
启动后,worker 会:
|
||||||
|
|
||||||
|
1. 加载 `.env.prod` 与 `accounts.yaml`。
|
||||||
|
2. 预热所有已配置账号的浏览器上下文。
|
||||||
|
3. 执行登录预检(`LOGIN_PRECHECK_ENABLED=true`)。
|
||||||
|
4. 启动 Redis Stream 监听器、延迟任务监听器、订单抓取监听器、通知定时调度器。
|
||||||
|
5. 启动截图查看 API(`http://0.0.0.0:8080/`)。
|
||||||
|
|
||||||
|
### 5.6 验证
|
||||||
|
|
||||||
|
1. 查看日志输出,确认无配置错误。
|
||||||
|
2. 打开浏览器访问 `http://127.0.0.1:8080/`,应能看到截图查看器。
|
||||||
|
3. 向 Redis Stream 投递一条测试任务:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
redis-cli XADD jp_surugaya_purchase_task * task_id "task_test_001" topic "surugaya_tasks" purchase_account "acc_01" site_id "surugaya" record_video "0" payload "{\"item_list\":[{\"id\":\"m72275988725\",\"price_limit\":6500,\"number\":1}]}"
|
||||||
|
```
|
||||||
|
|
||||||
|
> 如果 Redis 设置了密码,请使用 `redis-cli -a your_password ...`。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. 长期后台运行(可选)
|
||||||
|
|
||||||
|
Windows 裸机环境下,可采用以下方式让服务随系统启动并保持运行:
|
||||||
|
|
||||||
|
### 6.1 使用任务计划程序(Task Scheduler)
|
||||||
|
|
||||||
|
1. 创建基本任务,触发器选择“当用户登录时”或“启动时”。
|
||||||
|
2. 操作选择“启动程序”,程序路径填写 `powershell.exe`。
|
||||||
|
3. 参数示例(采购 worker):
|
||||||
|
|
||||||
|
```text
|
||||||
|
-WindowStyle Hidden -Command "cd C:\jp-deploy\jp_pruchase_zdh_5; $env:APP_ENV='prod'; uv run python -m src.main"
|
||||||
|
```
|
||||||
|
|
||||||
|
4. 在“常规”选项卡中勾选“不管用户是否登录都要运行”时,服务将运行在 Session 0,可能导致 `pyautogui`、模板匹配、录屏等依赖桌面的功能失败。因此:
|
||||||
|
|
||||||
|
> **建议**:采购 worker 必须运行在**交互式用户会话**中(本地登录或持续 RDP 会话)。抓取 API 服务可运行在非交互式账户下。
|
||||||
|
|
||||||
|
### 6.2 使用 NSSM
|
||||||
|
|
||||||
|
[NSSM](https://nssm.cc/) 可将任意程序封装为 Windows 服务。仅建议用于 `jp_surugaya`;采购 worker 因需要桌面交互,不建议使用 NSSM。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. 升级维护
|
||||||
|
|
||||||
|
1. 拉取最新代码(如使用 git):
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
cd C:\jp-deploy\jp_pruchase_zdh_5
|
||||||
|
git pull
|
||||||
|
```
|
||||||
|
|
||||||
|
2. 重新同步依赖:
|
||||||
|
|
||||||
|
```powershell
|
||||||
|
uv sync
|
||||||
|
```
|
||||||
|
|
||||||
|
3. 重启对应 PowerShell 窗口中的进程即可。
|
||||||
|
|
||||||
|
> 提示:采购 worker 在 Windows 上不支持通过内置 API(`POST /api/restart`)重启,修改 `accounts.yaml` 后请手动重启进程。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 8. 常见问题
|
||||||
|
|
||||||
|
### 8.1 `uv sync` 失败,提示索引无法访问
|
||||||
|
|
||||||
|
- 确认部署机能正常访问 `https://git.jerryyan.net/api/packages/jp/pypi/simple`。
|
||||||
|
- 检查系统代理或防火墙是否拦截了该地址。
|
||||||
|
- 若企业内部网络需通过代理访问外网,请在执行 `uv sync` 前配置好 `HTTP_PROXY` / `HTTPS_PROXY` 环境变量。
|
||||||
|
|
||||||
|
### 8.2 Playwright 启动浏览器失败
|
||||||
|
|
||||||
|
- 确认已执行 `uv run playwright install chromium`。
|
||||||
|
- 确认已安装 [Microsoft Visual C++ Redistributable](https://aka.ms/vs/17/release/vc_redist.x64.exe)。
|
||||||
|
- 若指定了 `BROWSER_EXECUTABLE_PATH` 或 `SURUGAYA_BROWSER_CHANNEL=chrome`,确认 Chrome 已安装且路径正确。
|
||||||
|
|
||||||
|
### 8.3 模板匹配(Google Pay 确认按钮等)失败
|
||||||
|
|
||||||
|
- 确认当前屏幕分辨率与 `resources/images/` 下模板图的采集分辨率一致(推荐 `1920x1080`)。
|
||||||
|
- 若分辨率不同,请在当前分辨率下重新截取模板图并替换原文件。
|
||||||
|
|
||||||
|
### 8.4 端口被占用
|
||||||
|
|
||||||
|
- `31106` 被占用:先结束旧的 `python.exe` / `uvicorn` 进程,或修改 `.env` 中的 `SURUGAYA_APP_PORT`。
|
||||||
|
- `8080` 被占用:采购 worker 内置截图 API 默认使用 8080,若被其他程序占用,目前需修改源码中 `src/api/server.py` 的默认端口。
|
||||||
|
|
||||||
|
### 8.5 调用 `/api/search` 返回 403
|
||||||
|
|
||||||
|
检查请求头中的 `Authorization: Bearer <token>` 与 `.env` 中的 `SURUGAYA_BEARER_TOKEN` 是否一致。
|
||||||
|
|
||||||
|
### 8.6 采购 worker 收不到 jp_surugaya 投递的任务
|
||||||
|
|
||||||
|
检查两边 Redis 配置是否指向同一主机/端口/DB,并确认 `APP_ENV`(采购侧)与 `SURUGAYA_APP_ENV`(抓取侧)一致(例如都是 `prod`)。
|
||||||
|
|
||||||
|
### 8.7 账号登录态失效
|
||||||
|
|
||||||
|
删除该账号对应的 `user_data_dir` 目录,重启 worker 后会重新走登录流程。
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 9. 安全建议
|
||||||
|
|
||||||
|
- **不要**将 `8080` 端口直接暴露到公网;截图查看器与账号配置接口均无额外应用层鉴权。
|
||||||
|
- `31106` 若需对外提供,请通过反向代理并加鉴权(如 Nginx / Cloudflare Access / 企业 VPN)。
|
||||||
|
- `SURUGAYA_BEARER_TOKEN` 应使用至少 32 位的强随机字符串。
|
||||||
|
- 敏感信息(密码、Token、SMTP 密码)只存放在 `.env` / `accounts.yaml` 中,不要提交到 Git。
|
||||||
|
- 建议为日本站点使用稳定的日本 IP 代理,降低 Cloudflare 挑战频率。
|
||||||
+2
-1
@@ -10,7 +10,7 @@ authors = [
|
|||||||
]
|
]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
# playwright / playwright-stealth 由 surugaya-common 的 [browser] extra 提供,不再直接声明。
|
# playwright / playwright-stealth 由 surugaya-common 的 [browser] extra 提供,不再直接声明。
|
||||||
"surugaya-common[browser]>=0.3.0",
|
"surugaya-common[browser]>=0.4.0",
|
||||||
"pydantic>=2.7.0",
|
"pydantic>=2.7.0",
|
||||||
"pydantic-settings>=2.2.0",
|
"pydantic-settings>=2.2.0",
|
||||||
"loguru>=0.7.2",
|
"loguru>=0.7.2",
|
||||||
@@ -22,6 +22,7 @@ dependencies = [
|
|||||||
"pyscreeze>=1.0.1",
|
"pyscreeze>=1.0.1",
|
||||||
"pillow>=12.2.0",
|
"pillow>=12.2.0",
|
||||||
"httpx>=0.27.0",
|
"httpx>=0.27.0",
|
||||||
|
"aiohttp>=3.9.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[build-system]
|
[build-system]
|
||||||
|
|||||||
@@ -0,0 +1,375 @@
|
|||||||
|
"""
|
||||||
|
轻量级 HTTP API 服务,为运维人员提供截图查看接口。
|
||||||
|
|
||||||
|
启动后与主守护进程共享 asyncio 事件循环,不阻塞 Redis 消费。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import os
|
||||||
|
import signal
|
||||||
|
import sys
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import yaml
|
||||||
|
from aiohttp import web
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from src.config.accounts import MultiAccountConfig
|
||||||
|
from src.config.settings import get_settings
|
||||||
|
|
||||||
|
# 默认只展示最近 N 天的截图
|
||||||
|
_DEFAULT_DAYS = 7
|
||||||
|
|
||||||
|
|
||||||
|
def _scan_screenshots(archives_root: str, days: int = _DEFAULT_DAYS) -> list[dict]:
|
||||||
|
"""
|
||||||
|
扫描 archives 目录,返回最近 `days` 天的截图信息列表。
|
||||||
|
|
||||||
|
按修改时间倒序排列(最新在前)。
|
||||||
|
"""
|
||||||
|
root = Path(archives_root)
|
||||||
|
if not root.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
results: list[dict] = []
|
||||||
|
|
||||||
|
# 筛选最近的日期目录
|
||||||
|
today = datetime.now()
|
||||||
|
valid_date_dirs: set[str] = set()
|
||||||
|
for i in range(days):
|
||||||
|
d = today - __import__("datetime").timedelta(days=i)
|
||||||
|
valid_date_dirs.add(d.strftime("%Y%m%d"))
|
||||||
|
|
||||||
|
for date_dir in sorted(root.iterdir(), reverse=True):
|
||||||
|
if not date_dir.is_dir():
|
||||||
|
continue
|
||||||
|
if date_dir.name not in valid_date_dirs:
|
||||||
|
continue
|
||||||
|
|
||||||
|
for topic_dir in date_dir.iterdir():
|
||||||
|
if not topic_dir.is_dir():
|
||||||
|
continue
|
||||||
|
for img_file in topic_dir.iterdir():
|
||||||
|
if not img_file.is_file():
|
||||||
|
continue
|
||||||
|
if img_file.suffix.lower() not in (".png", ".jpg", ".jpeg"):
|
||||||
|
continue
|
||||||
|
|
||||||
|
# 解析文件名:{task_id}_{scene}_{timestamp_ms}.png
|
||||||
|
name_parts = img_file.stem.split("_")
|
||||||
|
scene = name_parts[1] if len(name_parts) >= 2 else "unknown"
|
||||||
|
|
||||||
|
stat = img_file.stat()
|
||||||
|
results.append({
|
||||||
|
"filename": img_file.name,
|
||||||
|
"date": date_dir.name,
|
||||||
|
"topic": topic_dir.name,
|
||||||
|
"scene": scene,
|
||||||
|
"relative_path": f"{date_dir.name}/{topic_dir.name}/{img_file.name}",
|
||||||
|
"size_kb": round(stat.st_size / 1024, 1),
|
||||||
|
"modified_at": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
})
|
||||||
|
|
||||||
|
# 按修改时间倒序
|
||||||
|
results.sort(key=lambda x: x["modified_at"], reverse=True)
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_screenshots(request: web.Request) -> web.Response:
|
||||||
|
"""GET /api/screenshots — 返回截图列表 JSON。"""
|
||||||
|
days = int(request.query.get("days", _DEFAULT_DAYS))
|
||||||
|
scene_filter = request.query.get("scene", "").strip()
|
||||||
|
topic_filter = request.query.get("topic", "").strip()
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
screenshots = _scan_screenshots(settings.screenshot_dir, days=days)
|
||||||
|
|
||||||
|
# 可选过滤
|
||||||
|
if scene_filter:
|
||||||
|
screenshots = [s for s in screenshots if scene_filter in s["scene"]]
|
||||||
|
if topic_filter:
|
||||||
|
screenshots = [s for s in screenshots if topic_filter in s["topic"]]
|
||||||
|
|
||||||
|
return web.json_response({
|
||||||
|
"total": len(screenshots),
|
||||||
|
"days": days,
|
||||||
|
"screenshots": screenshots,
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_screenshot_image(request: web.Request) -> web.Response:
|
||||||
|
"""GET /api/screenshots/{date}/{topic}/{filename} — 返回截图图片文件。"""
|
||||||
|
date = request.match_info["date"]
|
||||||
|
topic = request.match_info["topic"]
|
||||||
|
filename = request.match_info["filename"]
|
||||||
|
|
||||||
|
# 安全校验:防止路径遍历
|
||||||
|
if ".." in date or ".." in topic or ".." in filename:
|
||||||
|
return web.json_response({"error": "非法路径"}, status=400)
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
filepath = Path(settings.screenshot_dir) / date / topic / filename
|
||||||
|
|
||||||
|
if not filepath.exists() or not filepath.is_file():
|
||||||
|
return web.json_response({"error": "文件不存在"}, status=404)
|
||||||
|
|
||||||
|
return web.FileResponse(filepath)
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_index(request: web.Request) -> web.Response:
|
||||||
|
"""GET / — 简单的截图浏览 HTML 页面。"""
|
||||||
|
return web.Response(text=_INDEX_HTML, content_type="text/html")
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_accounts_get(request: web.Request) -> web.Response:
|
||||||
|
"""GET /api/accounts — 返回 accounts.yaml 原始文本。"""
|
||||||
|
settings = get_settings()
|
||||||
|
config_path = Path(settings.accounts_config_path)
|
||||||
|
|
||||||
|
if not config_path.exists():
|
||||||
|
return web.Response(text="", content_type="text/plain", charset="utf-8")
|
||||||
|
|
||||||
|
return web.Response(
|
||||||
|
text=config_path.read_text(encoding="utf-8"),
|
||||||
|
content_type="text/plain",
|
||||||
|
charset="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_accounts_put(request: web.Request) -> web.Response:
|
||||||
|
"""PUT /api/accounts — 校验并保存 accounts.yaml(Pydantic 结构校验通过才写入)。"""
|
||||||
|
raw_text = await request.text()
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw_data = yaml.safe_load(raw_text) or {}
|
||||||
|
config = MultiAccountConfig.model_validate(raw_data)
|
||||||
|
except Exception as e:
|
||||||
|
return web.json_response({"ok": False, "error": f"配置校验失败: {e}"}, status=400)
|
||||||
|
|
||||||
|
settings = get_settings()
|
||||||
|
config_path = Path(settings.accounts_config_path)
|
||||||
|
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# 先写临时文件再原子替换,避免写入中途崩溃损坏配置
|
||||||
|
tmp_path = config_path.with_name(config_path.name + ".tmp")
|
||||||
|
tmp_path.write_text(raw_text, encoding="utf-8")
|
||||||
|
os.replace(tmp_path, config_path)
|
||||||
|
|
||||||
|
account_count = len(config.sites.surugaya)
|
||||||
|
logger.warning(f"accounts.yaml 已通过 API 更新(骏河屋账号 {account_count} 个),重启进程后生效。")
|
||||||
|
return web.json_response({
|
||||||
|
"ok": True,
|
||||||
|
"message": f"配置已保存(骏河屋账号 {account_count} 个),需重启进程后生效。",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_restart(request: web.Request) -> web.Response:
|
||||||
|
"""
|
||||||
|
POST /api/restart — 触发进程优雅重启。
|
||||||
|
|
||||||
|
向自身发送 SIGTERM,复用 main.py 的优雅停机流程(停监听、关浏览器);
|
||||||
|
进程退出后由 supervisord(容器内 autorestart=true)拉起新进程,
|
||||||
|
重新加载 accounts.yaml 并预热浏览器。
|
||||||
|
"""
|
||||||
|
if sys.platform == "win32":
|
||||||
|
return web.json_response(
|
||||||
|
{"ok": False, "error": "Windows 环境不支持 API 重启,请手动重启进程。"},
|
||||||
|
status=501,
|
||||||
|
)
|
||||||
|
|
||||||
|
logger.warning("收到 API 重启请求,0.5 秒后向自身发送 SIGTERM 触发优雅重启...")
|
||||||
|
loop = asyncio.get_running_loop()
|
||||||
|
loop.call_later(0.5, os.kill, os.getpid(), signal.SIGTERM)
|
||||||
|
return web.json_response({
|
||||||
|
"ok": True,
|
||||||
|
"message": "进程即将优雅退出,由 supervisord 自动重启并加载最新配置。",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_accounts_page(request: web.Request) -> web.Response:
|
||||||
|
"""GET /accounts — accounts.yaml 在线编辑 HTML 页面。"""
|
||||||
|
return web.Response(text=_ACCOUNTS_HTML, content_type="text/html")
|
||||||
|
|
||||||
|
|
||||||
|
# 简易 HTML 页面,运维人员可直接在浏览器中查看
|
||||||
|
_INDEX_HTML = """<!DOCTYPE html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>截图查看器 — JP Purchase Bot</title>
|
||||||
|
<style>
|
||||||
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 20px; background: #f5f5f5; }
|
||||||
|
h1 { color: #333; font-size: 1.4em; }
|
||||||
|
.filters { margin: 12px 0; display: flex; gap: 10px; flex-wrap: wrap; align-items: center; }
|
||||||
|
.filters label { font-size: 0.9em; color: #666; }
|
||||||
|
.filters input, .filters select { padding: 4px 8px; border: 1px solid #ccc; border-radius: 4px; }
|
||||||
|
.filters button { padding: 6px 16px; background: #4a90d9; color: #fff; border: none; border-radius: 4px; cursor: pointer; }
|
||||||
|
.filters button:hover { background: #357abd; }
|
||||||
|
table { width: 100%; border-collapse: collapse; background: #fff; box-shadow: 0 1px 3px rgba(0,0,0,0.1); }
|
||||||
|
th, td { padding: 8px 12px; text-align: left; border-bottom: 1px solid #eee; font-size: 0.85em; }
|
||||||
|
th { background: #fafafa; color: #555; font-weight: 600; }
|
||||||
|
tr:hover { background: #f0f7ff; }
|
||||||
|
.scene-tag { display: inline-block; padding: 2px 8px; border-radius: 3px; font-size: 0.8em; }
|
||||||
|
.scene-error { background: #ffe0e0; color: #c00; }
|
||||||
|
.scene-other { background: #e0f0ff; color: #369; }
|
||||||
|
.thumb { max-width: 60px; max-height: 40px; cursor: pointer; border: 1px solid #ddd; }
|
||||||
|
.thumb:hover { border-color: #4a90d9; }
|
||||||
|
#overlay { display: none; position: fixed; top: 0; left: 0; width: 100%; height: 100%; background: rgba(0,0,0,0.85); z-index: 999; cursor: pointer; }
|
||||||
|
#overlay img { max-width: 95%; max-height: 95%; position: absolute; top: 50%; left: 50%; transform: translate(-50%, -50%); }
|
||||||
|
.stats { font-size: 0.85em; color: #888; margin-bottom: 8px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>截图查看器 <a href="/accounts" style="font-size:0.6em;font-weight:normal;margin-left:12px;">账号配置 →</a></h1>
|
||||||
|
<div class="filters">
|
||||||
|
<label>最近 <input type="number" id="days" value="3" min="1" max="30" style="width:50px"> 天</label>
|
||||||
|
<label>场景 <input type="text" id="scene" placeholder="如:任务执行出错"></label>
|
||||||
|
<label>主题 <input type="text" id="topic" placeholder="如:surugaya"></label>
|
||||||
|
<button onclick="load()">查询</button>
|
||||||
|
</div>
|
||||||
|
<div class="stats" id="stats"></div>
|
||||||
|
<table>
|
||||||
|
<thead><tr><th>时间</th><th>主题</th><th>场景</th><th>文件</th><th>大小</th><th>预览</th></tr></thead>
|
||||||
|
<tbody id="tbody"></tbody>
|
||||||
|
</table>
|
||||||
|
<div id="overlay" onclick="this.style.display='none'"><img id="bigImg" src=""></div>
|
||||||
|
<script>
|
||||||
|
async function load() {
|
||||||
|
const days = document.getElementById('days').value;
|
||||||
|
const scene = document.getElementById('scene').value;
|
||||||
|
const topic = document.getElementById('topic').value;
|
||||||
|
let url = '/api/screenshots?days=' + days;
|
||||||
|
if (scene) url += '&scene=' + encodeURIComponent(scene);
|
||||||
|
if (topic) url += '&topic=' + encodeURIComponent(topic);
|
||||||
|
const resp = await fetch(url);
|
||||||
|
const data = await resp.json();
|
||||||
|
document.getElementById('stats').textContent = '共 ' + data.total + ' 张截图(最近 ' + data.days + ' 天)';
|
||||||
|
const tbody = document.getElementById('tbody');
|
||||||
|
tbody.innerHTML = '';
|
||||||
|
for (const s of data.screenshots) {
|
||||||
|
const cls = s.scene.includes('出错') || s.scene.includes('失败') || s.scene.includes('异常') ? 'scene-error' : 'scene-other';
|
||||||
|
const imgUrl = '/api/screenshots/' + s.relative_path;
|
||||||
|
tbody.innerHTML += '<tr>'
|
||||||
|
+ '<td>' + s.modified_at + '</td>'
|
||||||
|
+ '<td>' + s.topic + '</td>'
|
||||||
|
+ '<td><span class="scene-tag ' + cls + '">' + s.scene + '</span></td>'
|
||||||
|
+ '<td>' + s.filename + '</td>'
|
||||||
|
+ '<td>' + s.size_kb + ' KB</td>'
|
||||||
|
+ '<td><img class="thumb" src="' + imgUrl + '" onclick="showBig(\\'' + imgUrl + '\\')"></td>'
|
||||||
|
+ '</tr>';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
function showBig(url) {
|
||||||
|
document.getElementById('bigImg').src = url;
|
||||||
|
document.getElementById('overlay').style.display = 'block';
|
||||||
|
}
|
||||||
|
load();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>"""
|
||||||
|
|
||||||
|
|
||||||
|
# accounts.yaml 在线编辑页面,供运维人员修改账号配置后触发优雅重启
|
||||||
|
_ACCOUNTS_HTML = """<!DOCTYPE html>
|
||||||
|
<html lang="zh-CN">
|
||||||
|
<head>
|
||||||
|
<meta charset="UTF-8">
|
||||||
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
|
<title>账号配置 — JP Purchase Bot</title>
|
||||||
|
<style>
|
||||||
|
body { font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; margin: 20px; background: #f5f5f5; }
|
||||||
|
h1 { color: #333; font-size: 1.4em; }
|
||||||
|
textarea { width: 100%; height: 60vh; font-family: Consolas, Menlo, monospace; font-size: 13px;
|
||||||
|
padding: 10px; border: 1px solid #ccc; border-radius: 4px; box-sizing: border-box;
|
||||||
|
background: #fff; white-space: pre; }
|
||||||
|
.actions { margin: 12px 0; display: flex; gap: 10px; align-items: center; }
|
||||||
|
.actions button { padding: 8px 20px; border: none; border-radius: 4px; cursor: pointer; color: #fff; }
|
||||||
|
#btnSave { background: #4a90d9; }
|
||||||
|
#btnSave:hover { background: #357abd; }
|
||||||
|
#btnSaveRestart { background: #d9534f; }
|
||||||
|
#btnSaveRestart:hover { background: #c9302c; }
|
||||||
|
.actions button:disabled { background: #aaa; cursor: not-allowed; }
|
||||||
|
#msg { font-size: 0.9em; }
|
||||||
|
#msg.ok { color: #2a7a2a; }
|
||||||
|
#msg.err { color: #c00; white-space: pre-wrap; }
|
||||||
|
.tip { font-size: 0.85em; color: #888; margin-bottom: 8px; }
|
||||||
|
</style>
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
<h1>账号配置 (accounts.yaml) <a href="/" style="font-size:0.6em;font-weight:normal;margin-left:12px;">← 截图查看器</a></h1>
|
||||||
|
<div class="tip">保存时会做 YAML 结构校验;修改仅在进程重启后生效。"保存并重启"会优雅停机后由 supervisord 自动拉起进程并按新配置重新预热浏览器。</div>
|
||||||
|
<textarea id="editor" spellcheck="false"></textarea>
|
||||||
|
<div class="actions">
|
||||||
|
<button id="btnSave" onclick="save(false)">保存</button>
|
||||||
|
<button id="btnSaveRestart" onclick="save(true)">保存并重启</button>
|
||||||
|
<span id="msg"></span>
|
||||||
|
</div>
|
||||||
|
<script>
|
||||||
|
function setMsg(text, isErr) {
|
||||||
|
const el = document.getElementById('msg');
|
||||||
|
el.textContent = text;
|
||||||
|
el.className = isErr ? 'err' : 'ok';
|
||||||
|
}
|
||||||
|
function setBusy(busy) {
|
||||||
|
document.getElementById('btnSave').disabled = busy;
|
||||||
|
document.getElementById('btnSaveRestart').disabled = busy;
|
||||||
|
}
|
||||||
|
async function load() {
|
||||||
|
const resp = await fetch('/api/accounts');
|
||||||
|
document.getElementById('editor').value = await resp.text();
|
||||||
|
}
|
||||||
|
async function save(restart) {
|
||||||
|
if (restart && !confirm('确认保存并重启进程?进行中的任务会被优雅停机打断。')) return;
|
||||||
|
setBusy(true);
|
||||||
|
setMsg('保存中...', false);
|
||||||
|
try {
|
||||||
|
const resp = await fetch('/api/accounts', {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'Content-Type': 'text/plain; charset=utf-8' },
|
||||||
|
body: document.getElementById('editor').value,
|
||||||
|
});
|
||||||
|
const data = await resp.json();
|
||||||
|
if (!data.ok) { setMsg(data.error, true); return; }
|
||||||
|
if (!restart) { setMsg(data.message, false); return; }
|
||||||
|
|
||||||
|
setMsg('已保存,正在触发重启...', false);
|
||||||
|
const r = await fetch('/api/restart', { method: 'POST' });
|
||||||
|
const rd = await r.json();
|
||||||
|
if (!rd.ok) { setMsg(rd.error, true); return; }
|
||||||
|
setMsg('重启已触发,进程将在优雅停机后自动拉起(约数十秒),期间 API 短暂不可用。', false);
|
||||||
|
} catch (e) {
|
||||||
|
setMsg('请求失败: ' + e, true);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
load();
|
||||||
|
</script>
|
||||||
|
</body>
|
||||||
|
</html>"""
|
||||||
|
|
||||||
|
|
||||||
|
def create_app() -> web.Application:
|
||||||
|
"""创建 aiohttp Application 实例。"""
|
||||||
|
app = web.Application()
|
||||||
|
app.router.add_get("/", handle_index)
|
||||||
|
app.router.add_get("/accounts", handle_accounts_page)
|
||||||
|
app.router.add_get("/api/screenshots", handle_screenshots)
|
||||||
|
app.router.add_get("/api/screenshots/{date}/{topic}/{filename}", handle_screenshot_image)
|
||||||
|
app.router.add_get("/api/accounts", handle_accounts_get)
|
||||||
|
app.router.add_put("/api/accounts", handle_accounts_put)
|
||||||
|
app.router.add_post("/api/restart", handle_restart)
|
||||||
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
async def start_api_server(host: str = "0.0.0.0", port: int = 8080) -> web.AppRunner:
|
||||||
|
"""启动 API 服务,返回 AppRunner 以便后续优雅关闭。"""
|
||||||
|
app = create_app()
|
||||||
|
runner = web.AppRunner(app)
|
||||||
|
await runner.setup()
|
||||||
|
site = web.TCPSite(runner, host, port)
|
||||||
|
await site.start()
|
||||||
|
logger.info(f"截图查看 API 已启动 → http://{host}:{port}/")
|
||||||
|
return runner
|
||||||
+10
-1
@@ -40,12 +40,21 @@ class Settings(BaseSettings):
|
|||||||
redis_trade_fetch_stream_key: str = Field(default=redis_keys.TRADE_FETCH_STREAM, alias="REDIS_TRADE_FETCH_STREAM_KEY")
|
redis_trade_fetch_stream_key: str = Field(default=redis_keys.TRADE_FETCH_STREAM, alias="REDIS_TRADE_FETCH_STREAM_KEY")
|
||||||
redis_trade_fetch_consumer_group: str = Field(default="surugaya_trade_fetch_group", alias="REDIS_TRADE_FETCH_CONSUMER_GROUP")
|
redis_trade_fetch_consumer_group: str = Field(default="surugaya_trade_fetch_group", alias="REDIS_TRADE_FETCH_CONSUMER_GROUP")
|
||||||
redis_trade_fetch_consumer_name: str = Field(default="surugaya_trade_fetch_worker", alias="REDIS_TRADE_FETCH_CONSUMER_NAME")
|
redis_trade_fetch_consumer_name: str = Field(default="surugaya_trade_fetch_worker", alias="REDIS_TRADE_FETCH_CONSUMER_NAME")
|
||||||
|
# 通知(お知らせ)抓取结果写入的 Redis Hash,field=data_id,中心 API 据此读取。
|
||||||
|
# 默认值直接写字面量(不引用 redis_keys,兼容尚未发布该常量的旧版 surugaya-common)。
|
||||||
|
redis_news_hash_key: str = Field(default="jp_surugaya_news", alias="REDIS_NEWS_HASH_KEY")
|
||||||
|
|
||||||
|
# 通知定时抓取调度
|
||||||
|
news_fetch_enabled: bool = Field(default=True, alias="NEWS_FETCH_ENABLED")
|
||||||
|
news_fetch_interval_seconds: int = Field(default=300, alias="NEWS_FETCH_INTERVAL_SECONDS")
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# 浏览器控制
|
# 浏览器控制
|
||||||
browser_headless: bool = Field(default=True, alias="BROWSER_HEADLESS")
|
browser_headless: bool = Field(default=True, alias="BROWSER_HEADLESS")
|
||||||
browser_executable_path: str | None = Field(default=None, alias="BROWSER_EXECUTABLE_PATH")
|
browser_executable_path: str | None = Field(default=None, alias="BROWSER_EXECUTABLE_PATH")
|
||||||
browser_keep_open: bool = Field(default=False, alias="BROWSER_KEEP_OPEN")
|
browser_keep_open: bool = Field(default=False, alias="BROWSER_KEEP_OPEN")
|
||||||
|
# 启动时登录预检:预热浏览器后逐账号确认登录态,未登录则自动登录
|
||||||
|
login_precheck_enabled: bool = Field(default=True, alias="LOGIN_PRECHECK_ENABLED")
|
||||||
|
|
||||||
# 配置与资源路径
|
# 配置与资源路径
|
||||||
accounts_config_path: str = Field(default="./accounts.yaml", alias="ACCOUNTS_CONFIG_PATH")
|
accounts_config_path: str = Field(default="./accounts.yaml", alias="ACCOUNTS_CONFIG_PATH")
|
||||||
|
|||||||
+15
-1
@@ -20,7 +20,10 @@ from src.task_queue.dispatcher import TaskDispatcher
|
|||||||
from src.task_queue.delay_zset_listener import RedisZSetDelayListener
|
from src.task_queue.delay_zset_listener import RedisZSetDelayListener
|
||||||
from src.task_queue.listener import RedisStreamListener
|
from src.task_queue.listener import RedisStreamListener
|
||||||
from src.task_queue.trade_fetch_listener import TradeFetchStreamListener
|
from src.task_queue.trade_fetch_listener import TradeFetchStreamListener
|
||||||
|
from src.task_queue.news_scheduler import NewsFetchScheduler
|
||||||
from src.task_queue.redis_manager import get_redis_manager
|
from src.task_queue.redis_manager import get_redis_manager
|
||||||
|
from src.tasks.surugaya_auth import login_precheck
|
||||||
|
from src.api.server import start_api_server
|
||||||
|
|
||||||
async def main() -> None:
|
async def main() -> None:
|
||||||
"""
|
"""
|
||||||
@@ -50,11 +53,18 @@ async def main() -> None:
|
|||||||
# 3. 预热浏览器:为所有已配置账号提前启动常驻浏览器上下文,任务到达时即开即用
|
# 3. 预热浏览器:为所有已配置账号提前启动常驻浏览器上下文,任务到达时即开即用
|
||||||
await get_browser_manager().prewarm()
|
await get_browser_manager().prewarm()
|
||||||
|
|
||||||
|
# 3.1 登录预检:逐账号确认登录态,未登录则自动登录,避免任务到达时再耗时走登录流程
|
||||||
|
await login_precheck()
|
||||||
|
|
||||||
# 4. 实例化核心任务分发器与 Stream 监听器
|
# 4. 实例化核心任务分发器与 Stream 监听器
|
||||||
dispatcher = TaskDispatcher()
|
dispatcher = TaskDispatcher()
|
||||||
stream_listener = RedisStreamListener(dispatcher)
|
stream_listener = RedisStreamListener(dispatcher)
|
||||||
delay_listener = RedisZSetDelayListener(dispatcher)
|
delay_listener = RedisZSetDelayListener(dispatcher)
|
||||||
trade_fetch_listener = TradeFetchStreamListener(dispatcher)
|
trade_fetch_listener = TradeFetchStreamListener(dispatcher)
|
||||||
|
news_scheduler = NewsFetchScheduler(dispatcher)
|
||||||
|
|
||||||
|
# 4.1 启动截图查看 API 服务
|
||||||
|
api_runner = await start_api_server()
|
||||||
|
|
||||||
# 5. 优雅停机信号处理 (跨平台支持)
|
# 5. 优雅停机信号处理 (跨平台支持)
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
@@ -65,6 +75,7 @@ async def main() -> None:
|
|||||||
asyncio.create_task(stream_listener.stop())
|
asyncio.create_task(stream_listener.stop())
|
||||||
asyncio.create_task(delay_listener.stop())
|
asyncio.create_task(delay_listener.stop())
|
||||||
asyncio.create_task(trade_fetch_listener.stop())
|
asyncio.create_task(trade_fetch_listener.stop())
|
||||||
|
asyncio.create_task(news_scheduler.stop())
|
||||||
|
|
||||||
# Windows 默认的 Proactor 事件循环不支持 add_signal_handler。
|
# Windows 默认的 Proactor 事件循环不支持 add_signal_handler。
|
||||||
if sys.platform != "win32":
|
if sys.platform != "win32":
|
||||||
@@ -82,7 +93,8 @@ async def main() -> None:
|
|||||||
stream_task = asyncio.create_task(stream_listener.listen())
|
stream_task = asyncio.create_task(stream_listener.listen())
|
||||||
delay_task = asyncio.create_task(delay_listener.listen())
|
delay_task = asyncio.create_task(delay_listener.listen())
|
||||||
trade_fetch_task = asyncio.create_task(trade_fetch_listener.listen())
|
trade_fetch_task = asyncio.create_task(trade_fetch_listener.listen())
|
||||||
await asyncio.gather(stream_task, delay_task, trade_fetch_task)
|
news_task = asyncio.create_task(news_scheduler.listen())
|
||||||
|
await asyncio.gather(stream_task, delay_task, trade_fetch_task, news_task)
|
||||||
except asyncio.CancelledError:
|
except asyncio.CancelledError:
|
||||||
logger.warning("监听器轮询执行被事件循环取消。")
|
logger.warning("监听器轮询执行被事件循环取消。")
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
@@ -93,8 +105,10 @@ async def main() -> None:
|
|||||||
stream_listener.stop(),
|
stream_listener.stop(),
|
||||||
delay_listener.stop(),
|
delay_listener.stop(),
|
||||||
trade_fetch_listener.stop(),
|
trade_fetch_listener.stop(),
|
||||||
|
news_scheduler.stop(),
|
||||||
get_browser_manager().close_all(),
|
get_browser_manager().close_all(),
|
||||||
get_redis_manager().close(),
|
get_redis_manager().close(),
|
||||||
|
api_runner.cleanup(),
|
||||||
return_exceptions=True,
|
return_exceptions=True,
|
||||||
)
|
)
|
||||||
logger.success("日本采购自动化守护进程已安全优雅退出。")
|
logger.success("日本采购自动化守护进程已安全优雅退出。")
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
"""worker 本地页面解析器。"""
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
"""骏河屋「お知らせ一覧」(action_news)通知列表解析器。
|
||||||
|
|
||||||
|
页面地址:https://www.suruga-ya.jp/pcmypage/action_news
|
||||||
|
通知表格由 DataTables 异步渲染,每行结构为:
|
||||||
|
<tr><td>2026/06/12</td>
|
||||||
|
<td><a data-type="2"
|
||||||
|
data-url="/pcmypage/action_sell_search/detail?trade_code=S2606115133"
|
||||||
|
data-id="869431007">【販売】(取引番号:S2606115133)発送準備中です。</a></td></tr>
|
||||||
|
|
||||||
|
解析拆成几层:
|
||||||
|
- `extract_raw_rows(page)`:仅负责在浏览器侧抓取原始单元格(依赖 Playwright)。
|
||||||
|
- `build_news_items(raw_rows)`:纯函数,把原始行规整为通知字典并从 data-url 解析 trade_code,
|
||||||
|
无浏览器依赖,便于单元测试。
|
||||||
|
- `parse_news_page(page)`:解析「当前已渲染页」的通知。
|
||||||
|
- `collect_all_news(page)`:驱动 DataTables 逐页翻页,按 data_id 去重累积全部通知。
|
||||||
|
"""
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
from typing import Any
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from playwright.async_api import Page
|
||||||
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||||
|
|
||||||
|
# 通知列表所在的「お知らせ(取引)」标签页表格。
|
||||||
|
NEWS_TABLE_ROWS_SELECTOR = "#table_news tbody tr"
|
||||||
|
# 行内承载通知数据的链接(带 data-id 才是有效通知行,可借此过滤空占位行)。
|
||||||
|
NEWS_ROW_LINK_SELECTOR = "td a[data-id]"
|
||||||
|
# DataTables 翻页「次へ」按钮;最后一页时带 disabled 类。
|
||||||
|
NEWS_NEXT_BUTTON_SELECTOR = "#table_news_next"
|
||||||
|
|
||||||
|
# 等待 DataTables 异步渲染出首条通知链接的超时(毫秒)。
|
||||||
|
NEWS_TABLE_RENDER_TIMEOUT_MS = 15000
|
||||||
|
# 翻页后等待新一页渲染完成的超时(毫秒)。
|
||||||
|
NEWS_PAGE_TURN_TIMEOUT_MS = 15000
|
||||||
|
# 翻页安全上限,兜底防止异常情况下死循环(远高于真实页数)。
|
||||||
|
NEWS_MAX_PAGES = 100
|
||||||
|
|
||||||
|
|
||||||
|
def parse_trade_code(data_url: str | None) -> str | None:
|
||||||
|
"""从 data-url 的查询串中解析 trade_code,非交易类通知返回 None。"""
|
||||||
|
if not data_url:
|
||||||
|
return None
|
||||||
|
query = urlsplit(data_url).query
|
||||||
|
values = parse_qs(query).get("trade_code")
|
||||||
|
if not values:
|
||||||
|
return None
|
||||||
|
trade_code = values[0].strip()
|
||||||
|
return trade_code or None
|
||||||
|
|
||||||
|
|
||||||
|
def build_news_items(raw_rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||||
|
"""把浏览器侧抓取的原始行规整为通知字典列表。
|
||||||
|
|
||||||
|
缺少 data_id 的行视为无效(空占位/异常行)直接跳过;data_id 是站点侧
|
||||||
|
单条通知的唯一且单调递增标识,作为 Redis Hash 的 field 用于去重与排序。
|
||||||
|
"""
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
for row in raw_rows:
|
||||||
|
data_id = (row.get("data_id") or "").strip()
|
||||||
|
if not data_id:
|
||||||
|
continue
|
||||||
|
|
||||||
|
data_url = (row.get("data_url") or "").strip() or None
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"data_id": data_id,
|
||||||
|
"date": (row.get("date") or "").strip(),
|
||||||
|
"message": (row.get("message") or "").strip(),
|
||||||
|
"data_type": (row.get("data_type") or "").strip(),
|
||||||
|
"data_url": data_url,
|
||||||
|
"trade_code": parse_trade_code(data_url),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
async def extract_raw_rows(page: Page) -> list[dict[str, Any]]:
|
||||||
|
"""在浏览器侧抓取通知表格的原始单元格数据(不做业务清洗)。"""
|
||||||
|
return await page.locator(NEWS_TABLE_ROWS_SELECTOR).evaluate_all(
|
||||||
|
"""rows => rows.map((row) => {
|
||||||
|
const dateCell = row.querySelector(':scope > td');
|
||||||
|
const link = row.querySelector('td a[data-id]');
|
||||||
|
return {
|
||||||
|
date: dateCell ? (dateCell.innerText || '').trim() : '',
|
||||||
|
message: link ? (link.innerText || '').trim() : '',
|
||||||
|
data_type: link ? (link.getAttribute('data-type') || '') : '',
|
||||||
|
data_url: link ? (link.getAttribute('data-url') || '') : '',
|
||||||
|
data_id: link ? (link.getAttribute('data-id') || '') : '',
|
||||||
|
};
|
||||||
|
})"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def parse_news_page(page: Page) -> list[dict[str, Any]]:
|
||||||
|
"""解析「当前已渲染页」的通知字典列表。"""
|
||||||
|
raw_rows = await extract_raw_rows(page)
|
||||||
|
return build_news_items(raw_rows)
|
||||||
|
|
||||||
|
|
||||||
|
async def _first_data_id(page: Page) -> str | None:
|
||||||
|
"""当前页首行通知的 data-id;无有效行返回 None。"""
|
||||||
|
return await page.locator(NEWS_ROW_LINK_SELECTOR).first.get_attribute(
|
||||||
|
"data-id", timeout=1000
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _next_button_disabled(page: Page) -> bool:
|
||||||
|
"""「次へ」按钮是否处于禁用态(即已是最后一页)。"""
|
||||||
|
cls = await page.locator(NEWS_NEXT_BUTTON_SELECTOR).get_attribute("class") or ""
|
||||||
|
return "disabled" in cls.split()
|
||||||
|
|
||||||
|
|
||||||
|
async def collect_all_news(page: Page, max_pages: int = NEWS_MAX_PAGES) -> list[dict[str, Any]]:
|
||||||
|
"""驱动 DataTables 逐页翻页,按 data_id 去重累积全部通知。
|
||||||
|
|
||||||
|
通知表由 DataTables 异步渲染并分页(翻页走 ajax/JS,仅渲染当前页的 <tr>)。
|
||||||
|
逐页解析后点击「次へ」,以「首行 data-id 变化」作为新页渲染完成信号;
|
||||||
|
「次へ」带 disabled 类即最后一页,停止。始终无通知时返回空列表。
|
||||||
|
"""
|
||||||
|
# 等待首页渲染出有效通知行;超时按「无通知」处理,不报错。
|
||||||
|
try:
|
||||||
|
await page.locator(NEWS_ROW_LINK_SELECTOR).first.wait_for(
|
||||||
|
state="attached", timeout=NEWS_TABLE_RENDER_TIMEOUT_MS
|
||||||
|
)
|
||||||
|
except PlaywrightTimeoutError:
|
||||||
|
logger.warning("等待通知表格渲染超时,按无通知处理。")
|
||||||
|
return []
|
||||||
|
|
||||||
|
collected: dict[str, dict[str, Any]] = {}
|
||||||
|
for page_index in range(max_pages):
|
||||||
|
for item in await parse_news_page(page):
|
||||||
|
collected.setdefault(item["data_id"], item)
|
||||||
|
|
||||||
|
if await _next_button_disabled(page):
|
||||||
|
break
|
||||||
|
|
||||||
|
before_id = await _first_data_id(page)
|
||||||
|
await page.locator(NEWS_NEXT_BUTTON_SELECTOR).click()
|
||||||
|
|
||||||
|
# 等待新一页渲染:首行 data-id 与翻页前不同。
|
||||||
|
try:
|
||||||
|
await page.wait_for_function(
|
||||||
|
"""(beforeId) => {
|
||||||
|
const link = document.querySelector('#table_news tbody tr td a[data-id]');
|
||||||
|
const cur = link ? link.getAttribute('data-id') : null;
|
||||||
|
return !!cur && cur !== beforeId;
|
||||||
|
}""",
|
||||||
|
arg=before_id,
|
||||||
|
timeout=NEWS_PAGE_TURN_TIMEOUT_MS,
|
||||||
|
)
|
||||||
|
except PlaywrightTimeoutError:
|
||||||
|
logger.warning(f"翻页第 {page_index + 2} 页等待渲染超时,停止翻页。")
|
||||||
|
break
|
||||||
|
else:
|
||||||
|
logger.warning(f"通知翻页达到安全上限 {max_pages} 页,停止翻页。")
|
||||||
|
|
||||||
|
return list(collected.values())
|
||||||
@@ -5,11 +5,13 @@ from loguru import logger
|
|||||||
|
|
||||||
from src.tasks.surugaya import SurugayaPurchaseTask
|
from src.tasks.surugaya import SurugayaPurchaseTask
|
||||||
from src.tasks.surugaya_trade_monitor import SurugayaTradeMonitorTask
|
from src.tasks.surugaya_trade_monitor import SurugayaTradeMonitorTask
|
||||||
|
from src.tasks.surugaya_news_monitor import SurugayaNewsMonitorTask
|
||||||
|
|
||||||
# topic 到采购任务类的映射注册表
|
# topic 到采购任务类的映射注册表
|
||||||
TOPIC_TO_TASK_MAP: Dict[str, Type[Any]] = {
|
TOPIC_TO_TASK_MAP: Dict[str, Type[Any]] = {
|
||||||
"surugaya": SurugayaPurchaseTask, # 骏河屋采购任务
|
"surugaya": SurugayaPurchaseTask, # 骏河屋采购任务
|
||||||
"surugaya_trade_monitor": SurugayaTradeMonitorTask, # 骏河屋交易订单获取任务
|
"surugaya_trade_monitor": SurugayaTradeMonitorTask, # 骏河屋交易订单获取任务
|
||||||
|
"surugaya_news": SurugayaNewsMonitorTask, # 骏河屋通知(お知らせ)抓取任务
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,63 @@
|
|||||||
|
import asyncio
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from src.config.accounts import load_accounts_config
|
||||||
|
from src.config.settings import get_settings
|
||||||
|
from src.task_queue.dispatcher import TaskDispatcher
|
||||||
|
from src.task_queue.task_messages import build_news_monitor_task
|
||||||
|
|
||||||
|
|
||||||
|
class NewsFetchScheduler:
|
||||||
|
"""
|
||||||
|
通知(お知らせ)定时抓取调度器。
|
||||||
|
|
||||||
|
与 Redis 监听器不同,这是纯时间驱动:每隔 news_fetch_interval_seconds,
|
||||||
|
为每个已配置的骏河屋账号派发一条 surugaya_news 任务(启动后立即先跑一轮)。
|
||||||
|
任务经 TaskDispatcher 的同账号串行锁排队,不会与采购/订单任务争抢浏览器。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, dispatcher: TaskDispatcher) -> None:
|
||||||
|
self.settings = get_settings()
|
||||||
|
self.dispatcher = dispatcher
|
||||||
|
self.running = False
|
||||||
|
self.interval_seconds = self.settings.news_fetch_interval_seconds
|
||||||
|
|
||||||
|
async def listen(self) -> None:
|
||||||
|
if not self.settings.news_fetch_enabled:
|
||||||
|
logger.info("通知定时抓取已禁用(NEWS_FETCH_ENABLED=false),跳过。")
|
||||||
|
return
|
||||||
|
|
||||||
|
self.running = True
|
||||||
|
logger.info(
|
||||||
|
f"通知定时抓取调度器已启动,间隔 {self.interval_seconds}s。"
|
||||||
|
)
|
||||||
|
|
||||||
|
while self.running:
|
||||||
|
try:
|
||||||
|
await self._dispatch_round()
|
||||||
|
except Exception as e:
|
||||||
|
logger.exception(f"通知定时抓取调度发生未预期错误: {e}")
|
||||||
|
|
||||||
|
# 可中断的间隔等待:停机时尽快退出循环。
|
||||||
|
for _ in range(self.interval_seconds):
|
||||||
|
if not self.running:
|
||||||
|
break
|
||||||
|
await asyncio.sleep(1)
|
||||||
|
|
||||||
|
async def _dispatch_round(self) -> None:
|
||||||
|
accounts = load_accounts_config().sites.surugaya
|
||||||
|
if not accounts:
|
||||||
|
logger.debug("未配置骏河屋账号,本轮通知抓取跳过。")
|
||||||
|
return
|
||||||
|
|
||||||
|
for account in accounts:
|
||||||
|
message = build_news_monitor_task(account.account_id)
|
||||||
|
logger.debug(
|
||||||
|
f"派发通知抓取任务: account='{account.account_id}', task_id='{message['task_id']}'。"
|
||||||
|
)
|
||||||
|
await self.dispatcher.dispatch(message)
|
||||||
|
|
||||||
|
async def stop(self) -> None:
|
||||||
|
logger.info("正在停止通知定时抓取调度器...")
|
||||||
|
self.running = False
|
||||||
@@ -6,6 +6,23 @@ TRADE_MONITOR_ACCOUNT = "trdian023@tokugen.jp"
|
|||||||
TRADE_MONITOR_SITE_ID = "7"
|
TRADE_MONITOR_SITE_ID = "7"
|
||||||
|
|
||||||
|
|
||||||
|
def build_news_monitor_task(account_id: str) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
构造骏河屋通知抓取任务消息(TaskDispatcher 兼容结构)。
|
||||||
|
|
||||||
|
供通知定时调度器(NewsFetchScheduler)使用:每个配置账号一条任务,
|
||||||
|
与采购/订单任务共用同账号串行锁,避免并发占用同一浏览器上下文。
|
||||||
|
"""
|
||||||
|
return {
|
||||||
|
"task_id": f"news_{int(time.time())}",
|
||||||
|
"topic": "surugaya_news",
|
||||||
|
"purchase_account": account_id,
|
||||||
|
"site_id": TRADE_MONITOR_SITE_ID,
|
||||||
|
"record_video": "",
|
||||||
|
"payload": {},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def build_trade_monitor_task(trade_code: Any) -> Dict[str, Any]:
|
def build_trade_monitor_task(trade_code: Any) -> Dict[str, Any]:
|
||||||
"""
|
"""
|
||||||
构造骏河屋交易订单获取任务消息(TaskDispatcher 兼容结构)。
|
构造骏河屋交易订单获取任务消息(TaskDispatcher 兼容结构)。
|
||||||
|
|||||||
+9
-28
@@ -15,6 +15,7 @@ from playwright.async_api import Page
|
|||||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||||
|
|
||||||
from notifications import NotificationEvent
|
from notifications import NotificationEvent
|
||||||
|
from src.tasks import surugaya_auth
|
||||||
from src.tasks.base import BasePurchaseTask
|
from src.tasks.base import BasePurchaseTask
|
||||||
from src.config.accounts import get_account_config
|
from src.config.accounts import get_account_config
|
||||||
from src.task_queue.redis_manager import get_redis_client
|
from src.task_queue.redis_manager import get_redis_client
|
||||||
@@ -87,6 +88,7 @@ class SurugayaPurchaseTask(BasePurchaseTask):
|
|||||||
product_id = item.get("id")
|
product_id = item.get("id")
|
||||||
sku_id = item.get("sku_id")
|
sku_id = item.get("sku_id")
|
||||||
tenpo_cd = item.get("tenpo_cd")
|
tenpo_cd = item.get("tenpo_cd")
|
||||||
|
branch_number = item.get("branch_number")
|
||||||
price_limit = int(item.get("price_limit") or 0)
|
price_limit = int(item.get("price_limit") or 0)
|
||||||
if not product_id:
|
if not product_id:
|
||||||
logger.warning(f"[任务 {self.task_id}] 索引 {index} 的商品未提供 id,跳过。")
|
logger.warning(f"[任务 {self.task_id}] 索引 {index} 的商品未提供 id,跳过。")
|
||||||
@@ -97,7 +99,7 @@ class SurugayaPurchaseTask(BasePurchaseTask):
|
|||||||
logger.warning(f"[任务 {self.task_id}] 索引 {index} 的商品数量为 0,跳过。")
|
logger.warning(f"[任务 {self.task_id}] 索引 {index} 的商品数量为 0,跳过。")
|
||||||
continue
|
continue
|
||||||
|
|
||||||
detail_url = product_detail_url(product_id, tenpo_cd)
|
detail_url = product_detail_url(product_id, tenpo_cd, branch_number)
|
||||||
|
|
||||||
logger.debug(
|
logger.debug(
|
||||||
f"[任务 {self.task_id}] 进入商品 {product_id} 详情页,购买数量: {number}, URL: {detail_url}")
|
f"[任务 {self.task_id}] 进入商品 {product_id} 详情页,购买数量: {number}, URL: {detail_url}")
|
||||||
@@ -314,24 +316,13 @@ class SurugayaPurchaseTask(BasePurchaseTask):
|
|||||||
|
|
||||||
async def check_login_status(self, page: Page) -> bool:
|
async def check_login_status(self, page: Page) -> bool:
|
||||||
"""
|
"""
|
||||||
通过限定页头区域来精准判断登录状态
|
通过限定页头区域来精准判断登录状态(共享实现见 surugaya_auth)。
|
||||||
"""
|
"""
|
||||||
try:
|
return await surugaya_auth.check_login_status(page, log_prefix=f"[任务 {self.task_id}] ")
|
||||||
text_login = page.locator(".top_nav .text-login").first
|
|
||||||
if await text_login.is_visible():
|
|
||||||
logger.debug(f"[任务 {self.task_id}] 检测结果:[未登录]")
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
# logger.debug(f"[任务 {self.task_id}] 检测结果:[已登录] 用户名: {await text_login.inner_text()}")
|
|
||||||
return True
|
|
||||||
|
|
||||||
except PlaywrightTimeoutError as e:
|
|
||||||
logger.warning(f"[任务 {self.task_id}] 检测失败: {e}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
async def surugaya_login(self, page: Page) -> None:
|
async def surugaya_login(self, page: Page) -> None:
|
||||||
"""
|
"""
|
||||||
骏河屋 (Surugaya) 专用安全登录方法
|
骏河屋 (Surugaya) 专用安全登录方法(共享实现见 surugaya_auth)。
|
||||||
:param page: Playwright 的 Page 对象
|
:param page: Playwright 的 Page 对象
|
||||||
"""
|
"""
|
||||||
account_config = get_account_config("surugaya", self.account_id)
|
account_config = get_account_config("surugaya", self.account_id)
|
||||||
@@ -373,21 +364,11 @@ class SurugayaPurchaseTask(BasePurchaseTask):
|
|||||||
await password_input.press_sequentially(password, delay=100)
|
await password_input.press_sequentially(password, delay=100)
|
||||||
await asyncio.sleep(0.6)
|
await asyncio.sleep(0.6)
|
||||||
|
|
||||||
# 6. 点击登录按钮("サインイン")
|
|
||||||
logger.debug(f"[任务 {self.task_id}] 正在点击登录按钮...")
|
|
||||||
await submit_button.click()
|
|
||||||
|
|
||||||
# 7. 【防风控核心】等待登录结果跳转
|
|
||||||
# 骏河屋成功登录后通常会跳转到 mypage (个人中心)
|
|
||||||
logger.info(f"[任务 {self.task_id}] 正在等待登录结果...")
|
|
||||||
try:
|
try:
|
||||||
# 等待 URL 变成包含 pcmypage 的地址(代表成功进入个人中心)
|
await surugaya_auth.surugaya_login(page, account_config, log_prefix=f"[任务 {self.task_id}] ")
|
||||||
await page.wait_for_url("**/pcmypage*", timeout=15000)
|
except Exception:
|
||||||
logger.info(f"[任务 {self.task_id}] 骏河屋登录成功。")
|
|
||||||
except Exception as e:
|
|
||||||
logger.error(f"[任务 {self.task_id}] 骏河屋登录超时或失败,可能触发了验证码或账号凭据异常。")
|
|
||||||
await self.take_screenshot(scene="surugaya_login_failed")
|
await self.take_screenshot(scene="surugaya_login_failed")
|
||||||
raise e
|
raise
|
||||||
|
|
||||||
async def get_show_payment_methods(self, page: Page) -> list:
|
async def get_show_payment_methods(self, page: Page) -> list:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1,176 @@
|
|||||||
|
"""
|
||||||
|
骏河屋登录共享逻辑:登录状态检测、拟真表单登录、启动时登录预检。
|
||||||
|
|
||||||
|
供两处复用:
|
||||||
|
- SurugayaPurchaseTask:任务执行中的登录兜底(src/tasks/surugaya.py);
|
||||||
|
- main.py:启动时登录预检(浏览器预热后、消息监听启动前),
|
||||||
|
提前确认各账号登录态并自动登录,避免任务到达时再花时间走登录流程。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
from playwright.async_api import Page
|
||||||
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||||
|
|
||||||
|
from src.browser.manager import get_browser_manager
|
||||||
|
from src.config.accounts import AccountConfig, load_accounts_config
|
||||||
|
from src.config.settings import get_settings
|
||||||
|
|
||||||
|
MYPAGE_URL = "https://www.suruga-ya.jp/pcmypage"
|
||||||
|
|
||||||
|
# 登录预检失败截图的归档 topic(可在 8080 截图查看器中按此主题过滤)
|
||||||
|
_PRECHECK_TOPIC = "login_precheck"
|
||||||
|
|
||||||
|
|
||||||
|
async def check_login_status(page: Page, *, log_prefix: str = "") -> bool:
|
||||||
|
"""
|
||||||
|
通过限定页头区域来精准判断登录状态。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
text_login = page.locator(".top_nav .text-login").first
|
||||||
|
if await text_login.is_visible():
|
||||||
|
logger.debug(f"{log_prefix}检测结果:[未登录]")
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
# logger.debug(f"{log_prefix}检测结果:[已登录] 用户名: {await text_login.inner_text()}")
|
||||||
|
return True
|
||||||
|
|
||||||
|
except PlaywrightTimeoutError as e:
|
||||||
|
logger.warning(f"{log_prefix}检测失败: {e}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
async def surugaya_login(page: Page, account: AccountConfig, *, log_prefix: str = "") -> None:
|
||||||
|
"""
|
||||||
|
骏河屋 (Surugaya) 专用安全登录方法(高拟真输入)。
|
||||||
|
失败时抛出异常,截图等善后由调用方按各自场景处置。
|
||||||
|
"""
|
||||||
|
if not account.password:
|
||||||
|
raise ValueError(f"账号 '{account.account_id}' 缺少密码配置,无法执行登录。")
|
||||||
|
|
||||||
|
email = account.username
|
||||||
|
password = account.password
|
||||||
|
|
||||||
|
logger.info(f"{log_prefix}正在初始化骏河屋登录流程...")
|
||||||
|
await page.goto(MYPAGE_URL, wait_until="domcontentloaded", timeout=30000)
|
||||||
|
|
||||||
|
# 1. 确保表单和输入框在画面上完全可见
|
||||||
|
# 骏河屋的账号框 id 是 'edit-mail'
|
||||||
|
email_input = page.locator("#edit-mail")
|
||||||
|
await email_input.wait_for(state="visible", timeout=10000)
|
||||||
|
|
||||||
|
# 2. 密码框定位(id 是 'edit-password')
|
||||||
|
password_input = page.locator("#edit-password")
|
||||||
|
await password_input.wait_for(state="visible", timeout=10000)
|
||||||
|
|
||||||
|
# 3. 登录提交按钮定位(id 是 'edit-submit',值是 'サインイン')
|
||||||
|
submit_button = page.locator("#edit-submit")
|
||||||
|
await submit_button.wait_for(state="visible", timeout=10000)
|
||||||
|
|
||||||
|
# 4. 使用异步 API 输入凭据,不阻塞其他 topic 的任务
|
||||||
|
logger.debug(f"{log_prefix}正在输入骏河屋账号...")
|
||||||
|
await email_input.focus()
|
||||||
|
await email_input.clear()
|
||||||
|
await email_input.press_sequentially(email, delay=100)
|
||||||
|
await asyncio.sleep(0.4)
|
||||||
|
|
||||||
|
# 5. 【高拟真输入】模拟真人输入密码
|
||||||
|
logger.debug(f"{log_prefix}正在输入密码...")
|
||||||
|
await password_input.focus()
|
||||||
|
await password_input.clear()
|
||||||
|
await password_input.press_sequentially(password, delay=100)
|
||||||
|
await asyncio.sleep(0.6)
|
||||||
|
|
||||||
|
# 6. 点击登录按钮("サインイン")
|
||||||
|
logger.debug(f"{log_prefix}正在点击登录按钮...")
|
||||||
|
await submit_button.click()
|
||||||
|
|
||||||
|
# 7. 【防风控核心】等待登录结果跳转
|
||||||
|
# 骏河屋成功登录后通常会跳转到 mypage (个人中心)
|
||||||
|
logger.info(f"{log_prefix}正在等待登录结果...")
|
||||||
|
try:
|
||||||
|
# 等待 URL 变成包含 pcmypage 的地址(代表成功进入个人中心)
|
||||||
|
await page.wait_for_url("**/pcmypage*", timeout=15000)
|
||||||
|
logger.info(f"{log_prefix}骏河屋登录成功。")
|
||||||
|
except Exception:
|
||||||
|
logger.error(f"{log_prefix}骏河屋登录超时或失败,可能触发了验证码或账号凭据异常。")
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
async def login_precheck() -> None:
|
||||||
|
"""
|
||||||
|
启动时登录预检:逐个打开骏河屋账号的常驻浏览器页面确认登录态,
|
||||||
|
未登录则自动登录。串行执行以降低多账号并发登录的风控风险;
|
||||||
|
单账号失败不阻断启动(任务执行时仍有登录兜底)。
|
||||||
|
"""
|
||||||
|
settings = get_settings()
|
||||||
|
if not settings.login_precheck_enabled:
|
||||||
|
logger.info("登录预检已禁用(LOGIN_PRECHECK_ENABLED=false),跳过。")
|
||||||
|
return
|
||||||
|
|
||||||
|
accounts = load_accounts_config().sites.surugaya
|
||||||
|
if not accounts:
|
||||||
|
logger.info("未配置骏河屋账号,跳过登录预检。")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.info(f"开始启动登录预检,共 {len(accounts)} 个骏河屋账号...")
|
||||||
|
for account in accounts:
|
||||||
|
log_prefix = f"[登录预检 {account.account_id}] "
|
||||||
|
try:
|
||||||
|
await _precheck_account(account, log_prefix)
|
||||||
|
except Exception as e:
|
||||||
|
logger.error(f"{log_prefix}登录预检失败(任务执行时仍会自动登录兜底): {e}")
|
||||||
|
logger.success("启动登录预检完成。")
|
||||||
|
|
||||||
|
|
||||||
|
async def _precheck_account(account: AccountConfig, log_prefix: str) -> None:
|
||||||
|
"""
|
||||||
|
单账号登录预检:复用常驻浏览器上下文,仅开关自己的标签页。
|
||||||
|
"""
|
||||||
|
context = await get_browser_manager().get_context(account)
|
||||||
|
page = await context.new_page()
|
||||||
|
try:
|
||||||
|
await page.goto(MYPAGE_URL, wait_until="domcontentloaded", timeout=30000)
|
||||||
|
|
||||||
|
if await check_login_status(page, log_prefix=log_prefix):
|
||||||
|
logger.info(f"{log_prefix}登录态有效,无需登录。")
|
||||||
|
return
|
||||||
|
|
||||||
|
logger.warning(f"{log_prefix}未登录,开始自动登录...")
|
||||||
|
try:
|
||||||
|
await surugaya_login(page, account, log_prefix=log_prefix)
|
||||||
|
except Exception:
|
||||||
|
await _save_precheck_screenshot(page, account, log_prefix)
|
||||||
|
raise
|
||||||
|
|
||||||
|
if not await check_login_status(page, log_prefix=log_prefix):
|
||||||
|
await _save_precheck_screenshot(page, account, log_prefix)
|
||||||
|
raise RuntimeError("自动登录后仍未检测到登录态,请通过 noVNC 人工登录核查。")
|
||||||
|
|
||||||
|
logger.success(f"{log_prefix}自动登录成功,登录态已就绪。")
|
||||||
|
finally:
|
||||||
|
await page.close()
|
||||||
|
|
||||||
|
|
||||||
|
async def _save_precheck_screenshot(page: Page, account: AccountConfig, log_prefix: str) -> None:
|
||||||
|
"""
|
||||||
|
保存登录预检失败现场截图到归档目录(8080 截图查看器可见),失败不影响主流程。
|
||||||
|
"""
|
||||||
|
settings = get_settings()
|
||||||
|
account_tag = "".join(c if c.isalnum() else "_" for c in account.account_id)
|
||||||
|
timestamp_ms = int(datetime.now().timestamp() * 1000)
|
||||||
|
filepath = (
|
||||||
|
Path(settings.screenshot_dir)
|
||||||
|
/ datetime.now().strftime("%Y%m%d")
|
||||||
|
/ _PRECHECK_TOPIC
|
||||||
|
/ f"{account_tag}_登录预检失败_{timestamp_ms}.png"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
filepath.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
await page.screenshot(path=str(filepath), timeout=10000)
|
||||||
|
logger.info(f"{log_prefix}失败现场截图已保存: {filepath}")
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning(f"{log_prefix}失败现场截图保存失败: {e}")
|
||||||
@@ -0,0 +1,98 @@
|
|||||||
|
import json
|
||||||
|
import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from loguru import logger
|
||||||
|
|
||||||
|
from src.tasks import surugaya_auth
|
||||||
|
from src.tasks.base import BasePurchaseTask
|
||||||
|
from src.config.accounts import get_account_config
|
||||||
|
from src.parsers.surugaya_news import collect_all_news
|
||||||
|
from src.task_queue.redis_manager import get_redis_client
|
||||||
|
|
||||||
|
# 「お知らせ一覧」页面地址。
|
||||||
|
NEWS_URL = "https://www.suruga-ya.jp/pcmypage/action_news"
|
||||||
|
|
||||||
|
|
||||||
|
class SurugayaNewsMonitorTask(BasePurchaseTask):
|
||||||
|
"""
|
||||||
|
骏河屋「お知らせ一覧」通知抓取任务。
|
||||||
|
|
||||||
|
入参约定:
|
||||||
|
- topic: surugaya_news
|
||||||
|
- purchase_account: 抓取通知所属账号
|
||||||
|
- payload: 无额外字段
|
||||||
|
|
||||||
|
行为:打开 action_news 页面(必要时自动登录),解析首页最新通知,
|
||||||
|
按 data_id 去重写入 Redis Hash(field=data_id),供中心 API 读取。
|
||||||
|
"""
|
||||||
|
|
||||||
|
async def execute(self) -> dict[str, Any]:
|
||||||
|
self.report_event = "surugaya_news"
|
||||||
|
self.is_report = False
|
||||||
|
|
||||||
|
logger.debug(f"[任务 {self.task_id}] 骏河屋通知抓取工作流已启动。")
|
||||||
|
if not self.page:
|
||||||
|
raise ValueError("页面对象未初始化,无法执行通知抓取任务。")
|
||||||
|
|
||||||
|
page = self.page
|
||||||
|
|
||||||
|
await page.goto(NEWS_URL, wait_until="domcontentloaded", timeout=30000)
|
||||||
|
|
||||||
|
# 登录兜底:长时间运行登录态可能过期,未登录则自动登录后重新进入。
|
||||||
|
if not await self._ensure_login(page):
|
||||||
|
raise RuntimeError(f"账号 '{self.account_id}' 登录失败,无法抓取通知。")
|
||||||
|
|
||||||
|
news_items = await collect_all_news(page)
|
||||||
|
logger.info(f"[任务 {self.task_id}] 账号 '{self.account_id}' 翻页解析到 {len(news_items)} 条通知。")
|
||||||
|
|
||||||
|
written = await self._save_news_to_redis(news_items)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"account_id": self.account_id,
|
||||||
|
"parsed_count": len(news_items),
|
||||||
|
"written_count": written,
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _notify_task_failed(self) -> None:
|
||||||
|
"""通知抓取是每 5 分钟运行的常规轮询,单次失败下轮即重试,不发失败告警邮件,
|
||||||
|
避免登录抖动等偶发问题造成告警刷屏(失败现场截图仍由基类保留)。"""
|
||||||
|
logger.warning(
|
||||||
|
f"[任务 {self.task_id}] 通知抓取失败(不告警,等待下一轮重试): {self.error_message}"
|
||||||
|
)
|
||||||
|
|
||||||
|
async def _ensure_login(self, page) -> bool:
|
||||||
|
log_prefix = f"[任务 {self.task_id}] "
|
||||||
|
if await surugaya_auth.check_login_status(page, log_prefix=log_prefix):
|
||||||
|
return True
|
||||||
|
|
||||||
|
logger.warning(f"{log_prefix}账号 '{self.account_id}' 未登录,执行登录流程。")
|
||||||
|
account_config = get_account_config("surugaya", self.account_id)
|
||||||
|
if not account_config:
|
||||||
|
raise ValueError(f"未找到骏河屋账号配置: {self.account_id}")
|
||||||
|
|
||||||
|
await surugaya_auth.surugaya_login(page, account_config, log_prefix=log_prefix)
|
||||||
|
await page.goto(NEWS_URL, wait_until="domcontentloaded", timeout=30000)
|
||||||
|
return await surugaya_auth.check_login_status(page, log_prefix=log_prefix)
|
||||||
|
|
||||||
|
async def _save_news_to_redis(self, news_items: list[dict[str, Any]]) -> int:
|
||||||
|
"""按 data_id 作为 field 写入 Redis Hash;返回写入条数。"""
|
||||||
|
if not news_items:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
redis_client = get_redis_client()
|
||||||
|
hash_key = self.settings.redis_news_hash_key
|
||||||
|
fetched_at = int(time.time())
|
||||||
|
|
||||||
|
mapping = {
|
||||||
|
item["data_id"]: json.dumps(
|
||||||
|
{**item, "account_id": self.account_id, "fetched_at": fetched_at},
|
||||||
|
ensure_ascii=False,
|
||||||
|
)
|
||||||
|
for item in news_items
|
||||||
|
}
|
||||||
|
await redis_client.hset(hash_key, mapping=mapping)
|
||||||
|
logger.info(
|
||||||
|
f"[任务 {self.task_id}] 已将 {len(mapping)} 条通知写入 Redis Hash '{hash_key}'。"
|
||||||
|
)
|
||||||
|
return len(mapping)
|
||||||
+19
-15
@@ -1,5 +1,7 @@
|
|||||||
import asyncio
|
import asyncio
|
||||||
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import sys
|
||||||
from datetime import datetime
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import pyautogui
|
import pyautogui
|
||||||
@@ -10,9 +12,24 @@ def _sanitize_path_component(value: str) -> str:
|
|||||||
return sanitized or "unknown"
|
return sanitized or "unknown"
|
||||||
|
|
||||||
|
|
||||||
|
def _build_capture_args(fps: int, width: int, height: int) -> list[str]:
|
||||||
|
"""
|
||||||
|
按运行平台组装 FFmpeg 屏幕采集输入参数。
|
||||||
|
Windows 使用 gdigrab 抓取整个桌面;Linux(容器内 Xvfb)使用 x11grab 抓取 DISPLAY。
|
||||||
|
"""
|
||||||
|
common_args = [
|
||||||
|
"-draw_mouse", "1",
|
||||||
|
"-framerate", str(fps),
|
||||||
|
"-video_size", f"{width}x{height}",
|
||||||
|
]
|
||||||
|
if sys.platform == "win32":
|
||||||
|
return ["-f", "gdigrab", *common_args, "-offset_x", "0", "-offset_y", "0", "-i", "desktop"]
|
||||||
|
return ["-f", "x11grab", *common_args, "-i", os.environ.get("DISPLAY", ":0")]
|
||||||
|
|
||||||
|
|
||||||
class TaskVideoRecorder:
|
class TaskVideoRecorder:
|
||||||
"""
|
"""
|
||||||
使用 FFmpeg 进行 Windows 整桌面录制的轻量封装。
|
使用 FFmpeg 进行整桌面录制的轻量封装(Windows: gdigrab / Linux: x11grab)。
|
||||||
生命周期由任务基类控制:start() 开始录制,stop() 停止并返回产物路径。
|
生命周期由任务基类控制:start() 开始录制,stop() 停止并返回产物路径。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -61,20 +78,7 @@ class TaskVideoRecorder:
|
|||||||
"-y",
|
"-y",
|
||||||
"-loglevel",
|
"-loglevel",
|
||||||
"error",
|
"error",
|
||||||
"-f",
|
*_build_capture_args(self.fps, screen_width, screen_height),
|
||||||
"gdigrab",
|
|
||||||
"-draw_mouse",
|
|
||||||
"1",
|
|
||||||
"-framerate",
|
|
||||||
str(self.fps),
|
|
||||||
"-video_size",
|
|
||||||
f"{screen_width}x{screen_height}",
|
|
||||||
"-offset_x",
|
|
||||||
"0",
|
|
||||||
"-offset_y",
|
|
||||||
"0",
|
|
||||||
"-i",
|
|
||||||
"desktop",
|
|
||||||
"-c:v",
|
"-c:v",
|
||||||
"libx264",
|
"libx264",
|
||||||
"-preset",
|
"-preset",
|
||||||
|
|||||||
Reference in New Issue
Block a user