From 104d7fef6b21a629e6d1d059275af182df03109f Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Mon, 27 Jul 2026 15:05:01 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8B=86=E5=88=86=E6=8A=93=E5=8F=96=E4=B8=8E?= =?UTF-8?q?=E4=BA=A4=E6=98=93=E6=9C=8D=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) --- .env.example | 23 +- .gitignore | 3 + Dockerfile | 16 +- README.md | 115 +++++++- app/api/dependencies.py | 42 --- app/core/container.py | 28 -- app/parsers/rakuma/__init__.py | 8 - app/{api => scraping}/__init__.py | 0 app/{api/routes => scraping/api}/__init__.py | 0 app/{core => scraping/api/routes}/__init__.py | 0 app/{ => scraping}/api/routes/health.py | 14 +- app/{ => scraping}/api/routes/rakuma.py | 47 +++- app/{ => scraping}/api/routes/scrape.py | 17 +- app/scraping/container.py | 32 +++ app/{models => scraping/core}/__init__.py | 0 app/{ => scraping}/core/rakuma_site.py | 28 +- app/{ => scraping}/core/site.py | 28 +- app/scraping/main.py | 96 +++++++ app/{parsers => scraping/models}/__init__.py | 0 app/{ => scraping}/models/scrape.py | 66 ++++- .../parsers}/__init__.py | 0 app/{ => scraping}/parsers/genre.py | 8 +- app/{ => scraping}/parsers/item.py | 6 +- app/scraping/parsers/rakuma/__init__.py | 9 + app/{ => scraping}/parsers/rakuma/base.py | 0 app/scraping/parsers/rakuma/category.py | 155 ++++++++++ app/{ => scraping}/parsers/rakuma/item.py | 10 +- app/{ => scraping}/parsers/rakuma/search.py | 10 +- app/{ => scraping}/parsers/rakuma/shop.py | 10 +- app/{ => scraping}/parsers/search.py | 10 +- app/{ => scraping}/parsers/shop.py | 8 +- app/{ => scraping}/parsers/state.py | 2 +- .../parsers/subsites/__init__.py | 12 +- app/{ => scraping}/parsers/subsites/base.py | 2 +- .../parsers/subsites/biccamera.py | 8 +- app/{ => scraping}/parsers/subsites/books.py | 6 +- .../parsers/subsites/brandavenue.py | 10 +- app/{utils => scraping/services}/__init__.py | 0 .../services/browser_fallback.py | 4 +- app/{ => scraping}/services/rakuma_client.py | 43 ++- app/{ => scraping}/services/rakuma_session.py | 6 +- app/{ => scraping}/services/rakuten_client.py | 24 +- app/{ => scraping}/services/site_session.py | 10 +- app/scraping/utils/__init__.py | 0 app/{ => scraping}/utils/coerce.py | 0 app/{ => scraping}/utils/rakuma_urls.py | 16 +- app/{ => scraping}/utils/urls.py | 6 +- app/shared/__init__.py | 0 app/{main.py => shared/api.py} | 158 +++++------ app/{core => shared}/config.py | 48 +++- app/{core => shared}/errors.py | 56 ++++ app/shared/headers.py | 45 +++ app/{core => shared}/logging_setup.py | 2 +- app/trading/__init__.py | 0 app/trading/api/__init__.py | 0 app/trading/api/routes/__init__.py | 0 app/trading/api/routes/auth.py | 89 ++++++ app/trading/api/routes/health.py | 27 ++ app/trading/container.py | 23 ++ app/trading/core/__init__.py | 0 app/trading/core/auth_site.py | 115 ++++++++ app/trading/main.py | 84 ++++++ app/trading/models.py | 68 +++++ app/trading/services/__init__.py | 0 app/trading/services/auth_session.py | 260 +++++++++++++++++ scripts/login.py | 135 +++++++++ tests/fixtures/rakuma_category.html | 8 + tests/test_api.py | 42 ++- tests/test_architecture.py | 73 +++++ tests/test_auth_session.py | 264 ++++++++++++++++++ tests/test_genre.py | 8 +- tests/test_parsers.py | 8 +- tests/test_purchase.py | 8 +- tests/test_rakuma.py | 76 ++++- tests/test_rakuma_session.py | 8 +- tests/test_search_url.py | 6 +- tests/test_shop.py | 8 +- tests/test_site_session.py | 12 +- tests/test_subsites.py | 6 +- tests/test_trading_api.py | 157 +++++++++++ 80 files changed, 2330 insertions(+), 402 deletions(-) delete mode 100644 app/api/dependencies.py delete mode 100644 app/core/container.py delete mode 100644 app/parsers/rakuma/__init__.py rename app/{api => scraping}/__init__.py (100%) rename app/{api/routes => scraping/api}/__init__.py (100%) rename app/{core => scraping/api/routes}/__init__.py (100%) rename app/{ => scraping}/api/routes/health.py (70%) rename app/{ => scraping}/api/routes/rakuma.py (70%) rename app/{ => scraping}/api/routes/scrape.py (90%) create mode 100644 app/scraping/container.py rename app/{models => scraping/core}/__init__.py (100%) rename app/{ => scraping}/core/rakuma_site.py (83%) rename app/{ => scraping}/core/site.py (76%) create mode 100644 app/scraping/main.py rename app/{parsers => scraping/models}/__init__.py (100%) rename app/{ => scraping}/models/scrape.py (92%) rename app/{services => scraping/parsers}/__init__.py (100%) rename app/{ => scraping}/parsers/genre.py (93%) rename app/{ => scraping}/parsers/item.py (98%) create mode 100644 app/scraping/parsers/rakuma/__init__.py rename app/{ => scraping}/parsers/rakuma/base.py (100%) create mode 100644 app/scraping/parsers/rakuma/category.py rename app/{ => scraping}/parsers/rakuma/item.py (96%) rename app/{ => scraping}/parsers/rakuma/search.py (94%) rename app/{ => scraping}/parsers/rakuma/shop.py (96%) rename app/{ => scraping}/parsers/search.py (95%) rename app/{ => scraping}/parsers/shop.py (92%) rename app/{ => scraping}/parsers/state.py (97%) rename app/{ => scraping}/parsers/subsites/__init__.py (82%) rename app/{ => scraping}/parsers/subsites/base.py (97%) rename app/{ => scraping}/parsers/subsites/biccamera.py (95%) rename app/{ => scraping}/parsers/subsites/books.py (97%) rename app/{ => scraping}/parsers/subsites/brandavenue.py (96%) rename app/{utils => scraping/services}/__init__.py (100%) rename app/{ => scraping}/services/browser_fallback.py (99%) rename app/{ => scraping}/services/rakuma_client.py (73%) rename app/{ => scraping}/services/rakuma_session.py (97%) rename app/{ => scraping}/services/rakuten_client.py (91%) rename app/{ => scraping}/services/site_session.py (97%) create mode 100644 app/scraping/utils/__init__.py rename app/{ => scraping}/utils/coerce.py (100%) rename app/{ => scraping}/utils/rakuma_urls.py (90%) rename app/{ => scraping}/utils/urls.py (97%) create mode 100644 app/shared/__init__.py rename app/{main.py => shared/api.py} (55%) rename app/{core => shared}/config.py (64%) rename app/{core => shared}/errors.py (63%) create mode 100644 app/shared/headers.py rename app/{core => shared}/logging_setup.py (98%) create mode 100644 app/trading/__init__.py create mode 100644 app/trading/api/__init__.py create mode 100644 app/trading/api/routes/__init__.py create mode 100644 app/trading/api/routes/auth.py create mode 100644 app/trading/api/routes/health.py create mode 100644 app/trading/container.py create mode 100644 app/trading/core/__init__.py create mode 100644 app/trading/core/auth_site.py create mode 100644 app/trading/main.py create mode 100644 app/trading/models.py create mode 100644 app/trading/services/__init__.py create mode 100644 app/trading/services/auth_session.py create mode 100644 scripts/login.py create mode 100644 tests/fixtures/rakuma_category.html create mode 100644 tests/test_architecture.py create mode 100644 tests/test_auth_session.py create mode 100644 tests/test_trading_api.py diff --git a/.env.example b/.env.example index 5aceb2c..82d40ae 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,16 @@ -# 服务监听地址，通常本地用 127.0.0.1，容器/服务器用 0.0.0.0 +# 本仓库出两个服务，共用这一份配置： +# 抓取服务 python -m app.scraping.main —— 匿名、无状态、可多开实例 +# 交易服务 python -m app.trading.main —— 带账号登录态、有状态，只能单实例 +# 各自只读自己那部分，下面按用途分组标注。 + +# ---- 抓取服务监听地址，通常本地用 127.0.0.1，容器/服务器用 0.0.0.0 ---- RAKUTEN_APP_HOST=0.0.0.0 -# 服务监听端口 RAKUTEN_APP_PORT=31107 + +# ---- 交易服务监听地址（同机部署时端口必须与上面错开）---- +RAKUTEN_TRADING_HOST=0.0.0.0 +RAKUTEN_TRADING_PORT=31108 + # 运行环境：dev / prod / test RAKUTEN_APP_ENV=dev @@ -39,8 +48,18 @@ RAKUTEN_BROWSER_HEADLESS= RAKUTEN_BROWSER_CHANNEL= # 代理地址（需要日本 IP 时配置，例如 http://127.0.0.1:7890） +# 建议两个服务各配各的：抓取被限速换 IP 即可，交易带账号，出口 IP 频繁漂移 +# 反而会触发风控。 RAKUTEN_PROXY_SERVER= # 代理用户名（如代理需要认证则填写） RAKUTEN_PROXY_USERNAME= # 代理密码（如代理需要认证则填写） RAKUTEN_PROXY_PASSWORD= + +# ---- 以下仅交易服务使用 ---- +# 人工登录后落盘的 cookie 目录（相对项目根目录）。 +# 里面是可直接冒充账号的凭据，已在 .gitignore 排除，不要提交、不要外传。 +RAKUTEN_AUTH_STATE_DIR=.auth +# 下单金额上限（日元）：实际应付超过该值直接拒绝提交，防止解析出错或页面改版 +# 导致买到远超预期的订单。设为 0 表示不设上限（不建议）。 +RAKUTEN_ORDER_MAX_TOTAL_YEN=30000 diff --git a/.gitignore b/.gitignore index 3b377f9..86e64fc 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,6 @@ logs/ .mcp.json openapi.json + +# 登录态目录：含可冒充账号的 cookie，绝不可提交 +.auth/ diff --git a/Dockerfile b/Dockerfile index d127ef6..0545109 100644 --- a/Dockerfile +++ b/Dockerfile @@ -107,11 +107,17 @@ USER rakuten # 下载 Chromium（仅二进制，不再装系统依赖） RUN /app/.venv/bin/playwright install chromium -# 健康检查走应用自带的 /health（轻量、不会触发真实抓取） +# 健康检查走应用自带的 /health（轻量、不会触发真实抓取）。 +# 端口取 RAKUTEN_HEALTH_PORT，未设时用抓取服务的端口——交易容器启动时把它 +# 设成 31108 即可，两个服务共用这一个镜像。 HEALTHCHECK --interval=30s --timeout=5s --start-period=20s --retries=3 \ - CMD curl -fsS http://127.0.0.1:${RAKUTEN_APP_PORT}/health || exit 1 + CMD curl -fsS http://127.0.0.1:${RAKUTEN_HEALTH_PORT:-${RAKUTEN_APP_PORT}}/health || exit 1 -EXPOSE 31107 +# 31107 抓取服务，31108 交易服务（同一镜像，用 command 区分跑哪个） +EXPOSE 31107 31108 -# 配置走环境变量（生产请挂 .env 或用 k8s ConfigMap/Secret 注入） -CMD ["python", "-m", "app.main"] \ No newline at end of file +# 默认起抓取服务。交易服务覆盖 command 即可： +# docker run -e RAKUTEN_HEALTH_PORT=31108 -v ./.auth:/app/.auth \ +# python -m app.trading.main +# 注意交易服务持有登录态与订单监控，**只能起一个实例**，不要挂多副本。 +CMD ["python", "-m", "app.scraping.main"] \ No newline at end of file diff --git a/README.md b/README.md index 142e92b..779866e 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Rakuten Scraper Service -面向乐天集团两个购物站点的抓取 HTTP API 服务： +面向乐天集团两个购物站点的 HTTP API 服务： | 站点 | 域名 | 形态 | | --- | --- | --- | @@ -9,6 +9,34 @@ 两站均支持**搜索**、**商品详情**、**商家信息**与**商家名下商品**。 +## 两个部署单元 + +同一个仓库出**两个服务**，分进程运行： + +| | 抓取服务 `app.scraping` | 交易服务 `app.trading` | +| --- | --- | --- | +| 启动 | `python -m app.scraping.main`（:31107） | `python -m app.trading.main`（:31108） | +| 账号 | 全程匿名 | 必须带登录态 cookie | +| 状态 | 无状态，请求-响应 | 有状态：订单、页面证据、付款进度 | +| 失败重试 | 幂等，重试无代价 | **不可逆**，重复提交即重复下单 | +| 实例数 | 想开几个开几个 | **只能一个**（或按账号分片） | +| 出口 IP | 被限速换掉即可 | 频繁漂移会触发风控 | + +拆开的决定性理由是「实例数」那一行，而不是「要不要登录」：登录态 cookie 全局唯一、 +订单监控是常驻轮询，一旦与抓取同进程，抓取横向扩容就会把登录态和轮询任务复制 N 份， +让同一个账号被多个进程并发操作。 + +依赖方向固定为 `scraping → shared`、`trading → shared`，两侧互不 import +（`tests/test_architecture.py` 会守着）。交易侧需要商品信息时，走抓取服务的 HTTP 接口—— +下单要用的 `purchase` 块本来就是抓取服务的对外契约。 + +``` +app/ + shared/ 配置、错误码、日志、响应信封与鉴权（两侧共用，不认识两侧） + scraping/ 站点常量 / 会话 / 解析器 / 抓取路由（本 README 的绝大部分） + trading/ 登录态、（在建）加购、下单、付款与订单监控 +``` + ## 抓取原理 两站的页面形态与防护完全不同，因此各走一条独立链路。 @@ -63,15 +91,18 @@ PC UA 在搜索页、详情页、店铺页上都能拿到完整模板。因此 ## 接口 +抓取服务（:31107）： + | 接口 | 站点 | 说明 | | --- | --- | --- | -| `GET /health` | — | 健康检查，含各通道状态 | +| `GET /health` | — | 健康检查，含各抓取通道状态 | | `POST /api/search` | 乐天 | 商品搜索 | | `POST /api/genres` | 乐天 | 分类树 | | `POST /api/item_detail` | 乐天 | 商品详情 | | `POST /api/shop_detail` | 乐天 | 商家详情 | | `POST /api/shop_items` | 乐天 | 商家名下商品 | | `POST /api/rakuma/search` | ラクマ | 商品搜索 | +| `POST /api/rakuma/categories` | ラクマ | 分类树 | | `POST /api/rakuma/item_detail` | ラクマ | 商品详情 | | `POST /api/rakuma/shop_detail` | ラクマ | 卖家详情 | | `POST /api/rakuma/shop_items` | ラクマ | 卖家名下商品 | @@ -80,7 +111,16 @@ PC UA 在搜索页、详情页、店铺页上都能拿到完整模板。因此 （乐天有 `genre_id` / 成色 / SuperDEAL，ラクマ 有 `category_id` / `brand_id` / 匿名配送 / 鉴定服务）， 合并会让大半字段对另一站无效。 -启动后可访问 `http://127.0.0.1:31107/docs` 查看完整 OpenAPI 文档。 +交易服务（:31108）： + +| 接口 | 说明 | +| --- | --- | +| `GET /health` | 健康检查，含两站登录态（只读缓存，不打站点） | +| `POST /api/auth/status` | 查询登录态，默认真实探测一次 | +| `POST /api/auth/reload` | 人工重新登录后免重启换上新 cookie | + +两个服务共用同一个 Bearer Token，错误码表也是同一份。 +启动后分别在 `http://127.0.0.1:31107/docs` 与 `:31108/docs` 查看 OpenAPI 文档。 ## 安装 @@ -94,14 +134,26 @@ uv sync --extra dev --extra browser ## 启动 -默认监听 `0.0.0.0:31107`。启动前建议先按 `.env.example` 配置 `.env`。 +启动前建议先按 `.env.example` 配置 `.env`，两个服务共用这一份。 ```bash -.venv/Scripts/python.exe -m app.main -# 或 -uvicorn app.main:app --host 0.0.0.0 --port 31107 +# 抓取服务，默认 0.0.0.0:31107；可多开实例 +.venv/Scripts/python.exe -m app.scraping.main + +# 交易服务，默认 0.0.0.0:31108；只能起一个实例 +.venv/Scripts/python.exe -m app.trading.main ``` +只需要抓取时不必起交易服务。交易服务启动前要先人工登录一次： + +```bash +.venv/Scripts/python.exe scripts/login.py --site all +``` + +浏览器窗口打开后手动完成登录（账号密码只在浏览器与站点之间传递，脚本不读取）， +cookie 落在 `.auth/`（已 gitignore，内含可直接冒充账号的凭据，不要提交或外传）。 +后续重新登录后调 `POST /api/auth/reload` 换上新 cookie，不必重启服务。 + ## 测试 ```bash @@ -357,6 +409,31 @@ uvicorn app.main:app --host 0.0.0.0 --port 31107 每页固定 40 条。`total_count` 取自页面埋点里的精确值——页面上可见的「約1,190,000件」 是四舍五入后的展示值，不要拿它做分页计算；翻页以 `has_more` 为准。 +### 分类（ラクマ） + +`POST /api/rakuma/categories` + +```json +{ "category_id": "10007", "include_descendants": true } +``` + +不传 `category_id` 返回 14 个顶层分类；传入后返回该分类的名称、`full_name` 路径名、 +`ancestors` 祖先链与 `children` 直接子分类。分类共三层（14 顶层 / 169 二级 / 1503 三级）， +拿到的 `category_id` 可直接用于 `/api/rakuma/search`。 + +| 参数 | 类型 | 说明 | +| --- | --- | --- | +| `category_id` | string | 目标分类；不传取顶层列表。无效 ID 返回 404 | +| `include_descendants` | bool | `children` 里带完整子树而非只有直接子级，默认 `false` | + +站点的分类一览页一次就把整棵树写进页面（与 URL 上的 `category_id` 无关）， +因此不论查哪一层、要不要子树，服务端都只打一次请求——不像乐天 `/api/genres` 需要逐层下钻。 +响应里的 `total_count` 是站点分类树的节点总数（当前 1686），可用于确认取到的是全量树。 + +> **本接口不返回商品数。** 站点的分类数据里没有这一项，只有 `/category/{id}` 列表页的埋点上有， +> 要逐个分类多打一次请求，成本与收益不匹配。需要某分类的商品数时，用该 `category_id` +> 调一次 `/api/rakuma/search` 取 `total_count`。 + ### 商品详情（ラクマ） `POST /api/rakuma/item_detail` @@ -407,7 +484,7 @@ uvicorn app.main:app --host 0.0.0.0 --port 31107 | --- | --- | --- | | 商品标识 | `shop_code` + `item_code` 两段 | `item_id` 单个 hash | | 商家标识 | `shop_code`（如 `edion`）/ `shop_id`（数值） | `shop_id`（hash） | -| 分类 | `genre_id` + `/api/genres` 分类树 | `category_id`（无分类树接口） | +| 分类 | `genre_id` + `/api/genres` 分类树（逐层下钻，带商品数） | `category_id` + `/api/rakuma/categories` 分类树（一次取全，无商品数） | | 品牌 | — | `brand_id` | | 成色 | 3 档（`new`/`used`/`rental`） | 6 档卖家申告 | | 规格 | `sku.axis` + `sku.variants` | 无（单件商品，仅一个 `size` 字段） | @@ -419,7 +496,9 @@ uvicorn app.main:app --host 0.0.0.0 --port 31107 ## 加购与下单（仅乐天市场） 详情响应里的 `purchase` 块给出构造「加入购物车」请求所需的标识。 -**本服务只提供数据，不执行加购**——加购需要已登录的乐天账号会话，由上游采购流程持有。 +**抓取服务只提供数据，不执行加购**——加购需要账号登录态，属于交易服务 +（`app/trading/`，尚在建设中）。这个 `purchase` 块正是两个服务之间的契约： +交易服务调抓取服务的 `/api/item_detail` 取它，而不是直接 import 解析器。 ```jsonc "purchase": { @@ -470,16 +549,24 @@ uvicorn app.main:app --host 0.0.0.0 --port 31107 | 4001 | 页面解析失败（ラクマ 传了站点不认的筛选取值时也归此类） | 400 | | 4002 | 商品页跳转至未登记的乐天子站 | 400 | | 4004 | 商品/店铺不存在或已下架 | 404 | +| 5001 | 账号未登录或登录态失效（交易服务） | 401 | +| 5002 | 加购失败（交易服务） | 400 | +| 5003 | 下单失败（交易服务） | 400 | +| 5004 | 下单安全闸门未通过：未显式确认或金额超上限（交易服务） | 400 | -错误码在两站间通用。ラクマ 链路不会出现 `3002`（无反爬拦截行为）与 `4002`（无子站跳转）。 +错误码在两站、两个服务之间通用。ラクマ 链路不会出现 `3002`（无反爬拦截行为） +与 `4002`（无子站跳转）；`5xxx` 只会来自交易服务——抓取服务全程匿名，不会有登录态问题。 +`5001` 与 `5004` 都标记为不可重试：前者要人工重新登录，后者要调用方改入参。 ## 常用环境变量 -完整列表见 [.env.example](.env.example)。配置项前缀统一为 `RAKUTEN_`，两站共用同一套抓取参数 -（并发数、超时、重试次数）；`SESSION_TTL` 与浏览器兜底只对乐天链路生效。 +完整列表见 [.env.example](.env.example)。配置项前缀统一为 `RAKUTEN_`，两个服务共用同一份 +配置文件、各读各的那部分；两站共用同一套抓取参数（并发数、超时、重试次数）， +`SESSION_TTL` 与浏览器兜底只对乐天链路生效。 -- 服务：`RAKUTEN_APP_HOST`、`RAKUTEN_APP_PORT`（默认 `31107`）、`RAKUTEN_APP_ENV` -- 鉴权：`RAKUTEN_BEARER_TOKEN` +- 抓取服务：`RAKUTEN_APP_HOST`、`RAKUTEN_APP_PORT`（默认 `31107`）、`RAKUTEN_APP_ENV` +- 交易服务：`RAKUTEN_TRADING_HOST`、`RAKUTEN_TRADING_PORT`（默认 `31108`）、`RAKUTEN_AUTH_STATE_DIR`（默认 `.auth`）、`RAKUTEN_ORDER_MAX_TOTAL_YEN`（默认 `30000`） +- 鉴权：`RAKUTEN_BEARER_TOKEN`（两个服务共用） - 抓取：`RAKUTEN_MAX_SITE_CONCURRENCY`（默认 `8`，两站各自独立计数）、`RAKUTEN_HTTP_MAX_ATTEMPTS`（默认 `3`）、`RAKUTEN_SESSION_TTL_SECONDS`（默认 `1800`，仅乐天） - 浏览器兜底（仅乐天）：`RAKUTEN_BROWSER_FALLBACK_ENABLED`、`RAKUTEN_BROWSER_HEADLESS`、`RAKUTEN_BROWSER_CHANNEL` - 代理（需日本 IP 时）：`RAKUTEN_PROXY_SERVER`、`RAKUTEN_PROXY_USERNAME`、`RAKUTEN_PROXY_PASSWORD` diff --git a/app/api/dependencies.py b/app/api/dependencies.py deleted file mode 100644 index 9ff91ee..0000000 --- a/app/api/dependencies.py +++ /dev/null @@ -1,42 +0,0 @@ -"""FastAPI 依赖注入：提供容器获取和鉴权校验""" -import logging -import secrets - -from fastapi import Depends, Request - -from app.core.container import ServiceContainer -from app.core.errors import AuthenticationError - -logger = logging.getLogger(__name__) - - -def get_container(request: Request) -> ServiceContainer: - """从请求中获取服务容器""" - return request.app.state.container - - -def require_bearer_token( - request: Request, - container: ServiceContainer = Depends(get_container), -) -> None: - """Bearer Token 鉴权依赖 - - 从请求头 Authorization 中提取 Bearer Token， - 与服务端配置的 token 做安全比较（使用 secrets.compare_digest 防止时序攻击）。 - """ - authorization = request.headers.get("Authorization") - if not authorization: - logger.warning("鉴权失败：缺少 Authorization 请求头") - raise AuthenticationError("Missing Authorization header") - - token = authorization.replace("Bearer ", "", 1).strip() - if token == authorization: - logger.warning("鉴权失败：Authorization scheme 非 Bearer") - raise AuthenticationError("Invalid Authorization scheme") - if not token: - logger.warning("鉴权失败：Bearer token 为空") - raise AuthenticationError("Invalid token") - - if not secrets.compare_digest(token, container.settings.bearer_token): - logger.warning("鉴权失败：token 不匹配") - raise AuthenticationError("Invalid token") diff --git a/app/core/container.py b/app/core/container.py deleted file mode 100644 index 7bc01b1..0000000 --- a/app/core/container.py +++ /dev/null @@ -1,28 +0,0 @@ -"""服务容器：集中管理所有服务实例，用于依赖注入""" -from dataclasses import dataclass - -from app.core.config import Settings -from app.services.browser_fallback import BrowserFallback -from app.services.rakuma_client import RakumaClient -from app.services.rakuma_session import RakumaSession -from app.services.rakuten_client import RakutenClient -from app.services.site_session import SiteSession - - -@dataclass(slots=True) -class ServiceContainer: - """服务容器，持有所有核心服务实例 - - 通过 FastAPI 的 app.state.container 在请求间共享， - 各路由通过依赖注入获取容器中的服务。 - - 两个站点各自持有独立的会话与客户端：乐天需要 Akamai cookie 预热与双指纹 - 通道，ラクマ 不需要，抓取前提不同不便合并。 - """ - - settings: Settings - browser_fallback: BrowserFallback - site_session: SiteSession - rakuten_client: RakutenClient - rakuma_session: RakumaSession - rakuma_client: RakumaClient diff --git a/app/parsers/rakuma/__init__.py b/app/parsers/rakuma/__init__.py deleted file mode 100644 index 43ffe32..0000000 --- a/app/parsers/rakuma/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""ラクマ（fril.jp）页面解析器 - -站点是服务端渲染的 HTML，没有内联状态 JSON，因此各模块都走 DOM 解析： -- base — 埋点属性与文本取值的公共工具 -- search — 搜索页（商品卡片解析同时被店铺页复用） -- item — 商品详情页 -- shop — 店铺页与评价页 -""" diff --git a/app/api/__init__.py b/app/scraping/__init__.py similarity index 100% rename from app/api/__init__.py rename to app/scraping/__init__.py diff --git a/app/api/routes/__init__.py b/app/scraping/api/__init__.py similarity index 100% rename from app/api/routes/__init__.py rename to app/scraping/api/__init__.py diff --git a/app/core/__init__.py b/app/scraping/api/routes/__init__.py similarity index 100% rename from app/core/__init__.py rename to app/scraping/api/routes/__init__.py diff --git a/app/api/routes/health.py b/app/scraping/api/routes/health.py similarity index 70% rename from app/api/routes/health.py rename to app/scraping/api/routes/health.py index ce924de..78db8ce 100644 --- a/app/api/routes/health.py +++ b/app/scraping/api/routes/health.py @@ -1,21 +1,23 @@ -"""健康检查路由""" +"""抓取服务健康检查路由""" from fastapi import APIRouter, Depends -from app.api.dependencies import get_container -from app.core.container import ServiceContainer -from app.models.scrape import ApiResponse, HealthData +from app.scraping.container import ScrapingContainer +from app.scraping.models.scrape import HealthData +from app.shared.api import ApiResponse, get_container router = APIRouter(tags=["health"]) @router.get("/health", response_model=ApiResponse[HealthData]) -async def health(container: ServiceContainer = Depends(get_container)) -> ApiResponse[HealthData]: - """服务健康状态 +async def health(container: ScrapingContainer = Depends(get_container)) -> ApiResponse[HealthData]: + """抓取服务健康状态 sessions 里给出乐天的 PC / 手机两条抓取通道的 cookie 预热情况，以及 ラクマ 通道的就绪状态（ラクマ 无需预热，只报是否已初始化）； browser_fallback_* 反映浏览器兜底当前是否可用（未安装 playwright 时为不可用， 属于预期降级，不影响主链路）。 + + 账号登录态不在这里——它属于交易服务，查它请打交易服务的 /health。 """ return ApiResponse[HealthData]( success=True, diff --git a/app/api/routes/rakuma.py b/app/scraping/api/routes/rakuma.py similarity index 70% rename from app/api/routes/rakuma.py rename to app/scraping/api/routes/rakuma.py index bd5ce77..c98165e 100644 --- a/app/api/routes/rakuma.py +++ b/app/scraping/api/routes/rakuma.py @@ -1,4 +1,4 @@ -"""抓取路由：ラクマ（fril.jp）的搜索、商品详情与卖家 +"""抓取路由：ラクマ（fril.jp）的搜索、分类、商品详情与卖家 单独挂在 /api/rakuma 前缀下，不与乐天市场的接口合并：两站的筛选参数体系 差异很大（乐天有 genre_id / 成色 / SuperDEAL，ラクマ 有 category_id / @@ -6,10 +6,11 @@ brand_id / 匿名配送 / 鉴定服务），合并会让大半字段对另一站 """ from fastapi import APIRouter, Depends -from app.api.dependencies import get_container, require_bearer_token -from app.core.container import ServiceContainer -from app.models.scrape import ( - ApiResponse, +from app.shared.api import ApiResponse, get_container, require_bearer_token +from app.scraping.container import ScrapingContainer +from app.scraping.models.scrape import ( + RakumaCategoryData, + RakumaCategoryRequest, RakumaItemDetailData, RakumaItemDetailRequest, RakumaSearchRequest, @@ -30,7 +31,7 @@ router = APIRouter(prefix="/api/rakuma", tags=["rakuma"]) ) async def search( payload: RakumaSearchRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[RakumaSearchResultData]: """搜索 ラクマ 商品列表 @@ -49,6 +50,34 @@ async def search( ) +@router.post( + "/categories", + response_model=ApiResponse[RakumaCategoryData], + dependencies=[Depends(require_bearer_token)], +) +async def categories( + payload: RakumaCategoryRequest, + container: ScrapingContainer = Depends(get_container), +) -> ApiResponse[RakumaCategoryData]: + """获取 ラクマ 分类树，用于取得 /api/rakuma/search 需要的 category_id + + 不传 category_id 返回 14 个顶层分类；传入后返回该分类的名称、祖先路径与 + 直接子分类。分类共三层，逐层下钻即可定位到叶子分类。 + + 站点一次请求就返回整棵树，所以把 include_descendants 置为 true 可以直接 + 拿到该分类下的完整子树，不会多花请求。 + + 站点分类数据里没有商品数（要逐个分类另抓一次页面才有），因此本接口不返回。 + """ + data = await container.rakuma_client.categories(payload) + return ApiResponse[RakumaCategoryData]( + success=True, + msg="success", + data=data, + code=0, + ) + + @router.post( "/item_detail", response_model=ApiResponse[RakumaItemDetailData], @@ -56,7 +85,7 @@ async def search( ) async def item_detail( payload: RakumaItemDetailRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[RakumaItemDetailData]: """获取 ラクマ 商品详情 @@ -82,7 +111,7 @@ async def item_detail( ) async def shop_detail( payload: RakumaShopDetailRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[RakumaShopDetailData]: """获取 ラクマ 卖家（出品者）详情 @@ -109,7 +138,7 @@ async def shop_detail( ) async def shop_items( payload: RakumaShopItemsRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[RakumaShopItemsData]: """获取 ラクマ 卖家名下的商品列表 diff --git a/app/api/routes/scrape.py b/app/scraping/api/routes/scrape.py similarity index 90% rename from app/api/routes/scrape.py rename to app/scraping/api/routes/scrape.py index 4340336..f5fb517 100644 --- a/app/api/routes/scrape.py +++ b/app/scraping/api/routes/scrape.py @@ -1,10 +1,9 @@ """抓取路由：乐天市场的搜索、分类、商品详情与商家""" from fastapi import APIRouter, Depends -from app.api.dependencies import get_container, require_bearer_token -from app.core.container import ServiceContainer -from app.models.scrape import ( - ApiResponse, +from app.shared.api import ApiResponse, get_container, require_bearer_token +from app.scraping.container import ScrapingContainer +from app.scraping.models.scrape import ( GenreData, GenreRequest, ItemDetailData, @@ -26,7 +25,7 @@ router = APIRouter(prefix="/api", tags=["scrape"]) ) async def search( payload: SearchRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[SearchResultData]: """搜索商品列表 @@ -52,7 +51,7 @@ async def search( ) async def genres( payload: GenreRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[GenreData]: """获取乐天分类（genre）树，用于取得 /api/search 需要的 genre_id @@ -78,7 +77,7 @@ async def genres( ) async def item_detail( payload: ItemDetailRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[ItemDetailData]: """获取商品详情 @@ -102,7 +101,7 @@ async def item_detail( ) async def shop_detail( payload: ShopDetailRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[ShopDetailData]: """获取乐天商家（店铺）详情 @@ -128,7 +127,7 @@ async def shop_detail( ) async def shop_items( payload: ShopItemsRequest, - container: ServiceContainer = Depends(get_container), + container: ScrapingContainer = Depends(get_container), ) -> ApiResponse[SearchResultData]: """获取乐天商家名下的商品列表 diff --git a/app/scraping/container.py b/app/scraping/container.py new file mode 100644 index 0000000..a551b00 --- /dev/null +++ b/app/scraping/container.py @@ -0,0 +1,32 @@ +"""抓取服务容器：集中管理抓取侧服务实例，用于依赖注入""" +from dataclasses import dataclass + +from app.scraping.services.browser_fallback import BrowserFallback +from app.scraping.services.rakuma_client import RakumaClient +from app.scraping.services.rakuma_session import RakumaSession +from app.scraping.services.rakuten_client import RakutenClient +from app.scraping.services.site_session import SiteSession +from app.shared.config import Settings + + +@dataclass(slots=True) +class ScrapingContainer: + """抓取服务容器，持有抓取链路的全部服务实例 + + 通过 FastAPI 的 app.state.container 在请求间共享， + 各路由通过依赖注入获取容器中的服务。 + + 两个站点各自持有独立的会话与客户端：乐天需要 Akamai cookie 预热与双指纹 + 通道，ラクマ 不需要，抓取前提不同不便合并。 + + 这里**没有登录态**：抓取全程匿名，账号相关的一切在交易服务 + （app/trading/）里。这也是抓取服务可以随意多开实例的前提——一旦把 + 登录态放回来，多实例就会出现同一账号被多个进程并发操作的问题。 + """ + + settings: Settings + browser_fallback: BrowserFallback + site_session: SiteSession + rakuten_client: RakutenClient + rakuma_session: RakumaSession + rakuma_client: RakumaClient diff --git a/app/models/__init__.py b/app/scraping/core/__init__.py similarity index 100% rename from app/models/__init__.py rename to app/scraping/core/__init__.py diff --git a/app/core/rakuma_site.py b/app/scraping/core/rakuma_site.py similarity index 83% rename from app/core/rakuma_site.py rename to app/scraping/core/rakuma_site.py index 4f12002..dd000a5 100644 --- a/app/core/rakuma_site.py +++ b/app/scraping/core/rakuma_site.py @@ -15,6 +15,8 @@ from __future__ import annotations from typing import Final +from app.shared import headers + # ---- 站点入口 ---- HOME_URL: Final = "https://fril.jp/" SEARCH_BASE_URL: Final = "https://fril.jp/s" @@ -23,6 +25,12 @@ SHOP_BASE_URL: Final = "https://fril.jp/shop/" CATEGORY_BASE_URL: Final = "https://fril.jp/category/" BRAND_BASE_URL: Final = "https://fril.jp/brand/" +# 分类一览页。它是 Next.js App Router 页面，服务端把整棵分类树写进 RSC flight +# payload（`self.__next_f.push`）。实测这份数据与 `?category_id=` 无关：传任意 +# 合法分类、或完全不传，返回的 categoryList 都是同一份全量树（1686 条）， +# 因此取分类只需一次请求，不像乐天 genre 那样必须逐层下钻。 +CATEGORY_LIST_URL: Final = "https://fril.jp/category" + SEARCH_HOST: Final = "fril.jp" ITEM_HOST: Final = "item.fril.jp" @@ -39,28 +47,12 @@ USER_AGENT: Final = ( "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" ) -ACCEPT_LANGUAGE: Final = "ja,en-US;q=0.9,en;q=0.8" +ACCEPT_LANGUAGE: Final = headers.ACCEPT_LANGUAGE def default_headers() -> dict[str, str]: """构造一套完整的浏览器导航请求头""" - return { - "User-Agent": USER_AGENT, - "Accept": ( - "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/apng,*/*;q=0.8," - "application/signed-exchange;v=b3;q=0.7" - ), - "Accept-Language": ACCEPT_LANGUAGE, - "sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24", "Google Chrome";v="131"', - "sec-ch-ua-mobile": "?0", - "sec-ch-ua-platform": '"Windows"', - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - } + return headers.navigation_headers(USER_AGENT, mobile=False) # ---- 排序（搜索页 `sort=` + `order=` 两个参数）---- diff --git a/app/core/site.py b/app/scraping/core/site.py similarity index 76% rename from app/core/site.py rename to app/scraping/core/site.py index 03b274e..82a3c85 100644 --- a/app/core/site.py +++ b/app/scraping/core/site.py @@ -10,6 +10,8 @@ from __future__ import annotations from typing import Final +from app.shared import headers + # ---- 站点入口 ---- HOME_URL: Final = "https://www.rakuten.co.jp/" SEARCH_BASE_URL: Final = "https://search.rakuten.co.jp/search/mall/" @@ -47,30 +49,14 @@ SP_USER_AGENT: Final = ( "(KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1" ) -ACCEPT_LANGUAGE: Final = "ja,en-US;q=0.9,en;q=0.8" +ACCEPT_LANGUAGE: Final = headers.ACCEPT_LANGUAGE + -# 乐天前置 Akamai Bot Manager。请求头不完整时不会直接封禁，而是把响应 -# 拖到 ~11s（实测与响应体大小无关，22 字节的响应同样耗时 11s）；补齐 -# 下列头并复用 Akamai 下发的 cookie 后，稳定在 ~0.6-0.9s。 def default_headers(*, mobile: bool) -> dict[str, str]: """构造一套完整的浏览器导航请求头。""" - return { - "User-Agent": SP_USER_AGENT if mobile else PC_USER_AGENT, - "Accept": ( - "text/html,application/xhtml+xml,application/xml;q=0.9," - "image/avif,image/webp,image/apng,*/*;q=0.8," - "application/signed-exchange;v=b3;q=0.7" - ), - "Accept-Language": ACCEPT_LANGUAGE, - "sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24", "Google Chrome";v="131"', - "sec-ch-ua-mobile": "?1" if mobile else "?0", - "sec-ch-ua-platform": '"iOS"' if mobile else '"Windows"', - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - } + return headers.navigation_headers( + SP_USER_AGENT if mobile else PC_USER_AGENT, mobile=mobile + ) # Akamai Bot Manager 下发的 cookie：判断会话是否已预热完成的依据 diff --git a/app/scraping/main.py b/app/scraping/main.py new file mode 100644 index 0000000..e03c022 --- /dev/null +++ b/app/scraping/main.py @@ -0,0 +1,96 @@ +"""抓取服务入口：FastAPI 应用创建与生命周期管理 + +职责： +- 构建抓取服务容器（依赖注入） +- 管理应用生命周期（启动/关闭抓取会话与兜底浏览器） +- 注册抓取路由和全局异常处理器 + +这个进程全程匿名、无状态，可按需要多开实例。需要账号登录态的加购、下单与 +订单监控在交易服务里（`python -m app.trading.main`），两者独立部署。 +""" +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from app.scraping.api.routes.health import router as health_router +from app.scraping.api.routes.rakuma import router as rakuma_router +from app.scraping.api.routes.scrape import router as scrape_router +from app.scraping.container import ScrapingContainer +from app.scraping.services.browser_fallback import BrowserFallback +from app.scraping.services.rakuma_client import RakumaClient +from app.scraping.services.rakuma_session import RakumaSession +from app.scraping.services.rakuten_client import RakutenClient +from app.scraping.services.site_session import SiteSession +from app.shared.api import register_exception_handlers +from app.shared.config import get_settings +from app.shared.logging_setup import configure_logging + +logger = logging.getLogger(__name__) + + +def build_container() -> ScrapingContainer: + """构建抓取服务容器，组装所有依赖""" + settings = get_settings() + browser_fallback = BrowserFallback(settings) + site_session = SiteSession(settings, browser_fallback) + rakuten_client = RakutenClient(settings, site_session) + rakuma_session = RakumaSession(settings) + rakuma_client = RakumaClient(settings, rakuma_session) + return ScrapingContainer( + settings=settings, + browser_fallback=browser_fallback, + site_session=site_session, + rakuten_client=rakuten_client, + rakuma_session=rakuma_session, + rakuma_client=rakuma_client, + ) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """应用生命周期管理：启动时初始化各服务，关闭时释放资源""" + container = build_container() + app.state.container = container + + configure_logging(container.settings) + logger.info("抓取服务启动：%s:%s", container.settings.app_host, container.settings.app_port) + logger.info("日志级别：%s", container.settings.log_level) + logger.info("当前环境：%s", container.settings.app_env) + await container.site_session.start() + await container.rakuma_session.start() + try: + yield + finally: + await container.rakuma_session.close() + await container.site_session.close() + await container.browser_fallback.close() + + +def create_app() -> FastAPI: + """创建 FastAPI 应用实例，注册路由和异常处理器""" + app = FastAPI(title="Rakuten Scraper Service", lifespan=lifespan) + app.include_router(health_router) + app.include_router(scrape_router) + app.include_router(rakuma_router) + register_exception_handlers(app) + return app + + +app = create_app() + + +if __name__ == "__main__": + import uvicorn + + settings = get_settings() + configure_logging(settings) + uvicorn.run( + "app.scraping.main:app", + host=settings.app_host, + port=settings.app_port, + log_config=None, + timeout_keep_alive=120, + ) diff --git a/app/parsers/__init__.py b/app/scraping/models/__init__.py similarity index 100% rename from app/parsers/__init__.py rename to app/scraping/models/__init__.py diff --git a/app/models/scrape.py b/app/scraping/models/scrape.py similarity index 92% rename from app/models/scrape.py rename to app/scraping/models/scrape.py index 8ae159b..16944fe 100644 --- a/app/models/scrape.py +++ b/app/scraping/models/scrape.py @@ -1,26 +1,18 @@ -"""API 数据模型：请求体和响应体定义 +"""抓取侧 API 数据模型：请求体和响应体定义 字段命名贴合乐天站点自身的语义（item_code / shop_code / genre_id / sku 等）， 不做跨站点的字段名归一，避免解析层与对外契约之间反复翻译。 + +响应信封 `ApiResponse` 在 app/shared/api.py，与交易侧共用；登录态、订单等需要 +账号的模型在 app/trading/models.py，不在这里。 """ from __future__ import annotations from enum import StrEnum -from typing import Any, Generic, TypeVar +from typing import Any from pydantic import BaseModel, Field, HttpUrl, model_validator -T = TypeVar("T") - - -class ApiResponse(BaseModel, Generic[T]): - """统一 API 响应格式""" - - success: bool - msg: str - data: T | None = None - code: int - class SortOption(StrEnum): """搜索排序方式，对应搜索页 `s=` 参数""" @@ -433,7 +425,10 @@ class ItemDetailData(BaseModel): class HealthData(BaseModel): - """健康检查响应数据""" + """抓取服务健康检查响应数据 + + 这里不含登录态——登录态属于交易服务，查它请打交易服务的 /health。 + """ status: str browser_fallback_enabled: bool @@ -570,6 +565,49 @@ class RakumaShopItemsRequest(BaseModel): return self +class RakumaCategoryRequest(BaseModel): + """ラクマ 分类查询参数 + + 不传 category_id 时返回 14 个顶层分类；传入时返回该分类的名称、祖先路径与 + 直接子分类。站点一次请求就返回整棵树，因此 include_descendants 只是换一种 + 组织方式，不会多打请求。 + """ + + category_id: str | None = None + # 返回该分类下的完整子树（不止直接子级）。分类树共三层， + # 顶层分类的子树可达上百条。 + include_descendants: bool = False + + +class RakumaCategoryNode(BaseModel): + """ラクマ 分类树上的一个节点 + + 站点只给 id / parentId / name / hasChild 四个字段，没有商品数—— + 商品数要逐个分类去抓 `/category/{id}` 页面，成本过高，本接口不提供。 + """ + + category_id: str = "" + name: str = "" + parent_id: str = "" # "0" 表示顶层分类 + is_leaf: bool = False # 叶子分类，没有下级 + url: str = "" # 分类页地址 + # include_descendants=true 时填充下级分类，否则始终为空 + children: list[RakumaCategoryNode] = Field(default_factory=list) + + +class RakumaCategoryData(BaseModel): + """ラクマ 分类查询结果""" + + category_id: str = "" # 空串表示顶层 + name: str = "" + full_name: str = "" # 从顶层拼到自身的路径名，如「エンタメ/ホビー / ゲームソフト/ゲーム機本体 / 家庭用ゲームソフト」 + is_leaf: bool = False + url: str = "" + total_count: int = 0 # 站点分类树的节点总数，用于确认取到的是全量树 + ancestors: list[RakumaCategoryNode] = Field(default_factory=list) # 从顶层到父级，不含自身 + children: list[RakumaCategoryNode] = Field(default_factory=list) # 直接子分类；include_descendants=true 时带子树 + + class RakumaSeller(BaseModel): """ラクマ 卖家（出品者）摘要""" diff --git a/app/services/__init__.py b/app/scraping/parsers/__init__.py similarity index 100% rename from app/services/__init__.py rename to app/scraping/parsers/__init__.py diff --git a/app/parsers/genre.py b/app/scraping/parsers/genre.py similarity index 93% rename from app/parsers/genre.py rename to app/scraping/parsers/genre.py index c33a33b..711b432 100644 --- a/app/parsers/genre.py +++ b/app/scraping/parsers/genre.py @@ -10,10 +10,10 @@ from __future__ import annotations from typing import Any -from app.core import site -from app.core.errors import ScrapeParseError -from app.models.scrape import GenreData, GenreNode -from app.utils.coerce import as_dict, as_int, as_list, as_str +from app.scraping.core import site +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import GenreData, GenreNode +from app.scraping.utils.coerce import as_dict, as_int, as_list, as_str # 站点用 id=0 表示分类树的虚拟根，它不是一个真实分类 ROOT_GENRE_ID = 0 diff --git a/app/parsers/item.py b/app/scraping/parsers/item.py similarity index 98% rename from app/parsers/item.py rename to app/scraping/parsers/item.py index 2a835d0..497a636 100644 --- a/app/parsers/item.py +++ b/app/scraping/parsers/item.py @@ -10,8 +10,8 @@ from __future__ import annotations from typing import Any -from app.core.errors import ScrapeParseError -from app.models.scrape import ( +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ( Breadcrumb, ItemDetailData, PurchaseInfo, @@ -26,7 +26,7 @@ from app.models.scrape import ( SkuInfo, SkuVariant, ) -from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str +from app.scraping.utils.coerce import as_dict, as_float, as_int, as_list, as_str # purchase.sellType 下表示「可正常购买」的状态值 _PURCHASABLE_CONDITION = "enabled" diff --git a/app/scraping/parsers/rakuma/__init__.py b/app/scraping/parsers/rakuma/__init__.py new file mode 100644 index 0000000..d7ea2a1 --- /dev/null +++ b/app/scraping/parsers/rakuma/__init__.py @@ -0,0 +1,9 @@ +"""ラクマ（fril.jp）页面解析器 + +站点是服务端渲染的 HTML，没有内联状态 JSON，因此各模块都走 DOM 解析： +- base — 埋点属性与文本取值的公共工具 +- search — 搜索页（商品卡片解析同时被店铺页复用） +- item — 商品详情页 +- shop — 店铺页与评价页 +- category — 分类一览页（唯一的例外：Next.js 页面，数据在 RSC flight payload 里） +""" diff --git a/app/parsers/rakuma/base.py b/app/scraping/parsers/rakuma/base.py similarity index 100% rename from app/parsers/rakuma/base.py rename to app/scraping/parsers/rakuma/base.py diff --git a/app/scraping/parsers/rakuma/category.py b/app/scraping/parsers/rakuma/category.py new file mode 100644 index 0000000..3b67ec6 --- /dev/null +++ b/app/scraping/parsers/rakuma/category.py @@ -0,0 +1,155 @@ +"""ラクマ 分类一览页 → RakumaCategoryData + +站点其余页面都是 Rails 服务端渲染的老模板，只有 `/category` 换成了 Next.js +App Router：整棵分类树写在 RSC flight payload 里，一段段挂在 +`self.__next_f.push([1, "<字符串>"])` 上。把这些字符串按顺序拼回去，就能拿到 +`"categoryList":[{"id":...,"parentId":...,"name":...,"hasChild":...}, ...]`。 + +这份 categoryList 是**全量扁平树**（实测 1686 条：14 个顶层 + 169 个二级 + +1503 个三级），且与 URL 上的 `?category_id=` 无关——传任意分类或完全不传， +内容都一样。所以一次请求即可满足任意层级的查询，不需要像乐天 genre 那样 +逐层下钻。 + +站点只给 id / parentId / name / hasChild 四个字段，没有商品数：商品数只在 +`/category/{id}` 列表页的埋点属性上，要逐个分类多打一次请求，这里不做。 +""" +from __future__ import annotations + +import json +import re +from typing import Any + +from app.shared.errors import ItemNotFoundError, ScrapeParseError +from app.scraping.models.scrape import RakumaCategoryData, RakumaCategoryNode +from app.scraping.utils.rakuma_urls import build_category_url + +# flight payload 的分片：self.__next_f.push([1,"...JSON 字符串字面量..."]) +_FLIGHT_CHUNK_RE = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)') +# 站点用 parentId=0 表示顶层分类，0 本身不是一个真实分类 +ROOT_PARENT_ID = 0 + + +def _flight_payload(html: str) -> str: + """把 RSC flight payload 的所有分片按顺序拼成一整段文本 + + 每个分片是一个 JS 字符串字面量，转义规则与 JSON 一致，因此直接用 + json.loads 解码；单个分片解不开时跳过它而不是放弃整页——分类数据可能 + 落在其他分片上。 + """ + parts: list[str] = [] + for match in _FLIGHT_CHUNK_RE.finditer(html): + try: + parts.append(json.loads(match.group(1))) + except ValueError: + continue + return "".join(parts) + + +def _raw_categories(html: str) -> list[dict[str, Any]]: + """从页面里取出扁平分类列表 + + Raises: + ScrapeParseError: 页面里没有 categoryList，或它不是非空数组 + """ + payload = _flight_payload(html) + marker = payload.find('"categoryList":') + if marker < 0: + raise ScrapeParseError( + "分类页中缺少 categoryList 数据；站点可能改版或返回了非预期页面" + ) + + start = payload.find("[", marker) + if start < 0: + raise ScrapeParseError("分类页的 categoryList 不是数组") + + try: + raw, _ = json.JSONDecoder().raw_decode(payload[start:]) + except ValueError as exc: + raise ScrapeParseError(f"分类页的 categoryList 解析失败：{exc}") from exc + + items = [item for item in raw if isinstance(item, dict) and item.get("id") is not None] + if not items: + raise ScrapeParseError("分类页的 categoryList 为空") + return items + + +def _to_node(raw: dict[str, Any]) -> RakumaCategoryNode: + category_id = str(raw.get("id")) + return RakumaCategoryNode( + category_id=category_id, + name=str(raw.get("name") or ""), + parent_id=str(raw.get("parentId") if raw.get("parentId") is not None else ROOT_PARENT_ID), + is_leaf=not bool(raw.get("hasChild")), + url=build_category_url(category_id), + ) + + +def _build_subtree( + node: RakumaCategoryNode, + children_of: dict[str, list[RakumaCategoryNode]], +) -> RakumaCategoryNode: + """递归把下级分类挂到 children 上 + + 分类树只有三层，递归深度可控；节点在 categoryList 里 id 唯一， + 不会出现自环。 + """ + return node.model_copy( + update={ + "children": [ + _build_subtree(child, children_of) + for child in children_of.get(node.category_id, []) + ] + } + ) + + +def parse_categories( + html: str, *, category_id: str | None, include_descendants: bool +) -> RakumaCategoryData: + """解析分类树 + + Args: + category_id: 目标分类；None 表示取顶层分类列表 + include_descendants: children 里带上完整子树而非只有直接子级 + + Raises: + ScrapeParseError: 页面里没有分类树 + ItemNotFoundError: 目标分类不存在于站点分类树中 + """ + raw_items = _raw_categories(html) + nodes = {str(raw["id"]): _to_node(raw) for raw in raw_items} + + children_of: dict[str, list[RakumaCategoryNode]] = {} + for node in nodes.values(): + children_of.setdefault(node.parent_id, []).append(node) + + def resolve(node: RakumaCategoryNode) -> RakumaCategoryNode: + return _build_subtree(node, children_of) if include_descendants else node + + root_children = children_of.get(str(ROOT_PARENT_ID), []) + if category_id is None: + return RakumaCategoryData( + total_count=len(nodes), + children=[resolve(node) for node in root_children], + ) + + target = nodes.get(category_id.strip()) + if target is None: + raise ItemNotFoundError(f"分类树中未找到分类 {category_id}") + + ancestors: list[RakumaCategoryNode] = [] + parent = nodes.get(target.parent_id) + while parent is not None: + ancestors.insert(0, parent) + parent = nodes.get(parent.parent_id) + + return RakumaCategoryData( + category_id=target.category_id, + name=target.name, + full_name=" / ".join([node.name for node in ancestors] + [target.name]), + is_leaf=target.is_leaf, + url=target.url, + total_count=len(nodes), + ancestors=ancestors, + children=[resolve(node) for node in children_of.get(target.category_id, [])], + ) diff --git a/app/parsers/rakuma/item.py b/app/scraping/parsers/rakuma/item.py similarity index 96% rename from app/parsers/rakuma/item.py rename to app/scraping/parsers/rakuma/item.py index 012b274..206eaed 100644 --- a/app/parsers/rakuma/item.py +++ b/app/scraping/parsers/rakuma/item.py @@ -17,10 +17,10 @@ from typing import Any from selectolax.parser import HTMLParser -from app.core import rakuma_site as site -from app.core.errors import ScrapeParseError -from app.models.scrape import Breadcrumb, RakumaItemDetailData, RakumaSeller -from app.parsers.rakuma.base import ( +from app.scraping.core import rakuma_site as site +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import Breadcrumb, RakumaItemDetailData, RakumaSeller +from app.scraping.parsers.rakuma.base import ( attr, find_item_payload, image_url, @@ -29,7 +29,7 @@ from app.parsers.rakuma.base import ( parse_int, rat_params, ) -from app.utils.rakuma_urls import split_shop_url +from app.scraping.utils.rakuma_urls import split_shop_url _LD_JSON_RE = re.compile( r']*type="application/ld\+json"[^>]*>(.*?)', re.S diff --git a/app/parsers/rakuma/search.py b/app/scraping/parsers/rakuma/search.py similarity index 94% rename from app/parsers/rakuma/search.py rename to app/scraping/parsers/rakuma/search.py index fab289b..2ce583b 100644 --- a/app/parsers/rakuma/search.py +++ b/app/scraping/parsers/rakuma/search.py @@ -13,10 +13,10 @@ import re from selectolax.parser import HTMLParser, Node -from app.core import rakuma_site as site -from app.core.errors import ScrapeParseError -from app.models.scrape import RakumaSearchItem, RakumaSearchResultData -from app.parsers.rakuma.base import ( +from app.scraping.core import rakuma_site as site +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import RakumaSearchItem, RakumaSearchResultData +from app.scraping.parsers.rakuma.base import ( attr, event_payload, image_url, @@ -24,7 +24,7 @@ from app.parsers.rakuma.base import ( parse_int, parse_total_count, ) -from app.utils.rakuma_urls import item_id_from_url +from app.scraping.utils.rakuma_urls import item_id_from_url # 埋点属性里的精确命中总数 _TOTAL_RESULTS_RE = re.compile(r'data-rat-cp-totalresults="(\d+)"') diff --git a/app/parsers/rakuma/shop.py b/app/scraping/parsers/rakuma/shop.py similarity index 96% rename from app/parsers/rakuma/shop.py rename to app/scraping/parsers/rakuma/shop.py index 1d627e4..892bac3 100644 --- a/app/parsers/rakuma/shop.py +++ b/app/scraping/parsers/rakuma/shop.py @@ -13,15 +13,15 @@ import re from selectolax.parser import HTMLParser -from app.core import rakuma_site as site -from app.core.errors import ScrapeParseError -from app.models.scrape import ( +from app.scraping.core import rakuma_site as site +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ( RakumaRatingBreakdown, RakumaReview, RakumaShopDetailData, RakumaShopItemsData, ) -from app.parsers.rakuma.base import ( +from app.scraping.parsers.rakuma.base import ( attr, event_payload, image_url, @@ -30,7 +30,7 @@ from app.parsers.rakuma.base import ( parse_int, parse_total_count, ) -from app.parsers.rakuma.search import parse_item_cards +from app.scraping.parsers.rakuma.search import parse_item_cards # 评价条目标题左侧的图标 class → 评价档位 _RATING_ICONS = { diff --git a/app/parsers/search.py b/app/scraping/parsers/search.py similarity index 95% rename from app/parsers/search.py rename to app/scraping/parsers/search.py index f0a6ab2..627a9f9 100644 --- a/app/parsers/search.py +++ b/app/scraping/parsers/search.py @@ -11,11 +11,11 @@ from __future__ import annotations from typing import Any -from app.core import site -from app.core.errors import ScrapeParseError -from app.models.scrape import ReviewSummary, SearchItem, SearchResultData, ShopSummary -from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str -from app.utils.urls import item_url_parts +from app.scraping.core import site +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ReviewSummary, SearchItem, SearchResultData, ShopSummary +from app.scraping.utils.coerce import as_dict, as_float, as_int, as_list, as_str +from app.scraping.utils.urls import item_url_parts def parse_search_item(raw: dict[str, Any]) -> SearchItem: diff --git a/app/parsers/shop.py b/app/scraping/parsers/shop.py similarity index 92% rename from app/parsers/shop.py rename to app/scraping/parsers/shop.py index 0c06c39..61079dc 100644 --- a/app/parsers/shop.py +++ b/app/scraping/parsers/shop.py @@ -12,10 +12,10 @@ import json import re from typing import Any -from app.core import site -from app.core.errors import ScrapeParseError -from app.models.scrape import ShopDetailData -from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str +from app.scraping.core import site +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ShopDetailData +from app.scraping.utils.coerce import as_dict, as_float, as_int, as_list, as_str _LD_JSON_RE = re.compile( r']*type="application/ld\+json"[^>]*>(.*?)', re.S diff --git a/app/parsers/state.py b/app/scraping/parsers/state.py similarity index 97% rename from app/parsers/state.py rename to app/scraping/parsers/state.py index f633532..c53f357 100644 --- a/app/parsers/state.py +++ b/app/scraping/parsers/state.py @@ -13,7 +13,7 @@ import re from collections.abc import Callable from typing import Any -from app.core.errors import ScrapeParseError +from app.shared.errors import ScrapeParseError # 页面必须包含的服务端渲染数据标记；缺失说明拿到的不是正常页面 STATE_MARKER = "window.__INITIAL_STATE__" diff --git a/app/parsers/subsites/__init__.py b/app/scraping/parsers/subsites/__init__.py similarity index 82% rename from app/parsers/subsites/__init__.py rename to app/scraping/parsers/subsites/__init__.py index 53db44d..062ea95 100644 --- a/app/parsers/subsites/__init__.py +++ b/app/scraping/parsers/subsites/__init__.py @@ -8,12 +8,12 @@ from __future__ import annotations from urllib.parse import urlsplit -from app.core import site -from app.core.errors import OffIchibaRedirectError, ScrapeParseError -from app.models.scrape import ItemDetailData -from app.parsers.state import PageValidator, require_state_marker -from app.parsers.subsites import biccamera, books, brandavenue -from app.parsers.subsites.base import SubsitePage, SubsiteParser +from app.scraping.core import site +from app.shared.errors import OffIchibaRedirectError, ScrapeParseError +from app.scraping.models.scrape import ItemDetailData +from app.scraping.parsers.state import PageValidator, require_state_marker +from app.scraping.parsers.subsites import biccamera, books, brandavenue +from app.scraping.parsers.subsites.base import SubsitePage, SubsiteParser SUBSITE_PARSERS: dict[str, SubsiteParser] = { module.HOST: SubsiteParser( diff --git a/app/parsers/subsites/base.py b/app/scraping/parsers/subsites/base.py similarity index 97% rename from app/parsers/subsites/base.py rename to app/scraping/parsers/subsites/base.py index b8af12c..861b07d 100644 --- a/app/parsers/subsites/base.py +++ b/app/scraping/parsers/subsites/base.py @@ -10,7 +10,7 @@ import re from collections.abc import Callable from dataclasses import dataclass -from app.models.scrape import ItemDetailData +from app.scraping.models.scrape import ItemDetailData # 售罄判定关键词：日文页面上表示不可购买的常见措辞 SOLD_OUT_MARKERS = ("在庫なし", "在庫切れ", "品切れ", "入荷未定", "販売終了", "取扱終了") diff --git a/app/parsers/subsites/biccamera.py b/app/scraping/parsers/subsites/biccamera.py similarity index 95% rename from app/parsers/subsites/biccamera.py rename to app/scraping/parsers/subsites/biccamera.py index b6aaacd..2a7a6b0 100644 --- a/app/parsers/subsites/biccamera.py +++ b/app/scraping/parsers/subsites/biccamera.py @@ -11,8 +11,8 @@ from __future__ import annotations import json import re -from app.core.errors import ScrapeParseError -from app.models.scrape import ( +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ( Breadcrumb, ItemDetailData, PurchaseInfo, @@ -20,8 +20,8 @@ from app.models.scrape import ( ShopSummary, SkuInfo, ) -from app.parsers.subsites.base import SubsitePage -from app.utils.coerce import as_dict, as_int, as_list, as_str +from app.scraping.parsers.subsites.base import SubsitePage +from app.scraping.utils.coerce import as_dict, as_int, as_list, as_str HOST = "biccamera.rakuten.co.jp" SOURCE = "biccamera" diff --git a/app/parsers/subsites/books.py b/app/scraping/parsers/subsites/books.py similarity index 97% rename from app/parsers/subsites/books.py rename to app/scraping/parsers/subsites/books.py index 4b8a932..392baa0 100644 --- a/app/parsers/subsites/books.py +++ b/app/scraping/parsers/subsites/books.py @@ -13,8 +13,8 @@ import re from selectolax.parser import HTMLParser -from app.core.errors import ScrapeParseError -from app.models.scrape import ( +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ( Breadcrumb, ItemDetailData, PurchaseInfo, @@ -24,7 +24,7 @@ from app.models.scrape import ( SkuAttribute, SkuInfo, ) -from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price +from app.scraping.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price HOST = "books.rakuten.co.jp" SOURCE = "books" diff --git a/app/parsers/subsites/brandavenue.py b/app/scraping/parsers/subsites/brandavenue.py similarity index 96% rename from app/parsers/subsites/brandavenue.py rename to app/scraping/parsers/subsites/brandavenue.py index 35faf7e..7aecc30 100644 --- a/app/parsers/subsites/brandavenue.py +++ b/app/scraping/parsers/subsites/brandavenue.py @@ -14,8 +14,8 @@ from __future__ import annotations import re -from app.core.errors import ScrapeParseError -from app.models.scrape import ( +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ( Breadcrumb, ItemDetailData, PurchaseInfo, @@ -26,9 +26,9 @@ from app.models.scrape import ( SkuInfo, SkuVariant, ) -from app.parsers.state import extract_initial_state -from app.parsers.subsites.base import SubsitePage, parse_price -from app.utils.coerce import as_dict, as_int, as_list, as_str +from app.scraping.parsers.state import extract_initial_state +from app.scraping.parsers.subsites.base import SubsitePage, parse_price +from app.scraping.utils.coerce import as_dict, as_int, as_list, as_str HOST = "brandavenue.rakuten.co.jp" SOURCE = "brandavenue" diff --git a/app/utils/__init__.py b/app/scraping/services/__init__.py similarity index 100% rename from app/utils/__init__.py rename to app/scraping/services/__init__.py diff --git a/app/services/browser_fallback.py b/app/scraping/services/browser_fallback.py similarity index 99% rename from app/services/browser_fallback.py rename to app/scraping/services/browser_fallback.py index b93a46d..90c52a7 100644 --- a/app/services/browser_fallback.py +++ b/app/scraping/services/browser_fallback.py @@ -15,8 +15,8 @@ import logging from dataclasses import dataclass, field from typing import Any -from app.core.config import Settings -from app.core import site +from app.shared.config import Settings +from app.scraping.core import site logger = logging.getLogger(__name__) diff --git a/app/services/rakuma_client.py b/app/scraping/services/rakuma_client.py similarity index 73% rename from app/services/rakuma_client.py rename to app/scraping/services/rakuma_client.py index 8c02193..9d0bae0 100644 --- a/app/services/rakuma_client.py +++ b/app/scraping/services/rakuma_client.py @@ -1,15 +1,18 @@ """ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据 -四个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）： -搜索、商品详情、卖家详情、卖家商品列表。 +五个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）： +搜索、分类、商品详情、卖家详情、卖家商品列表。 """ from __future__ import annotations import asyncio import logging -from app.core.config import Settings -from app.models.scrape import ( +from app.scraping.core import rakuma_site as site +from app.shared.config import Settings +from app.scraping.models.scrape import ( + RakumaCategoryData, + RakumaCategoryRequest, RakumaItemDetailData, RakumaItemDetailRequest, RakumaSearchRequest, @@ -19,11 +22,12 @@ from app.models.scrape import ( RakumaShopItemsData, RakumaShopItemsRequest, ) -from app.parsers.rakuma.item import parse_item_detail -from app.parsers.rakuma.search import parse_search -from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items -from app.services.rakuma_session import RakumaSession -from app.utils.rakuma_urls import ( +from app.scraping.parsers.rakuma.category import parse_categories +from app.scraping.parsers.rakuma.item import parse_item_detail +from app.scraping.parsers.rakuma.search import parse_search +from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items +from app.scraping.services.rakuma_session import RakumaSession +from app.scraping.utils.rakuma_urls import ( build_item_url, build_search_url, build_shop_url, @@ -63,6 +67,27 @@ class RakumaClient: ) return result + async def categories(self, payload: RakumaCategoryRequest) -> RakumaCategoryData: + """抓取分类树 + + 分类一览页一次就返回整棵树（与 URL 上的 category_id 无关），因此不论 + 查哪一层都只打一次请求，页面地址也固定不带参数。 + """ + url = site.CATEGORY_LIST_URL + + logger.info("抓取 ラクマ 分类页：category_id=%s", payload.category_id) + html = await self._session.fetch_html(url) + data = parse_categories( + html, + category_id=payload.category_id, + include_descendants=payload.include_descendants, + ) + logger.info( + "ラクマ 分类完成：category_id=%s name=%s children=%s tree=%s", + data.category_id, data.name, len(data.children), data.total_count, + ) + return data + async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData: """抓取商品详情""" if payload.item_url is not None: diff --git a/app/services/rakuma_session.py b/app/scraping/services/rakuma_session.py similarity index 97% rename from app/services/rakuma_session.py rename to app/scraping/services/rakuma_session.py index 6dbc40d..3cf7dee 100644 --- a/app/services/rakuma_session.py +++ b/app/scraping/services/rakuma_session.py @@ -19,9 +19,9 @@ from typing import Any import httpx -from app.core import rakuma_site as site -from app.core.config import Settings -from app.core.errors import ( +from app.scraping.core import rakuma_site as site +from app.shared.config import Settings +from app.shared.errors import ( ItemNotFoundError, ResourceBusyError, UpstreamBlockedError, diff --git a/app/services/rakuten_client.py b/app/scraping/services/rakuten_client.py similarity index 91% rename from app/services/rakuten_client.py rename to app/scraping/services/rakuten_client.py index 8d5a5a2..9f2be17 100644 --- a/app/services/rakuten_client.py +++ b/app/scraping/services/rakuten_client.py @@ -8,10 +8,10 @@ from __future__ import annotations import logging -from app.core import site -from app.core.config import Settings -from app.core.errors import ScrapeParseError -from app.models.scrape import ( +from app.scraping.core import site +from app.shared.config import Settings +from app.shared.errors import ScrapeParseError +from app.scraping.models.scrape import ( GenreData, GenreRequest, ItemDetailData, @@ -22,19 +22,19 @@ from app.models.scrape import ( ShopDetailRequest, ShopItemsRequest, ) -from app.parsers.genre import parse_genres -from app.parsers.item import parse_item_detail -from app.parsers.search import parse_search -from app.parsers.shop import parse_shop_detail -from app.parsers.state import extract_initial_state -from app.parsers.subsites import ( +from app.scraping.parsers.genre import parse_genres +from app.scraping.parsers.item import parse_item_detail +from app.scraping.parsers.search import parse_search +from app.scraping.parsers.shop import parse_shop_detail +from app.scraping.parsers.state import extract_initial_state +from app.scraping.parsers.subsites import ( SubsitePage, build_item_page_validator, host_of, parse_subsite_item, ) -from app.services.site_session import SiteSession -from app.utils.urls import ( +from app.scraping.services.site_session import SiteSession +from app.scraping.utils.urls import ( build_genre_url, build_item_url, build_search_url, diff --git a/app/services/site_session.py b/app/scraping/services/site_session.py similarity index 97% rename from app/services/site_session.py rename to app/scraping/services/site_session.py index c98417a..7a069ea 100644 --- a/app/services/site_session.py +++ b/app/scraping/services/site_session.py @@ -20,16 +20,16 @@ from typing import Any import httpx -from app.core import site -from app.core.config import Settings -from app.core.errors import ( +from app.scraping.core import site +from app.shared.config import Settings +from app.shared.errors import ( ItemNotFoundError, ResourceBusyError, UpstreamBlockedError, UpstreamRequestError, ) -from app.parsers.state import PageValidator, require_state_marker -from app.services.browser_fallback import BrowserFallback +from app.scraping.parsers.state import PageValidator, require_state_marker +from app.scraping.services.browser_fallback import BrowserFallback logger = logging.getLogger(__name__) diff --git a/app/scraping/utils/__init__.py b/app/scraping/utils/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/utils/coerce.py b/app/scraping/utils/coerce.py similarity index 100% rename from app/utils/coerce.py rename to app/scraping/utils/coerce.py diff --git a/app/utils/rakuma_urls.py b/app/scraping/utils/rakuma_urls.py similarity index 90% rename from app/utils/rakuma_urls.py rename to app/scraping/utils/rakuma_urls.py index 383e743..95f6988 100644 --- a/app/utils/rakuma_urls.py +++ b/app/scraping/utils/rakuma_urls.py @@ -14,9 +14,9 @@ from __future__ import annotations from urllib.parse import quote, urlencode, urlsplit, urlunsplit, parse_qsl -from app.core import rakuma_site as site -from app.core.errors import InvalidRequestError -from app.models.scrape import RakumaSearchRequest +from app.scraping.core import rakuma_site as site +from app.shared.errors import InvalidRequestError +from app.scraping.models.scrape import RakumaSearchRequest def build_search_url(payload: RakumaSearchRequest) -> str: @@ -85,6 +85,16 @@ def normalize_search_url(raw_url: str, page: int) -> str: return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query), parsed.fragment)) +def build_category_url(category_id: str) -> str: + """由分类 ID 拼出分类商品列表页 URL（`/category/{id}`） + + 这是对外返回的分类页地址；抓取分类树用的是 `/category` 一览页 + （见 rakuma_site.CATEGORY_LIST_URL），两者不是同一个页面。 + """ + category = quote(str(category_id).strip().strip("/"), safe="") + return f"{site.CATEGORY_BASE_URL}{category}" if category else "" + + def build_item_url(item_id: str) -> str: """由商品 hash 拼出商品详情页 URL""" item = quote(item_id.strip().strip("/"), safe="") diff --git a/app/utils/urls.py b/app/scraping/utils/urls.py similarity index 97% rename from app/utils/urls.py rename to app/scraping/utils/urls.py index 30cb250..afd7597 100644 --- a/app/utils/urls.py +++ b/app/scraping/utils/urls.py @@ -11,9 +11,9 @@ from __future__ import annotations from urllib.parse import quote, urlencode, urlparse, urlsplit, urlunsplit, parse_qsl -from app.core import site -from app.core.errors import InvalidRequestError -from app.models.scrape import SearchRequest +from app.scraping.core import site +from app.shared.errors import InvalidRequestError +from app.scraping.models.scrape import SearchRequest # 只按分类检索（无关键词）时，关键词段的占位符 _KEYWORD_PLACEHOLDER = "-" diff --git a/app/shared/__init__.py b/app/shared/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main.py b/app/shared/api.py similarity index 55% rename from app/main.py rename to app/shared/api.py index f7d8d39..0f27d62 100644 --- a/app/main.py +++ b/app/shared/api.py @@ -1,37 +1,80 @@ -"""应用入口：FastAPI 应用创建与生命周期管理 +"""两个入口共用的 HTTP 层：响应信封、鉴权依赖、异常处理器 -职责： -- 构建服务容器（依赖注入） -- 管理应用生命周期（启动/关闭抓取会话与兜底浏览器） -- 注册路由和全局异常处理器 +抓取服务与交易服务是两个独立进程（见 README「两个部署单元」），但对外契约必须 +一致：同一套 `ApiResponse` 信封、同一份错误码表、同一个 Bearer token。共用的部分 +集中在这里，两侧的差异只体现在各自注册的路由与容器。 + +这里刻意不放任何站点或业务知识——`get_container` 不标注具体容器类型，shared 因此 +不需要认识 `ScrapingContainer` / `TradingContainer`，避免共用层反向依赖两侧。 """ from __future__ import annotations import logging -from contextlib import asynccontextmanager +import secrets +from typing import Any, Generic, TypeVar -from fastapi import FastAPI, Request +from fastapi import Depends, FastAPI, Request from fastapi.exceptions import RequestValidationError from fastapi.responses import JSONResponse -from pydantic import ValidationError +from pydantic import BaseModel, ValidationError from starlette.exceptions import HTTPException as StarletteHTTPException -from app.api.routes.health import router as health_router -from app.api.routes.rakuma import router as rakuma_router -from app.api.routes.scrape import router as scrape_router -from app.core.config import get_settings -from app.core.container import ServiceContainer -from app.core.errors import AppError -from app.core.logging_setup import configure_logging -from app.models.scrape import ApiResponse -from app.services.browser_fallback import BrowserFallback -from app.services.rakuma_client import RakumaClient -from app.services.rakuma_session import RakumaSession -from app.services.rakuten_client import RakutenClient -from app.services.site_session import SiteSession +from app.shared.errors import AppError, AuthenticationError logger = logging.getLogger(__name__) +T = TypeVar("T") + + +class ApiResponse(BaseModel, Generic[T]): + """统一 API 响应格式""" + + success: bool + msg: str + data: T | None = None + code: int + + +# ---- 依赖注入 ---- + + +def get_container(request: Request) -> Any: + """从请求中获取服务容器 + + 返回类型故意留成 Any：抓取侧与交易侧的容器结构不同，由各自路由标注具体类型。 + """ + return request.app.state.container + + +def require_bearer_token( + request: Request, + container: Any = Depends(get_container), +) -> None: + """Bearer Token 鉴权依赖 + + 从请求头 Authorization 中提取 Bearer Token， + 与服务端配置的 token 做安全比较（使用 secrets.compare_digest 防止时序攻击）。 + """ + authorization = request.headers.get("Authorization") + if not authorization: + logger.warning("鉴权失败：缺少 Authorization 请求头") + raise AuthenticationError("Missing Authorization header") + + token = authorization.replace("Bearer ", "", 1).strip() + if token == authorization: + logger.warning("鉴权失败：Authorization scheme 非 Bearer") + raise AuthenticationError("Invalid Authorization scheme") + if not token: + logger.warning("鉴权失败：Bearer token 为空") + raise AuthenticationError("Invalid token") + + if not secrets.compare_digest(token, container.settings.bearer_token): + logger.warning("鉴权失败：token 不匹配") + raise AuthenticationError("Invalid token") + + +# ---- 异常处理 ---- + def _format_validation_msg(errors: list[dict]) -> str: """将校验错误整理为便于前端展示的消息。""" @@ -47,50 +90,17 @@ def _format_validation_msg(errors: list[dict]) -> str: return "; ".join(messages) -def build_container() -> ServiceContainer: - """构建服务容器，组装所有依赖""" - settings = get_settings() - browser_fallback = BrowserFallback(settings) - site_session = SiteSession(settings, browser_fallback) - rakuten_client = RakutenClient(settings, site_session) - rakuma_session = RakumaSession(settings) - rakuma_client = RakumaClient(settings, rakuma_session) - return ServiceContainer( - settings=settings, - browser_fallback=browser_fallback, - site_session=site_session, - rakuten_client=rakuten_client, - rakuma_session=rakuma_session, - rakuma_client=rakuma_client, - ) +def jsonable_errors(errors: list[dict]) -> list[dict]: + """剔除校验错误里不可 JSON 序列化的 ctx（如原始异常对象）""" + return [{key: value for key, value in error.items() if key != "ctx"} for error in errors] -@asynccontextmanager -async def lifespan(app: FastAPI): - """应用生命周期管理：启动时初始化各服务，关闭时释放资源""" - container = build_container() - app.state.container = container +def register_exception_handlers(app: FastAPI) -> None: + """给应用挂上全套异常处理器 - configure_logging(container.settings) - logger.info("应用启动：%s:%s", container.settings.app_host, container.settings.app_port) - logger.info("日志级别：%s", container.settings.log_level) - logger.info("当前环境：%s", container.settings.app_env) - await container.site_session.start() - await container.rakuma_session.start() - try: - yield - finally: - await container.rakuma_session.close() - await container.site_session.close() - await container.browser_fallback.close() - - -def create_app() -> FastAPI: - """创建 FastAPI 应用实例，注册路由和异常处理器""" - app = FastAPI(title="Rakuten Scraper Service", lifespan=lifespan) - app.include_router(health_router) - app.include_router(scrape_router) - app.include_router(rakuma_router) + 两个入口都调用它，保证抓取失败与下单失败返回的错误结构完全一致， + 上游只需要按 code 分支，不必区分是哪个服务回的。 + """ @app.exception_handler(AppError) async def app_error_handler(_: Request, exc: AppError) -> JSONResponse: @@ -163,27 +173,3 @@ def create_app() -> FastAPI: code=1500, ).model_dump(), ) - - return app - - -def jsonable_errors(errors: list[dict]) -> list[dict]: - """剔除校验错误里不可 JSON 序列化的 ctx（如原始异常对象）""" - return [{key: value for key, value in error.items() if key != "ctx"} for error in errors] - - -app = create_app() - - -if __name__ == "__main__": - import uvicorn - - settings = get_settings() - configure_logging(settings) - uvicorn.run( - "app.main:app", - host=settings.app_host, - port=settings.app_port, - log_config=None, - timeout_keep_alive=120, - ) diff --git a/app/core/config.py b/app/shared/config.py similarity index 64% rename from app/core/config.py rename to app/shared/config.py index a79d4c8..917c9f4 100644 --- a/app/core/config.py +++ b/app/shared/config.py @@ -2,6 +2,10 @@ 配置项统一使用 RAKUTEN_ 前缀，例如 RAKUTEN_APP_PORT=31107。 支持 .env 文件自动加载。 + +抓取服务与交易服务是两个进程，但共用这一个 Settings 类：两边都要日志、代理、 +超时与同一个 Bearer token，拆成两份配置只会让部署时多维护一套。下面按 +「通用 / 仅抓取 / 仅交易」分区标注，各进程只读自己那部分。 """ from functools import lru_cache from pathlib import Path @@ -9,10 +13,12 @@ from typing import Literal from pydantic_settings import BaseSettings, SettingsConfigDict -from app.core import site - BASE_DIR = Path(__file__).resolve().parent.parent.parent +# 乐天市场首页。这里不 import app.scraping —— shared 不能反向依赖两侧任何一方， +# 否则交易服务也会被迫加载整套抓取模块。 +DEFAULT_HOME_URL = "https://www.rakuten.co.jp/" + class Settings(BaseSettings): """应用全局配置 @@ -28,12 +34,19 @@ class Settings(BaseSettings): extra="ignore", ) - # ---- 服务基本配置 ---- + # ---- 服务基本配置（通用）---- app_name: str = "Rakuten Scraper Service" app_env: Literal["dev", "prod", "test"] = "dev" + + # 抓取服务监听地址；交易服务用下面的 trading_host / trading_port app_host: str = "0.0.0.0" app_port: int = 31107 + # 交易服务监听地址。两个服务同机部署时端口必须错开；交易服务只能单实例， + # 不要在它前面挂多副本负载均衡。 + trading_host: str = "0.0.0.0" + trading_port: int = 31108 + # ---- 日志配置 ---- log_level: str = "INFO" log_to_file: bool | None = None # None 表示根据环境自动决定 @@ -44,16 +57,16 @@ class Settings(BaseSettings): log_format: str = "{time:YYYY-MM-DD HH:mm:ss} {level} {message}" log_enqueue: bool = True - # ---- 鉴权配置 ---- + # ---- 鉴权配置（通用：两个服务共用同一个对外 token）---- bearer_token: str = "REPLACE_WITH_TOKEN_32CHARS" - # ---- HTTP 抓取配置 ---- + # ---- HTTP 抓取配置（仅抓取服务）---- request_timeout_seconds: float = 30.0 max_site_concurrency: int = 8 # 对站点的最大并发请求数 http_max_attempts: int = 3 # 单次抓取的最大尝试次数（含首次） session_ttl_seconds: float = 1800.0 # Akamai cookie 会话最长复用时长，超时后重新预热 - # ---- 浏览器兜底配置 ---- + # ---- 浏览器兜底配置（仅抓取服务）---- # 纯 HTTP 被 Akamai 拦截时，用 Playwright 打开页面取回 cookie 再回灌给 # HTTP 客户端重试。日常流量不会触发；未安装 playwright 时自动降级为不兜底。 browser_fallback_enabled: bool = True @@ -62,13 +75,23 @@ class Settings(BaseSettings): browser_launch_timeout_seconds: float = 60.0 browser_nav_timeout_seconds: float = 60.0 - # ---- 代理配置（可选，用于日本 IP）---- + # ---- 代理配置（通用，可选，用于日本 IP）---- + # 两个服务同机部署时通常各配各的：抓取高频匿名，出口 IP 被限速换掉即可； + # 交易带账号，出口 IP 频繁漂移反而会触发风控。 proxy_server: str | None = None proxy_username: str | None = None proxy_password: str | None = None + # ---- 登录态与下单配置（仅交易服务）---- + # 人工登录一次后落盘的 Playwright storage_state 目录（相对项目根目录）。 + # 目录里是可直接冒充账号的 cookie，务必不要提交到版本库。 + auth_state_dir: str = ".auth" + # 下单金额上限（日元）。实际应付金额超过该值时拒绝提交，防止解析出错或 + # 页面改版导致买到远超预期的订单。设为 0 表示不设上限（不建议）。 + order_max_total_yen: int = 30000 + # ---- 目标站点 ---- - home_url: str = site.HOME_URL + home_url: str = DEFAULT_HOME_URL @property def browser_headless_effective(self) -> bool: @@ -111,6 +134,15 @@ class Settings(BaseSettings): credentials = f"{self.proxy_username}:{self.proxy_password or ''}" return f"{scheme}://{credentials}@{rest}" + @property + def auth_state_path(self) -> Path: + """登录态目录的绝对路径，不存在时创建""" + path = Path(self.auth_state_dir) + if not path.is_absolute(): + path = BASE_DIR / path + path.mkdir(parents=True, exist_ok=True) + return path + @lru_cache(maxsize=1) def get_settings() -> Settings: diff --git a/app/core/errors.py b/app/shared/errors.py similarity index 63% rename from app/core/errors.py rename to app/shared/errors.py index f3fff7d..a46cfbe 100644 --- a/app/core/errors.py +++ b/app/shared/errors.py @@ -5,6 +5,7 @@ - 2xxx: 抓取资源错误 - 3xxx: 反爬/上游阻断相关错误 - 4xxx: 页面解析错误 +- 5xxx: 加购/下单错误（需要账号登录态的写操作） """ @@ -113,3 +114,58 @@ class OffIchibaRedirectError(AppError): ) self.requested_url = requested_url self.final_url = final_url + + +class NotLoggedInError(AppError): + """账号登录态缺失或已失效 + + 加购与下单必须带已登录的账号会话。两站登录都要过 reCAPTCHA / 设备验证， + 无法自动恢复，因此这里明确标记 retryable=False，让上游停下来走一次 + `scripts/login.py` 重新人工登录，而不是原地重试。 + """ + + def __init__(self, site: str, detail: str = ""): + suffix = f"（{detail}）" if detail else "" + super().__init__( + message=( + f"{site} 账号未登录或登录态已失效{suffix}，" + f"请运行 scripts/login.py --site {site} 重新登录" + ), + code="NOT_LOGGED_IN", + err_code=5001, + retryable=False, + status_code=401, + ) + self.site = site + self.detail = detail + + +class CartOperationError(AppError): + """加购失败 + + 站点对加购请求几乎不返回结构化错误：缺必填选项、SKU 已售罄、商品下架 + 都可能返回 200 并把用户导回商品页。因此判定依据是「加购后购物车里有没有 + 这件商品」，而不是 HTTP 状态码。 + """ + + def __init__(self, message: str = "加入购物车失败"): + super().__init__(message=message, code="CART_FAILED", err_code=5002, retryable=False) + + +class OrderOperationError(AppError): + """下单流程失败（确认页解析不出、金额校验不通过、提交被拒等）""" + + def __init__(self, message: str = "下单失败"): + super().__init__(message=message, code="ORDER_FAILED", err_code=5003, retryable=False) + + +class OrderGuardError(AppError): + """下单安全闸门未通过 + + 真实付款不可逆，因此把「调用方没有显式确认」「实际金额超出上限」这类拦截 + 单独成一类错误，与站点侧失败区分开——前者是本服务主动拒绝，重试无意义， + 需要调用方修改入参后再来。 + """ + + def __init__(self, message: str): + super().__init__(message=message, code="ORDER_GUARD", err_code=5004, retryable=False) diff --git a/app/shared/headers.py b/app/shared/headers.py new file mode 100644 index 0000000..e3ec12c --- /dev/null +++ b/app/shared/headers.py @@ -0,0 +1,45 @@ +"""浏览器导航请求头构造 + +抓取链路与登录态链路都要把 httpx 请求伪装成一次正常的浏览器页面导航，头部结构 +完全一致，只有 User-Agent 以及随之联动的 `sec-ch-ua-mobile` / `sec-ch-ua-platform` +不同。结构留在这里共用，UA 常量则各自持有： + +- 抓取侧的 UA（`scraping/core/site.py`、`scraping/core/rakuma_site.py`）为反爬表现服务， + 换了只影响抓取成功率。 +- 登录态侧的 UA（`trading/core/auth_site.py`）必须与人工登录时浏览器用的那一个一致， + 换了可能触发站点的设备校验，使已落盘的 cookie 直接失效。 + +值今天相同，变更理由不同，因此不合并成一份常量。 +""" +from __future__ import annotations + +from typing import Final + +ACCEPT_LANGUAGE: Final = "ja,en-US;q=0.9,en;q=0.8" + + +def navigation_headers(user_agent: str, *, mobile: bool) -> dict[str, str]: + """构造一套完整的浏览器导航请求头 + + 乐天前置 Akamai Bot Manager，请求头不完整时不会直接封禁，而是把响应拖到 + ~11s（实测与响应体大小无关，22 字节的响应同样耗时 11s）；补齐下列头并复用 + Akamai 下发的 cookie 后，稳定在 ~0.6-0.9s。ラクマ 无此限速，但沿用同一套头 + 没有代价，两侧保持一致更省心。 + """ + return { + "User-Agent": user_agent, + "Accept": ( + "text/html,application/xhtml+xml,application/xml;q=0.9," + "image/avif,image/webp,image/apng,*/*;q=0.8," + "application/signed-exchange;v=b3;q=0.7" + ), + "Accept-Language": ACCEPT_LANGUAGE, + "sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24", "Google Chrome";v="131"', + "sec-ch-ua-mobile": "?1" if mobile else "?0", + "sec-ch-ua-platform": '"iOS"' if mobile else '"Windows"', + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", + } diff --git a/app/core/logging_setup.py b/app/shared/logging_setup.py similarity index 98% rename from app/core/logging_setup.py rename to app/shared/logging_setup.py index bad8971..008d36d 100644 --- a/app/core/logging_setup.py +++ b/app/shared/logging_setup.py @@ -13,7 +13,7 @@ from types import FrameType from loguru import logger as loguru_logger -from app.core.config import Settings +from app.shared.config import Settings class InterceptHandler(logging.Handler): diff --git a/app/trading/__init__.py b/app/trading/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/trading/api/__init__.py b/app/trading/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/trading/api/routes/__init__.py b/app/trading/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/trading/api/routes/auth.py b/app/trading/api/routes/auth.py new file mode 100644 index 0000000..8c06a2c --- /dev/null +++ b/app/trading/api/routes/auth.py @@ -0,0 +1,89 @@ +"""登录态路由：查询与重新加载账号登录态 + +抓取接口全部匿名，只有加购与下单需要账号。登录本身不在这里做——两站登录都要过 +reCAPTCHA 与设备验证，由 `scripts/login.py` 起有头浏览器人工完成一次， +本服务只读取落盘的 cookie。这里提供的是运维视角的两个动作： + +- `/api/auth/status`：现在还登录着吗（默认真实打一次请求探测，不看缓存） +- `/api/auth/reload`：人工重新登录后，免重启服务重新读取登录态 +""" +from fastapi import APIRouter, Depends + +from app.shared.api import ApiResponse, get_container, require_bearer_token +from app.trading.container import TradingContainer +from app.trading.models import ( + AuthReloadData, + AuthReloadRequest, + AuthSiteStatus, + AuthStatusData, + AuthStatusRequest, +) +from app.trading.services.auth_session import AuthStatus + +router = APIRouter(prefix="/api/auth", tags=["auth"]) + + +def _to_model(status: AuthStatus) -> AuthSiteStatus: + return AuthSiteStatus(**status.to_dict()) + + +@router.post( + "/status", + response_model=ApiResponse[AuthStatusData], + dependencies=[Depends(require_bearer_token)], +) +async def auth_status( + payload: AuthStatusRequest, + container: TradingContainer = Depends(get_container), +) -> ApiResponse[AuthStatusData]: + """查询账号登录态 + + `refresh=true`（默认）时会真实访问站点探测——登录态过期没有可靠的本地判据， + cookie 上的 expires 与服务端会话不是一回事，只能问站点。 + 不需要这次网络往返时传 `refresh=false`，此时返回上一次探测的缓存结果 + （`logged_in` 为 null 表示从未探测过）。 + + `logged_in=false` 时，加购与下单接口会直接返回 5001，需要重新跑 + `scripts/login.py --site ` 后调用 `/api/auth/reload`。 + """ + sites = [payload.site.value] if payload.site else list(container.auth_session.sites) + statuses = [] + for site in sites: + status = ( + await container.auth_session.check(site) + if payload.refresh + else container.auth_session.status(site) + ) + statuses.append(_to_model(status)) + + return ApiResponse[AuthStatusData]( + success=True, + msg="success", + data=AuthStatusData(sites=statuses), + code=0, + ) + + +@router.post( + "/reload", + response_model=ApiResponse[AuthReloadData], + dependencies=[Depends(require_bearer_token)], +) +async def auth_reload( + payload: AuthReloadRequest, + container: TradingContainer = Depends(get_container), +) -> ApiResponse[AuthReloadData]: + """重新从磁盘加载登录态并立即探测 + + 人工跑完 `scripts/login.py` 后调用，避免为了换一套 cookie 重启整个服务。 + """ + sites = [payload.site.value] if payload.site else list(container.auth_session.sites) + reloaded = {site: container.auth_session.reload(site) for site in sites} + statuses = [_to_model(await container.auth_session.check(site)) for site in sites] + + return ApiResponse[AuthReloadData]( + success=True, + msg="success", + data=AuthReloadData(reloaded=reloaded, sites=statuses), + code=0, + ) diff --git a/app/trading/api/routes/health.py b/app/trading/api/routes/health.py new file mode 100644 index 0000000..937215d --- /dev/null +++ b/app/trading/api/routes/health.py @@ -0,0 +1,27 @@ +"""交易服务健康检查路由""" +from fastapi import APIRouter, Depends + +from app.shared.api import ApiResponse, get_container +from app.trading.container import TradingContainer +from app.trading.models import TradingHealthData + +router = APIRouter(tags=["health"]) + + +@router.get("/health", response_model=ApiResponse[TradingHealthData]) +async def health( + container: TradingContainer = Depends(get_container), +) -> ApiResponse[TradingHealthData]: + """交易服务健康状态 + + auth 给出两站账号登录态。这里只读缓存、不触发网络探测，避免健康检查被 + 上游高频轮询时反复打站点；要实时结果请用 POST /api/auth/status。 + + 注意 `logged_in=null` 表示服务启动后还没探测过，不等于未登录。 + """ + return ApiResponse[TradingHealthData]( + success=True, + msg="success", + data=TradingHealthData(status="ok", auth=container.auth_session.status_all()), + code=0, + ) diff --git a/app/trading/container.py b/app/trading/container.py new file mode 100644 index 0000000..d0a9176 --- /dev/null +++ b/app/trading/container.py @@ -0,0 +1,23 @@ +"""交易服务容器：集中管理交易侧服务实例，用于依赖注入 + +目前只有登录态会话。加购、下单、付款、订单监控与页面证据留痕的服务实例 +后续挂在这里，它们共享同一份 auth_session——同一个账号的写操作必须走同一条 +cookie 通道，且必须串行，不能各建各的客户端。 +""" +from dataclasses import dataclass + +from app.shared.config import Settings +from app.trading.services.auth_session import AuthSession + + +@dataclass(slots=True) +class TradingContainer: + """交易服务容器 + + 与抓取容器最本质的差别不是字段多少，而是**这个进程有状态**：登录态、 + 订单、页面证据都属于某个具体账号，因此交易服务只能单实例运行 + （或按账号分片），不能像抓取服务那样随意横向扩容。 + """ + + settings: Settings + auth_session: AuthSession diff --git a/app/trading/core/__init__.py b/app/trading/core/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/trading/core/auth_site.py b/app/trading/core/auth_site.py new file mode 100644 index 0000000..3fe50f6 --- /dev/null +++ b/app/trading/core/auth_site.py @@ -0,0 +1,115 @@ +"""登录态相关的站点常量 + +与抓取用的 `scraping/core/site.py` / `rakuma_site.py` 分开：那两个模块描述的是 +「匿名抓取怎么拿到页面」，这里描述的是「怎么判断这套 cookie 还登录着」以及登录 +入口在哪。 + +登录态探针的选取原则：挑一个**未登录时行为明确可辨**的页面，而不是靠首页上有没有 +用户名这类会随改版漂移的文案。两站各自的信号（均为实测）： + +- 乐天：手机版购物车页始终返回 200，未登录时正文含「現在ログインしていません」， + 登录后该串消失。购物车页同时是加购结果的校验页，一页两用。 +- ラクマ：`/mypage` 未登录时 302 到 `/users/sign_in`，登录后停在 `/mypage`。 + 用落地 URL 判断比翻文案稳。 + +每站的这组事实收在 `PROFILES` 里，`scripts/login.py`（起浏览器人工登录）与 +`AuthSession`（在 httpx 里探测）共用同一份，避免两处各写一遍判据后悄悄漂移。 +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Final + +from app.shared import headers + +# ---- 乐天市场 ---- +# 手机版购物车。加购走的是 sp.basket 集群，校验也用手机版，保持同一套指纹。 +RAKUTEN_CART_URL: Final = "https://sp.cart.step.rakuten.co.jp/cart" + +# 登录入口。人工登录时打开这个地址，站点会在登录成功后跳回 my.rakuten.co.jp。 +RAKUTEN_LOGIN_URL: Final = "https://www.rakuten.co.jp/myrakuten/" + +# 购物车页上表示「当前会话未登录」的文案。出现即判定登录态失效。 +RAKUTEN_LOGGED_OUT_MARKER: Final = "現在ログインしていません" + +# ---- ラクマ ---- +RAKUMA_MYPAGE_URL: Final = "https://fril.jp/mypage" +RAKUMA_LOGIN_URL: Final = "https://fril.jp/users/sign_in" + +# 未登录时会被重定向到的登录页路径特征 +RAKUMA_SIGN_IN_PATH: Final = "/users/sign_in" + +# ---- 登录态请求指纹 ---- +# 这两个 UA 必须与 scripts/login.py 起浏览器时用的一致：cookie 是在那个 UA 下拿到的， +# 服务端再拿它发请求时换了 UA，可能触发站点的设备校验让登录态提前失效。 +# +# 值与抓取侧当前相同，但**刻意不复用**抓取侧的常量：抓取 UA 是为绕反爬服务的， +# 随时可能为了成功率被调整，那种调整不该波及已落盘的账号 cookie。 +RAKUTEN_USER_AGENT: Final = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 " + "(KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1" +) +RAKUMA_USER_AGENT: Final = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) + + +@dataclass(frozen=True, slots=True) +class SiteAuthProfile: + """一个站点的登录态配置：登录入口、探针、指纹与落盘文件名""" + + name: str + label: str + login_url: str + probe_url: str # 判断登录态是否仍有效的页面 + user_agent: str + mobile: bool + state_filename: str # storage_state 落盘文件名（放在 settings.auth_state_dir 下） + + def headers(self) -> dict[str, str]: + """该站登录态请求用的完整导航请求头""" + return headers.navigation_headers(self.user_agent, mobile=self.mobile) + + +PROFILES: Final[dict[str, SiteAuthProfile]] = { + "rakuten": SiteAuthProfile( + name="rakuten", + label="楽天市場", + login_url=RAKUTEN_LOGIN_URL, + probe_url=RAKUTEN_CART_URL, + user_agent=RAKUTEN_USER_AGENT, + mobile=True, + state_filename="rakuten_state.json", + ), + "rakuma": SiteAuthProfile( + name="rakuma", + label="ラクマ", + login_url=RAKUMA_LOGIN_URL, + probe_url=RAKUMA_MYPAGE_URL, + user_agent=RAKUMA_USER_AGENT, + mobile=False, + state_filename="rakuma_state.json", + ), +} + +SITES: Final = tuple(PROFILES) + + +def profile(site: str) -> SiteAuthProfile: + """取某站的登录态配置，未知站点直接报错""" + try: + return PROFILES[site] + except KeyError: + raise ValueError(f"未知站点：{site}") from None + + +def is_logged_in(site: str, *, final_url: str, body: str) -> bool: + """按该站判据，从探针页的落地 URL 与正文判断是否仍登录着 + + 两处共用：`AuthSession` 传 httpx 响应的 URL 与文本，`scripts/login.py` 传 + 浏览器页面的 URL 与内容。判据只写一遍，两条链路不会各判各的。 + """ + if site == "rakuten": + return RAKUTEN_LOGGED_OUT_MARKER not in body + return RAKUMA_SIGN_IN_PATH not in final_url diff --git a/app/trading/main.py b/app/trading/main.py new file mode 100644 index 0000000..ea1d29b --- /dev/null +++ b/app/trading/main.py @@ -0,0 +1,84 @@ +"""交易服务入口：FastAPI 应用创建与生命周期管理 + +与抓取服务（app/scraping/main.py）分成两个进程运行，理由不是「要不要登录」 +这一条，而是运行特性根本不同： + +- 抓取无状态、可重试、可多开实例；这里的写操作**不可逆**，重复提交就是重复下单。 +- 登录态 cookie 全局唯一，订单监控是常驻轮询；多开实例会让同一个账号被多个 + 进程并发操作，也会让轮询重复触发。 +- 抓取被限速最多是慢，账号被风控是封号；两者不该共用出口 IP 与请求节奏。 + +因此本服务只能单实例运行（或按账号分片），扩容靠抓取服务那一侧。 + +当前只提供登录态的查询与重载；加购、下单、付款与订单监控在此基础上叠加。 +""" +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from app.shared.api import register_exception_handlers +from app.shared.config import get_settings +from app.shared.logging_setup import configure_logging +from app.trading.api.routes.auth import router as auth_router +from app.trading.api.routes.health import router as health_router +from app.trading.container import TradingContainer +from app.trading.services.auth_session import AuthSession + +logger = logging.getLogger(__name__) + + +def build_container() -> TradingContainer: + """构建交易服务容器，组装所有依赖""" + settings = get_settings() + return TradingContainer(settings=settings, auth_session=AuthSession(settings)) + + +@asynccontextmanager +async def lifespan(app: FastAPI): + """应用生命周期管理：启动时加载登录态，关闭时释放 HTTP 客户端""" + container = build_container() + app.state.container = container + + configure_logging(container.settings) + logger.info( + "交易服务启动：%s:%s", + container.settings.trading_host, + container.settings.trading_port, + ) + logger.info("当前环境：%s", container.settings.app_env) + await container.auth_session.start() + try: + yield + finally: + await container.auth_session.close() + + +def create_app() -> FastAPI: + """创建 FastAPI 应用实例，注册路由和异常处理器""" + app = FastAPI(title="Rakuten Trading Service", lifespan=lifespan) + app.include_router(health_router) + app.include_router(auth_router) + register_exception_handlers(app) + return app + + +app = create_app() + + +if __name__ == "__main__": + import uvicorn + + settings = get_settings() + configure_logging(settings) + uvicorn.run( + "app.trading.main:app", + host=settings.trading_host, + port=settings.trading_port, + log_config=None, + timeout_keep_alive=120, + # 单进程：登录态与后续的订单监控都不能有第二份 + workers=1, + ) diff --git a/app/trading/models.py b/app/trading/models.py new file mode 100644 index 0000000..4ac4bdf --- /dev/null +++ b/app/trading/models.py @@ -0,0 +1,68 @@ +"""交易侧 API 数据模型：登录态、（后续的）加购、下单与订单监控 + +与抓取侧模型（app/scraping/models/scrape.py）分开的理由和服务本身拆开的理由 +一致：抓取模型描述的是「站点上有什么」，这里描述的是「我们对某个账号做了什么、 +现在处于哪一步」——后者有生命周期、有状态迁移，会持久化，不是一次请求的产物。 + +响应信封 `ApiResponse` 与抓取侧共用，在 app/shared/api.py。 +""" +from __future__ import annotations + +from enum import StrEnum +from typing import Any + +from pydantic import BaseModel, Field + + +class AuthSite(StrEnum): + """支持登录的站点""" + + RAKUTEN = "rakuten" + RAKUMA = "rakuma" + + +class AuthStatusRequest(BaseModel): + """登录态查询请求""" + + site: AuthSite | None = None # 不传则返回两站 + refresh: bool = True # 是否真实打一次请求探测；false 时只读缓存 + + +class AuthSiteStatus(BaseModel): + """单站登录态""" + + site: str + state_file_exists: bool # 是否已跑过 scripts/login.py + logged_in: bool | None # None 表示尚未探测 + checked_age_seconds: float | None = None + detail: str = "" + + +class AuthStatusData(BaseModel): + """登录态查询响应""" + + sites: list[AuthSiteStatus] = Field(default_factory=list) + + +class AuthReloadRequest(BaseModel): + """重新加载登录态请求（人工登录完成后调用，免重启服务）""" + + site: AuthSite | None = None # 不传则两站都重载 + + +class AuthReloadData(BaseModel): + """重新加载登录态响应""" + + reloaded: dict[str, int] = Field(default_factory=dict) # site -> cookie 条数 + sites: list[AuthSiteStatus] = Field(default_factory=list) + + +class TradingHealthData(BaseModel): + """交易服务健康检查响应数据 + + 只读缓存的登录态，不触发网络探测——健康检查会被高频轮询，实时结果请用 + POST /api/auth/status。 + """ + + status: str + auth: dict[str, Any] = Field(default_factory=dict) diff --git a/app/trading/services/__init__.py b/app/trading/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/trading/services/auth_session.py b/app/trading/services/auth_session.py new file mode 100644 index 0000000..33c4611 --- /dev/null +++ b/app/trading/services/auth_session.py @@ -0,0 +1,260 @@ +"""登录态会话：持有已登录账号的 cookie，供加购与下单链路使用 + +与抓取链路（app/scraping 的 site_session / rakuma_session）不只是分模块， +而是分进程运行，原因： + +- 抓取是**匿名**的，cookie 只用来过 Akamai 限速，丢了重新预热即可，无状态可言。 +- 加购下单必须**带账号**，cookie 一旦失效不能自动恢复——乐天与 ラクマ 登录都有 + reCAPTCHA 与设备验证，只能由人重新登录一次。因此这里的失效处理是「明确报错让 + 上游停下」，而不是像抓取那样静默重试。 +- 这份登录态在整个系统里只能有一份。跟抓取同进程的话，抓取一扩容就会复制出 N 份 + 登录态与 N 个订单轮询，同一个账号被并发操作。 + +登录态来源是 Playwright 的 storage_state：由 `scripts/login.py` 起一个有头浏览器 +让人工登录一次后落盘，本服务只读取、不生产。账号密码不经过本服务，也不写日志。 + +cookie 会被同时喂给两处： +- httpx 客户端 —— 加购这类纯表单提交走 HTTP 更快 +- Playwright context —— 下单确认页有 JS 参与，必要时用浏览器走完 +""" +from __future__ import annotations + +import asyncio +import json +import logging +import time +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +import httpx + +from app.shared.config import Settings +from app.shared.errors import NotLoggedInError, UpstreamRequestError +from app.trading.core import auth_site + +logger = logging.getLogger(__name__) + + +@dataclass(slots=True) +class AuthStatus: + """一个站点的登录态快照""" + + site: str + state_file_exists: bool + logged_in: bool | None # None 表示尚未探测过 + checked_at: float | None = None + detail: str = "" + + def to_dict(self) -> dict[str, Any]: + return { + "site": self.site, + "state_file_exists": self.state_file_exists, + "logged_in": self.logged_in, + "checked_age_seconds": ( + round(time.monotonic() - self.checked_at, 1) if self.checked_at else None + ), + "detail": self.detail, + } + + +@dataclass(slots=True) +class _SiteAuth: + """一个站点的登录通道：独立的 httpx 客户端与登录态缓存""" + + name: str + client: httpx.AsyncClient + lock: asyncio.Lock = field(default_factory=asyncio.Lock) + logged_in: bool | None = None + checked_at: float | None = None + detail: str = "" + + +class AuthSession: + """持有两站登录态的会话 + + 职责边界：只负责「有没有登录态、cookie 是什么、还有效吗」， + 具体加购/下单的业务请求由各自的 client 组装后借这里的 HTTP 客户端发出。 + """ + + def __init__(self, settings: Settings): + self._settings = settings + self._sites: dict[str, _SiteAuth] = {} + + # ---- 生命周期 ---- + + async def start(self) -> None: + """为两站创建带登录 cookie 的 HTTP 客户端 + + 登录态文件不存在时同样创建客户端（只是没有 cookie），这样 /health 与 + /api/auth/status 能如实回报「未登录」，而不是整个服务起不来。 + """ + for name, profile in auth_site.PROFILES.items(): + if name in self._sites: + continue + client = httpx.AsyncClient( + headers=profile.headers(), + timeout=self._settings.request_timeout_seconds, + follow_redirects=True, + proxy=self._settings.httpx_proxy, + http2=True, + ) + self._sites[name] = _SiteAuth(name=name, client=client) + loaded = self._load_cookies(name) + logger.info("登录通道已就绪：site=%s cookies=%s", name, loaded) + + async def close(self) -> None: + for auth in self._sites.values(): + try: + await auth.client.aclose() + except Exception: + logger.debug("关闭登录 HTTP 客户端失败：site=%s", auth.name, exc_info=True) + self._sites.clear() + + # ---- 登录态文件 ---- + + def state_path(self, site: str) -> Path: + """某站点 storage_state 文件的落盘路径""" + return self._settings.auth_state_path / auth_site.profile(site).state_filename + + def _load_cookies(self, site: str) -> int: + """把 storage_state 里的 cookie 灌进该站的 httpx 客户端，返回条数""" + path = self.state_path(site) + if not path.exists(): + return 0 + + try: + state = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("登录态文件读取失败：site=%s path=%s err=%s", site, path, exc) + return 0 + + auth = self._sites[site] + auth.client.cookies.clear() + count = 0 + for cookie in state.get("cookies", []): + name = cookie.get("name") + value = cookie.get("value") + if not name or value is None: + continue + auth.client.cookies.set( + name, + value, + domain=cookie.get("domain") or "", + path=cookie.get("path") or "/", + ) + count += 1 + return count + + def reload(self, site: str) -> int: + """重新从磁盘加载登录态（人工登录完成后调用），返回 cookie 条数""" + auth = self._sites.get(site) + if auth is None: + raise ValueError(f"未知站点：{site}") + count = self._load_cookies(site) + auth.logged_in = None + auth.checked_at = None + auth.detail = "已重新加载登录态，尚未探测" + logger.info("登录态已重新加载：site=%s cookies=%s", site, count) + return count + + # ---- 登录态探测 ---- + + async def check(self, site: str) -> AuthStatus: + """探测某站登录态是否仍然有效 + + 每次都真实打一次请求——登录态过期没有可靠的本地判据（cookie 的 expires + 与服务端会话不是一回事），只能问站点。 + """ + auth = self._require_site(site) + async with auth.lock: + try: + if site == "rakuten": + logged_in, detail = await self._check_rakuten(auth) + else: + logged_in, detail = await self._check_rakuma(auth) + except httpx.HTTPError as exc: + raise UpstreamRequestError( + f"登录态探测请求失败：site={site} err={type(exc).__name__}: {exc}" + ) from exc + + auth.logged_in = logged_in + auth.checked_at = time.monotonic() + auth.detail = detail + logger.info("登录态探测：site=%s logged_in=%s detail=%s", site, logged_in, detail) + return self.status(site) + + async def _check_rakuten(self, auth: _SiteAuth) -> tuple[bool, str]: + """购物车页含「現在ログインしていません」即未登录""" + response = await auth.client.get(auth_site.PROFILES["rakuten"].probe_url) + if response.status_code >= 400: + return False, f"购物车页返回 status {response.status_code}" + if not auth_site.is_logged_in( + "rakuten", final_url=str(response.url), body=response.text + ): + return False, "购物车页显示未登录" + return True, "购物车页未出现未登录标记" + + async def _check_rakuma(self, auth: _SiteAuth) -> tuple[bool, str]: + """/mypage 被重定向到 /users/sign_in 即未登录""" + response = await auth.client.get(auth_site.PROFILES["rakuma"].probe_url) + if response.status_code >= 400: + return False, f"mypage 返回 status {response.status_code}" + if not auth_site.is_logged_in( + "rakuma", final_url=str(response.url), body=response.text + ): + return False, "mypage 被重定向至登录页" + return True, "mypage 正常返回" + + async def require_logged_in(self, site: str) -> None: + """确保某站处于登录态，否则抛错 + + 加购与下单前的统一入口。登录态失效时不做任何自动恢复尝试——两站登录都需要 + 人工过验证码，只能让上游停下来重新跑一次 scripts/login.py。 + """ + status = await self.check(site) + if not status.logged_in: + raise NotLoggedInError(site=site, detail=status.detail) + + # ---- 对外访问 ---- + + @property + def sites(self) -> tuple[str, ...]: + """已初始化的站点名""" + return tuple(self._sites) + + def client(self, site: str) -> httpx.AsyncClient: + """取该站带登录 cookie 的 HTTP 客户端""" + return self._require_site(site).client + + def cookies_for_browser(self, site: str) -> list[dict[str, Any]]: + """导出 cookie 供 Playwright context 使用""" + path = self.state_path(site) + if not path.exists(): + return [] + try: + state = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return [] + return list(state.get("cookies", [])) + + def status(self, site: str) -> AuthStatus: + """该站登录态快照（不触发探测，用缓存结果）""" + auth = self._require_site(site) + return AuthStatus( + site=site, + state_file_exists=self.state_path(site).exists(), + logged_in=auth.logged_in, + checked_at=auth.checked_at, + detail=auth.detail, + ) + + def status_all(self) -> dict[str, dict[str, Any]]: + """两站登录态快照，供健康检查展示""" + return {name: self.status(name).to_dict() for name in self._sites} + + def _require_site(self, site: str) -> _SiteAuth: + auth = self._sites.get(site) + if auth is None: + raise ValueError(f"未知站点或登录会话未初始化：{site}") + return auth diff --git a/scripts/login.py b/scripts/login.py new file mode 100644 index 0000000..a81d168 --- /dev/null +++ b/scripts/login.py @@ -0,0 +1,135 @@ +"""人工登录：起一个有头浏览器，由人完成登录，落盘 cookie 供服务复用 + +为什么必须人工：乐天与 ラクマ 的登录都带 reCAPTCHA 与设备验证（短信/邮箱 OTP）， +自动填表的成功率既低又不稳定，还要在本地存明文密码。这里的取舍是——**密码只经过 +你和站点，不经过本服务**：脚本只负责把浏览器打开、等你登录完，然后把 cookie 存下来。 + +用法： + + .venv/Scripts/python.exe scripts/login.py --site rakuten + .venv/Scripts/python.exe scripts/login.py --site rakuma + .venv/Scripts/python.exe scripts/login.py --site all + +浏览器窗口打开后手动完成登录，脚本会自动轮询登录态；检测到已登录即保存并退出。 +也可以登录完成后回到终端按回车立即保存。 + +产物落在 settings.auth_state_dir（默认 .auth/），内含可直接冒充账号的 cookie， +已在 .gitignore 里排除，不要提交、不要外传。 +""" +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +from pathlib import Path + +# 允许以 `python scripts/login.py` 直接运行 +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from app.shared.config import get_settings # noqa: E402 +from app.trading.core import auth_site # noqa: E402 + +# 轮询间隔与总时长：给足人工过验证码、收短信的时间 +_POLL_INTERVAL_SECONDS = 5 +_POLL_TIMEOUT_SECONDS = 600 + + +async def _is_logged_in(page, site: str) -> bool: + """在浏览器里探测登录态 + + 判据与 AuthSession 共用 auth_site.is_logged_in，只是这里喂浏览器页面内容、 + 那里喂 httpx 响应，避免两条链路对「算不算登录」各判各的。 + """ + await page.goto( + auth_site.profile(site).probe_url, wait_until="domcontentloaded", timeout=60_000 + ) + return auth_site.is_logged_in(site, final_url=page.url, body=await page.content()) + + +async def login(site: str) -> bool: + """打开浏览器让用户登录指定站点，成功则保存 storage_state""" + from playwright.async_api import async_playwright + + settings = get_settings() + profile = auth_site.profile(site) + state_path = settings.auth_state_path / profile.state_filename + + print(f"\n=== {profile.label}（{site}）登录 ===") + print(f"即将打开浏览器：{profile.login_url}") + print("请在浏览器窗口里完成登录（账号密码只在浏览器与站点之间传递，本脚本不读取）。") + print(f"登录完成后脚本会自动检测，最长等待 {_POLL_TIMEOUT_SECONDS // 60} 分钟。\n") + + async with async_playwright() as playwright: + browser = await playwright.chromium.launch( + headless=False, # 人工登录必须有头 + channel=settings.browser_channel or None, + proxy=settings.playwright_proxy, + args=["--no-first-run", "--disable-blink-features=AutomationControlled"], + ) + try: + context = await browser.new_context( + # UA 取自 auth_site：服务端后续用同一个 UA 发请求，换了可能触发 + # 站点的设备校验，让这次辛苦登来的 cookie 提前失效。 + user_agent=profile.user_agent, + locale="ja-JP", + timezone_id="Asia/Tokyo", + viewport=( + {"width": 390, "height": 844} + if profile.mobile + else {"width": 1440, "height": 900} + ), + is_mobile=profile.mobile, + has_touch=profile.mobile, + ) + page = await context.new_page() + await page.goto(profile.login_url, wait_until="domcontentloaded", timeout=60_000) + + waited = 0 + while waited < _POLL_TIMEOUT_SECONDS: + await asyncio.sleep(_POLL_INTERVAL_SECONDS) + waited += _POLL_INTERVAL_SECONDS + try: + if await _is_logged_in(page, site): + break + except Exception as exc: # 页面正在跳转时探测可能失败，继续等 + print(f" 探测中（{waited}s）：{type(exc).__name__}") + continue + print(f" 等待登录中…（{waited}s）") + else: + print(f"✗ {profile.label} 等待超时，未检测到登录态。未保存。") + return False + + state = await context.storage_state() + state_path.write_text( + json.dumps(state, ensure_ascii=False, indent=2), encoding="utf-8" + ) + print(f"✓ {profile.label} 登录成功，已保存 {len(state.get('cookies', []))} 条 cookie") + print(f" → {state_path}") + return True + finally: + await browser.close() + + +async def main() -> int: + parser = argparse.ArgumentParser(description="人工登录并保存乐天/ラクマ 登录态") + parser.add_argument( + "--site", + choices=[*auth_site.SITES, "all"], + default="all", + help="要登录的站点，默认两站都登录", + ) + args = parser.parse_args() + + targets = list(auth_site.SITES) if args.site == "all" else [args.site] + results = {site: await login(site) for site in targets} + + print("\n=== 结果 ===") + for site, ok in results.items(): + print(f" {site}: {'已登录' if ok else '失败'}") + print("\n登录态已就绪，可启动服务并用 POST /api/auth/status 复核。") + return 0 if all(results.values()) else 1 + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(main())) diff --git a/tests/fixtures/rakuma_category.html b/tests/fixtures/rakuma_category.html new file mode 100644 index 0000000..79d6c4e --- /dev/null +++ b/tests/fixtures/rakuma_category.html @@ -0,0 +1,8 @@ + +カテゴリー一覧 | ラクマ + + + + + + diff --git a/tests/test_api.py b/tests/test_api.py index 5e9b54a..f2118d5 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -7,13 +7,15 @@ from __future__ import annotations import pytest from fastapi.testclient import TestClient -from app.core.config import get_settings -from app.core.errors import ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError -from app.main import create_app -from app.models.scrape import ( +from app.shared.config import get_settings +from app.shared.errors import ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError +from app.scraping.main import create_app +from app.scraping.models.scrape import ( GenreData, GenreNode, ItemDetailData, + RakumaCategoryData, + RakumaCategoryNode, RakumaItemDetailData, RakumaSearchItem, RakumaSearchResultData, @@ -90,6 +92,7 @@ class StubRakumaClient: def __init__(self) -> None: self.search_payload = None + self.category_payload = None self.detail_payload = None self.shop_detail_payload = None self.shop_items_payload = None @@ -107,6 +110,15 @@ class StubRakumaClient: items=[RakumaSearchItem(item_id="abc", item_name="商品", price=6299)], ) + async def categories(self, payload) -> RakumaCategoryData: + self.category_payload = payload + return RakumaCategoryData( + category_id=payload.category_id or "", + name="エンタメ/ホビー" if payload.category_id else "", + total_count=1686, + children=[RakumaCategoryNode(category_id="786", name="ゲームソフト/ゲーム機本体")], + ) + async def item_detail(self, payload) -> RakumaItemDetailData: self.detail_payload = payload if self.raise_on_detail: @@ -346,6 +358,7 @@ def test_shop_items_requires_an_identifier(client): "path", [ "/api/rakuma/search", + "/api/rakuma/categories", "/api/rakuma/item_detail", "/api/rakuma/shop_detail", "/api/rakuma/shop_items", @@ -405,6 +418,27 @@ def test_rakuma_search_rejects_rakuten_only_sort(client): assert response.status_code == 422 +def test_rakuma_categories_accepts_empty_body_for_top_level(client, rakuma_stub): + """不传 category_id 时取顶层分类，不应因缺参数被拦下""" + response = client.post("/api/rakuma/categories", json={}, headers=AUTH) + assert response.status_code == 200 + assert rakuma_stub.category_payload.category_id is None + assert rakuma_stub.category_payload.include_descendants is False + assert response.json()["data"]["total_count"] == 1686 + + +def test_rakuma_categories_passes_options_through(client, rakuma_stub): + response = client.post( + "/api/rakuma/categories", + json={"category_id": "10007", "include_descendants": True}, + headers=AUTH, + ) + assert response.status_code == 200 + assert rakuma_stub.category_payload.category_id == "10007" + assert rakuma_stub.category_payload.include_descendants is True + assert response.json()["data"]["children"][0]["category_id"] == "786" + + def test_rakuma_item_detail_accepts_item_id(client, rakuma_stub): response = client.post("/api/rakuma/item_detail", json={"item_id": "abc"}, headers=AUTH) assert response.status_code == 200 diff --git a/tests/test_architecture.py b/tests/test_architecture.py new file mode 100644 index 0000000..f845a4a --- /dev/null +++ b/tests/test_architecture.py @@ -0,0 +1,73 @@ +"""架构测试：守住抓取侧与交易侧的依赖方向 + +拆成两个进程之后，最容易悄悄退化的不是功能而是边界——某天为了省事在交易侧 +`from app.scraping.parsers...` 一句，两个服务就重新长回一起：抓取的解析改动会 +牵动下单链路，交易服务也被迫加载整套抓取依赖（包括 Playwright）。 + +允许的方向只有两条：scraping → shared、trading → shared。 +交易侧要用抓取的能力，走抓取服务的 HTTP 接口（`purchase` 块本来就是它的对外契约）， +不直接 import。 +""" +from __future__ import annotations + +import ast +from pathlib import Path + +import pytest + +APP_DIR = Path(__file__).resolve().parent.parent / "app" + + +def _imported_modules(path: Path) -> set[str]: + """取一个源文件里 import 到的模块名（只看真实 import，不看注释与文档字符串）""" + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + modules: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + modules.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0: + modules.add(node.module) + return modules + + +def _python_files(package: str) -> list[Path]: + return sorted((APP_DIR / package).rglob("*.py")) + + +@pytest.mark.parametrize( + ("package", "forbidden"), + [ + ("scraping", "app.trading"), + ("trading", "app.scraping"), + # shared 是两侧的共同底座，反向依赖任何一侧都会形成环 + ("shared", "app.scraping"), + ("shared", "app.trading"), + ], +) +def test_package_does_not_import(package: str, forbidden: str): + offenders = [ + f"{path.relative_to(APP_DIR)} -> {module}" + for path in _python_files(package) + for module in _imported_modules(path) + if module == forbidden or module.startswith(f"{forbidden}.") + ] + assert not offenders, f"{package} 不应依赖 {forbidden}：{offenders}" + + +def test_both_entrypoints_build(): + """两个入口都要能独立创建应用——这是「两个部署单元」的最低验收""" + from app.scraping.main import create_app as create_scraping_app + from app.trading.main import create_app as create_trading_app + + # 用 OpenAPI 里的路径而不是 app.routes：新版 FastAPI 把 include_router 的结果 + # 包成 _IncludedRouter 而非摊平，OpenAPI 反映的才是真正对外暴露的契约 + scraping_paths = set(create_scraping_app().openapi()["paths"]) + trading_paths = set(create_trading_app().openapi()["paths"]) + + assert "/api/search" in scraping_paths + assert "/api/auth/status" in trading_paths + # 登录态接口不该出现在抓取服务上：抓取实例可以多开，多份登录态就是重复下单的温床 + assert not any(path.startswith("/api/auth") for path in scraping_paths) + assert not any(path.startswith("/api/rakuma") for path in trading_paths) + # /health 两边都有，各报各的 + assert "/health" in scraping_paths and "/health" in trading_paths diff --git a/tests/test_auth_session.py b/tests/test_auth_session.py new file mode 100644 index 0000000..201bda4 --- /dev/null +++ b/tests/test_auth_session.py @@ -0,0 +1,264 @@ +"""登录态会话测试：cookie 加载、失效探测、重新加载 + +全部用 httpx.MockTransport 拦截，不触达真实站点、不需要真实账号。 +登录态文件写在 tmp_path 下，不碰仓库里的 .auth/。 +""" +from __future__ import annotations + +import json +from pathlib import Path + +import httpx +import pytest + +from app.shared.config import Settings +from app.shared.errors import NotLoggedInError, UpstreamRequestError +from app.trading.core import auth_site +from app.trading.services.auth_session import AuthSession + +# 购物车页的两种形态：含未登录标记 = 未登录，不含 = 已登录 +CART_LOGGED_OUT = f"買い物かご {auth_site.RAKUTEN_LOGGED_OUT_MARKER}" +CART_LOGGED_IN = "買い物かご 商品が1点入っています" + +MYPAGE_HTML = "マイページ" + + +def make_settings(tmp_path: Path, **overrides) -> Settings: + base = { + "auth_state_dir": str(tmp_path / "auth"), + "request_timeout_seconds": 5.0, + } + base.update(overrides) + return Settings(**base) + + +def write_state(settings: Settings, site: str, cookies: list[dict]) -> Path: + """伪造一份 Playwright storage_state 落盘""" + path = settings.auth_state_path / auth_site.profile(site).state_filename + path.write_text( + json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False), + encoding="utf-8", + ) + return path + + +async def build_session(settings: Settings, handler) -> AuthSession: + """构建 AuthSession 并把两站 client 换成 MockTransport 版本 + + 换掉 client 会丢掉 start() 时灌进去的 cookie，因此重新走一次 reload + 把登录态读回新客户端；reload 同时清空探测缓存，正好是测试想要的干净起点。 + """ + session = AuthSession(settings) + await session.start() + for name in session.sites: + auth = session._sites[name] + await auth.client.aclose() + auth.client = httpx.AsyncClient( + transport=httpx.MockTransport(handler), + follow_redirects=True, + ) + session.reload(name) + return session + + +# ---- cookie 加载 ---- + + +async def test_loads_cookies_from_state_file(tmp_path): + """storage_state 里的 cookie 应被灌进 httpx 客户端""" + settings = make_settings(tmp_path) + write_state( + settings, + "rakuten", + [{"name": "SESSION", "value": "abc", "domain": ".rakuten.co.jp", "path": "/"}], + ) + + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN)) + try: + names = {c.name for c in session.client("rakuten").cookies.jar} + assert "SESSION" in names + # 没有 state 文件的那一站应为空，而不是报错 + assert not list(session.client("rakuma").cookies.jar) + finally: + await session.close() + + +async def test_starts_without_state_file(tmp_path): + """登录态文件不存在时仍能启动，只是报告未登录——服务不应因此起不来""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT)) + try: + status = session.status("rakuten") + assert status.state_file_exists is False + assert status.logged_in is None # 尚未探测 + finally: + await session.close() + + +async def test_corrupted_state_file_is_tolerated(tmp_path): + """登录态文件损坏时降级为无 cookie，不抛异常""" + settings = make_settings(tmp_path) + path = settings.auth_state_path / auth_site.profile("rakuten").state_filename + path.write_text("{ not json", encoding="utf-8") + + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT)) + try: + assert not list(session.client("rakuten").cookies.jar) + finally: + await session.close() + + +# ---- 登录态探测 ---- + + +async def test_rakuten_detects_logged_out_by_marker(tmp_path): + """购物车页出现未登录文案即判定未登录""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT)) + try: + status = await session.check("rakuten") + assert status.logged_in is False + assert "未登录" in status.detail + finally: + await session.close() + + +async def test_rakuten_detects_logged_in(tmp_path): + """购物车页没有未登录文案即判定已登录""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN)) + try: + status = await session.check("rakuten") + assert status.logged_in is True + assert status.checked_at is not None + # 序列化后暴露给 API 的是相对时长，不是单调时钟原值 + assert status.to_dict()["checked_age_seconds"] is not None + finally: + await session.close() + + +async def test_rakuma_detects_logged_out_by_redirect(tmp_path): + """/mypage 被重定向到登录页即判定未登录 + + ラクマ 未登录时返回 302 而非改文案，因此判据是落地 URL 不是页面内容。 + """ + settings = make_settings(tmp_path) + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path == "/mypage": + return httpx.Response(302, headers={"Location": auth_site.RAKUMA_LOGIN_URL}) + return httpx.Response(200, text="ログイン") + + session = await build_session(settings, handler) + try: + status = await session.check("rakuma") + assert status.logged_in is False + assert "登录页" in status.detail + finally: + await session.close() + + +async def test_rakuma_detects_logged_in(tmp_path): + """/mypage 正常返回即判定已登录""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=MYPAGE_HTML)) + try: + status = await session.check("rakuma") + assert status.logged_in is True + finally: + await session.close() + + +async def test_error_status_counts_as_logged_out(tmp_path): + """探测页返回 4xx/5xx 时保守判定为未登录，不放行下单""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(503, text="oops")) + try: + status = await session.check("rakuten") + assert status.logged_in is False + assert "503" in status.detail + finally: + await session.close() + + +async def test_network_error_raises_upstream(tmp_path): + """网络异常与「确实未登录」是两回事，应抛错而不是静默判未登录""" + settings = make_settings(tmp_path) + + def handler(request: httpx.Request) -> httpx.Response: + raise httpx.ConnectError("boom") + + session = await build_session(settings, handler) + try: + with pytest.raises(UpstreamRequestError): + await session.check("rakuten") + finally: + await session.close() + + +# ---- 下单前置校验 ---- + + +async def test_require_logged_in_raises_when_logged_out(tmp_path): + """未登录时 require_logged_in 抛 5001，且标记为不可重试 + + 登录需要人工过验证码，自动重试没有意义，必须让上游停下来。 + """ + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT)) + try: + with pytest.raises(NotLoggedInError) as excinfo: + await session.require_logged_in("rakuten") + assert excinfo.value.err_code == 5001 + assert excinfo.value.retryable is False + assert excinfo.value.status_code == 401 + assert "scripts/login.py" in excinfo.value.message + finally: + await session.close() + + +async def test_require_logged_in_passes_when_logged_in(tmp_path): + """已登录时放行，不抛异常""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN)) + try: + await session.require_logged_in("rakuten") # 不应抛出 + finally: + await session.close() + + +# ---- 重新加载 ---- + + +async def test_reload_picks_up_new_cookies(tmp_path): + """人工重新登录后 reload 应换上新 cookie 并清掉旧的探测结论""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN)) + try: + await session.check("rakuten") + assert session.status("rakuten").logged_in is True + + write_state( + settings, + "rakuten", + [{"name": "FRESH", "value": "xyz", "domain": ".rakuten.co.jp", "path": "/"}], + ) + count = session.reload("rakuten") + + assert count == 1 + assert "FRESH" in {c.name for c in session.client("rakuten").cookies.jar} + # 重载后旧结论必须作废，避免拿过期判断放行下单 + assert session.status("rakuten").logged_in is None + finally: + await session.close() + + +async def test_unknown_site_rejected(tmp_path): + """未知站点名应明确报错，不静默返回空状态""" + settings = make_settings(tmp_path) + session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN)) + try: + with pytest.raises(ValueError): + session.status("mercari") + finally: + await session.close() diff --git a/tests/test_genre.py b/tests/test_genre.py index 76c35ee..9fda1ca 100644 --- a/tests/test_genre.py +++ b/tests/test_genre.py @@ -1,10 +1,10 @@ """分类树解析测试：基于真实页面状态样本""" import pytest -from app.core import site -from app.core.errors import ScrapeParseError -from app.parsers.genre import parse_genres -from app.utils.urls import build_genre_url +from app.scraping.core import site +from app.shared.errors import ScrapeParseError +from app.scraping.parsers.genre import parse_genres +from app.scraping.utils.urls import build_genre_url # ---- URL 构建 ---- diff --git a/tests/test_parsers.py b/tests/test_parsers.py index 13aff2b..a03bbc5 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -1,10 +1,10 @@ """解析器测试：基于真实页面状态样本""" import pytest -from app.core.errors import ScrapeParseError -from app.parsers.item import parse_item_detail -from app.parsers.search import parse_search -from app.parsers.state import extract_initial_state +from app.shared.errors import ScrapeParseError +from app.scraping.parsers.item import parse_item_detail +from app.scraping.parsers.search import parse_search +from app.scraping.parsers.state import extract_initial_state # ---- __INITIAL_STATE__ 抽取 ---- diff --git a/tests/test_purchase.py b/tests/test_purchase.py index 875f0bf..4497f08 100644 --- a/tests/test_purchase.py +++ b/tests/test_purchase.py @@ -8,10 +8,10 @@ from pathlib import Path import pytest -from app.parsers.item import parse_item_detail, parse_purchase_options -from app.parsers.subsites import parse_subsite_item -from app.parsers.subsites.base import SubsitePage -from app.parsers.subsites.brandavenue import _resolve_cart_url +from app.scraping.parsers.item import parse_item_detail, parse_purchase_options +from app.scraping.parsers.subsites import parse_subsite_item +from app.scraping.parsers.subsites.base import SubsitePage +from app.scraping.parsers.subsites.brandavenue import _resolve_cart_url FIXTURES = Path(__file__).parent / "fixtures" diff --git a/tests/test_rakuma.py b/tests/test_rakuma.py index a55a73e..c639c9a 100644 --- a/tests/test_rakuma.py +++ b/tests/test_rakuma.py @@ -7,19 +7,20 @@ from pathlib import Path import pytest -from app.core.errors import InvalidRequestError, ScrapeParseError -from app.models.scrape import ( +from app.shared.errors import InvalidRequestError, ItemNotFoundError, ScrapeParseError +from app.scraping.models.scrape import ( RakumaAuthenticity, RakumaCondition, RakumaSearchRequest, RakumaSortOption, RakumaTransaction, ) -from app.parsers.rakuma.base import parse_int, parse_total_count -from app.parsers.rakuma.item import parse_item_detail -from app.parsers.rakuma.search import parse_search -from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items -from app.utils.rakuma_urls import ( +from app.scraping.parsers.rakuma.base import parse_int, parse_total_count +from app.scraping.parsers.rakuma.category import parse_categories +from app.scraping.parsers.rakuma.item import parse_item_detail +from app.scraping.parsers.rakuma.search import parse_search +from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items +from app.scraping.utils.rakuma_urls import ( build_item_url, build_search_url, build_shop_url, @@ -269,6 +270,67 @@ def test_parse_shop_pages_reject_non_shop_page(): parse_shop_items("x", shop_id="S", request_url="u", page=1) +# ---- 分类树解析 ---- + +def test_parse_categories_returns_top_level_by_default(): + data = parse_categories( + fixture("rakuma_category.html"), category_id=None, include_descendants=False + ) + assert data.category_id == "" + assert len(data.children) == 14 # 站点顶层分类固定 14 个 + top = data.children[0] + assert top.category_id == "10001" + assert top.name == "レディース" + assert top.parent_id == "0" + assert top.is_leaf is False + assert top.url == "https://fril.jp/category/10001" + assert top.children == [] # 未要求子树时不展开 + + +def test_parse_categories_reads_ancestors_and_children(): + data = parse_categories( + fixture("rakuma_category.html"), category_id="786", include_descendants=False + ) + assert data.name == "ゲームソフト/ゲーム機本体" + assert [node.category_id for node in data.ancestors] == ["10007"] + assert data.full_name == "エンタメ/ホビー / ゲームソフト/ゲーム機本体" + assert [node.category_id for node in data.children] == ["787", "788", "789"] + assert all(node.is_leaf for node in data.children) + assert data.is_leaf is False + + +def test_parse_categories_marks_leaf_without_children(): + data = parse_categories( + fixture("rakuma_category.html"), category_id="788", include_descendants=False + ) + assert data.is_leaf is True + assert data.children == [] + assert data.full_name == "エンタメ/ホビー / ゲームソフト/ゲーム機本体 / 家庭用ゲームソフト" + + +def test_parse_categories_can_expand_full_subtree(): + """整棵树本来就在一次响应里，展开子树不需要多打请求""" + data = parse_categories( + fixture("rakuma_category.html"), category_id="10007", include_descendants=True + ) + branch = data.children[0] + assert branch.category_id == "786" + assert [node.category_id for node in branch.children] == ["787", "788", "789"] + + +def test_parse_categories_rejects_unknown_category(): + """无效分类要报 404，而不是当成「没有子分类」返回空结果""" + with pytest.raises(ItemNotFoundError): + parse_categories( + fixture("rakuma_category.html"), category_id="99999999", include_descendants=False + ) + + +def test_parse_categories_rejects_page_without_tree(): + with pytest.raises(ScrapeParseError): + parse_categories("home", category_id=None, include_descendants=False) + + # ---- 取值工具 ---- @pytest.mark.parametrize( diff --git a/tests/test_rakuma_session.py b/tests/test_rakuma_session.py index c917dfe..1251edd 100644 --- a/tests/test_rakuma_session.py +++ b/tests/test_rakuma_session.py @@ -7,10 +7,10 @@ from __future__ import annotations import httpx import pytest -from app.core import rakuma_site as site -from app.core.config import Settings -from app.core.errors import ItemNotFoundError, UpstreamRequestError -from app.services.rakuma_session import RakumaSession +from app.scraping.core import rakuma_site as site +from app.shared.config import Settings +from app.shared.errors import ItemNotFoundError, UpstreamRequestError +from app.scraping.services.rakuma_session import RakumaSession TARGET = "https://fril.jp/s?query=switch" GOOD_PAGE = '
21件中 1 - 21件
' diff --git a/tests/test_search_url.py b/tests/test_search_url.py index 998ff11..4d61023 100644 --- a/tests/test_search_url.py +++ b/tests/test_search_url.py @@ -3,9 +3,9 @@ from urllib.parse import parse_qsl, urlsplit import pytest -from app.core.errors import InvalidRequestError -from app.models.scrape import ItemCondition, SearchRequest, SortOption -from app.utils.urls import ( +from app.shared.errors import InvalidRequestError +from app.scraping.models.scrape import ItemCondition, SearchRequest, SortOption +from app.scraping.utils.urls import ( build_item_url, build_search_url, item_url_parts, diff --git a/tests/test_shop.py b/tests/test_shop.py index c7784ba..bb316e1 100644 --- a/tests/test_shop.py +++ b/tests/test_shop.py @@ -4,10 +4,10 @@ from pathlib import Path import pytest -from app.core.errors import InvalidRequestError, ScrapeParseError -from app.models.scrape import ShopItemsRequest, SortOption -from app.parsers.shop import parse_shop_detail -from app.utils.urls import build_search_url, build_shop_url, split_shop_url +from app.shared.errors import InvalidRequestError, ScrapeParseError +from app.scraping.models.scrape import ShopItemsRequest, SortOption +from app.scraping.parsers.shop import parse_shop_detail +from app.scraping.utils.urls import build_search_url, build_shop_url, split_shop_url FIXTURES = Path(__file__).parent / "fixtures" diff --git a/tests/test_site_session.py b/tests/test_site_session.py index 7c586d0..cb85c59 100644 --- a/tests/test_site_session.py +++ b/tests/test_site_session.py @@ -7,17 +7,17 @@ from __future__ import annotations import httpx import pytest -from app.core import site -from app.core.config import Settings -from app.core.errors import ( +from app.scraping.core import site +from app.shared.config import Settings +from app.shared.errors import ( ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError, UpstreamRequestError, ) -from app.parsers.subsites import build_item_page_validator -from app.services.browser_fallback import BrowserVisit -from app.services.site_session import SiteSession +from app.scraping.parsers.subsites import build_item_page_validator +from app.scraping.services.browser_fallback import BrowserVisit +from app.scraping.services.site_session import SiteSession GOOD_PAGE = '' BLOCK_PAGE = "Access Denied. Reference #18.abc" diff --git a/tests/test_subsites.py b/tests/test_subsites.py index 6808614..67edbda 100644 --- a/tests/test_subsites.py +++ b/tests/test_subsites.py @@ -7,14 +7,14 @@ from pathlib import Path import pytest -from app.core.errors import OffIchibaRedirectError, ScrapeParseError -from app.parsers.subsites import ( +from app.shared.errors import OffIchibaRedirectError, ScrapeParseError +from app.scraping.parsers.subsites import ( SUBSITE_PARSERS, build_item_page_validator, host_of, parse_subsite_item, ) -from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price +from app.scraping.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price FIXTURES = Path(__file__).parent / "fixtures" diff --git a/tests/test_trading_api.py b/tests/test_trading_api.py new file mode 100644 index 0000000..eeb5d67 --- /dev/null +++ b/tests/test_trading_api.py @@ -0,0 +1,157 @@ +"""交易服务 API 测试：健康检查、登录态查询与重载、鉴权 + +登录态会话被替换为桩，不触达真实站点、不需要真实账号。 +AuthSession 自身的行为（cookie 加载、探测判据）在 tests/test_auth_session.py。 +""" +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient + +from app.shared.config import get_settings +from app.shared.errors import NotLoggedInError +from app.trading.main import create_app +from app.trading.services.auth_session import AuthStatus + +TOKEN = get_settings().bearer_token +AUTH = {"Authorization": f"Bearer {TOKEN}"} + + +class StubAuthSession: + """记录调用并返回固定登录态的桩""" + + def __init__(self) -> None: + self.checked: list[str] = [] + self.reloaded: list[str] = [] + self.logged_in = True + + @property + def sites(self) -> tuple[str, ...]: + return ("rakuten", "rakuma") + + def _status(self, site: str) -> AuthStatus: + return AuthStatus( + site=site, + state_file_exists=True, + logged_in=self.logged_in, + checked_at=None, + detail="stub", + ) + + async def check(self, site: str) -> AuthStatus: + self.checked.append(site) + return self._status(site) + + def status(self, site: str) -> AuthStatus: + return self._status(site) + + def status_all(self) -> dict[str, dict]: + return {site: self._status(site).to_dict() for site in self.sites} + + def reload(self, site: str) -> int: + self.reloaded.append(site) + return 3 + + async def require_logged_in(self, site: str) -> None: + if not self.logged_in: + raise NotLoggedInError(site=site, detail="stub") + + async def close(self) -> None: + """lifespan 收尾会调用；桩没有真实客户端要关""" + + +@pytest.fixture +def client_and_stub(): + app = create_app() + with TestClient(app) as client: + stub = StubAuthSession() + app.state.container.auth_session = stub + yield client, stub + + +@pytest.fixture +def client(client_and_stub): + return client_and_stub[0] + + +@pytest.fixture +def stub(client_and_stub): + return client_and_stub[1] + + +# ---- 健康检查 ---- + + +def test_health_needs_no_token(client): + response = client.get("/health") + assert response.status_code == 200 + body = response.json() + assert body["data"]["status"] == "ok" + assert set(body["data"]["auth"]) == {"rakuten", "rakuma"} + + +def test_health_does_not_probe_the_site(client, stub): + """健康检查只读缓存：它会被高频轮询，不能每次都去打站点""" + client.get("/health") + assert stub.checked == [] + + +# ---- 鉴权 ---- + + +@pytest.mark.parametrize("path", ["/api/auth/status", "/api/auth/reload"]) +def test_auth_endpoints_reject_missing_token(client, path): + response = client.post(path, json={}) + assert response.status_code == 401 + assert response.json()["code"] == 1001 + + +def test_auth_endpoints_reject_wrong_token(client): + response = client.post( + "/api/auth/status", json={}, headers={"Authorization": "Bearer wrong-token"} + ) + assert response.status_code == 401 + + +# ---- 登录态查询 ---- + + +def test_status_probes_both_sites_by_default(client, stub): + response = client.post("/api/auth/status", json={}, headers=AUTH) + assert response.status_code == 200 + body = response.json() + assert [item["site"] for item in body["data"]["sites"]] == ["rakuten", "rakuma"] + assert stub.checked == ["rakuten", "rakuma"] + + +def test_status_can_skip_the_probe(client, stub): + """refresh=false 时读缓存，不打站点""" + response = client.post("/api/auth/status", json={"refresh": False}, headers=AUTH) + assert response.status_code == 200 + assert stub.checked == [] + + +def test_status_accepts_a_single_site(client, stub): + response = client.post("/api/auth/status", json={"site": "rakuma"}, headers=AUTH) + assert response.status_code == 200 + assert stub.checked == ["rakuma"] + + +def test_status_rejects_unknown_site(client): + """站点名是枚举，未知值应在校验层就被挡下""" + response = client.post("/api/auth/status", json={"site": "mercari"}, headers=AUTH) + assert response.status_code == 422 + assert response.json()["code"] == 1002 + + +# ---- 登录态重载 ---- + + +def test_reload_reloads_then_probes(client, stub): + response = client.post("/api/auth/reload", json={"site": "rakuten"}, headers=AUTH) + assert response.status_code == 200 + body = response.json() + assert body["data"]["reloaded"] == {"rakuten": 3} + # 重载后必须立刻探测一次，否则调用方拿不到「这次登录到底成没成」 + assert stub.reloaded == ["rakuten"] + assert stub.checked == ["rakuten"]