拆分抓取与交易服务

把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」,
而是抓取无状态、幂等、可多开实例,而交易的写操作不可逆、登录态全局唯一、
订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询,
同一账号会被并发操作。

- app/shared:配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、
  导航请求头构造器
- app/scraping:站点常量、会话、解析器与 10 个抓取接口,:31107,可多开
- app/trading:登录态查询/重载与健康检查,:31108,只能单实例
- 依赖方向锁为 scraping→shared、trading→shared,两侧互不 import;
  tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串
- 登录态 UA 在 trading 独立持有:与抓取 UA 值相同但变更理由不同,抓取 UA 为绕
  反爬可随时调整,登录 UA 一改可能触发设备校验使已落盘 cookie 失效
- scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in,判据只写一遍
- 同一镜像两个启动命令,交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT

同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。

验证:239 个离线用例全绿;两个入口真实启动,/health 与鉴权正常。
未验证:真实探测登录态(当前开发机无外网,对站点的连接全部超时)。

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-27 15:05:01 +08:00
co-authored by Claude Opus 5
parent 4250388762
commit 104d7fef6b
80 changed files with 2330 additions and 402 deletions
+38 -4
View File
@@ -7,13 +7,15 @@ from __future__ import annotations
import pytest
from fastapi.testclient import TestClient
from app.core.config import get_settings
from app.core.errors import ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError
from app.main import create_app
from app.models.scrape import (
from app.shared.config import get_settings
from app.shared.errors import ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError
from app.scraping.main import create_app
from app.scraping.models.scrape import (
GenreData,
GenreNode,
ItemDetailData,
RakumaCategoryData,
RakumaCategoryNode,
RakumaItemDetailData,
RakumaSearchItem,
RakumaSearchResultData,
@@ -90,6 +92,7 @@ class StubRakumaClient:
def __init__(self) -> None:
self.search_payload = None
self.category_payload = None
self.detail_payload = None
self.shop_detail_payload = None
self.shop_items_payload = None
@@ -107,6 +110,15 @@ class StubRakumaClient:
items=[RakumaSearchItem(item_id="abc", item_name="商品", price=6299)],
)
async def categories(self, payload) -> RakumaCategoryData:
self.category_payload = payload
return RakumaCategoryData(
category_id=payload.category_id or "",
name="エンタメ/ホビー" if payload.category_id else "",
total_count=1686,
children=[RakumaCategoryNode(category_id="786", name="ゲームソフト/ゲーム機本体")],
)
async def item_detail(self, payload) -> RakumaItemDetailData:
self.detail_payload = payload
if self.raise_on_detail:
@@ -346,6 +358,7 @@ def test_shop_items_requires_an_identifier(client):
"path",
[
"/api/rakuma/search",
"/api/rakuma/categories",
"/api/rakuma/item_detail",
"/api/rakuma/shop_detail",
"/api/rakuma/shop_items",
@@ -405,6 +418,27 @@ def test_rakuma_search_rejects_rakuten_only_sort(client):
assert response.status_code == 422
def test_rakuma_categories_accepts_empty_body_for_top_level(client, rakuma_stub):
"""不传 category_id 时取顶层分类,不应因缺参数被拦下"""
response = client.post("/api/rakuma/categories", json={}, headers=AUTH)
assert response.status_code == 200
assert rakuma_stub.category_payload.category_id is None
assert rakuma_stub.category_payload.include_descendants is False
assert response.json()["data"]["total_count"] == 1686
def test_rakuma_categories_passes_options_through(client, rakuma_stub):
response = client.post(
"/api/rakuma/categories",
json={"category_id": "10007", "include_descendants": True},
headers=AUTH,
)
assert response.status_code == 200
assert rakuma_stub.category_payload.category_id == "10007"
assert rakuma_stub.category_payload.include_descendants is True
assert response.json()["data"]["children"][0]["category_id"] == "786"
def test_rakuma_item_detail_accepts_item_id(client, rakuma_stub):
response = client.post("/api/rakuma/item_detail", json={"item_id": "abc"}, headers=AUTH)
assert response.status_code == 200