拆分抓取与交易服务
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
Vendored
+8
@@ -0,0 +1,8 @@
|
||||
<!DOCTYPE html>
|
||||
<html><head><title>カテゴリー一覧 | ラクマ</title></head>
|
||||
<body>
|
||||
<script>self.__next_f=self.__next_f||[]</script>
|
||||
<script>self.__next_f.push([1,"3:I[12345,[\"static/chunk.js\"],\"CategoryPage\"]\n2:{\"unrelated\":true}\n"])</script>
|
||||
<script>self.__next_f.push([1,"{\"categoryList\": [{\"id\": 10001, \"parentId\": 0, \"name\": \"レディース\", \"hasChild\": true}, {\"id\": 10005, \"parentId\": 0, \"name\": \"メンズ\", \"hasChild\": true}, {\"id\": 10004, \"parentId\": 0, \"name\": \"コスメ/美容\", \"hasChild\": true}, {\"id\": 10003, \"parentId\": 0, \"name\": \"キッズ/ベビー/マタニティ\", \"hasChild\": true}, {\"id\": 10007, \"parentId\": 0, \"name\": \"エンタメ/ホビー\", \"hasChild\": true}, {\"id\": 10013, \"parentId\": 0, \"name\": \"楽器\", \"hasChild\": true}, {\"id\": 10008, \"parentId\": 0, \"name\": \"チケット\", \"hasChild\": true}, {\"id\": 10009, \"parentId\": 0, \"name\": \"インテリア/住まい/日用品\", \"hasChild\": true}, {\"id\": 10006, \"parentId\": 0, \"name\": \"スマホ/家電/カメラ\", \"hasChild\": true}, {\"id\": 10010, \"parentId\": 0, \"name\": \"ハンドメイド\", \"hasChild\": true},"])</script>
|
||||
<script>self.__next_f.push([1," {\"id\": 10012, \"parentId\": 0, \"name\": \"食品/飲料/酒\", \"hasChild\": true}, {\"id\": 10014, \"parentId\": 0, \"name\": \"スポーツ/アウトドア\", \"hasChild\": true}, {\"id\": 10011, \"parentId\": 0, \"name\": \"自動車/バイク\", \"hasChild\": true}, {\"id\": 10002, \"parentId\": 0, \"name\": \"その他\", \"hasChild\": true}, {\"id\": 1, \"parentId\": 10001, \"name\": \"トップス\", \"hasChild\": true}, {\"id\": 2, \"parentId\": 10001, \"name\": \"ジャケット/アウター\", \"hasChild\": true}, {\"id\": 786, \"parentId\": 10007, \"name\": \"ゲームソフト/ゲーム機本体\", \"hasChild\": true}, {\"id\": 787, \"parentId\": 786, \"name\": \"家庭用ゲーム機本体\", \"hasChild\": false}, {\"id\": 788, \"parentId\": 786, \"name\": \"家庭用ゲームソフト\", \"hasChild\": false}, {\"id\": 789, \"parentId\": 786, \"name\": \"携帯用ゲーム機本体\", \"hasChild\": false}]}"])</script>
|
||||
</body></html>
|
||||
+38
-4
@@ -7,13 +7,15 @@ from __future__ import annotations
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError
|
||||
from app.main import create_app
|
||||
from app.models.scrape import (
|
||||
from app.shared.config import get_settings
|
||||
from app.shared.errors import ItemNotFoundError, OffIchibaRedirectError, UpstreamBlockedError
|
||||
from app.scraping.main import create_app
|
||||
from app.scraping.models.scrape import (
|
||||
GenreData,
|
||||
GenreNode,
|
||||
ItemDetailData,
|
||||
RakumaCategoryData,
|
||||
RakumaCategoryNode,
|
||||
RakumaItemDetailData,
|
||||
RakumaSearchItem,
|
||||
RakumaSearchResultData,
|
||||
@@ -90,6 +92,7 @@ class StubRakumaClient:
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.search_payload = None
|
||||
self.category_payload = None
|
||||
self.detail_payload = None
|
||||
self.shop_detail_payload = None
|
||||
self.shop_items_payload = None
|
||||
@@ -107,6 +110,15 @@ class StubRakumaClient:
|
||||
items=[RakumaSearchItem(item_id="abc", item_name="商品", price=6299)],
|
||||
)
|
||||
|
||||
async def categories(self, payload) -> RakumaCategoryData:
|
||||
self.category_payload = payload
|
||||
return RakumaCategoryData(
|
||||
category_id=payload.category_id or "",
|
||||
name="エンタメ/ホビー" if payload.category_id else "",
|
||||
total_count=1686,
|
||||
children=[RakumaCategoryNode(category_id="786", name="ゲームソフト/ゲーム機本体")],
|
||||
)
|
||||
|
||||
async def item_detail(self, payload) -> RakumaItemDetailData:
|
||||
self.detail_payload = payload
|
||||
if self.raise_on_detail:
|
||||
@@ -346,6 +358,7 @@ def test_shop_items_requires_an_identifier(client):
|
||||
"path",
|
||||
[
|
||||
"/api/rakuma/search",
|
||||
"/api/rakuma/categories",
|
||||
"/api/rakuma/item_detail",
|
||||
"/api/rakuma/shop_detail",
|
||||
"/api/rakuma/shop_items",
|
||||
@@ -405,6 +418,27 @@ def test_rakuma_search_rejects_rakuten_only_sort(client):
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
def test_rakuma_categories_accepts_empty_body_for_top_level(client, rakuma_stub):
|
||||
"""不传 category_id 时取顶层分类，不应因缺参数被拦下"""
|
||||
response = client.post("/api/rakuma/categories", json={}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
assert rakuma_stub.category_payload.category_id is None
|
||||
assert rakuma_stub.category_payload.include_descendants is False
|
||||
assert response.json()["data"]["total_count"] == 1686
|
||||
|
||||
|
||||
def test_rakuma_categories_passes_options_through(client, rakuma_stub):
|
||||
response = client.post(
|
||||
"/api/rakuma/categories",
|
||||
json={"category_id": "10007", "include_descendants": True},
|
||||
headers=AUTH,
|
||||
)
|
||||
assert response.status_code == 200
|
||||
assert rakuma_stub.category_payload.category_id == "10007"
|
||||
assert rakuma_stub.category_payload.include_descendants is True
|
||||
assert response.json()["data"]["children"][0]["category_id"] == "786"
|
||||
|
||||
|
||||
def test_rakuma_item_detail_accepts_item_id(client, rakuma_stub):
|
||||
response = client.post("/api/rakuma/item_detail", json={"item_id": "abc"}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""架构测试：守住抓取侧与交易侧的依赖方向
|
||||
|
||||
拆成两个进程之后，最容易悄悄退化的不是功能而是边界——某天为了省事在交易侧
|
||||
`from app.scraping.parsers...` 一句，两个服务就重新长回一起：抓取的解析改动会
|
||||
牵动下单链路，交易服务也被迫加载整套抓取依赖（包括 Playwright）。
|
||||
|
||||
允许的方向只有两条：scraping → shared、trading → shared。
|
||||
交易侧要用抓取的能力，走抓取服务的 HTTP 接口（`purchase` 块本来就是它的对外契约），
|
||||
不直接 import。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
APP_DIR = Path(__file__).resolve().parent.parent / "app"
|
||||
|
||||
|
||||
def _imported_modules(path: Path) -> set[str]:
|
||||
"""取一个源文件里 import 到的模块名（只看真实 import，不看注释与文档字符串）"""
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
modules: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
modules.update(alias.name for alias in node.names)
|
||||
elif isinstance(node, ast.ImportFrom) and node.module and node.level == 0:
|
||||
modules.add(node.module)
|
||||
return modules
|
||||
|
||||
|
||||
def _python_files(package: str) -> list[Path]:
|
||||
return sorted((APP_DIR / package).rglob("*.py"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("package", "forbidden"),
|
||||
[
|
||||
("scraping", "app.trading"),
|
||||
("trading", "app.scraping"),
|
||||
# shared 是两侧的共同底座，反向依赖任何一侧都会形成环
|
||||
("shared", "app.scraping"),
|
||||
("shared", "app.trading"),
|
||||
],
|
||||
)
|
||||
def test_package_does_not_import(package: str, forbidden: str):
|
||||
offenders = [
|
||||
f"{path.relative_to(APP_DIR)} -> {module}"
|
||||
for path in _python_files(package)
|
||||
for module in _imported_modules(path)
|
||||
if module == forbidden or module.startswith(f"{forbidden}.")
|
||||
]
|
||||
assert not offenders, f"{package} 不应依赖 {forbidden}：{offenders}"
|
||||
|
||||
|
||||
def test_both_entrypoints_build():
|
||||
"""两个入口都要能独立创建应用——这是「两个部署单元」的最低验收"""
|
||||
from app.scraping.main import create_app as create_scraping_app
|
||||
from app.trading.main import create_app as create_trading_app
|
||||
|
||||
# 用 OpenAPI 里的路径而不是 app.routes：新版 FastAPI 把 include_router 的结果
|
||||
# 包成 _IncludedRouter 而非摊平，OpenAPI 反映的才是真正对外暴露的契约
|
||||
scraping_paths = set(create_scraping_app().openapi()["paths"])
|
||||
trading_paths = set(create_trading_app().openapi()["paths"])
|
||||
|
||||
assert "/api/search" in scraping_paths
|
||||
assert "/api/auth/status" in trading_paths
|
||||
# 登录态接口不该出现在抓取服务上：抓取实例可以多开，多份登录态就是重复下单的温床
|
||||
assert not any(path.startswith("/api/auth") for path in scraping_paths)
|
||||
assert not any(path.startswith("/api/rakuma") for path in trading_paths)
|
||||
# /health 两边都有，各报各的
|
||||
assert "/health" in scraping_paths and "/health" in trading_paths
|
||||
@@ -0,0 +1,264 @@
|
||||
"""登录态会话测试：cookie 加载、失效探测、重新加载
|
||||
|
||||
全部用 httpx.MockTransport 拦截，不触达真实站点、不需要真实账号。
|
||||
登录态文件写在 tmp_path 下，不碰仓库里的 .auth/。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import NotLoggedInError, UpstreamRequestError
|
||||
from app.trading.core import auth_site
|
||||
from app.trading.services.auth_session import AuthSession
|
||||
|
||||
# 购物车页的两种形态：含未登录标记 = 未登录，不含 = 已登录
|
||||
CART_LOGGED_OUT = f"<html><body>買い物かご {auth_site.RAKUTEN_LOGGED_OUT_MARKER}</body></html>"
|
||||
CART_LOGGED_IN = "<html><body>買い物かご 商品が1点入っています</body></html>"
|
||||
|
||||
MYPAGE_HTML = "<html><body>マイページ</body></html>"
|
||||
|
||||
|
||||
def make_settings(tmp_path: Path, **overrides) -> Settings:
|
||||
base = {
|
||||
"auth_state_dir": str(tmp_path / "auth"),
|
||||
"request_timeout_seconds": 5.0,
|
||||
}
|
||||
base.update(overrides)
|
||||
return Settings(**base)
|
||||
|
||||
|
||||
def write_state(settings: Settings, site: str, cookies: list[dict]) -> Path:
|
||||
"""伪造一份 Playwright storage_state 落盘"""
|
||||
path = settings.auth_state_path / auth_site.profile(site).state_filename
|
||||
path.write_text(
|
||||
json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
async def build_session(settings: Settings, handler) -> AuthSession:
|
||||
"""构建 AuthSession 并把两站 client 换成 MockTransport 版本
|
||||
|
||||
换掉 client 会丢掉 start() 时灌进去的 cookie，因此重新走一次 reload
|
||||
把登录态读回新客户端；reload 同时清空探测缓存，正好是测试想要的干净起点。
|
||||
"""
|
||||
session = AuthSession(settings)
|
||||
await session.start()
|
||||
for name in session.sites:
|
||||
auth = session._sites[name]
|
||||
await auth.client.aclose()
|
||||
auth.client = httpx.AsyncClient(
|
||||
transport=httpx.MockTransport(handler),
|
||||
follow_redirects=True,
|
||||
)
|
||||
session.reload(name)
|
||||
return session
|
||||
|
||||
|
||||
# ---- cookie 加载 ----
|
||||
|
||||
|
||||
async def test_loads_cookies_from_state_file(tmp_path):
|
||||
"""storage_state 里的 cookie 应被灌进 httpx 客户端"""
|
||||
settings = make_settings(tmp_path)
|
||||
write_state(
|
||||
settings,
|
||||
"rakuten",
|
||||
[{"name": "SESSION", "value": "abc", "domain": ".rakuten.co.jp", "path": "/"}],
|
||||
)
|
||||
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN))
|
||||
try:
|
||||
names = {c.name for c in session.client("rakuten").cookies.jar}
|
||||
assert "SESSION" in names
|
||||
# 没有 state 文件的那一站应为空，而不是报错
|
||||
assert not list(session.client("rakuma").cookies.jar)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_starts_without_state_file(tmp_path):
|
||||
"""登录态文件不存在时仍能启动，只是报告未登录——服务不应因此起不来"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT))
|
||||
try:
|
||||
status = session.status("rakuten")
|
||||
assert status.state_file_exists is False
|
||||
assert status.logged_in is None # 尚未探测
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_corrupted_state_file_is_tolerated(tmp_path):
|
||||
"""登录态文件损坏时降级为无 cookie，不抛异常"""
|
||||
settings = make_settings(tmp_path)
|
||||
path = settings.auth_state_path / auth_site.profile("rakuten").state_filename
|
||||
path.write_text("{ not json", encoding="utf-8")
|
||||
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT))
|
||||
try:
|
||||
assert not list(session.client("rakuten").cookies.jar)
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
# ---- 登录态探测 ----
|
||||
|
||||
|
||||
async def test_rakuten_detects_logged_out_by_marker(tmp_path):
|
||||
"""购物车页出现未登录文案即判定未登录"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT))
|
||||
try:
|
||||
status = await session.check("rakuten")
|
||||
assert status.logged_in is False
|
||||
assert "未登录" in status.detail
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_rakuten_detects_logged_in(tmp_path):
|
||||
"""购物车页没有未登录文案即判定已登录"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN))
|
||||
try:
|
||||
status = await session.check("rakuten")
|
||||
assert status.logged_in is True
|
||||
assert status.checked_at is not None
|
||||
# 序列化后暴露给 API 的是相对时长，不是单调时钟原值
|
||||
assert status.to_dict()["checked_age_seconds"] is not None
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_rakuma_detects_logged_out_by_redirect(tmp_path):
|
||||
"""/mypage 被重定向到登录页即判定未登录
|
||||
|
||||
ラクマ 未登录时返回 302 而非改文案，因此判据是落地 URL 不是页面内容。
|
||||
"""
|
||||
settings = make_settings(tmp_path)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if request.url.path == "/mypage":
|
||||
return httpx.Response(302, headers={"Location": auth_site.RAKUMA_LOGIN_URL})
|
||||
return httpx.Response(200, text="<html>ログイン</html>")
|
||||
|
||||
session = await build_session(settings, handler)
|
||||
try:
|
||||
status = await session.check("rakuma")
|
||||
assert status.logged_in is False
|
||||
assert "登录页" in status.detail
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_rakuma_detects_logged_in(tmp_path):
|
||||
"""/mypage 正常返回即判定已登录"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=MYPAGE_HTML))
|
||||
try:
|
||||
status = await session.check("rakuma")
|
||||
assert status.logged_in is True
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_error_status_counts_as_logged_out(tmp_path):
|
||||
"""探测页返回 4xx/5xx 时保守判定为未登录，不放行下单"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(503, text="oops"))
|
||||
try:
|
||||
status = await session.check("rakuten")
|
||||
assert status.logged_in is False
|
||||
assert "503" in status.detail
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_network_error_raises_upstream(tmp_path):
|
||||
"""网络异常与「确实未登录」是两回事，应抛错而不是静默判未登录"""
|
||||
settings = make_settings(tmp_path)
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
raise httpx.ConnectError("boom")
|
||||
|
||||
session = await build_session(settings, handler)
|
||||
try:
|
||||
with pytest.raises(UpstreamRequestError):
|
||||
await session.check("rakuten")
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
# ---- 下单前置校验 ----
|
||||
|
||||
|
||||
async def test_require_logged_in_raises_when_logged_out(tmp_path):
|
||||
"""未登录时 require_logged_in 抛 5001，且标记为不可重试
|
||||
|
||||
登录需要人工过验证码，自动重试没有意义，必须让上游停下来。
|
||||
"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_OUT))
|
||||
try:
|
||||
with pytest.raises(NotLoggedInError) as excinfo:
|
||||
await session.require_logged_in("rakuten")
|
||||
assert excinfo.value.err_code == 5001
|
||||
assert excinfo.value.retryable is False
|
||||
assert excinfo.value.status_code == 401
|
||||
assert "scripts/login.py" in excinfo.value.message
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_require_logged_in_passes_when_logged_in(tmp_path):
|
||||
"""已登录时放行，不抛异常"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN))
|
||||
try:
|
||||
await session.require_logged_in("rakuten") # 不应抛出
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
# ---- 重新加载 ----
|
||||
|
||||
|
||||
async def test_reload_picks_up_new_cookies(tmp_path):
|
||||
"""人工重新登录后 reload 应换上新 cookie 并清掉旧的探测结论"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN))
|
||||
try:
|
||||
await session.check("rakuten")
|
||||
assert session.status("rakuten").logged_in is True
|
||||
|
||||
write_state(
|
||||
settings,
|
||||
"rakuten",
|
||||
[{"name": "FRESH", "value": "xyz", "domain": ".rakuten.co.jp", "path": "/"}],
|
||||
)
|
||||
count = session.reload("rakuten")
|
||||
|
||||
assert count == 1
|
||||
assert "FRESH" in {c.name for c in session.client("rakuten").cookies.jar}
|
||||
# 重载后旧结论必须作废，避免拿过期判断放行下单
|
||||
assert session.status("rakuten").logged_in is None
|
||||
finally:
|
||||
await session.close()
|
||||
|
||||
|
||||
async def test_unknown_site_rejected(tmp_path):
|
||||
"""未知站点名应明确报错，不静默返回空状态"""
|
||||
settings = make_settings(tmp_path)
|
||||
session = await build_session(settings, lambda r: httpx.Response(200, text=CART_LOGGED_IN))
|
||||
try:
|
||||
with pytest.raises(ValueError):
|
||||
session.status("mercari")
|
||||
finally:
|
||||
await session.close()
|
||||
+4
-4
@@ -1,10 +1,10 @@
|
||||
"""分类树解析测试：基于真实页面状态样本"""
|
||||
import pytest
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.parsers.genre import parse_genres
|
||||
from app.utils.urls import build_genre_url
|
||||
from app.scraping.core import site
|
||||
from app.shared.errors import ScrapeParseError
|
||||
from app.scraping.parsers.genre import parse_genres
|
||||
from app.scraping.utils.urls import build_genre_url
|
||||
|
||||
|
||||
# ---- URL 构建 ----
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
"""解析器测试：基于真实页面状态样本"""
|
||||
import pytest
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.parsers.item import parse_item_detail
|
||||
from app.parsers.search import parse_search
|
||||
from app.parsers.state import extract_initial_state
|
||||
from app.shared.errors import ScrapeParseError
|
||||
from app.scraping.parsers.item import parse_item_detail
|
||||
from app.scraping.parsers.search import parse_search
|
||||
from app.scraping.parsers.state import extract_initial_state
|
||||
|
||||
|
||||
# ---- __INITIAL_STATE__ 抽取 ----
|
||||
|
||||
@@ -8,10 +8,10 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.parsers.item import parse_item_detail, parse_purchase_options
|
||||
from app.parsers.subsites import parse_subsite_item
|
||||
from app.parsers.subsites.base import SubsitePage
|
||||
from app.parsers.subsites.brandavenue import _resolve_cart_url
|
||||
from app.scraping.parsers.item import parse_item_detail, parse_purchase_options
|
||||
from app.scraping.parsers.subsites import parse_subsite_item
|
||||
from app.scraping.parsers.subsites.base import SubsitePage
|
||||
from app.scraping.parsers.subsites.brandavenue import _resolve_cart_url
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
+69
-7
@@ -7,19 +7,20 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import InvalidRequestError, ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
from app.shared.errors import InvalidRequestError, ItemNotFoundError, ScrapeParseError
|
||||
from app.scraping.models.scrape import (
|
||||
RakumaAuthenticity,
|
||||
RakumaCondition,
|
||||
RakumaSearchRequest,
|
||||
RakumaSortOption,
|
||||
RakumaTransaction,
|
||||
)
|
||||
from app.parsers.rakuma.base import parse_int, parse_total_count
|
||||
from app.parsers.rakuma.item import parse_item_detail
|
||||
from app.parsers.rakuma.search import parse_search
|
||||
from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
||||
from app.utils.rakuma_urls import (
|
||||
from app.scraping.parsers.rakuma.base import parse_int, parse_total_count
|
||||
from app.scraping.parsers.rakuma.category import parse_categories
|
||||
from app.scraping.parsers.rakuma.item import parse_item_detail
|
||||
from app.scraping.parsers.rakuma.search import parse_search
|
||||
from app.scraping.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
||||
from app.scraping.utils.rakuma_urls import (
|
||||
build_item_url,
|
||||
build_search_url,
|
||||
build_shop_url,
|
||||
@@ -269,6 +270,67 @@ def test_parse_shop_pages_reject_non_shop_page():
|
||||
parse_shop_items("<html><body>x</body></html>", shop_id="S", request_url="u", page=1)
|
||||
|
||||
|
||||
# ---- 分类树解析 ----
|
||||
|
||||
def test_parse_categories_returns_top_level_by_default():
|
||||
data = parse_categories(
|
||||
fixture("rakuma_category.html"), category_id=None, include_descendants=False
|
||||
)
|
||||
assert data.category_id == ""
|
||||
assert len(data.children) == 14 # 站点顶层分类固定 14 个
|
||||
top = data.children[0]
|
||||
assert top.category_id == "10001"
|
||||
assert top.name == "レディース"
|
||||
assert top.parent_id == "0"
|
||||
assert top.is_leaf is False
|
||||
assert top.url == "https://fril.jp/category/10001"
|
||||
assert top.children == [] # 未要求子树时不展开
|
||||
|
||||
|
||||
def test_parse_categories_reads_ancestors_and_children():
|
||||
data = parse_categories(
|
||||
fixture("rakuma_category.html"), category_id="786", include_descendants=False
|
||||
)
|
||||
assert data.name == "ゲームソフト/ゲーム機本体"
|
||||
assert [node.category_id for node in data.ancestors] == ["10007"]
|
||||
assert data.full_name == "エンタメ/ホビー / ゲームソフト/ゲーム機本体"
|
||||
assert [node.category_id for node in data.children] == ["787", "788", "789"]
|
||||
assert all(node.is_leaf for node in data.children)
|
||||
assert data.is_leaf is False
|
||||
|
||||
|
||||
def test_parse_categories_marks_leaf_without_children():
|
||||
data = parse_categories(
|
||||
fixture("rakuma_category.html"), category_id="788", include_descendants=False
|
||||
)
|
||||
assert data.is_leaf is True
|
||||
assert data.children == []
|
||||
assert data.full_name == "エンタメ/ホビー / ゲームソフト/ゲーム機本体 / 家庭用ゲームソフト"
|
||||
|
||||
|
||||
def test_parse_categories_can_expand_full_subtree():
|
||||
"""整棵树本来就在一次响应里，展开子树不需要多打请求"""
|
||||
data = parse_categories(
|
||||
fixture("rakuma_category.html"), category_id="10007", include_descendants=True
|
||||
)
|
||||
branch = data.children[0]
|
||||
assert branch.category_id == "786"
|
||||
assert [node.category_id for node in branch.children] == ["787", "788", "789"]
|
||||
|
||||
|
||||
def test_parse_categories_rejects_unknown_category():
|
||||
"""无效分类要报 404，而不是当成「没有子分类」返回空结果"""
|
||||
with pytest.raises(ItemNotFoundError):
|
||||
parse_categories(
|
||||
fixture("rakuma_category.html"), category_id="99999999", include_descendants=False
|
||||
)
|
||||
|
||||
|
||||
def test_parse_categories_rejects_page_without_tree():
|
||||
with pytest.raises(ScrapeParseError):
|
||||
parse_categories("<html><body>home</body></html>", category_id=None, include_descendants=False)
|
||||
|
||||
|
||||
# ---- 取值工具 ----
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
|
||||
@@ -7,10 +7,10 @@ from __future__ import annotations
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.core import rakuma_site as site
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import ItemNotFoundError, UpstreamRequestError
|
||||
from app.services.rakuma_session import RakumaSession
|
||||
from app.scraping.core import rakuma_site as site
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import ItemNotFoundError, UpstreamRequestError
|
||||
from app.scraping.services.rakuma_session import RakumaSession
|
||||
|
||||
TARGET = "https://fril.jp/s?query=switch"
|
||||
GOOD_PAGE = '<html><body><div class="page-count">21件中 1 - 21件</div></body></html>'
|
||||
|
||||
@@ -3,9 +3,9 @@ from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import InvalidRequestError
|
||||
from app.models.scrape import ItemCondition, SearchRequest, SortOption
|
||||
from app.utils.urls import (
|
||||
from app.shared.errors import InvalidRequestError
|
||||
from app.scraping.models.scrape import ItemCondition, SearchRequest, SortOption
|
||||
from app.scraping.utils.urls import (
|
||||
build_item_url,
|
||||
build_search_url,
|
||||
item_url_parts,
|
||||
|
||||
+4
-4
@@ -4,10 +4,10 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import InvalidRequestError, ScrapeParseError
|
||||
from app.models.scrape import ShopItemsRequest, SortOption
|
||||
from app.parsers.shop import parse_shop_detail
|
||||
from app.utils.urls import build_search_url, build_shop_url, split_shop_url
|
||||
from app.shared.errors import InvalidRequestError, ScrapeParseError
|
||||
from app.scraping.models.scrape import ShopItemsRequest, SortOption
|
||||
from app.scraping.parsers.shop import parse_shop_detail
|
||||
from app.scraping.utils.urls import build_search_url, build_shop_url, split_shop_url
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
@@ -7,17 +7,17 @@ from __future__ import annotations
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.core import site
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import (
|
||||
from app.scraping.core import site
|
||||
from app.shared.config import Settings
|
||||
from app.shared.errors import (
|
||||
ItemNotFoundError,
|
||||
OffIchibaRedirectError,
|
||||
UpstreamBlockedError,
|
||||
UpstreamRequestError,
|
||||
)
|
||||
from app.parsers.subsites import build_item_page_validator
|
||||
from app.services.browser_fallback import BrowserVisit
|
||||
from app.services.site_session import SiteSession
|
||||
from app.scraping.parsers.subsites import build_item_page_validator
|
||||
from app.scraping.services.browser_fallback import BrowserVisit
|
||||
from app.scraping.services.site_session import SiteSession
|
||||
|
||||
GOOD_PAGE = '<html><script>window.__INITIAL_STATE__ = {"ok":1};</script></html>'
|
||||
BLOCK_PAGE = "<html><body>Access Denied. Reference #18.abc</body></html>"
|
||||
|
||||
@@ -7,14 +7,14 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import OffIchibaRedirectError, ScrapeParseError
|
||||
from app.parsers.subsites import (
|
||||
from app.shared.errors import OffIchibaRedirectError, ScrapeParseError
|
||||
from app.scraping.parsers.subsites import (
|
||||
SUBSITE_PARSERS,
|
||||
build_item_page_validator,
|
||||
host_of,
|
||||
parse_subsite_item,
|
||||
)
|
||||
from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
|
||||
from app.scraping.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
|
||||
|
||||
FIXTURES = Path(__file__).parent / "fixtures"
|
||||
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
"""交易服务 API 测试：健康检查、登录态查询与重载、鉴权
|
||||
|
||||
登录态会话被替换为桩，不触达真实站点、不需要真实账号。
|
||||
AuthSession 自身的行为（cookie 加载、探测判据）在 tests/test_auth_session.py。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.shared.config import get_settings
|
||||
from app.shared.errors import NotLoggedInError
|
||||
from app.trading.main import create_app
|
||||
from app.trading.services.auth_session import AuthStatus
|
||||
|
||||
TOKEN = get_settings().bearer_token
|
||||
AUTH = {"Authorization": f"Bearer {TOKEN}"}
|
||||
|
||||
|
||||
class StubAuthSession:
|
||||
"""记录调用并返回固定登录态的桩"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.checked: list[str] = []
|
||||
self.reloaded: list[str] = []
|
||||
self.logged_in = True
|
||||
|
||||
@property
|
||||
def sites(self) -> tuple[str, ...]:
|
||||
return ("rakuten", "rakuma")
|
||||
|
||||
def _status(self, site: str) -> AuthStatus:
|
||||
return AuthStatus(
|
||||
site=site,
|
||||
state_file_exists=True,
|
||||
logged_in=self.logged_in,
|
||||
checked_at=None,
|
||||
detail="stub",
|
||||
)
|
||||
|
||||
async def check(self, site: str) -> AuthStatus:
|
||||
self.checked.append(site)
|
||||
return self._status(site)
|
||||
|
||||
def status(self, site: str) -> AuthStatus:
|
||||
return self._status(site)
|
||||
|
||||
def status_all(self) -> dict[str, dict]:
|
||||
return {site: self._status(site).to_dict() for site in self.sites}
|
||||
|
||||
def reload(self, site: str) -> int:
|
||||
self.reloaded.append(site)
|
||||
return 3
|
||||
|
||||
async def require_logged_in(self, site: str) -> None:
|
||||
if not self.logged_in:
|
||||
raise NotLoggedInError(site=site, detail="stub")
|
||||
|
||||
async def close(self) -> None:
|
||||
"""lifespan 收尾会调用；桩没有真实客户端要关"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client_and_stub():
|
||||
app = create_app()
|
||||
with TestClient(app) as client:
|
||||
stub = StubAuthSession()
|
||||
app.state.container.auth_session = stub
|
||||
yield client, stub
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(client_and_stub):
|
||||
return client_and_stub[0]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def stub(client_and_stub):
|
||||
return client_and_stub[1]
|
||||
|
||||
|
||||
# ---- 健康检查 ----
|
||||
|
||||
|
||||
def test_health_needs_no_token(client):
|
||||
response = client.get("/health")
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["data"]["status"] == "ok"
|
||||
assert set(body["data"]["auth"]) == {"rakuten", "rakuma"}
|
||||
|
||||
|
||||
def test_health_does_not_probe_the_site(client, stub):
|
||||
"""健康检查只读缓存：它会被高频轮询，不能每次都去打站点"""
|
||||
client.get("/health")
|
||||
assert stub.checked == []
|
||||
|
||||
|
||||
# ---- 鉴权 ----
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/api/auth/status", "/api/auth/reload"])
|
||||
def test_auth_endpoints_reject_missing_token(client, path):
|
||||
response = client.post(path, json={})
|
||||
assert response.status_code == 401
|
||||
assert response.json()["code"] == 1001
|
||||
|
||||
|
||||
def test_auth_endpoints_reject_wrong_token(client):
|
||||
response = client.post(
|
||||
"/api/auth/status", json={}, headers={"Authorization": "Bearer wrong-token"}
|
||||
)
|
||||
assert response.status_code == 401
|
||||
|
||||
|
||||
# ---- 登录态查询 ----
|
||||
|
||||
|
||||
def test_status_probes_both_sites_by_default(client, stub):
|
||||
response = client.post("/api/auth/status", json={}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert [item["site"] for item in body["data"]["sites"]] == ["rakuten", "rakuma"]
|
||||
assert stub.checked == ["rakuten", "rakuma"]
|
||||
|
||||
|
||||
def test_status_can_skip_the_probe(client, stub):
|
||||
"""refresh=false 时读缓存，不打站点"""
|
||||
response = client.post("/api/auth/status", json={"refresh": False}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
assert stub.checked == []
|
||||
|
||||
|
||||
def test_status_accepts_a_single_site(client, stub):
|
||||
response = client.post("/api/auth/status", json={"site": "rakuma"}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
assert stub.checked == ["rakuma"]
|
||||
|
||||
|
||||
def test_status_rejects_unknown_site(client):
|
||||
"""站点名是枚举，未知值应在校验层就被挡下"""
|
||||
response = client.post("/api/auth/status", json={"site": "mercari"}, headers=AUTH)
|
||||
assert response.status_code == 422
|
||||
assert response.json()["code"] == 1002
|
||||
|
||||
|
||||
# ---- 登录态重载 ----
|
||||
|
||||
|
||||
def test_reload_reloads_then_probes(client, stub):
|
||||
response = client.post("/api/auth/reload", json={"site": "rakuten"}, headers=AUTH)
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert body["data"]["reloaded"] == {"rakuten": 3}
|
||||
# 重载后必须立刻探测一次，否则调用方拿不到「这次登录到底成没成」
|
||||
assert stub.reloaded == ["rakuten"]
|
||||
assert stub.checked == ["rakuten"]
|
||||
Reference in New Issue
Block a user