q792602257andClaude Opus 5
104d7fef6b
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
117 lines
3.5 KiB
Python
117 lines
3.5 KiB
Python
"""ラクマ 会话层测试：并发限流下的重试与错误映射
|
|
|
|
全部用 httpx.MockTransport 拦截，不触达真实站点。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
from app.scraping.core import rakuma_site as site
|
|
from app.shared.config import Settings
|
|
from app.shared.errors import ItemNotFoundError, UpstreamRequestError
|
|
from app.scraping.services.rakuma_session import RakumaSession
|
|
|
|
TARGET = "https://fril.jp/s?query=switch"
|
|
GOOD_PAGE = '<html><body><div class="page-count">21件中 1 - 21件</div></body></html>'
|
|
|
|
|
|
def make_settings(**overrides) -> Settings:
|
|
base = {
|
|
"http_max_attempts": 3,
|
|
"request_timeout_seconds": 5.0,
|
|
"max_site_concurrency": 4,
|
|
}
|
|
base.update(overrides)
|
|
return Settings(**base)
|
|
|
|
|
|
async def build_session(handler, *, settings=None) -> RakumaSession:
|
|
"""构建 RakumaSession 并把 client 换成 MockTransport 版本"""
|
|
session = RakumaSession(settings or make_settings())
|
|
await session.start()
|
|
await session._client.aclose()
|
|
session._client = httpx.AsyncClient(
|
|
headers=site.default_headers(),
|
|
transport=httpx.MockTransport(handler),
|
|
follow_redirects=True,
|
|
)
|
|
return session
|
|
|
|
|
|
async def test_fetch_returns_page_without_warmup():
|
|
"""ラクマ 无 Akamai 限速，不应像乐天那样先打一次首页预热"""
|
|
seen: list[str] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
seen.append(str(request.url))
|
|
return httpx.Response(200, text=GOOD_PAGE)
|
|
|
|
session = await build_session(handler)
|
|
try:
|
|
assert await session.fetch_html(TARGET) == GOOD_PAGE
|
|
finally:
|
|
await session.close()
|
|
|
|
assert seen == [TARGET]
|
|
|
|
|
|
async def test_404_fails_fast_without_retrying():
|
|
"""商品下架或 ID 不存在时重试没有意义"""
|
|
attempts = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
attempts["n"] += 1
|
|
return httpx.Response(404, text="not found")
|
|
|
|
session = await build_session(handler)
|
|
try:
|
|
with pytest.raises(ItemNotFoundError):
|
|
await session.fetch_html("https://item.fril.jp/deadbeef")
|
|
finally:
|
|
await session.close()
|
|
assert attempts["n"] == 1
|
|
|
|
|
|
async def test_retry_recovers_when_a_later_attempt_succeeds():
|
|
attempts = {"n": 0}
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
attempts["n"] += 1
|
|
if attempts["n"] == 1:
|
|
return httpx.Response(503, text="unavailable")
|
|
return httpx.Response(200, text=GOOD_PAGE)
|
|
|
|
session = await build_session(handler)
|
|
try:
|
|
assert await session.fetch_html(TARGET) == GOOD_PAGE
|
|
finally:
|
|
await session.close()
|
|
assert attempts["n"] == 2
|
|
|
|
|
|
async def test_persistent_5xx_surfaces_as_request_error():
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(503, text="unavailable")
|
|
|
|
session = await build_session(handler)
|
|
try:
|
|
with pytest.raises(UpstreamRequestError):
|
|
await session.fetch_html(TARGET)
|
|
finally:
|
|
await session.close()
|
|
|
|
|
|
async def test_status_reports_readiness():
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(200, text=GOOD_PAGE)
|
|
|
|
session = RakumaSession(make_settings())
|
|
assert session.status()["ready"] is False
|
|
session = await build_session(handler)
|
|
try:
|
|
assert session.status()["ready"] is True
|
|
finally:
|
|
await session.close()
|
|
assert session.status()["ready"] is False
|