q792602257andClaude Opus 5
104d7fef6b
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
102 lines
3.4 KiB
Python
102 lines
3.4 KiB
Python
"""乐天商家（店铺）解析与请求转换测试"""
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.shared.errors import InvalidRequestError, ScrapeParseError
|
|
from app.scraping.models.scrape import ShopItemsRequest, SortOption
|
|
from app.scraping.parsers.shop import parse_shop_detail
|
|
from app.scraping.utils.urls import build_search_url, build_shop_url, split_shop_url
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
@pytest.fixture
|
|
def shop_state() -> dict:
|
|
return json.loads((FIXTURES / "shop_state.json").read_text(encoding="utf-8"))
|
|
|
|
|
|
@pytest.fixture
|
|
def shop_html() -> str:
|
|
return (FIXTURES / "shop_page.html").read_text(encoding="utf-8")
|
|
|
|
|
|
# ---- 解析 ----
|
|
|
|
def test_parse_shop_detail_reads_state_and_logo(shop_state, shop_html):
|
|
detail = parse_shop_detail(shop_state, shop_code="edion", html=shop_html)
|
|
assert detail.shop_id == 272415
|
|
assert detail.shop_code == "edion"
|
|
assert detail.shop_name == "エディオン 楽天市場店"
|
|
assert detail.review_score == 4.51
|
|
assert detail.review_count == 128136
|
|
assert detail.review_displayed is True
|
|
assert detail.is_39_shop is True
|
|
assert detail.status == 1
|
|
assert detail.introduction
|
|
assert detail.signboard_url.startswith("https://")
|
|
# logo 不在 state 里，只能从页面的 ld+json 微数据取
|
|
assert detail.logo_url.startswith("https://thumbnail.image.rakuten.co.jp/")
|
|
|
|
|
|
def test_parse_shop_detail_normalizes_shop_url(shop_state):
|
|
"""state 里的 shopUrl 是 http 且不带尾斜杠，对外统一成规范形式"""
|
|
detail = parse_shop_detail(shop_state, shop_code="edion")
|
|
assert detail.shop_url == "https://www.rakuten.co.jp/edion/"
|
|
|
|
|
|
def test_parse_shop_detail_without_html_leaves_logo_empty(shop_state):
|
|
assert parse_shop_detail(shop_state, shop_code="edion").logo_url == ""
|
|
|
|
|
|
def test_parse_shop_detail_rejects_page_without_shop_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_shop_detail({"state": {}}, shop_code="edion")
|
|
|
|
|
|
# ---- URL ----
|
|
|
|
def test_shop_url_round_trip():
|
|
assert split_shop_url(build_shop_url("edion")) == "edion"
|
|
|
|
|
|
def test_split_shop_url_rejects_reserved_paths():
|
|
"""/category/ 之类的站点自有路径不是店铺页，别把 category 当成店铺代码"""
|
|
with pytest.raises(InvalidRequestError):
|
|
split_shop_url("https://www.rakuten.co.jp/category/101205/")
|
|
|
|
|
|
def test_split_shop_url_rejects_foreign_host():
|
|
with pytest.raises(InvalidRequestError):
|
|
split_shop_url("https://item.rakuten.co.jp/edion/123/")
|
|
|
|
|
|
# ---- 商家商品：转成限定店铺的搜索 ----
|
|
|
|
def test_shop_items_request_converts_to_shop_scoped_search():
|
|
payload = ShopItemsRequest(
|
|
shop_code="edion", page=2, keyword="テレビ", sort=SortOption.PRICE_ASC, max_price=50000
|
|
)
|
|
search = payload.to_search_request(272415)
|
|
assert search.shop_id == 272415
|
|
assert search.keyword == "テレビ"
|
|
assert search.page == 2
|
|
|
|
url = build_search_url(search)
|
|
assert "sid=272415" in url
|
|
assert "p=2" in url
|
|
assert "s=2" in url # price_asc
|
|
assert "max=50000" in url
|
|
|
|
|
|
def test_shop_items_request_needs_an_identifier():
|
|
with pytest.raises(ValueError):
|
|
ShopItemsRequest()
|
|
|
|
|
|
def test_search_accepts_shop_id_alone_as_target():
|
|
"""只按店铺取商品是合法查询：站点的 sid-only 搜索能正常返回"""
|
|
url = build_search_url(ShopItemsRequest(shop_id=272415).to_search_request(272415))
|
|
assert "sid=272415" in url
|