q792602257andClaude Opus 5
104d7fef6b
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
291 lines
10 KiB
Python
291 lines
10 KiB
Python
"""子站商品页解析测试：基于真实页面样本
|
|
|
|
夹具说明：books 的解析依赖 DOM 结构，保留了完整真实页面；brandavenue 与
|
|
biccamera 只读页面内联的 JSON，夹具做了裁剪以免把上百 KB 的噪音带进仓库。
|
|
"""
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.shared.errors import OffIchibaRedirectError, ScrapeParseError
|
|
from app.scraping.parsers.subsites import (
|
|
SUBSITE_PARSERS,
|
|
build_item_page_validator,
|
|
host_of,
|
|
parse_subsite_item,
|
|
)
|
|
from app.scraping.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
def make_page(filename: str, *, host: str, shop_code: str, item_code: str, variants: bool = True):
|
|
return SubsitePage(
|
|
html=(FIXTURES / filename).read_text(encoding="utf-8"),
|
|
requested_url=f"https://item.rakuten.co.jp/{shop_code}/{item_code}/",
|
|
final_url=f"https://{host}/item/{item_code}/",
|
|
shop_code=shop_code,
|
|
item_code=item_code,
|
|
include_sku_variants=variants,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def books_page():
|
|
return make_page("books_item.html", host="books.rakuten.co.jp", shop_code="book", item_code="17065211")
|
|
|
|
|
|
@pytest.fixture
|
|
def brandavenue_page():
|
|
return make_page(
|
|
"brandavenue_item.html", host="brandavenue.rakuten.co.jp", shop_code="stylife", item_code="md7606"
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def biccamera_page():
|
|
return make_page(
|
|
"biccamera_item.html", host="biccamera.rakuten.co.jp", shop_code="biccamera", item_code="4548736147041"
|
|
)
|
|
|
|
|
|
# ---- 通用工具 ----
|
|
|
|
@pytest.mark.parametrize(
|
|
("raw", "expected"),
|
|
[("3,091円", 3091), ("1760", 1760), (1760, 1760), ("", 0), (None, 0), ("価格未定", 0)],
|
|
)
|
|
def test_parse_price_handles_site_formats(raw, expected):
|
|
assert parse_price(raw) == expected
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("status", "expected"),
|
|
[("在庫あり", False), ("在庫なし", True), ("品切れ", True), ("販売終了", True), ("", False)],
|
|
)
|
|
def test_looks_sold_out_reads_japanese_stock_wording(status, expected):
|
|
assert looks_sold_out(status) is expected
|
|
|
|
|
|
# ---- 分派 ----
|
|
|
|
def test_registry_covers_the_three_known_subsites():
|
|
assert set(SUBSITE_PARSERS) == {
|
|
"books.rakuten.co.jp",
|
|
"brandavenue.rakuten.co.jp",
|
|
"biccamera.rakuten.co.jp",
|
|
}
|
|
|
|
|
|
def test_validator_accepts_ichiba_page_with_state():
|
|
validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/")
|
|
ok = validate("window.__INITIAL_STATE__ = {}", "https://item.rakuten.co.jp/s/c/")
|
|
assert ok is None
|
|
|
|
|
|
def test_validator_rejects_ichiba_page_without_state_as_retryable():
|
|
validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/")
|
|
reason = validate("<html>blocked</html>", "https://item.rakuten.co.jp/s/c/")
|
|
assert reason and "INITIAL_STATE" in reason
|
|
|
|
|
|
def test_validator_raises_for_unregistered_subsite():
|
|
"""未登记的站点是「不支持」而非「抓取失败」，不能走重试"""
|
|
validate = build_item_page_validator("https://item.rakuten.co.jp/s/c/")
|
|
with pytest.raises(OffIchibaRedirectError):
|
|
validate("<html>whatever</html>", "https://unknown.rakuten.co.jp/x/")
|
|
|
|
|
|
def test_validator_delegates_to_subsite_rules(books_page):
|
|
validate = build_item_page_validator(books_page.requested_url)
|
|
assert validate(books_page.html, books_page.final_url) is None
|
|
assert validate("<html>empty</html>", books_page.final_url) is not None
|
|
|
|
|
|
def test_parse_subsite_item_rejects_unknown_host(books_page):
|
|
books_page.final_url = "https://unknown.rakuten.co.jp/x/"
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_subsite_item(books_page)
|
|
|
|
|
|
def test_host_of_extracts_hostname():
|
|
assert host_of("https://books.rakuten.co.jp/rb/1/?a=b") == "books.rakuten.co.jp"
|
|
|
|
|
|
# ---- 楽天ブックス ----
|
|
|
|
def test_books_extracts_core_fields(books_page):
|
|
d = parse_subsite_item(books_page)
|
|
assert d.source == "books"
|
|
assert d.source_url == books_page.final_url
|
|
assert d.item_url == books_page.requested_url
|
|
assert d.item_name == "12歳までに知っておきたい語彙力図鑑"
|
|
assert d.price == 1760
|
|
assert d.shop.shop_code == "book"
|
|
assert d.shop.shop_name == "楽天ブックス"
|
|
|
|
|
|
def test_books_maps_review_and_stock(books_page):
|
|
d = parse_subsite_item(books_page)
|
|
assert d.review.score == pytest.approx(4.54)
|
|
assert d.review.count == 218
|
|
assert d.purchase_condition == "在庫あり"
|
|
assert d.is_sold_out is False
|
|
|
|
|
|
def test_books_uses_rms_genre_ids_for_breadcrumbs(books_page):
|
|
"""站内分类 ID 对上游没用，要取 data_genres 里的 rmsGenreId（市场侧 genre_id）"""
|
|
d = parse_subsite_item(books_page)
|
|
assert [b.name for b in d.breadcrumbs] == ["本", "絵本・児童書・図鑑", "図鑑・ちしき"]
|
|
assert d.genre_id == "208880"
|
|
assert d.breadcrumbs[0].url == "https://www.rakuten.co.jp/category/200162/"
|
|
|
|
|
|
def test_books_spec_table_becomes_sku_attributes(books_page):
|
|
d = parse_subsite_item(books_page)
|
|
spec = {a.title: a.value for a in d.sku.attributes}
|
|
assert spec["ISBN"] == "9784800590039"
|
|
assert spec["出版社"] == "日本能率協会マネジメントセンター"
|
|
assert "齋藤 孝" in spec["著者／編集"]
|
|
|
|
|
|
def test_books_description_collects_flat_sibling_sections(books_page):
|
|
"""标题与正文是兄弟节点而非父子，按容器取会漏掉绝大部分内容"""
|
|
d = parse_subsite_item(books_page)
|
|
assert len(d.description) > 1000
|
|
assert "内容紹介（JPROより）" in d.description
|
|
assert "目次" in d.description
|
|
|
|
|
|
def test_books_images_are_absolute_and_deduplicated(books_page):
|
|
d = parse_subsite_item(books_page)
|
|
assert d.images
|
|
assert all(url.startswith("https://") for url in d.images)
|
|
assert len(d.images) == len(set(d.images))
|
|
|
|
|
|
def test_books_has_no_sku_variants(books_page):
|
|
"""图书没有规格轴，SKU 只承载规格表"""
|
|
d = parse_subsite_item(books_page)
|
|
assert d.sku.inventory_type == "single"
|
|
assert d.sku.variant_count == 0
|
|
assert d.sku.axis == []
|
|
|
|
|
|
# ---- Rakuten Fashion ----
|
|
|
|
def test_brandavenue_extracts_core_fields(brandavenue_page):
|
|
d = parse_subsite_item(brandavenue_page)
|
|
assert d.source == "brandavenue"
|
|
assert d.item_name == "プリント ロゴ 刺繍 エンブロイダリー 半袖 Tシャツ"
|
|
assert d.price == 3091
|
|
assert d.catch_copy == "andme" # 品牌名
|
|
assert d.is_sold_out is False
|
|
|
|
|
|
def test_brandavenue_reads_ichiba_ids_from_rms_info(brandavenue_page):
|
|
"""市场侧的商品 ID 与 genre_id 藏在 rms_info 里，不是站内编码"""
|
|
d = parse_subsite_item(brandavenue_page)
|
|
assert d.item_id == "14674748"
|
|
assert d.genre_id == "303656"
|
|
assert d.shop.shop_id == 279405
|
|
assert d.shop.shop_code == "stylife"
|
|
|
|
|
|
def test_brandavenue_builds_color_and_size_axes(brandavenue_page):
|
|
d = parse_subsite_item(brandavenue_page)
|
|
assert [a.label for a in d.sku.axis] == ["カラー", "サイズ"]
|
|
assert d.sku.inventory_type == "multiple"
|
|
assert d.sku.variant_count == len(d.sku.variants) > 1
|
|
|
|
|
|
def test_brandavenue_variant_joins_stock_from_inventory_list(brandavenue_page):
|
|
"""售价来自 product_sku，库存与 variant_id 来自 rms_info.inventory_list，按尺码+颜色对齐"""
|
|
d = parse_subsite_item(brandavenue_page)
|
|
by_selector = {tuple(v.selector_values): v for v in d.sku.variants}
|
|
variant = by_selector[("オフホワイト×レッド", "Sサイズ")]
|
|
assert variant.price == 2810
|
|
assert variant.quantity == 90
|
|
assert variant.variant_id
|
|
|
|
|
|
def test_brandavenue_variant_without_matching_inventory_stays_purchasable(brandavenue_page):
|
|
"""站点两侧的颜色命名偶有出入导致对不上；此时库存未知，但不能据此判成售罄"""
|
|
import json
|
|
import re
|
|
|
|
html = brandavenue_page.html
|
|
state = json.loads(re.search(r"__INITIAL_STATE__ = (\{.*?\});window", html, re.S).group(1))
|
|
state["itemDetail"]["data"]["product"]["rms_info"]["inventory_list"] = []
|
|
brandavenue_page.html = re.sub(
|
|
r"__INITIAL_STATE__ = \{.*?\};window",
|
|
lambda _: f"__INITIAL_STATE__ = {json.dumps(state, ensure_ascii=False)};window",
|
|
html,
|
|
flags=re.S,
|
|
)
|
|
|
|
d = parse_subsite_item(brandavenue_page)
|
|
variant = d.sku.variants[0]
|
|
assert variant.quantity == 0
|
|
assert variant.variant_id == ""
|
|
assert variant.is_sold_out is False # 以 inventory_exist_flg 为准
|
|
|
|
|
|
def test_brandavenue_can_omit_variants_but_keeps_count(brandavenue_page):
|
|
brandavenue_page.include_sku_variants = False
|
|
d = parse_subsite_item(brandavenue_page)
|
|
assert d.sku.variants == []
|
|
assert d.sku.variant_count > 0
|
|
assert d.sku.axis
|
|
|
|
|
|
def test_brandavenue_takes_images_rendered_in_page(brandavenue_page):
|
|
"""图片按商品编号做了目录分片，分片规则不外露，只能取页面已渲染的地址"""
|
|
d = parse_subsite_item(brandavenue_page)
|
|
assert d.images
|
|
assert all("stylife/cabinet/item" in url for url in d.images)
|
|
# 主图应排在最前
|
|
assert d.images[0].lower().endswith("md7606-13_1.jpg")
|
|
|
|
|
|
def test_brandavenue_breadcrumbs_have_names_without_urls(brandavenue_page):
|
|
"""站内分类编码不是市场 genre_id，拼不出可用链接，只给名称"""
|
|
d = parse_subsite_item(brandavenue_page)
|
|
assert [b.name for b in d.breadcrumbs] == ["トップス", "カットソー・Tシャツ"]
|
|
assert all(b.url == "" for b in d.breadcrumbs)
|
|
|
|
|
|
# ---- ビックカメラ ----
|
|
|
|
def test_biccamera_extracts_core_fields(biccamera_page):
|
|
d = parse_subsite_item(biccamera_page)
|
|
assert d.source == "biccamera"
|
|
assert d.item_name.startswith("SONY")
|
|
assert d.price == 73760
|
|
assert d.item_id == "13873289"
|
|
assert d.item_code == "4548736147041"
|
|
assert d.shop.shop_id == 269553
|
|
|
|
|
|
def test_biccamera_maps_inventory_and_delivery(biccamera_page):
|
|
d = parse_subsite_item(biccamera_page)
|
|
assert d.is_sold_out is False
|
|
assert d.purchase_condition == "enabled"
|
|
assert d.sku.quantity == 28
|
|
assert d.sku.show_inventory is True
|
|
assert "出荷予定" in d.shipping.delivery_message
|
|
assert d.shipping.is_shipping_free is True
|
|
|
|
|
|
def test_biccamera_breadcrumbs_use_ichiba_genre_ids(biccamera_page):
|
|
d = parse_subsite_item(biccamera_page)
|
|
assert [b.name for b in d.breadcrumbs][0] == "TV・オーディオ・カメラ"
|
|
assert d.breadcrumbs[0].url == "https://www.rakuten.co.jp/category/211742/"
|
|
assert d.genre_id == "110110"
|
|
|
|
|
|
def test_biccamera_rejects_page_without_nuxt_state(biccamera_page):
|
|
biccamera_page.html = "<html><script>window.__NUXT__={\"state\":{}}</script></html>"
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_subsite_item(biccamera_page)
|