q792602257andClaude Opus 5
104d7fef6b
把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」， 而是抓取无状态、幂等、可多开实例，而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询， 同一账号会被并发操作。 - app/shared：配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping：站点常量、会话、解析器与 10 个抓取接口，:31107，可多开 - app/trading：登录态查询/重载与健康检查，:31108，只能单实例 - 依赖方向锁为 scraping→shared、trading→shared，两侧互不 import； tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有：与抓取 UA 值相同但变更理由不同，抓取 UA 为绕 反爬可随时调整，登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in，判据只写一遍 - 同一镜像两个启动命令，交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证：239 个离线用例全绿；两个入口真实启动，/health 与鉴权正常。 未验证：真实探测登录态（当前开发机无外网，对站点的连接全部超时）。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
148 lines
6.1 KiB
Python
148 lines
6.1 KiB
Python
"""解析器测试：基于真实页面状态样本"""
|
|
import pytest
|
|
|
|
from app.shared.errors import ScrapeParseError
|
|
from app.scraping.parsers.item import parse_item_detail
|
|
from app.scraping.parsers.search import parse_search
|
|
from app.scraping.parsers.state import extract_initial_state
|
|
|
|
|
|
# ---- __INITIAL_STATE__ 抽取 ----
|
|
|
|
def test_extract_initial_state_stops_at_end_of_json_object():
|
|
"""赋值语句后面紧跟其他脚本代码，不能整段吃掉"""
|
|
html = '<script>window.__INITIAL_STATE__ = {"a": {"b": 1}};window.__LOCALE__="ja";</script>'
|
|
assert extract_initial_state(html) == {"a": {"b": 1}}
|
|
|
|
|
|
def test_extract_initial_state_reports_missing_marker():
|
|
with pytest.raises(ScrapeParseError):
|
|
extract_initial_state("<html><body>blocked</body></html>")
|
|
|
|
|
|
def test_extract_initial_state_reports_broken_json():
|
|
with pytest.raises(ScrapeParseError):
|
|
extract_initial_state("window.__INITIAL_STATE__ = {oops")
|
|
|
|
|
|
# ---- 搜索结果 ----
|
|
|
|
def test_search_excludes_ads_by_default_but_still_counts_them(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
assert result.ad_count == 1
|
|
assert all(not item.is_ad for item in result.items)
|
|
assert len(result.items) == 2
|
|
|
|
|
|
def test_search_can_keep_ads_and_resolves_their_real_item_url(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=False)
|
|
ads = [item for item in result.items if item.is_ad]
|
|
assert len(ads) == 1
|
|
# 广告位的 url 是跳转域，必须还原成 originalItemUrl 才能拿到商品编号
|
|
assert ads[0].item_url.startswith("https://item.rakuten.co.jp/")
|
|
assert ads[0].shop.shop_code and ads[0].item_code
|
|
|
|
|
|
def test_search_item_carries_fields_needed_to_call_item_detail(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
item = result.items[0]
|
|
assert item.item_name
|
|
assert item.price > 0
|
|
# shop.shop_code + item_code 就是 /api/item_detail 的入参
|
|
assert item.shop.shop_code and item.item_code
|
|
assert item.image_url.startswith("https://")
|
|
assert item.genre_id.isdigit()
|
|
|
|
|
|
def test_search_pagination_is_capped_by_site_subset(search_state):
|
|
result = parse_search(search_state, request_url="https://x", page=1, exclude_ads=True)
|
|
assert result.total_count > 0
|
|
# 站点声明命中数很大，但实际只能翻到 subset 条为止
|
|
assert result.reachable_count <= result.total_count
|
|
assert result.reachable_count <= 6750
|
|
|
|
|
|
def test_search_reports_request_url(search_state):
|
|
result = parse_search(search_state, request_url="https://search.rakuten.co.jp/x", page=1, exclude_ads=True)
|
|
assert result.request_url == "https://search.rakuten.co.jp/x"
|
|
|
|
|
|
def test_search_flags_out_of_range_page_and_drops_wrapped_items(search_state):
|
|
"""越过可达窗口时站点会静默回绕到第 1 页，必须识别出来而不是把重复数据交上去"""
|
|
assert search_state["state"]["data"]["effectiveUiQuestion"]["page"] == 1
|
|
result = parse_search(search_state, request_url="https://x", page=10, exclude_ads=True)
|
|
assert result.out_of_range is True
|
|
assert result.items == []
|
|
assert result.has_more is False
|
|
assert result.page == 10 # 回报请求页码，而不是站点回绕后的 1
|
|
|
|
|
|
def test_search_within_range_is_not_flagged_out_of_range(search_state):
|
|
search_state["state"]["data"]["effectiveUiQuestion"]["page"] = 3
|
|
result = parse_search(search_state, request_url="https://x", page=3, exclude_ads=True)
|
|
assert result.out_of_range is False
|
|
assert result.items
|
|
|
|
|
|
def test_search_rejects_state_without_search_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_search({"state": {"data": {}}}, request_url="https://x", page=1, exclude_ads=True)
|
|
|
|
|
|
# ---- 商品详情 ----
|
|
|
|
def test_item_detail_extracts_core_fields(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://item.rakuten.co.jp/fafachai/tx-300/",
|
|
shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.source == "ichiba"
|
|
assert detail.source_url == "https://item.rakuten.co.jp/fafachai/tx-300/"
|
|
assert detail.item_name
|
|
assert detail.item_code
|
|
assert detail.price > 0
|
|
assert detail.images and all(url.startswith("https://") for url in detail.images)
|
|
assert detail.shop.shop_id and detail.shop.shop_code
|
|
assert detail.genre_id.isdigit()
|
|
assert detail.breadcrumbs
|
|
|
|
|
|
def test_item_detail_purchasable_item_is_not_sold_out(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.purchase_condition == "enabled"
|
|
assert detail.is_sold_out is False
|
|
|
|
|
|
def test_item_detail_parses_sku_axis_and_variants(item_state):
|
|
detail = parse_item_detail(
|
|
item_state, item_url="https://x", shop_code="fafachai", include_sku_variants=True,
|
|
)
|
|
assert detail.sku.inventory_type == "multiple"
|
|
assert detail.sku.axis and detail.sku.axis[0].values
|
|
assert detail.sku.variants
|
|
variant = detail.sku.variants[0]
|
|
assert variant.variant_id and variant.price > 0
|
|
assert len(variant.selector_values) == len(detail.sku.axis)
|
|
|
|
|
|
def test_item_detail_can_omit_variants_but_keeps_their_count(item_state):
|
|
"""SKU 组合可能上百条，关闭后应只省略明细、不丢失规模信息"""
|
|
full = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=True)
|
|
slim = parse_item_detail(item_state, item_url="https://x", shop_code="s", include_sku_variants=False)
|
|
assert slim.sku.variants == []
|
|
assert slim.sku.variant_count == full.sku.variant_count > 0
|
|
assert slim.sku.axis == full.sku.axis
|
|
|
|
|
|
def test_item_detail_falls_back_to_request_shop_code_when_state_lacks_it(item_state):
|
|
item_state["shop"]["information"]["shopUrl"] = ""
|
|
detail = parse_item_detail(item_state, item_url="https://x", shop_code="fallback", include_sku_variants=False)
|
|
assert detail.shop.shop_code == "fallback"
|
|
|
|
|
|
def test_item_detail_rejects_state_without_item_node():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_item_detail({}, item_url="https://x", shop_code="s", include_sku_variants=True)
|