把需要账号登录态的链路从抓取服务里拆出成独立进程。分界线不是「要不要登录」, 而是抓取无状态、幂等、可多开实例,而交易的写操作不可逆、登录态全局唯一、 订单监控是常驻轮询——同进程时抓取一扩容就会复制出 N 份登录态与 N 个轮询, 同一账号会被并发操作。 - app/shared:配置、错误码、日志、ApiResponse 信封 + Bearer 鉴权 + 异常处理器、 导航请求头构造器 - app/scraping:站点常量、会话、解析器与 10 个抓取接口,:31107,可多开 - app/trading:登录态查询/重载与健康检查,:31108,只能单实例 - 依赖方向锁为 scraping→shared、trading→shared,两侧互不 import; tests/test_architecture.py 用 AST 检查 import 并校验两个 app 的路径不串 - 登录态 UA 在 trading 独立持有:与抓取 UA 值相同但变更理由不同,抓取 UA 为绕 反爬可随时调整,登录 UA 一改可能触发设备校验使已落盘 cookie 失效 - scripts/login.py 与 AuthSession 共用 auth_site.PROFILES 与 is_logged_in,判据只写一遍 - 同一镜像两个启动命令,交易容器覆盖 command 并设 RAKUTEN_HEALTH_PORT 同时带上此前未提交的 ラクマ 分类接口与登录态基础设施。 验证:239 个离线用例全绿;两个入口真实启动,/health 与鉴权正常。 未验证:真实探测登录态(当前开发机无外网,对站点的连接全部超时)。 Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
188 lines
7.9 KiB
Python
188 lines
7.9 KiB
Python
"""加购信息(purchase 块)解析测试
|
|
|
|
契约取自各站点前端构造加购请求的逻辑,四个来源的端点与字段名互不相同,
|
|
因此这里逐来源验证「提交到哪里、带哪些固定字段、数量/规格/选项各用哪个字段名」。
|
|
本服务只提供这些数据,不执行加购。
|
|
"""
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.scraping.parsers.item import parse_item_detail, parse_purchase_options
|
|
from app.scraping.parsers.subsites import parse_subsite_item
|
|
from app.scraping.parsers.subsites.base import SubsitePage
|
|
from app.scraping.parsers.subsites.brandavenue import _resolve_cart_url
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
ICHIBA_URL = "https://item.rakuten.co.jp/fafachai/tx-300/"
|
|
|
|
|
|
def subsite(filename: str, host: str, shop_code: str, item_code: str):
|
|
return parse_subsite_item(
|
|
SubsitePage(
|
|
html=(FIXTURES / filename).read_text(encoding="utf-8"),
|
|
requested_url=f"https://item.rakuten.co.jp/{shop_code}/{item_code}/",
|
|
final_url=f"https://{host}/item/{item_code}/",
|
|
shop_code=shop_code,
|
|
item_code=item_code,
|
|
include_sku_variants=True,
|
|
)
|
|
)
|
|
|
|
|
|
# ---- 市场 ----
|
|
|
|
def test_ichiba_cart_url_comes_from_item_not_hardcoded(item_state):
|
|
"""basketDomain 逐商品不同(不同店铺落在不同 basket 集群),不能写死"""
|
|
d = parse_item_detail(item_state, item_url=ICHIBA_URL, shop_code="fafachai", include_sku_variants=True)
|
|
assert d.purchase.cart_url == "https://ts.sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set"
|
|
|
|
|
|
def test_ichiba_form_fields_match_site_contract(item_state):
|
|
d = parse_item_detail(item_state, item_url=ICHIBA_URL, shop_code="fafachai", include_sku_variants=True)
|
|
fields = d.purchase.form_fields
|
|
assert fields["shop_bid"] == str(d.shop.shop_id)
|
|
assert fields["item_id"] == d.item_id
|
|
assert fields["__event"] == "ES01_003_001"
|
|
assert d.purchase.quantity_field == "units"
|
|
assert d.purchase.variant_field == "variant_id"
|
|
|
|
|
|
def test_ichiba_multi_sku_reports_inventory_flag_2_without_prefilled_variant(item_state):
|
|
"""多规格商品必须由调用方选定 SKU,不能预填"""
|
|
d = parse_item_detail(item_state, item_url=ICHIBA_URL, shop_code="fafachai", include_sku_variants=True)
|
|
assert d.sku.inventory_type == "multiple"
|
|
assert d.purchase.form_fields["inventory_flag"] == "2"
|
|
assert "variant_id" not in d.purchase.form_fields
|
|
|
|
|
|
def test_ichiba_single_sku_prefills_its_only_variant(item_state):
|
|
"""单一库存商品规格固定,直接填好,调用方无需再选"""
|
|
item_state["purchase"]["sku"]["inventoryType"] = "single"
|
|
item_state["item"]["variantId"] = "4902370549263"
|
|
d = parse_item_detail(item_state, item_url=ICHIBA_URL, shop_code="fafachai", include_sku_variants=True)
|
|
assert d.purchase.form_fields["inventory_flag"] == "1"
|
|
assert d.purchase.form_fields["variant_id"] == "4902370549263"
|
|
|
|
|
|
def test_ichiba_selected_variant_id_is_available_from_sku(item_state):
|
|
"""调用方要往 variant_field 里填的值,来自 sku.variants[].variant_id"""
|
|
d = parse_item_detail(item_state, item_url=ICHIBA_URL, shop_code="fafachai", include_sku_variants=True)
|
|
assert d.sku.variants[0].variant_id
|
|
|
|
|
|
def test_item_without_options_has_no_options_field(item_state):
|
|
d = parse_item_detail(item_state, item_url=ICHIBA_URL, shop_code="fafachai", include_sku_variants=True)
|
|
assert d.purchase.options == []
|
|
assert d.purchase.options_field == ""
|
|
assert d.purchase.has_required_options is False
|
|
|
|
|
|
# ---- 商品选项 ----
|
|
|
|
def test_options_are_parsed_with_types_and_required_flag(item_with_options_state):
|
|
d = parse_item_detail(
|
|
item_with_options_state,
|
|
item_url="https://item.rakuten.co.jp/kizamu/set-beermug-031/",
|
|
shop_code="kizamu",
|
|
include_sku_variants=True,
|
|
)
|
|
assert d.purchase.options_field == "choice"
|
|
assert d.purchase.has_required_options is True
|
|
types = {o.type for o in d.purchase.options}
|
|
assert {"select", "text"} <= types
|
|
required = [o for o in d.purchase.options if o.is_required]
|
|
assert required and required[0].name
|
|
|
|
|
|
def test_select_option_carries_candidate_values(item_with_options_state):
|
|
d = parse_item_detail(
|
|
item_with_options_state, item_url="https://x", shop_code="kizamu", include_sku_variants=True
|
|
)
|
|
select = next(o for o in d.purchase.options if o.type == "select")
|
|
assert select.values
|
|
assert all(v.name for v in select.values)
|
|
|
|
|
|
def test_text_option_has_no_candidate_values(item_with_options_state):
|
|
"""自由文本选项由买家填写,没有候选值"""
|
|
d = parse_item_detail(
|
|
item_with_options_state, item_url="https://x", shop_code="kizamu", include_sku_variants=True
|
|
)
|
|
text = next(o for o in d.purchase.options if o.type == "text")
|
|
assert text.values == []
|
|
|
|
|
|
def test_parse_purchase_options_tolerates_malformed_input():
|
|
assert parse_purchase_options(None) == []
|
|
assert parse_purchase_options([None, "x", 1]) == []
|
|
assert parse_purchase_options([{"id": 1, "name": "a", "type": "select", "values": [None]}])[0].values == []
|
|
|
|
|
|
# ---- 楽天ブックス ----
|
|
|
|
def test_books_item_id_comes_from_cart_form_not_url():
|
|
"""加购要用表单里的 item_id;URL 上的编号是市场侧商品编号,拿去下单会失败"""
|
|
d = subsite("books_item.html", "books.rakuten.co.jp", "book", "17065211")
|
|
assert d.item_code == "17065211"
|
|
assert d.item_id == "20600328"
|
|
assert d.purchase.form_fields["item_id"] == "20600328"
|
|
|
|
|
|
def test_books_cart_endpoint_and_shop_bid():
|
|
d = subsite("books_item.html", "books.rakuten.co.jp", "book", "17065211")
|
|
assert d.purchase.cart_url == "https://sp.books.step.rakuten.co.jp/rms/mall/book/bs/Cart"
|
|
assert d.purchase.form_fields["shop_bid"] == "213310"
|
|
|
|
|
|
def test_books_cart_form_has_no_quantity_or_variant_field():
|
|
"""该站表单不支持指定件数与规格,字段名留空以示不可用"""
|
|
d = subsite("books_item.html", "books.rakuten.co.jp", "book", "17065211")
|
|
assert d.purchase.quantity_field == ""
|
|
assert d.purchase.variant_field == ""
|
|
|
|
|
|
# ---- Rakuten Fashion ----
|
|
|
|
@pytest.mark.parametrize(
|
|
("cart_url_type", "expected"),
|
|
[
|
|
("6", "https://sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set"),
|
|
("5", "https://basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set"),
|
|
("1", "https://ts.basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set"),
|
|
("https://custom.example.com/cart", "https://custom.example.com/cart"),
|
|
("99", ""),
|
|
("", ""),
|
|
],
|
|
)
|
|
def test_brandavenue_cart_url_type_mapping(cart_url_type, expected):
|
|
"""端点由 cart_url_type 编号映射得到;已经是 URL 时原样使用"""
|
|
assert _resolve_cart_url(cart_url_type) == expected
|
|
|
|
|
|
def test_brandavenue_uses_same_contract_as_ichiba():
|
|
d = subsite("brandavenue_item.html", "brandavenue.rakuten.co.jp", "stylife", "md7606")
|
|
assert d.purchase.cart_url == "https://sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set"
|
|
assert d.purchase.form_fields["shop_bid"] == "279405"
|
|
assert d.purchase.form_fields["item_id"] == "14674748"
|
|
assert d.purchase.form_fields["inventory_flag"] == "2"
|
|
assert d.purchase.quantity_field == "units"
|
|
assert d.purchase.variant_field == "variant_id"
|
|
|
|
|
|
def test_brandavenue_variant_ids_are_usable_for_cart():
|
|
"""variant_field 要填的值就是 sku.variants[].variant_id(来自 inventory_list)"""
|
|
d = subsite("brandavenue_item.html", "brandavenue.rakuten.co.jp", "stylife", "md7606")
|
|
assert all(v.variant_id for v in d.sku.variants if v.quantity)
|
|
|
|
|
|
# ---- ビックカメラ ----
|
|
|
|
def test_biccamera_uses_its_own_json_endpoint_without_shop_bid():
|
|
d = subsite("biccamera_item.html", "biccamera.rakuten.co.jp", "biccamera", "4548736147041")
|
|
assert d.purchase.cart_url == "https://biccamera.step.rakuten.co.jp/cart/item"
|
|
assert d.purchase.form_fields == {"item_id": "13873289"}
|
|
assert d.purchase.quantity_field == "units"
|
|
assert "shop_bid" not in d.purchase.form_fields
|