286 lines
10 KiB
Python
286 lines
10 KiB
Python
"""ラクマ（fril.jp）解析测试：基于真实页面裁剪出的夹具
|
|
|
|
夹具保留了解析所依赖的结构（微数据、规格表、埋点属性、商品卡片），
|
|
去掉了广告位与推荐位等噪音，全部离线运行。
|
|
"""
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.core.errors import InvalidRequestError, ScrapeParseError
|
|
from app.models.scrape import (
|
|
RakumaAuthenticity,
|
|
RakumaCondition,
|
|
RakumaSearchRequest,
|
|
RakumaSortOption,
|
|
RakumaTransaction,
|
|
)
|
|
from app.parsers.rakuma.base import parse_int, parse_total_count
|
|
from app.parsers.rakuma.item import parse_item_detail
|
|
from app.parsers.rakuma.search import parse_search
|
|
from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
|
from app.utils.rakuma_urls import (
|
|
build_item_url,
|
|
build_search_url,
|
|
build_shop_url,
|
|
normalize_search_url,
|
|
split_item_url,
|
|
split_shop_url,
|
|
)
|
|
|
|
FIXTURES = Path(__file__).parent / "fixtures"
|
|
|
|
|
|
def fixture(name: str) -> str:
|
|
return (FIXTURES / name).read_text(encoding="utf-8")
|
|
|
|
|
|
# ---- URL 构建 ----
|
|
|
|
def test_search_url_carries_keyword_and_default_sort():
|
|
url = build_search_url(RakumaSearchRequest(keyword="switch"))
|
|
assert "query=switch" in url
|
|
# 站点的排序由 sort + order 两个参数共同决定，默认是相关度降序
|
|
assert "sort=relevance" in url and "order=desc" in url
|
|
|
|
|
|
def test_search_url_maps_filters_to_site_codes():
|
|
"""筛选码必须与站点码表一致——站点对不认识的取值会静默返回首页"""
|
|
url = build_search_url(
|
|
RakumaSearchRequest(
|
|
keyword="switch",
|
|
category_id="788",
|
|
brand_id="5296",
|
|
min_price=3000,
|
|
max_price=20000,
|
|
conditions=[RakumaCondition.NEW, RakumaCondition.ALMOST_NEW],
|
|
transaction=RakumaTransaction.ON_SALE,
|
|
free_shipping=True,
|
|
anonymous_shipping=True,
|
|
authenticity_types=[RakumaAuthenticity.BEFORE_DELIVERY],
|
|
sort=RakumaSortOption.PRICE_ASC,
|
|
page=3,
|
|
)
|
|
)
|
|
assert "category_id=788" in url
|
|
assert "brand_id=5296" in url
|
|
assert "min=3000" in url and "max=20000" in url
|
|
assert "statuses=5%2C4" in url # 多选成色用逗号连接
|
|
assert "transaction=selling" in url
|
|
assert "carriage=1" in url
|
|
assert "anonymous_shipping=true" in url
|
|
assert "authenticity_types=pre" in url
|
|
assert "sort=sell_price" in url and "order=asc" in url
|
|
assert "page=3" in url
|
|
|
|
|
|
def test_search_url_except_for_no_brand_overrides_brand_id():
|
|
"""站点前端把两者做成互斥分支，勾选排除无品牌时不下发 brand_id"""
|
|
url = build_search_url(
|
|
RakumaSearchRequest(keyword="a", brand_id="5296", except_for_no_brand=True)
|
|
)
|
|
assert "except_for_no_brand=true" in url
|
|
assert "brand_id" not in url
|
|
|
|
|
|
def test_search_url_omits_page_on_first_page():
|
|
assert "page=" not in build_search_url(RakumaSearchRequest(keyword="switch"))
|
|
|
|
|
|
def test_search_requires_a_target():
|
|
with pytest.raises(ValueError):
|
|
RakumaSearchRequest()
|
|
|
|
|
|
def test_exclude_keyword_requires_a_keyword():
|
|
"""站点在无关键词时不接受排除词，提前拦下而不是打一次无效请求"""
|
|
with pytest.raises(ValueError):
|
|
RakumaSearchRequest(category_id="788", exclude_keyword="本体")
|
|
|
|
|
|
def test_normalize_search_url_overrides_page():
|
|
url = normalize_search_url("https://fril.jp/s?query=switch&page=2", 5)
|
|
assert "page=5" in url and "page=2" not in url
|
|
|
|
|
|
def test_normalize_search_url_rejects_foreign_host():
|
|
with pytest.raises(InvalidRequestError):
|
|
normalize_search_url("https://example.com/s?query=x", 1)
|
|
|
|
|
|
def test_item_and_shop_url_round_trip():
|
|
item_id = "4aca1d6db3e422f3a251a8a8b61e1eff"
|
|
assert split_item_url(build_item_url(item_id)) == item_id
|
|
shop_id = "422750cb7921557bc8dba2416915d968"
|
|
assert split_shop_url(build_shop_url(shop_id)) == shop_id
|
|
|
|
|
|
def test_shop_url_supports_paging_and_review_page():
|
|
shop_id = "abc123"
|
|
assert build_shop_url(shop_id, page=2).endswith("?page=2")
|
|
assert build_shop_url(shop_id, review=True).endswith("/review")
|
|
|
|
|
|
def test_split_urls_reject_wrong_host_or_path():
|
|
with pytest.raises(InvalidRequestError):
|
|
split_item_url("https://fril.jp/item/abc") # 商品页在 item.fril.jp 子域
|
|
with pytest.raises(InvalidRequestError):
|
|
split_shop_url("https://fril.jp/brand/5296")
|
|
|
|
|
|
# ---- 搜索页解析 ----
|
|
|
|
def test_parse_search_reads_items_and_exact_total():
|
|
result = parse_search(
|
|
fixture("rakuma_search.html"), request_url="u", page=1, keyword="switch"
|
|
)
|
|
assert len(result.items) == 3
|
|
# 页面上展示的是四舍五入值（約1,190,000件），精确值取自埋点属性
|
|
assert result.total_count == 1196774
|
|
assert result.request_url == "u"
|
|
|
|
item = result.items[0]
|
|
assert item.item_id == "61b174d42b58216828a24b86b7903ced"
|
|
assert item.item_number == "844649627"
|
|
assert item.price == 6299
|
|
assert item.is_sold_out is True
|
|
assert item.brand_id == "5296"
|
|
assert item.brand_name == "Nintendo Switch"
|
|
assert item.category_id == "788"
|
|
assert item.seller_user_id == "12073120"
|
|
assert item.image_url.startswith("https://img.fril.jp/")
|
|
|
|
|
|
def test_parse_search_rejects_non_search_page():
|
|
"""站点对无法识别的筛选取值会静默返回首页，必须识别出来而不是当成空结果"""
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_search("<html><body>home</body></html>", request_url="u", page=1, keyword="x")
|
|
|
|
|
|
def test_parse_search_has_more_stops_at_site_page_limit():
|
|
"""站点 page>100 直接 404，第 100 页不应再报 has_more"""
|
|
result = parse_search(
|
|
fixture("rakuma_search.html"), request_url="u", page=100, keyword="switch"
|
|
)
|
|
assert result.has_more is False
|
|
|
|
|
|
# ---- 商品详情解析 ----
|
|
|
|
def test_parse_item_detail_reads_all_blocks():
|
|
detail = parse_item_detail(fixture("rakuma_item.html"), item_id="X", item_url="u")
|
|
assert detail.item_id == "X"
|
|
assert detail.item_number == "844877033"
|
|
assert detail.item_name == "NintendoSwitch ONE PIECE 海賊無双4"
|
|
assert detail.price == 2460
|
|
assert detail.is_sold_out is False
|
|
assert detail.condition == "目立った傷や汚れなし"
|
|
assert detail.brand_id == "6454"
|
|
assert detail.category_id == "788"
|
|
assert detail.shipping_from == "北海道"
|
|
assert detail.shipping_method == "かんたんラクマパック(ヤマト運輸)"
|
|
assert detail.is_anonymous_shipping is True
|
|
assert [crumb.name for crumb in detail.breadcrumbs] == [
|
|
"エンタメ/ホビー",
|
|
"ゲームソフト/ゲーム機本体",
|
|
"家庭用ゲームソフト",
|
|
]
|
|
assert detail.images and all("img.fril.jp" in url for url in detail.images)
|
|
assert detail.seller.shop_id == "83b56a796a72011dc58b6e469270dfb7"
|
|
assert detail.seller.user_id == "3850223"
|
|
assert detail.seller.is_verified is True
|
|
|
|
|
|
def test_parse_item_detail_falls_back_to_tracking_attrs_when_sold_out():
|
|
"""已售出商品的页面换了套布局、规格表消失，需退回页面级埋点属性取值"""
|
|
detail = parse_item_detail(fixture("rakuma_item_sold.html"), item_id="Y", item_url="u")
|
|
assert detail.is_sold_out is True
|
|
assert detail.item_number == "844649627"
|
|
assert detail.condition == "新品、未使用" # 规格表已消失，取自埋点
|
|
assert detail.shipping_from == "千葉県"
|
|
assert detail.brand_id == "5296"
|
|
# 价格与名称仍由 ld+json 提供，不受布局变化影响
|
|
assert detail.price == 6299
|
|
assert detail.item_name.startswith("スプラトゥーン")
|
|
|
|
|
|
def test_parse_item_detail_rejects_non_item_page():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_item_detail("<html><body>nope</body></html>", item_id="X", item_url="u")
|
|
|
|
|
|
# ---- 店铺解析 ----
|
|
|
|
def test_parse_shop_detail_reads_profile_and_rating():
|
|
detail = parse_shop_detail(
|
|
fixture("rakuma_shop.html"), shop_id="S", shop_url="u"
|
|
)
|
|
assert detail.shop_name == "(^.^)"
|
|
assert detail.user_id == "12073120"
|
|
# 评价数只有 ld+json 的 Store 节点给得出来，可见 DOM 上没有
|
|
assert detail.review_score == 5.0
|
|
assert detail.review_count == 1886
|
|
assert detail.item_count == 21
|
|
assert detail.is_verified is True
|
|
assert detail.verification_label == "本人確認済"
|
|
assert detail.introduction
|
|
|
|
|
|
def test_parse_shop_detail_reads_reviews_when_supplied():
|
|
detail = parse_shop_detail(
|
|
fixture("rakuma_shop.html"),
|
|
shop_id="S",
|
|
shop_url="u",
|
|
review_html=fixture("rakuma_shop_review.html"),
|
|
)
|
|
assert detail.rating_breakdown.good == 118
|
|
assert detail.rating_breakdown.bad == 0
|
|
assert detail.seller_rating_breakdown.good == 110
|
|
assert len(detail.reviews) == 3
|
|
first = detail.reviews[0]
|
|
assert first.rating == "good"
|
|
assert first.reviewer_name == "トシちゃん"
|
|
assert first.reviewed_at == "2026/05/04"
|
|
|
|
|
|
def test_parse_shop_detail_leaves_reviews_empty_by_default():
|
|
"""不传评价页时不应臆造分档数据"""
|
|
detail = parse_shop_detail(fixture("rakuma_shop.html"), shop_id="S", shop_url="u")
|
|
assert detail.reviews == []
|
|
assert detail.rating_breakdown.good == 0
|
|
|
|
|
|
def test_parse_shop_items_reads_cards_and_total():
|
|
result = parse_shop_items(
|
|
fixture("rakuma_shop.html"), shop_id="S", request_url="u", page=1
|
|
)
|
|
assert result.shop_id == "S"
|
|
assert result.shop_name == "(^.^)"
|
|
assert result.total_count == 21
|
|
assert len(result.items) == 2
|
|
assert result.items[0].item_id == "61546447a7f13c34a5d015f37a7f3319"
|
|
assert result.items[0].price == 6399
|
|
|
|
|
|
def test_parse_shop_pages_reject_non_shop_page():
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_shop_detail("<html><body>x</body></html>", shop_id="S", shop_url="u")
|
|
with pytest.raises(ScrapeParseError):
|
|
parse_shop_items("<html><body>x</body></html>", shop_id="S", request_url="u", page=1)
|
|
|
|
|
|
# ---- 取值工具 ----
|
|
|
|
@pytest.mark.parametrize(
|
|
"text,expected",
|
|
[("¥6,299", 6299), ("いいね 12件", 12), ("6399", 6399), ("", 0), (None, 0)],
|
|
)
|
|
def test_parse_int_handles_site_formats(text, expected):
|
|
assert parse_int(text) == expected
|
|
|
|
|
|
def test_parse_total_count_handles_nbsp_and_rounding_prefix():
|
|
assert parse_total_count("約1,190,000件中 1\xa0-\xa040件") == (1190000, 1, 40)
|
|
assert parse_total_count("21件中 1 - 21件") == (21, 1, 21)
|
|
assert parse_total_count("no numbers here") == (0, 0, 0)
|