165 lines
6.0 KiB
Python
165 lines
6.0 KiB
Python
"""搜索页 __INITIAL_STATE__ → SearchResultData
|
|
|
|
数据位于 state.data.ichibaSearch，含 pagination 与 items 两部分。
|
|
|
|
需要注意的两个站点行为：
|
|
- items 里会混入 CPC 广告位，其 url 指向 grp07.ias.rakuten.co.jp 跳转域，
|
|
真实商品地址在 originalItemUrl；判定依据是 itemOptions.cpc 非空。
|
|
- pagination.numFound 是命中总数，但实际只能翻到 pagination.subset 条为止。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from typing import Any
|
|
|
|
from app.core import site
|
|
from app.core.errors import ScrapeParseError
|
|
from app.models.scrape import ReviewSummary, SearchItem, SearchResultData, ShopSummary
|
|
from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str
|
|
from app.utils.urls import item_url_parts
|
|
|
|
|
|
def parse_search_item(raw: dict[str, Any]) -> SearchItem:
|
|
"""解析单个搜索结果条目"""
|
|
item_options = as_dict(raw.get("itemOptions"))
|
|
is_ad = bool(item_options.get("cpc"))
|
|
|
|
# 广告位的 url 是跳转链接，真实商品地址只在 originalItemUrl 里
|
|
original_url = as_str(raw.get("originalItemUrl"))
|
|
fallback_url = as_str(raw.get("url"))
|
|
item_url = original_url or fallback_url
|
|
shop_code, item_code = item_url_parts(item_url)
|
|
if not item_code and fallback_url and fallback_url != item_url:
|
|
shop_code, item_code = item_url_parts(fallback_url)
|
|
|
|
images = [
|
|
as_str(image.get("url"))
|
|
for image in as_list(raw.get("images"))
|
|
if isinstance(image, dict) and as_str(image.get("url"))
|
|
]
|
|
|
|
raw_shop = as_dict(raw.get("shop"))
|
|
shop_review = as_dict(raw_shop.get("review"))
|
|
shop = ShopSummary(
|
|
shop_id=as_int(raw_shop.get("id")) or None,
|
|
shop_code=as_str(raw_shop.get("urlCode")) or shop_code,
|
|
shop_name=as_str(raw_shop.get("name")),
|
|
shop_url=as_str(raw_shop.get("url")),
|
|
review_score=as_float(shop_review.get("score")),
|
|
review_count=as_int(shop_review.get("count")),
|
|
)
|
|
|
|
raw_review = as_dict(raw.get("review"))
|
|
review = ReviewSummary(
|
|
score=as_float(raw_review.get("score")),
|
|
count=as_int(raw_review.get("numReviews")),
|
|
url=as_str(raw_review.get("url")),
|
|
)
|
|
|
|
genre_path = as_str(raw.get("genreIdList"))
|
|
genre_names = [
|
|
as_str(genre.get("name"))
|
|
for genre in as_list(raw.get("genres"))
|
|
if isinstance(genre, dict) and as_str(genre.get("name"))
|
|
]
|
|
# genreIdList 形如 /0/101205/565950/566404，末段为最具体的分类；根节点 0 不算
|
|
genre_id = next(
|
|
(segment for segment in reversed(genre_path.split("/")) if segment and segment != "0"),
|
|
"",
|
|
)
|
|
|
|
raw_shipping = as_dict(raw.get("shipping"))
|
|
shipping_price = raw_shipping.get("price")
|
|
sku_info = as_dict(raw.get("skuInfo"))
|
|
|
|
return SearchItem(
|
|
item_id=as_str(raw.get("code")),
|
|
item_code=item_code,
|
|
item_name=as_str(raw.get("name")),
|
|
item_url=item_url,
|
|
catch_copy=as_str(raw.get("subtitle")),
|
|
price=as_int(raw.get("price")),
|
|
price_range=as_str(sku_info.get("priceRange")),
|
|
has_price_range=bool(raw.get("hasPriceRange")),
|
|
image_url=images[0] if images else "",
|
|
image_urls=images,
|
|
shop=shop,
|
|
review=review,
|
|
genre_id=genre_id,
|
|
genre_path=genre_path,
|
|
genre_names=genre_names,
|
|
shipping_fee=as_int(shipping_price) if shipping_price is not None else None,
|
|
delivery_message=as_str(raw_shipping.get("estimateDeliveryDay")),
|
|
point_count=as_int(as_dict(raw.get("point")).get("count")),
|
|
is_sold_out=bool(raw.get("isSoldOut")),
|
|
is_ad=is_ad,
|
|
has_multi_sku=bool(sku_info.get("hasMultiSku")),
|
|
variant_id=as_str(raw.get("variantId")),
|
|
item_options=item_options,
|
|
)
|
|
|
|
|
|
def parse_search(
|
|
state: dict[str, Any],
|
|
*,
|
|
request_url: str,
|
|
page: int,
|
|
exclude_ads: bool,
|
|
) -> SearchResultData:
|
|
"""把搜索页状态解析为搜索结果
|
|
|
|
Raises:
|
|
ScrapeParseError: 状态中不存在 ichibaSearch 节点，或站点返回了错误
|
|
"""
|
|
data = as_dict(as_dict(state.get("state")).get("data"))
|
|
search = data.get("ichibaSearch")
|
|
if not isinstance(search, dict):
|
|
raise ScrapeParseError("搜索结果缺少 ichibaSearch 节点")
|
|
|
|
error = search.get("error")
|
|
if error:
|
|
raise ScrapeParseError(f"站点返回搜索错误：{error}")
|
|
|
|
pagination = as_dict(search.get("pagination"))
|
|
ui_question = as_dict(data.get("effectiveUiQuestion"))
|
|
|
|
# 站点声明的每页条数；items 实际长度会因为混入广告位而略大于它
|
|
page_size = as_int(pagination.get("pageSize"))
|
|
total_count = as_int(pagination.get("numFound"))
|
|
subset = as_int(pagination.get("subset")) or site.DEFAULT_SUBSET_LIMIT
|
|
reachable_count = min(total_count, subset) if total_count else 0
|
|
start = as_int(pagination.get("start"))
|
|
|
|
# 请求页超出可达窗口时，站点会静默回绕到第 1 页并把 start/page 重置为 0/1。
|
|
# 拿站点回报的页码与请求页码对账即可识别，比自行按 subset 推算更可靠。
|
|
site_page = as_int(ui_question.get("page"), page)
|
|
out_of_range = page > 1 and site_page != page
|
|
|
|
raw_items = [item for item in as_list(search.get("items")) if isinstance(item, dict)]
|
|
items = [parse_search_item(item) for item in raw_items]
|
|
ad_count = sum(1 for item in items if item.is_ad)
|
|
if exclude_ads:
|
|
items = [item for item in items if not item.is_ad]
|
|
if out_of_range:
|
|
# 这一页的内容是第 1 页的副本，交给上游只会造成重复入库
|
|
items = []
|
|
ad_count = 0
|
|
|
|
has_more = (
|
|
not out_of_range
|
|
and bool(raw_items)
|
|
and (start + (page_size or len(raw_items))) < reachable_count
|
|
)
|
|
|
|
return SearchResultData(
|
|
keyword=as_str(ui_question.get("keywords")),
|
|
page=page,
|
|
page_size=page_size,
|
|
total_count=total_count,
|
|
reachable_count=reachable_count,
|
|
has_more=has_more,
|
|
out_of_range=out_of_range,
|
|
ad_count=ad_count,
|
|
request_url=request_url,
|
|
items=items,
|
|
)
|