Init
This commit is contained in:
@@ -0,0 +1,57 @@
|
||||
"""子站解析器注册表与商品页分派
|
||||
|
||||
乐天部分官方旗舰店的商品页会 302 跳出 item.rakuten.co.jp，落到各自独立的站点。
|
||||
这里按落地域名把页面分派给对应解析器；落到未登记的站点时抛 OffIchibaRedirectError，
|
||||
让上游能明确区分「站点不支持」与「被反爬拦截」，而不是白白重试。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import OffIchibaRedirectError, ScrapeParseError
|
||||
from app.models.scrape import ItemDetailData
|
||||
from app.parsers.state import PageValidator, require_state_marker
|
||||
from app.parsers.subsites import biccamera, books, brandavenue
|
||||
from app.parsers.subsites.base import SubsitePage, SubsiteParser
|
||||
|
||||
SUBSITE_PARSERS: dict[str, SubsiteParser] = {
|
||||
module.HOST: SubsiteParser(
|
||||
host=module.HOST,
|
||||
source=module.SOURCE,
|
||||
shop_name=module.SHOP_NAME,
|
||||
validate=module.validate,
|
||||
parse=module.parse,
|
||||
)
|
||||
for module in (books, brandavenue, biccamera)
|
||||
}
|
||||
|
||||
|
||||
def host_of(url: str) -> str:
|
||||
return urlsplit(url).hostname or ""
|
||||
|
||||
|
||||
def build_item_page_validator(requested_url: str) -> PageValidator:
|
||||
"""构造商品页校验器：按落地域名选用对应的页面校验规则
|
||||
|
||||
落到未登记的站点时直接抛错——换 cookie 或上浏览器都改变不了页面归属。
|
||||
"""
|
||||
|
||||
def validate(html: str, final_url: str) -> str | None:
|
||||
host = host_of(final_url)
|
||||
if host == site.ITEM_HOST:
|
||||
return require_state_marker(html, final_url)
|
||||
parser = SUBSITE_PARSERS.get(host)
|
||||
if parser is None:
|
||||
raise OffIchibaRedirectError(requested_url, final_url)
|
||||
return parser.validate(html)
|
||||
|
||||
return validate
|
||||
|
||||
|
||||
def parse_subsite_item(page: SubsitePage) -> ItemDetailData:
|
||||
"""把子站页面交给对应解析器"""
|
||||
parser = SUBSITE_PARSERS.get(host_of(page.final_url))
|
||||
if parser is None:
|
||||
raise ScrapeParseError(f"没有匹配的子站解析器：{page.final_url}")
|
||||
return parser.parse(page)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""子站解析器的公共契约与工具
|
||||
|
||||
乐天部分官方旗舰店的商品页会跳出市场域名，落到各自独立的站点上。这些站点
|
||||
技术栈各不相同（微数据 / Vue SSR state / Nuxt state），但对外要收敛成同一个
|
||||
ItemDetailData，因此这里定义统一的输入结构与注册契约。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.models.scrape import ItemDetailData
|
||||
|
||||
# 售罄判定关键词：日文页面上表示不可购买的常见措辞
|
||||
SOLD_OUT_MARKERS = ("在庫なし", "在庫切れ", "品切れ", "入荷未定", "販売終了", "取扱終了")
|
||||
|
||||
_DIGITS_RE = re.compile(r"\d+")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SubsitePage:
|
||||
"""一个待解析的子站页面及其上下文"""
|
||||
|
||||
html: str
|
||||
requested_url: str # 请求时的 item.rakuten.co.jp 地址
|
||||
final_url: str # 跳转后的实际地址
|
||||
shop_code: str # 市场侧店铺代码，如 book / stylife / biccamera
|
||||
item_code: str # 市场侧商品编号
|
||||
include_sku_variants: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SubsiteParser:
|
||||
"""一个子站的解析能力"""
|
||||
|
||||
host: str # 落地域名
|
||||
source: str # 写入 ItemDetailData.source 的标识
|
||||
shop_name: str
|
||||
validate: Callable[[str], str | None] # 返回 None 表示页面正常
|
||||
parse: Callable[[SubsitePage], ItemDetailData]
|
||||
|
||||
|
||||
def parse_price(text: str | int | float | None) -> int:
|
||||
"""从 "3,091円" / "1760" / 1760 这类值里取出整数日元金额"""
|
||||
if isinstance(text, bool) or text is None:
|
||||
return 0
|
||||
if isinstance(text, (int, float)):
|
||||
return int(text)
|
||||
digits = _DIGITS_RE.findall(text.replace(",", ""))
|
||||
return int(digits[0]) if digits else 0
|
||||
|
||||
|
||||
def looks_sold_out(status_text: str) -> bool:
|
||||
"""按页面上的库存措辞判断是否不可购买"""
|
||||
return any(marker in status_text for marker in SOLD_OUT_MARKERS)
|
||||
@@ -0,0 +1,132 @@
|
||||
"""ビックカメラ楽天市場店（biccamera.rakuten.co.jp）商品页解析
|
||||
|
||||
Nuxt 应用，整页数据内联在 `window.__NUXT__`（纯 JSON 对象，可直接增量解析）。
|
||||
商品主体在 `state.item`，字段命名已经很接近市场侧语义，且直接给出市场的
|
||||
shop_id / genre_id / 分类路径与库存数。
|
||||
|
||||
站点不提供：SKU 组合（该店商品都是单一规格）、商品评分（异步加载）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
ShippingInfo,
|
||||
ShopSummary,
|
||||
SkuInfo,
|
||||
)
|
||||
from app.parsers.subsites.base import SubsitePage
|
||||
from app.utils.coerce import as_dict, as_int, as_list, as_str
|
||||
|
||||
HOST = "biccamera.rakuten.co.jp"
|
||||
SOURCE = "biccamera"
|
||||
SHOP_NAME = "ビックカメラ楽天市場店"
|
||||
|
||||
_NUXT_RE = re.compile(r"window\.__NUXT__\s*=\s*")
|
||||
_CATEGORY_URL = "https://www.rakuten.co.jp/category/{}/"
|
||||
|
||||
|
||||
def validate(html: str) -> str | None:
|
||||
if _NUXT_RE.search(html):
|
||||
return None
|
||||
return f"biccamera nuxt state not found (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def _extract_nuxt(html: str) -> dict:
|
||||
match = _NUXT_RE.search(html)
|
||||
if match is None:
|
||||
raise ScrapeParseError("ビックカメラ页面未找到 window.__NUXT__")
|
||||
try:
|
||||
state, _ = json.JSONDecoder().raw_decode(html, match.end())
|
||||
except ValueError as exc:
|
||||
raise ScrapeParseError(f"window.__NUXT__ 解析失败：{exc}") from exc
|
||||
if not isinstance(state, dict):
|
||||
raise ScrapeParseError("window.__NUXT__ 不是 JSON 对象")
|
||||
return state
|
||||
|
||||
|
||||
def _purchase_info(state: dict, item: dict) -> PurchaseInfo:
|
||||
"""该站加购走自己的 JSON 接口，字段与市场完全不同（没有 shop_bid）
|
||||
|
||||
选项（choices）的取值结构未取到样本验证，因此只如实回报「有没有选项」，
|
||||
不给出可能不准确的选项定义——带选项的商品需要调用方另行处理。
|
||||
"""
|
||||
choices = as_list(state.get("choices"))
|
||||
return PurchaseInfo(
|
||||
cart_url=as_str(item.get("add_cart_api_url")),
|
||||
form_fields={"item_id": str(as_int(item.get("item_id")))},
|
||||
quantity_field="units",
|
||||
options_field="choice" if choices else "",
|
||||
has_required_options=bool(choices),
|
||||
)
|
||||
|
||||
|
||||
def parse(page: SubsitePage) -> ItemDetailData:
|
||||
"""解析ビックカメラ商品页"""
|
||||
nuxt = _extract_nuxt(page.html)
|
||||
item = as_dict(as_dict(nuxt.get("state")).get("item"))
|
||||
if not item or not as_str(item.get("item_name")):
|
||||
raise ScrapeParseError("ビックカメラ页面缺少 state.item")
|
||||
|
||||
sold_out = bool(item.get("sold_out_flag"))
|
||||
inventory = as_int(item.get("inventory"))
|
||||
delivery = as_str(item.get("delivery_schedule_text"))
|
||||
|
||||
breadcrumbs = [
|
||||
Breadcrumb(
|
||||
name=as_str(genre.get("genre_name")),
|
||||
url=_CATEGORY_URL.format(as_str(genre.get("genre_id"))),
|
||||
)
|
||||
for genre in as_list(item.get("genres"))
|
||||
if isinstance(genre, dict) and as_str(genre.get("genre_name"))
|
||||
]
|
||||
|
||||
images = [
|
||||
as_str(image.get("url"))
|
||||
for image in as_list(item.get("images"))
|
||||
if isinstance(image, dict) and as_str(image.get("url"))
|
||||
]
|
||||
|
||||
shop_code = as_str(item.get("shop_url")) or page.shop_code
|
||||
return ItemDetailData(
|
||||
source=SOURCE,
|
||||
source_url=page.final_url,
|
||||
item_id=str(as_int(item.get("item_id"))) if item.get("item_id") is not None else "",
|
||||
item_code=as_str(item.get("item_number")) or page.item_code,
|
||||
item_name=as_str(item.get("item_name")),
|
||||
catch_copy=as_str(item.get("catch_copy")),
|
||||
description=as_str(item.get("caption")),
|
||||
item_url=page.requested_url,
|
||||
price=as_int(item.get("price_with_tax")),
|
||||
pre_tax_price=as_int(item.get("original_price")),
|
||||
tax_flag=bool(item.get("included_tax_flag")),
|
||||
purchase_condition="soldOut" if sold_out else "enabled",
|
||||
is_sold_out=sold_out,
|
||||
purchase_unit=as_int(item.get("units")),
|
||||
images=images,
|
||||
shop=ShopSummary(
|
||||
shop_id=as_int(item.get("shop_id")) or None,
|
||||
shop_code=shop_code,
|
||||
shop_name=SHOP_NAME,
|
||||
shop_url=f"https://www.rakuten.co.jp/{shop_code}/",
|
||||
),
|
||||
genre_id=str(as_int(item.get("genre_id"))) if item.get("genre_id") is not None else "",
|
||||
breadcrumbs=breadcrumbs,
|
||||
shipping=ShippingInfo(
|
||||
# 该店商品价格含运费时站点会置位此标记，不再单独给运费金额
|
||||
is_shipping_free=bool(item.get("included_shipping_fee_flag")),
|
||||
delivery_message=delivery,
|
||||
),
|
||||
sku=SkuInfo(
|
||||
inventory_type="single",
|
||||
quantity=inventory,
|
||||
show_inventory=inventory > 0,
|
||||
delivery_message=delivery,
|
||||
),
|
||||
purchase=_purchase_info(as_dict(nuxt.get("state")), item),
|
||||
)
|
||||
@@ -0,0 +1,187 @@
|
||||
"""楽天ブックス（books.rakuten.co.jp）商品页解析
|
||||
|
||||
该站是传统服务端渲染页面，商品数据以 schema.org 微数据标注（Product / Offer /
|
||||
AggregateRating），规格与简介在 `.sec-item` 分节里，分类路径则通过页面内联的
|
||||
`var data_genres` 给出——其中的 rmsGenreId 就是市场侧的 genre_id。
|
||||
|
||||
站点不提供：SKU 组合（图书没有规格轴）、运费明细、店铺评分。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
ReviewSummary,
|
||||
ShippingInfo,
|
||||
ShopSummary,
|
||||
SkuAttribute,
|
||||
SkuInfo,
|
||||
)
|
||||
from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
|
||||
|
||||
HOST = "books.rakuten.co.jp"
|
||||
SOURCE = "books"
|
||||
SHOP_NAME = "楽天ブックス"
|
||||
|
||||
_GENRES_RE = re.compile(r"var\s+data_genres\s*=\s*")
|
||||
# 折叠正文里由展开控件（checkbox/label）留下的连续空行
|
||||
_BLANK_LINES_RE = re.compile(r"\n{2,}")
|
||||
_IMAGE_RE = re.compile(r"//tshop\.r10s\.jp/book/cabinet/[^\"'\s?]+\.(?:jpg|jpeg|png)", re.I)
|
||||
_CATEGORY_URL = "https://www.rakuten.co.jp/category/{}/"
|
||||
|
||||
|
||||
def validate(html: str) -> str | None:
|
||||
"""页面须带 Product 微数据，否则不是正常的商品页"""
|
||||
if 'itemprop="price"' in html and "schema.org/Product" in html:
|
||||
return None
|
||||
return f"books item page markup not found (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def _attr(tree: HTMLParser, selector: str, name: str) -> str:
|
||||
node = tree.css_first(selector)
|
||||
return (node.attributes.get(name) or "") if node else ""
|
||||
|
||||
|
||||
def _text(tree: HTMLParser, selector: str) -> str:
|
||||
node = tree.css_first(selector)
|
||||
return node.text(strip=True) if node else ""
|
||||
|
||||
|
||||
def _parse_spec(tree: HTMLParser) -> list[SkuAttribute]:
|
||||
"""商品情報分节：每个 ul 内首个 li.product-title 是字段名，其后是取值"""
|
||||
attributes: list[SkuAttribute] = []
|
||||
for row in tree.css(".sec-item__identifier__list ul"):
|
||||
cells = row.css("li")
|
||||
if len(cells) < 2:
|
||||
continue
|
||||
title = cells[0]
|
||||
if "product-title" not in (title.attributes.get("class") or ""):
|
||||
continue
|
||||
value = " ".join(cell.text(strip=True) for cell in cells[1:] if cell.text(strip=True))
|
||||
if value:
|
||||
attributes.append(SkuAttribute(title=title.text(strip=True), value=value))
|
||||
return attributes
|
||||
|
||||
|
||||
def _parse_description(tree: HTMLParser) -> str:
|
||||
"""商品説明分节：把「内容紹介」「目次」等若干小节拼起来
|
||||
|
||||
这些小节在 DOM 里是平铺的——标题 h3 与正文 div 互为兄弟节点而非父子，
|
||||
所以要从标题往后找同级的正文块，不能直接按容器取。
|
||||
"""
|
||||
parts: list[str] = []
|
||||
for title in tree.css(".sec-item__extra__title"):
|
||||
node = title.next
|
||||
while node is not None:
|
||||
classes = node.attributes.get("class") or "" if node.tag != "-text" else ""
|
||||
if "sec-item__extra__content" in classes:
|
||||
body = _BLANK_LINES_RE.sub("\n", node.text(separator="\n", strip=True)).strip()
|
||||
if body:
|
||||
parts.append(f"{title.text(strip=True)}\n{body}")
|
||||
break
|
||||
# 只在紧邻的兄弟里找，遇到下一个标题就说明这一节没有正文
|
||||
if node.tag != "-text" and "sec-item__extra__title" in classes:
|
||||
break
|
||||
node = node.next
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _parse_genres(html: str) -> tuple[str, list[Breadcrumb]]:
|
||||
"""内联的 data_genres 给出分类路径，其中 rmsGenreId 对应市场侧 genre_id"""
|
||||
match = _GENRES_RE.search(html)
|
||||
if match is None:
|
||||
return "", []
|
||||
try:
|
||||
raw, _ = json.JSONDecoder().raw_decode(html, match.end())
|
||||
except ValueError:
|
||||
return "", []
|
||||
|
||||
# 结构是 [[{...}, {...}]]，取第一条路径
|
||||
path = raw[0] if isinstance(raw, list) and raw and isinstance(raw[0], list) else raw
|
||||
crumbs: list[Breadcrumb] = []
|
||||
genre_id = ""
|
||||
for node in path if isinstance(path, list) else []:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
rms_id = str(node.get("rmsGenreId") or "")
|
||||
name = str(node.get("genreName") or "")
|
||||
if not name:
|
||||
continue
|
||||
crumbs.append(Breadcrumb(name=name, url=_CATEGORY_URL.format(rms_id) if rms_id else ""))
|
||||
if rms_id:
|
||||
genre_id = rms_id
|
||||
return genre_id, crumbs
|
||||
|
||||
|
||||
def _parse_purchase(tree: HTMLParser) -> PurchaseInfo:
|
||||
"""加购信息直接取页面上的购物车表单
|
||||
|
||||
注意表单里的 item_id 与商品 URL 上的编号不是一回事，加购必须用表单里的值。
|
||||
该表单没有数量字段，无法在加购时指定件数。
|
||||
"""
|
||||
for form in tree.css("form"):
|
||||
action = form.attributes.get("action") or ""
|
||||
if "/bs/Cart" not in action:
|
||||
continue
|
||||
fields = {
|
||||
name: inp.attributes.get("value") or ""
|
||||
for inp in form.css("input")
|
||||
if (name := inp.attributes.get("name"))
|
||||
}
|
||||
return PurchaseInfo(
|
||||
cart_url=action,
|
||||
cart_method=(form.attributes.get("method") or "POST").upper(),
|
||||
form_fields=fields,
|
||||
)
|
||||
return PurchaseInfo()
|
||||
|
||||
|
||||
def parse(page: SubsitePage) -> ItemDetailData:
|
||||
"""解析楽天ブックス商品页"""
|
||||
tree = HTMLParser(page.html)
|
||||
|
||||
name = _text(tree, "#productTitle") or _text(tree, '[itemprop="name"]')
|
||||
if not name:
|
||||
raise ScrapeParseError("楽天ブックス页面未找到商品名")
|
||||
|
||||
images = ["https:" + url if url.startswith("//") else url for url in _IMAGE_RE.findall(page.html)]
|
||||
# 同一张图可能带不同裁剪参数重复出现，去重但保留出现顺序
|
||||
images = list(dict.fromkeys(images))
|
||||
|
||||
status = _text(tree, ".status")
|
||||
genre_id, breadcrumbs = _parse_genres(page.html)
|
||||
review_count = _text(tree, '[itemprop="reviewCount"]')
|
||||
purchase = _parse_purchase(tree)
|
||||
|
||||
return ItemDetailData(
|
||||
source=SOURCE,
|
||||
source_url=page.final_url,
|
||||
# 站内商品 ID 与 URL 上的编号不同，以购物车表单里的为准（下单要用它）
|
||||
item_id=purchase.form_fields.get("item_id", "") or page.item_code,
|
||||
item_code=page.item_code,
|
||||
item_name=name,
|
||||
description=_parse_description(tree),
|
||||
item_url=page.requested_url,
|
||||
price=parse_price(_attr(tree, '[itemprop="price"]', "content")),
|
||||
purchase_condition=status,
|
||||
is_sold_out=looks_sold_out(status),
|
||||
images=images,
|
||||
shop=ShopSummary(shop_code=page.shop_code, shop_name=SHOP_NAME),
|
||||
review=ReviewSummary(
|
||||
score=float(_attr(tree, '[itemprop="ratingValue"]', "content") or 0) or 0.0,
|
||||
count=parse_price(review_count),
|
||||
),
|
||||
genre_id=genre_id,
|
||||
breadcrumbs=breadcrumbs,
|
||||
# 图书统一由楽天ブックス发货，页面只给库存措辞，不给运费明细
|
||||
shipping=ShippingInfo(delivery_message=status),
|
||||
sku=SkuInfo(inventory_type="single", attributes=_parse_spec(tree), delivery_message=status),
|
||||
purchase=purchase,
|
||||
)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Rakuten Fashion / BRAND AVENUE（brandavenue.rakuten.co.jp）商品页解析
|
||||
|
||||
该站同样把整页数据内联在 `window.__INITIAL_STATE__`，但结构与市场页完全不同：
|
||||
商品主体在 `itemDetail.data.product`，店铺信息在 `env`，市场侧的 genre_id 与
|
||||
商品 ID 则藏在 `product.rms_info` 里。
|
||||
|
||||
SKU 有两个轴（颜色 / 尺码）：product_sku 给出可售组合与售价，rms_info.inventory_list
|
||||
给出各组合库存，两者按「尺码 + 颜色名」对齐。
|
||||
|
||||
站点不提供：商品评分（异步加载）、运费明细。面包屑用的是站内分类编码而非市场
|
||||
genre_id，因此只给分类名不给链接。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
ShopSummary,
|
||||
SkuAttribute,
|
||||
SkuAxis,
|
||||
SkuAxisValue,
|
||||
SkuInfo,
|
||||
SkuVariant,
|
||||
)
|
||||
from app.parsers.state import extract_initial_state
|
||||
from app.parsers.subsites.base import SubsitePage, parse_price
|
||||
from app.utils.coerce import as_dict, as_int, as_list, as_str
|
||||
|
||||
HOST = "brandavenue.rakuten.co.jp"
|
||||
SOURCE = "brandavenue"
|
||||
SHOP_NAME = "Rakuten Fashion"
|
||||
|
||||
_COLOR_AXIS = "カラー"
|
||||
_SIZE_AXIS = "サイズ"
|
||||
|
||||
# cart_info.cart_url_type → 加购端点。站点前端用同名映射表（resolveCartUrl）解析；
|
||||
# 若该字段本身已经是一个 URL，则直接使用。
|
||||
_CART_URL_BY_TYPE = {
|
||||
"1": "https://ts.basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set",
|
||||
"2": "https://ts.sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set",
|
||||
"3": "https://t2.basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set",
|
||||
"4": "https://t2.sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set",
|
||||
"5": "https://basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set",
|
||||
"6": "https://sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set",
|
||||
}
|
||||
_DEFAULT_PURCHASE_EVENT = "ES01_003_001"
|
||||
|
||||
|
||||
def _resolve_cart_url(cart_url_type: str) -> str:
|
||||
if cart_url_type.startswith("http"):
|
||||
return cart_url_type
|
||||
return _CART_URL_BY_TYPE.get(cart_url_type, "")
|
||||
|
||||
|
||||
def _purchase_info(product: dict) -> PurchaseInfo:
|
||||
"""加购字段与市场是同一套契约，差别只在端点由 cart_url_type 映射得到"""
|
||||
cart_info = as_dict(as_dict(product.get("rms_info")).get("cart_info"))
|
||||
if not cart_info:
|
||||
return PurchaseInfo()
|
||||
return PurchaseInfo(
|
||||
cart_url=_resolve_cart_url(as_str(cart_info.get("cart_url_type"))),
|
||||
form_fields={
|
||||
"shop_bid": as_str(cart_info.get("shop_bid")),
|
||||
"item_id": as_str(cart_info.get("item_id")),
|
||||
"inventory_flag": as_str(cart_info.get("inventory_type")),
|
||||
"__event": as_str(cart_info.get("event")) or _DEFAULT_PURCHASE_EVENT,
|
||||
"encode": "utf8",
|
||||
},
|
||||
quantity_field="units",
|
||||
variant_field="variant_id",
|
||||
)
|
||||
|
||||
|
||||
def validate(html: str) -> str | None:
|
||||
if "window.__INITIAL_STATE__" in html:
|
||||
return None
|
||||
return f"brandavenue state not found (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def _images(html: str, image_folder: str, main_filename: str) -> list[str]:
|
||||
"""从页面直出的图片地址中收集商品图
|
||||
|
||||
图片按商品编号做了目录分片，分片规则不对外暴露，所以直接取页面里已经渲染好的
|
||||
地址，而不是自行拼接，避免规则变化导致图片全错。
|
||||
"""
|
||||
if not image_folder:
|
||||
return []
|
||||
pattern = re.compile(
|
||||
rf"https://[a-z0-9.\-]+/{re.escape(image_folder)}/[^\"'\s]+\.(?:jpg|jpeg|png)", re.I
|
||||
)
|
||||
urls = list(dict.fromkeys(pattern.findall(html)))
|
||||
if not main_filename:
|
||||
return urls
|
||||
# 主图排在最前，便于调用方直接取 images[0] 当封面
|
||||
main = main_filename.lower()
|
||||
urls.sort(key=lambda url: 0 if url.lower().endswith("/" + main) else 1)
|
||||
return urls
|
||||
|
||||
|
||||
def _breadcrumbs(product: dict) -> list[Breadcrumb]:
|
||||
"""category_l_m_cd_name 是 [大类ID, 大类名, 中类ID, 中类名] 的扁平数组"""
|
||||
flat = [as_str(value) for value in as_list(product.get("category_l_m_cd_name"))]
|
||||
crumbs: list[Breadcrumb] = []
|
||||
for index in range(0, len(flat) - 1, 2):
|
||||
name = flat[index + 1]
|
||||
if name:
|
||||
crumbs.append(Breadcrumb(name=name))
|
||||
return crumbs
|
||||
|
||||
|
||||
def _sku(product: dict, *, include_variants: bool) -> SkuInfo:
|
||||
entries = [entry for entry in as_list(product.get("product_sku")) if isinstance(entry, dict)]
|
||||
inventory = {
|
||||
(as_str(row.get("size")), as_str(row.get("color_name"))): row
|
||||
for row in as_list(as_dict(product.get("rms_info")).get("inventory_list"))
|
||||
if isinstance(row, dict)
|
||||
}
|
||||
|
||||
variants: list[SkuVariant] = []
|
||||
colors: dict[str, bool] = {}
|
||||
sizes: dict[str, bool] = {}
|
||||
for entry in entries:
|
||||
color = as_str(entry.get("product_color_name"))
|
||||
size = as_str(entry.get("product_size_name"))
|
||||
in_stock = as_str(entry.get("inventory_exist_flg")) == "1"
|
||||
row = as_dict(inventory.get((size, color)))
|
||||
attributes = [
|
||||
SkuAttribute(title=title, value=as_str(entry.get(key)))
|
||||
for title, key in (("素材", "material"), ("お手入れ", "cleaning"), ("お届け目安", "inventory_status_message"))
|
||||
if as_str(entry.get(key))
|
||||
]
|
||||
variants.append(
|
||||
SkuVariant(
|
||||
variant_id=as_str(row.get("variant_id")),
|
||||
selector_values=[color, size],
|
||||
price=parse_price(entry.get("selling_price")),
|
||||
quantity=as_int(row.get("stock")),
|
||||
is_sold_out=not in_stock,
|
||||
delivery_message=as_str(entry.get("inventory_status_message")),
|
||||
attributes=attributes,
|
||||
)
|
||||
)
|
||||
# 任一组合可售即认为该取值可选
|
||||
colors[color] = colors.get(color, False) or in_stock
|
||||
sizes[size] = sizes.get(size, False) or in_stock
|
||||
|
||||
axis = [
|
||||
SkuAxis(
|
||||
key=key,
|
||||
label=key,
|
||||
values=[
|
||||
SkuAxisValue(value=value, label=value, is_sold_out=not available)
|
||||
for value, available in mapping.items()
|
||||
if value
|
||||
],
|
||||
)
|
||||
for key, mapping in ((_COLOR_AXIS, colors), (_SIZE_AXIS, sizes))
|
||||
if any(mapping)
|
||||
]
|
||||
|
||||
return SkuInfo(
|
||||
inventory_type="multiple" if len(variants) > 1 else "single",
|
||||
quantity=sum(variant.quantity for variant in variants),
|
||||
delivery_message=as_str(entries[0].get("inventory_status_message")) if entries else "",
|
||||
axis=axis,
|
||||
variants=variants if include_variants else [],
|
||||
variant_count=len(variants),
|
||||
)
|
||||
|
||||
|
||||
def parse(page: SubsitePage) -> ItemDetailData:
|
||||
"""解析 Rakuten Fashion 商品页"""
|
||||
state = extract_initial_state(page.html)
|
||||
product = as_dict(as_dict(as_dict(state.get("itemDetail")).get("data")).get("product"))
|
||||
if not product:
|
||||
raise ScrapeParseError("Rakuten Fashion 页面缺少 itemDetail.data.product")
|
||||
|
||||
env = as_dict(state.get("env"))
|
||||
rms = as_dict(product.get("rms_info"))
|
||||
sold_out = as_int(product.get("soldout_flg")) == 1
|
||||
|
||||
return ItemDetailData(
|
||||
source=SOURCE,
|
||||
source_url=page.final_url,
|
||||
item_id=as_str(rms.get("rms_item_id")),
|
||||
item_code=page.item_code,
|
||||
item_name=as_str(product.get("product_name")),
|
||||
catch_copy=as_str(product.get("brand_name")),
|
||||
description=as_str(product.get("product_exp")),
|
||||
item_url=page.requested_url,
|
||||
price=parse_price(product.get("selling_price_no_format")),
|
||||
pre_tax_price=parse_price(product.get("fixed_price_no_format")),
|
||||
purchase_condition="soldOut" if sold_out else "enabled",
|
||||
is_sold_out=sold_out,
|
||||
images=_images(page.html, as_str(env.get("product_image_folder")), as_str(product.get("product_img_path"))),
|
||||
shop=ShopSummary(
|
||||
shop_id=as_int(env.get("shop_id")) or None,
|
||||
shop_code=as_str(env.get("shop_url")) or page.shop_code,
|
||||
shop_name=as_str(env.get("shop_name")) or SHOP_NAME,
|
||||
shop_url=f"https://www.rakuten.co.jp/{as_str(env.get('shop_url')) or page.shop_code}/",
|
||||
),
|
||||
genre_id=as_str(rms.get("genre_id")),
|
||||
breadcrumbs=_breadcrumbs(product),
|
||||
sku=_sku(product, include_variants=page.include_sku_variants),
|
||||
purchase=_purchase_info(product),
|
||||
)
|
||||
Reference in New Issue
Block a user