Init
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
"""FastAPI 依赖注入：提供容器获取和鉴权校验"""
|
||||
import logging
|
||||
import secrets
|
||||
|
||||
from fastapi import Depends, Request
|
||||
|
||||
from app.core.container import ServiceContainer
|
||||
from app.core.errors import AuthenticationError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_container(request: Request) -> ServiceContainer:
|
||||
"""从请求中获取服务容器"""
|
||||
return request.app.state.container
|
||||
|
||||
|
||||
def require_bearer_token(
|
||||
request: Request,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> None:
|
||||
"""Bearer Token 鉴权依赖
|
||||
|
||||
从请求头 Authorization 中提取 Bearer Token，
|
||||
与服务端配置的 token 做安全比较（使用 secrets.compare_digest 防止时序攻击）。
|
||||
"""
|
||||
authorization = request.headers.get("Authorization")
|
||||
if not authorization:
|
||||
logger.warning("鉴权失败：缺少 Authorization 请求头")
|
||||
raise AuthenticationError("Missing Authorization header")
|
||||
|
||||
token = authorization.replace("Bearer ", "", 1).strip()
|
||||
if token == authorization:
|
||||
logger.warning("鉴权失败：Authorization scheme 非 Bearer")
|
||||
raise AuthenticationError("Invalid Authorization scheme")
|
||||
if not token:
|
||||
logger.warning("鉴权失败：Bearer token 为空")
|
||||
raise AuthenticationError("Invalid token")
|
||||
|
||||
if not secrets.compare_digest(token, container.settings.bearer_token):
|
||||
logger.warning("鉴权失败：token 不匹配")
|
||||
raise AuthenticationError("Invalid token")
|
||||
@@ -0,0 +1,34 @@
|
||||
"""健康检查路由"""
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.api.dependencies import get_container
|
||||
from app.core.container import ServiceContainer
|
||||
from app.models.scrape import ApiResponse, HealthData
|
||||
|
||||
router = APIRouter(tags=["health"])
|
||||
|
||||
|
||||
@router.get("/health", response_model=ApiResponse[HealthData])
|
||||
async def health(container: ServiceContainer = Depends(get_container)) -> ApiResponse[HealthData]:
|
||||
"""服务健康状态
|
||||
|
||||
sessions 里给出乐天的 PC / 手机两条抓取通道的 cookie 预热情况，以及
|
||||
ラクマ 通道的就绪状态（ラクマ 无需预热，只报是否已初始化）；
|
||||
browser_fallback_* 反映浏览器兜底当前是否可用（未安装 playwright 时为不可用，
|
||||
属于预期降级，不影响主链路）。
|
||||
"""
|
||||
return ApiResponse[HealthData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=HealthData(
|
||||
status="ok",
|
||||
browser_fallback_enabled=container.browser_fallback.enabled,
|
||||
browser_fallback_ready=container.browser_fallback.ready,
|
||||
browser_fallback_error=container.browser_fallback.unavailable_reason,
|
||||
sessions={
|
||||
**container.site_session.profile_status(),
|
||||
"rakuma": container.rakuma_session.status(),
|
||||
},
|
||||
),
|
||||
code=0,
|
||||
)
|
||||
@@ -0,0 +1,128 @@
|
||||
"""抓取路由：ラクマ（fril.jp）的搜索、商品详情与卖家
|
||||
|
||||
单独挂在 /api/rakuma 前缀下，不与乐天市场的接口合并：两站的筛选参数体系
|
||||
差异很大（乐天有 genre_id / 成色 / SuperDEAL，ラクマ 有 category_id /
|
||||
brand_id / 匿名配送 / 鉴定服务），合并会让大半字段对另一站无效。
|
||||
"""
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.api.dependencies import get_container, require_bearer_token
|
||||
from app.core.container import ServiceContainer
|
||||
from app.models.scrape import (
|
||||
ApiResponse,
|
||||
RakumaItemDetailData,
|
||||
RakumaItemDetailRequest,
|
||||
RakumaSearchRequest,
|
||||
RakumaSearchResultData,
|
||||
RakumaShopDetailData,
|
||||
RakumaShopDetailRequest,
|
||||
RakumaShopItemsData,
|
||||
RakumaShopItemsRequest,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/rakuma", tags=["rakuma"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/search",
|
||||
response_model=ApiResponse[RakumaSearchResultData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def search(
|
||||
payload: RakumaSearchRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[RakumaSearchResultData]:
|
||||
"""搜索 ラクマ 商品列表
|
||||
|
||||
支持关键词、分类、品牌、价格区间、商品状态（6 档成色，可多选）、
|
||||
在售/售罄、免运费、匿名配送等筛选；也可以直接传 search_url 透传一条
|
||||
fril.jp 搜索页地址。
|
||||
|
||||
每页固定 40 条，站点侧最多翻到第 100 页（page > 100 直接 404）。
|
||||
"""
|
||||
data = await container.rakuma_client.search(payload)
|
||||
return ApiResponse[RakumaSearchResultData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/item_detail",
|
||||
response_model=ApiResponse[RakumaItemDetailData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def item_detail(
|
||||
payload: RakumaItemDetailRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[RakumaItemDetailData]:
|
||||
"""获取 ラクマ 商品详情
|
||||
|
||||
传 item_id（商品页 URL 的最后一段 hash），或直接传 item_url。
|
||||
返回名称、价格、描述、图片、成色、配送信息与出品者摘要。
|
||||
|
||||
ラクマ 是 C2C 集市，每件商品都是单件的：没有 SKU 组合，也没有库存数量。
|
||||
响应里的 seller.shop_id 可直接用于 /api/rakuma/shop_detail。
|
||||
"""
|
||||
data = await container.rakuma_client.item_detail(payload)
|
||||
return ApiResponse[RakumaItemDetailData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/shop_detail",
|
||||
response_model=ApiResponse[RakumaShopDetailData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def shop_detail(
|
||||
payload: RakumaShopDetailRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[RakumaShopDetailData]:
|
||||
"""获取 ラクマ 卖家（出品者）详情
|
||||
|
||||
传 shop_id（店铺页 URL 的最后一段 hash），或直接传 shop_url。
|
||||
返回店铺名、昵称、头像与封面、简介、评分与评价数、本人确认状态、
|
||||
以及该卖家的商品总数。
|
||||
|
||||
评价明细（最新 100 条）与好评/普通/差评分档计数在站点的单独子页上，
|
||||
需要多打一次请求，把 include_reviews 置为 true 才会返回。
|
||||
"""
|
||||
data = await container.rakuma_client.shop_detail(payload)
|
||||
return ApiResponse[RakumaShopDetailData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/shop_items",
|
||||
response_model=ApiResponse[RakumaShopItemsData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def shop_items(
|
||||
payload: RakumaShopItemsRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[RakumaShopItemsData]:
|
||||
"""获取 ラクマ 卖家名下的商品列表
|
||||
|
||||
传 shop_id 或 shop_url，按页取该卖家的全部商品（含已售出，
|
||||
用每条的 is_sold_out 区分）。
|
||||
|
||||
站点在店铺页不提供排序与筛选参数，因此这里只有页码；需要筛选请改用
|
||||
/api/rakuma/search。
|
||||
"""
|
||||
data = await container.rakuma_client.shop_items(payload)
|
||||
return ApiResponse[RakumaShopItemsData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
@@ -0,0 +1,147 @@
|
||||
"""抓取路由：乐天市场的搜索、分类、商品详情与商家"""
|
||||
from fastapi import APIRouter, Depends
|
||||
|
||||
from app.api.dependencies import get_container, require_bearer_token
|
||||
from app.core.container import ServiceContainer
|
||||
from app.models.scrape import (
|
||||
ApiResponse,
|
||||
GenreData,
|
||||
GenreRequest,
|
||||
ItemDetailData,
|
||||
ItemDetailRequest,
|
||||
SearchRequest,
|
||||
SearchResultData,
|
||||
ShopDetailData,
|
||||
ShopDetailRequest,
|
||||
ShopItemsRequest,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["scrape"])
|
||||
|
||||
|
||||
@router.post(
|
||||
"/search",
|
||||
response_model=ApiResponse[SearchResultData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def search(
|
||||
payload: SearchRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[SearchResultData]:
|
||||
"""搜索商品列表
|
||||
|
||||
支持关键词、分类、排序、价格区间、店铺、成色、免运费等筛选；也可以直接传
|
||||
search_url 透传一条乐天搜索页地址。返回结果默认已剔除混入的 CPC 广告位
|
||||
（剔除数量见 ad_count）。
|
||||
|
||||
翻页上限受站点限制：最多只能取到 reachable_count 条（通常 6750）。
|
||||
"""
|
||||
data = await container.rakuten_client.search(payload)
|
||||
return ApiResponse[SearchResultData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/genres",
|
||||
response_model=ApiResponse[GenreData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def genres(
|
||||
payload: GenreRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[GenreData]:
|
||||
"""获取乐天分类（genre）树，用于取得 /api/search 需要的 genre_id
|
||||
|
||||
不传 genre_id 返回 39 个顶层分类；传入后返回该分类的名称、描述、祖先路径
|
||||
与直接子分类，逐层下钻即可定位到叶子分类。
|
||||
|
||||
子分类的 item_count 是该分类下的商品数；顶层列表不返回该值，因为站点给出的
|
||||
是「当前查询在该分类下的命中数」，并非分类自身的商品总量。
|
||||
"""
|
||||
data = await container.rakuten_client.genres(payload)
|
||||
return ApiResponse[GenreData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/item_detail",
|
||||
response_model=ApiResponse[ItemDetailData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def item_detail(
|
||||
payload: ItemDetailRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[ItemDetailData]:
|
||||
"""获取商品详情
|
||||
|
||||
传 shop_code + item_code（即商品 URL 的两段路径），或直接传 item_url。
|
||||
返回名称、价格、图片、店铺、评价、配送与全部 SKU 组合；SKU 组合可能多达
|
||||
数百条，不需要时可将 include_sku_variants 置为 false。
|
||||
"""
|
||||
data = await container.rakuten_client.item_detail(payload)
|
||||
return ApiResponse[ItemDetailData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/shop_detail",
|
||||
response_model=ApiResponse[ShopDetailData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def shop_detail(
|
||||
payload: ShopDetailRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[ShopDetailData]:
|
||||
"""获取乐天商家（店铺）详情
|
||||
|
||||
传 shop_code（店铺首页 URL 的路径段，如 edion），或直接传 shop_url。
|
||||
返回店铺名称、简介、评分与评价数、招牌图、是否 39ショップ 与休息日。
|
||||
|
||||
评价数过少时站点不展示评分，此时 review_displayed 为 false，
|
||||
review_score 不可信。
|
||||
"""
|
||||
data = await container.rakuten_client.shop_detail(payload)
|
||||
return ApiResponse[ShopDetailData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/shop_items",
|
||||
response_model=ApiResponse[SearchResultData],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def shop_items(
|
||||
payload: ShopItemsRequest,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[SearchResultData]:
|
||||
"""获取乐天商家名下的商品列表
|
||||
|
||||
传 shop_id（取自搜索结果或 /api/shop_detail）或 shop_code；只给 shop_code
|
||||
时服务端会先取一次店铺详情换出 shop_id，多花一次请求。
|
||||
|
||||
支持在店铺内按关键词、分类、价格、成色等继续筛选，翻页与返回结构同
|
||||
/api/search。
|
||||
"""
|
||||
data = await container.rakuten_client.shop_items(payload)
|
||||
return ApiResponse[SearchResultData](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=data,
|
||||
code=0,
|
||||
)
|
||||
@@ -0,0 +1,118 @@
|
||||
"""应用配置：通过环境变量和 .env 文件加载所有配置项
|
||||
|
||||
配置项统一使用 RAKUTEN_ 前缀，例如 RAKUTEN_APP_PORT=31107。
|
||||
支持 .env 文件自动加载。
|
||||
"""
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
from app.core import site
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""应用全局配置
|
||||
|
||||
所有配置项均可通过环境变量覆盖，前缀为 RAKUTEN_。
|
||||
例如：RAKUTEN_APP_PORT=31107 对应 app_port 配置项。
|
||||
"""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=str(BASE_DIR / ".env"),
|
||||
env_file_encoding="utf-8",
|
||||
env_prefix="RAKUTEN_",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# ---- 服务基本配置 ----
|
||||
app_name: str = "Rakuten Scraper Service"
|
||||
app_env: Literal["dev", "prod", "test"] = "dev"
|
||||
app_host: str = "0.0.0.0"
|
||||
app_port: int = 31107
|
||||
|
||||
# ---- 日志配置 ----
|
||||
log_level: str = "INFO"
|
||||
log_to_file: bool | None = None # None 表示根据环境自动决定
|
||||
log_dir: str = "logs"
|
||||
log_rotation: str = "100 MB"
|
||||
log_retention: str = "14 days"
|
||||
log_compression: str = "zip"
|
||||
log_format: str = "{time:YYYY-MM-DD HH:mm:ss} {level} {message}"
|
||||
log_enqueue: bool = True
|
||||
|
||||
# ---- 鉴权配置 ----
|
||||
bearer_token: str = "REPLACE_WITH_TOKEN_32CHARS"
|
||||
|
||||
# ---- HTTP 抓取配置 ----
|
||||
request_timeout_seconds: float = 30.0
|
||||
max_site_concurrency: int = 8 # 对站点的最大并发请求数
|
||||
http_max_attempts: int = 3 # 单次抓取的最大尝试次数（含首次）
|
||||
session_ttl_seconds: float = 1800.0 # Akamai cookie 会话最长复用时长，超时后重新预热
|
||||
|
||||
# ---- 浏览器兜底配置 ----
|
||||
# 纯 HTTP 被 Akamai 拦截时，用 Playwright 打开页面取回 cookie 再回灌给
|
||||
# HTTP 客户端重试。日常流量不会触发；未安装 playwright 时自动降级为不兜底。
|
||||
browser_fallback_enabled: bool = True
|
||||
browser_headless: bool | None = None # None 表示根据环境自动决定
|
||||
browser_channel: str | None = None # 例如 chrome；留空使用 bundled chromium
|
||||
browser_launch_timeout_seconds: float = 60.0
|
||||
browser_nav_timeout_seconds: float = 60.0
|
||||
|
||||
# ---- 代理配置（可选，用于日本 IP）----
|
||||
proxy_server: str | None = None
|
||||
proxy_username: str | None = None
|
||||
proxy_password: str | None = None
|
||||
|
||||
# ---- 目标站点 ----
|
||||
home_url: str = site.HOME_URL
|
||||
|
||||
@property
|
||||
def browser_headless_effective(self) -> bool:
|
||||
"""浏览器无头模式：显式配置优先，否则开发环境使用有头模式方便调试"""
|
||||
if self.browser_headless is not None:
|
||||
return self.browser_headless
|
||||
return self.app_env != "dev"
|
||||
|
||||
@property
|
||||
def log_to_file_effective(self) -> bool:
|
||||
"""是否写日志文件：显式配置优先，否则生产环境默认写文件"""
|
||||
if self.log_to_file is not None:
|
||||
return self.log_to_file
|
||||
return self.app_env == "prod"
|
||||
|
||||
@property
|
||||
def playwright_proxy(self) -> dict[str, str] | None:
|
||||
"""构建 Playwright 代理配置字典"""
|
||||
if not self.proxy_server:
|
||||
return None
|
||||
|
||||
proxy: dict[str, str] = {"server": self.proxy_server}
|
||||
if self.proxy_username:
|
||||
proxy["username"] = self.proxy_username
|
||||
if self.proxy_password:
|
||||
proxy["password"] = self.proxy_password
|
||||
return proxy
|
||||
|
||||
@property
|
||||
def httpx_proxy(self) -> str | None:
|
||||
"""构建 httpx 代理 URL（含认证信息）"""
|
||||
if not self.proxy_server:
|
||||
return None
|
||||
if not self.proxy_username:
|
||||
return self.proxy_server
|
||||
|
||||
scheme, _, rest = self.proxy_server.partition("://")
|
||||
if not rest:
|
||||
return self.proxy_server
|
||||
credentials = f"{self.proxy_username}:{self.proxy_password or ''}"
|
||||
return f"{scheme}://{credentials}@{rest}"
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_settings() -> Settings:
|
||||
"""获取全局配置单例"""
|
||||
return Settings()
|
||||
@@ -0,0 +1,28 @@
|
||||
"""服务容器：集中管理所有服务实例，用于依赖注入"""
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.services.browser_fallback import BrowserFallback
|
||||
from app.services.rakuma_client import RakumaClient
|
||||
from app.services.rakuma_session import RakumaSession
|
||||
from app.services.rakuten_client import RakutenClient
|
||||
from app.services.site_session import SiteSession
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ServiceContainer:
|
||||
"""服务容器，持有所有核心服务实例
|
||||
|
||||
通过 FastAPI 的 app.state.container 在请求间共享，
|
||||
各路由通过依赖注入获取容器中的服务。
|
||||
|
||||
两个站点各自持有独立的会话与客户端：乐天需要 Akamai cookie 预热与双指纹
|
||||
通道，ラクマ 不需要，抓取前提不同不便合并。
|
||||
"""
|
||||
|
||||
settings: Settings
|
||||
browser_fallback: BrowserFallback
|
||||
site_session: SiteSession
|
||||
rakuten_client: RakutenClient
|
||||
rakuma_session: RakumaSession
|
||||
rakuma_client: RakumaClient
|
||||
@@ -0,0 +1,115 @@
|
||||
"""应用异常定义：所有业务异常均继承自 AppError
|
||||
|
||||
错误码规范：
|
||||
- 1xxx: 请求/鉴权错误
|
||||
- 2xxx: 抓取资源错误
|
||||
- 3xxx: 反爬/上游阻断相关错误
|
||||
- 4xxx: 页面解析错误
|
||||
"""
|
||||
|
||||
|
||||
class AppError(Exception):
|
||||
"""应用基础异常，携带错误码、HTTP 状态码和是否可重试信息"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
message: str,
|
||||
code: str,
|
||||
err_code: int,
|
||||
retryable: bool = False,
|
||||
status_code: int = 400,
|
||||
headers: dict[str, str] | None = None,
|
||||
):
|
||||
super().__init__(message)
|
||||
self.message = message
|
||||
self.code = code
|
||||
self.err_code = err_code
|
||||
self.retryable = retryable
|
||||
self.status_code = status_code
|
||||
self.headers = headers
|
||||
|
||||
|
||||
class AuthenticationError(AppError):
|
||||
"""鉴权失败（Bearer Token 无效或缺失）"""
|
||||
|
||||
def __init__(self, message: str = "Invalid credentials"):
|
||||
super().__init__(
|
||||
message=message,
|
||||
code="AUTH_INVALID",
|
||||
err_code=1001,
|
||||
retryable=False,
|
||||
status_code=401,
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
class InvalidRequestError(AppError):
|
||||
"""请求参数不合法（如 URL 非乐天站点、缺少必填标识）"""
|
||||
|
||||
def __init__(self, message: str = "Invalid request"):
|
||||
super().__init__(message=message, code="INVALID_REQUEST", err_code=1003, retryable=False)
|
||||
|
||||
|
||||
class ResourceBusyError(AppError):
|
||||
"""抓取资源繁忙（等待并发槽位超时）"""
|
||||
|
||||
def __init__(self, message: str = "Timed out while waiting for a scrape slot"):
|
||||
super().__init__(message=message, code="RESOURCE_BUSY", err_code=2002, retryable=True)
|
||||
|
||||
|
||||
class UpstreamRequestError(AppError):
|
||||
"""上游请求失败（网络异常、超时、5xx）"""
|
||||
|
||||
def __init__(self, message: str = "The upstream request failed"):
|
||||
super().__init__(message=message, code="UPSTREAM_ERROR", err_code=3001, retryable=True)
|
||||
|
||||
|
||||
class UpstreamBlockedError(AppError):
|
||||
"""上游请求被反爬阻断（Akamai 挑战页 / 403 / 页面缺少渲染数据）"""
|
||||
|
||||
def __init__(self, message: str = "The upstream request was blocked"):
|
||||
super().__init__(message=message, code="UPSTREAM_BLOCKED", err_code=3002, retryable=True)
|
||||
|
||||
|
||||
class ItemNotFoundError(AppError):
|
||||
"""商品不存在或已下架（详情页 404）"""
|
||||
|
||||
def __init__(self, message: str = "Item not found"):
|
||||
super().__init__(
|
||||
message=message,
|
||||
code="ITEM_NOT_FOUND",
|
||||
err_code=4004,
|
||||
retryable=False,
|
||||
status_code=404,
|
||||
)
|
||||
|
||||
|
||||
class ScrapeParseError(AppError):
|
||||
"""页面解析失败"""
|
||||
|
||||
def __init__(self, message: str = "Failed to parse the target page"):
|
||||
super().__init__(message=message, code="PARSE_ERROR", err_code=4001, retryable=False)
|
||||
|
||||
|
||||
class OffIchibaRedirectError(AppError):
|
||||
"""商品页跳转到了市场之外的乐天官方子站
|
||||
|
||||
楽天ブックス（book → books.rakuten.co.jp）、ビックカメラ
|
||||
（biccamera → biccamera.rakuten.co.jp）等官方旗舰店有各自独立的站点与
|
||||
页面结构，不返回市场统一模板，本服务的详情解析不适用。
|
||||
|
||||
单独成一类错误是为了让上游能把这些商品路由到别处，而不是当成被反爬拦截去重试。
|
||||
"""
|
||||
|
||||
def __init__(self, requested_url: str, final_url: str):
|
||||
super().__init__(
|
||||
message=(
|
||||
f"商品页跳转至乐天市场以外的站点，当前不支持解析："
|
||||
f"{requested_url} -> {final_url}"
|
||||
),
|
||||
code="OFF_ICHIBA_REDIRECT",
|
||||
err_code=4002,
|
||||
retryable=False,
|
||||
)
|
||||
self.requested_url = requested_url
|
||||
self.final_url = final_url
|
||||
@@ -0,0 +1,86 @@
|
||||
"""日志配置：使用 loguru 替代标准 logging，统一管理日志输出
|
||||
|
||||
- 控制台输出：始终启用
|
||||
- 文件输出：生产环境默认启用，支持日志轮转、压缩和保留策略
|
||||
- 拦截标准 logging：将 uvicorn 等第三方库的日志转发到 loguru
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from types import FrameType
|
||||
|
||||
from loguru import logger as loguru_logger
|
||||
|
||||
from app.core.config import Settings
|
||||
|
||||
|
||||
class InterceptHandler(logging.Handler):
|
||||
"""拦截标准 logging 日志，转发到 loguru"""
|
||||
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
try:
|
||||
level: str | int = loguru_logger.level(record.levelname).name
|
||||
except ValueError:
|
||||
level = record.levelno
|
||||
|
||||
frame: FrameType | None = logging.currentframe()
|
||||
depth = 2
|
||||
while frame and frame.f_code.co_filename == logging.__file__:
|
||||
frame = frame.f_back
|
||||
depth += 1
|
||||
|
||||
loguru_logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
|
||||
|
||||
|
||||
def configure_logging(settings: Settings) -> None:
|
||||
"""根据配置初始化日志系统
|
||||
|
||||
- 移除 loguru 默认 handler
|
||||
- 添加控制台输出
|
||||
- 生产环境添加文件输出（轮转、压缩、保留策略）
|
||||
- 拦截标准 logging，统一走 loguru 输出
|
||||
"""
|
||||
loguru_logger.remove()
|
||||
loguru_logger.add(
|
||||
sys.stdout,
|
||||
level=settings.log_level.upper(),
|
||||
format=settings.log_format,
|
||||
enqueue=settings.log_enqueue,
|
||||
backtrace=False,
|
||||
diagnose=False,
|
||||
)
|
||||
|
||||
if settings.log_to_file_effective:
|
||||
log_dir = Path(settings.log_dir)
|
||||
log_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_path = log_dir / f"app_{settings.app_env}.log"
|
||||
loguru_logger.add(
|
||||
str(log_path),
|
||||
level=settings.log_level.upper(),
|
||||
format=settings.log_format,
|
||||
rotation=settings.log_rotation,
|
||||
retention=settings.log_retention,
|
||||
compression=settings.log_compression,
|
||||
enqueue=settings.log_enqueue,
|
||||
encoding="utf-8",
|
||||
backtrace=False,
|
||||
diagnose=False,
|
||||
)
|
||||
|
||||
intercept = InterceptHandler()
|
||||
logging.basicConfig(
|
||||
handlers=[intercept],
|
||||
level=getattr(logging, settings.log_level.upper(), logging.INFO),
|
||||
force=True,
|
||||
)
|
||||
|
||||
for name in ("uvicorn", "uvicorn.error", "uvicorn.access"):
|
||||
logger = logging.getLogger(name)
|
||||
logger.handlers = [intercept]
|
||||
logger.propagate = False
|
||||
|
||||
logging.getLogger("asyncio").setLevel(logging.WARNING)
|
||||
# httpx 每次请求都会打一条 INFO，抓取服务下噪音过大
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
@@ -0,0 +1,110 @@
|
||||
"""ラクマ（fril.jp）站点常量
|
||||
|
||||
集中维护站点入口 URL、浏览器指纹参数，以及搜索页 URL 的排序码 / 筛选码映射。
|
||||
|
||||
排序与筛选参数并非猜测，而是从站点前端 bundle
|
||||
（asset.fril.jp/assets/v2/application-*.js）中 SearchPanel 组件的 `_url()`
|
||||
方法里提取的——那段代码逐条拼出搜索 URL，因此参数名与取值与站点行为一一对应。
|
||||
|
||||
与乐天市场（app/core/site.py）的关键差异：
|
||||
- 页面是服务端渲染的 HTML，**没有** `window.__INITIAL_STATE__`，只能解析 DOM
|
||||
- 前置的不是 Akamai Bot Manager，无限速行为，冷请求即 ~0.6-1.1s，不需要 cookie 预热
|
||||
- 商品详情页用 PC UA 即可，不需要像乐天那样切手机 UA
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
# ---- 站点入口 ----
|
||||
HOME_URL: Final = "https://fril.jp/"
|
||||
SEARCH_BASE_URL: Final = "https://fril.jp/s"
|
||||
ITEM_BASE_URL: Final = "https://item.fril.jp/"
|
||||
SHOP_BASE_URL: Final = "https://fril.jp/shop/"
|
||||
CATEGORY_BASE_URL: Final = "https://fril.jp/category/"
|
||||
BRAND_BASE_URL: Final = "https://fril.jp/brand/"
|
||||
|
||||
SEARCH_HOST: Final = "fril.jp"
|
||||
ITEM_HOST: Final = "item.fril.jp"
|
||||
|
||||
# 站点每页固定返回 40 条（实测第 1 页与深翻页均为 40）
|
||||
PAGE_SIZE: Final = 40
|
||||
|
||||
# 站点侧翻页上限：page=100 正常返回，page=101 起直接 404。
|
||||
MAX_PAGE: Final = 100
|
||||
|
||||
# ---- 浏览器指纹 ----
|
||||
# 两类页面都用 PC UA：详情页 PC 版即为完整模板，无需像乐天那样切手机 UA。
|
||||
USER_AGENT: Final = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
ACCEPT_LANGUAGE: Final = "ja,en-US;q=0.9,en;q=0.8"
|
||||
|
||||
|
||||
def default_headers() -> dict[str, str]:
|
||||
"""构造一套完整的浏览器导航请求头"""
|
||||
return {
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,"
|
||||
"image/avif,image/webp,image/apng,*/*;q=0.8,"
|
||||
"application/signed-exchange;v=b3;q=0.7"
|
||||
),
|
||||
"Accept-Language": ACCEPT_LANGUAGE,
|
||||
"sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24", "Google Chrome";v="131"',
|
||||
"sec-ch-ua-mobile": "?0",
|
||||
"sec-ch-ua-platform": '"Windows"',
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
|
||||
# ---- 排序（搜索页 `sort=` + `order=` 两个参数）----
|
||||
# 键为对外暴露的语义化排序名，值为 (sort, order)。
|
||||
# 站点把排序字段与升降序拆成两个参数，因此这里成对给出。
|
||||
SORT_CODES: Final[dict[str, tuple[str, str]]] = {
|
||||
"standard": ("relevance", "desc"), # 站点默认的おすすめ順
|
||||
"newest": ("created_at", "desc"),
|
||||
"price_asc": ("sell_price", "asc"),
|
||||
"price_desc": ("sell_price", "desc"),
|
||||
"like_count": ("like_count", "desc"), # いいね数順
|
||||
}
|
||||
|
||||
# ---- 商品状态（搜索页 `statuses=`，逗号分隔可多选）----
|
||||
# 取值来自 SearchPanel 组件的 statusOptions 定义。
|
||||
CONDITION_CODES: Final[dict[str, str]] = {
|
||||
"new": "5", # 新品、未使用
|
||||
"almost_new": "4", # 未使用に近い
|
||||
"no_damage": "6", # 目立った傷や汚れなし
|
||||
"slight_damage": "3", # やや傷や汚れあり
|
||||
"damaged": "2", # 傷や汚れあり
|
||||
"poor": "1", # 全体的に状態が悪い
|
||||
}
|
||||
|
||||
# ---- 售卖状态（搜索页 `transaction=`）----
|
||||
# 站点默认 all（不带该参数），仅在筛选时下发。
|
||||
TRANSACTION_CODES: Final[dict[str, str]] = {
|
||||
"on_sale": "selling", # 販売中のみ
|
||||
"sold_out": "soldout", # 売切れのみ
|
||||
}
|
||||
|
||||
# ---- 鉴定服务（搜索页 `authenticity_types=`）----
|
||||
AUTHENTICITY_CODES: Final[dict[str, str]] = {
|
||||
"before_delivery": "pre", # お届け前鑑定
|
||||
"after_delivery": "post", # 後から鑑定
|
||||
}
|
||||
|
||||
# 运费负担：站点用 carriage=1 表示「送料込みのみ」（卖家承担），0 表示不限
|
||||
CARRIAGE_INCLUDED: Final = "1"
|
||||
|
||||
# 商品详情页上表示已售出的标记文案
|
||||
SOLD_OUT_MARKERS: Final = ("SOLD OUT", "SOLDOUT", "売り切れました")
|
||||
|
||||
# 页面校验用的结构标记：这几类页面各自必须出现的 DOM 特征
|
||||
SEARCH_PAGE_MARKER: Final = "page-count"
|
||||
ITEM_PAGE_MARKER: Final = "item-info"
|
||||
SHOP_PAGE_MARKER: Final = "profile-area__shop-name"
|
||||
@@ -0,0 +1,107 @@
|
||||
"""乐天市场（rakuten.co.jp）站点常量
|
||||
|
||||
集中维护站点入口 URL、浏览器指纹参数，以及搜索页 URL 的排序码 / 筛选码映射。
|
||||
|
||||
排序码（`s=`）与筛选码（`f=`）并非猜测所得，而是从搜索页前端 bundle
|
||||
（r.r10s.jp/com/assets/app/pages/search/javascript/pc-*.bundle.js）中的
|
||||
URL→UiQuestion 转换逻辑里提取的枚举，与站点行为一一对应。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Final
|
||||
|
||||
# ---- 站点入口 ----
|
||||
HOME_URL: Final = "https://www.rakuten.co.jp/"
|
||||
SEARCH_BASE_URL: Final = "https://search.rakuten.co.jp/search/mall/"
|
||||
ITEM_BASE_URL: Final = "https://item.rakuten.co.jp/"
|
||||
|
||||
CATEGORY_BASE_URL: Final = "https://www.rakuten.co.jp/category/"
|
||||
|
||||
# 店铺首页形如 https://www.rakuten.co.jp/edion/
|
||||
WWW_BASE_URL: Final = "https://www.rakuten.co.jp/"
|
||||
|
||||
SEARCH_HOST: Final = "search.rakuten.co.jp"
|
||||
ITEM_HOST: Final = "item.rakuten.co.jp"
|
||||
WWW_HOST: Final = "www.rakuten.co.jp"
|
||||
|
||||
# 取顶层分类列表用的哨兵关键词。
|
||||
# 站点的分类分面（genreTree）与查询内容无关，任何关键词都会返回同一套 39 个顶层
|
||||
# 分类；刻意用一个搜不到东西的词，可以拿到 count 全为 null 的干净列表，避免把
|
||||
# 「某个关键词下的命中数」误当成分类的商品总数返回。
|
||||
# 注意不能用单字母（如 a），站点对过短的拉丁关键词直接返回 503。
|
||||
GENRE_FACET_PROBE_KEYWORD: Final = "zzzqqqxyz123"
|
||||
|
||||
# 站点侧限制：搜索结果最多只能翻到 pagination.subset 条（实测 6750），
|
||||
# 再往后翻页返回空列表。用于计算 has_more，避免上游无意义地深翻。
|
||||
DEFAULT_SUBSET_LIMIT: Final = 6750
|
||||
|
||||
# ---- 浏览器指纹 ----
|
||||
# 搜索页用 PC UA；商品详情页必须用手机 UA，否则返回的是各店铺自定义的
|
||||
# EUC-JP 老模板（无 __INITIAL_STATE__，只有面包屑 JSON-LD，无法结构化解析）。
|
||||
PC_USER_AGENT: Final = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
|
||||
)
|
||||
SP_USER_AGENT: Final = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 "
|
||||
"(KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1"
|
||||
)
|
||||
|
||||
ACCEPT_LANGUAGE: Final = "ja,en-US;q=0.9,en;q=0.8"
|
||||
|
||||
# 乐天前置 Akamai Bot Manager。请求头不完整时不会直接封禁，而是把响应
|
||||
# 拖到 ~11s（实测与响应体大小无关，22 字节的响应同样耗时 11s）；补齐
|
||||
# 下列头并复用 Akamai 下发的 cookie 后，稳定在 ~0.6-0.9s。
|
||||
def default_headers(*, mobile: bool) -> dict[str, str]:
|
||||
"""构造一套完整的浏览器导航请求头。"""
|
||||
return {
|
||||
"User-Agent": SP_USER_AGENT if mobile else PC_USER_AGENT,
|
||||
"Accept": (
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,"
|
||||
"image/avif,image/webp,image/apng,*/*;q=0.8,"
|
||||
"application/signed-exchange;v=b3;q=0.7"
|
||||
),
|
||||
"Accept-Language": ACCEPT_LANGUAGE,
|
||||
"sec-ch-ua": '"Chromium";v="131", "Not_A Brand";v="24", "Google Chrome";v="131"',
|
||||
"sec-ch-ua-mobile": "?1" if mobile else "?0",
|
||||
"sec-ch-ua-platform": '"iOS"' if mobile else '"Windows"',
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
|
||||
|
||||
# Akamai Bot Manager 下发的 cookie：判断会话是否已预热完成的依据
|
||||
AKAMAI_COOKIE_NAMES: Final = ("ak_bmsc", "bm_sv", "bm_mi", "_abck")
|
||||
|
||||
# ---- 排序（搜索页 `s=` 参数）----
|
||||
# 键为对外暴露的语义化排序名，值为站点侧排序码；standard 不带 s 参数。
|
||||
SORT_CODES: Final[dict[str, str | None]] = {
|
||||
"standard": None, # 站点默认的相关度排序（relevancy）
|
||||
"price_asc": "2",
|
||||
"price_desc": "3",
|
||||
"newest": "4",
|
||||
"review_count": "5",
|
||||
"review_score": "6",
|
||||
"price_with_shipping_asc": "11",
|
||||
"price_with_shipping_desc": "12",
|
||||
}
|
||||
|
||||
# ---- 成色（搜索页 `f=` 参数中的 condition 段）----
|
||||
CONDITION_CODES: Final[dict[str, str]] = {
|
||||
"new": "101",
|
||||
"used": "100",
|
||||
"rental": "102",
|
||||
}
|
||||
|
||||
# ---- 布尔筛选（搜索页 `f=` 参数，可重复出现）----
|
||||
# 字段名 -> 筛选码
|
||||
BOOL_FILTER_CODES: Final[dict[str, str]] = {
|
||||
"include_sold_out": "0",
|
||||
"free_shipping": "2",
|
||||
"has_review": "4",
|
||||
"next_day_delivery": "12",
|
||||
"super_deal": "13",
|
||||
}
|
||||
+189
@@ -0,0 +1,189 @@
|
||||
"""应用入口：FastAPI 应用创建与生命周期管理
|
||||
|
||||
职责：
|
||||
- 构建服务容器（依赖注入）
|
||||
- 管理应用生命周期（启动/关闭抓取会话与兜底浏览器）
|
||||
- 注册路由和全局异常处理器
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import ValidationError
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
|
||||
from app.api.routes.health import router as health_router
|
||||
from app.api.routes.rakuma import router as rakuma_router
|
||||
from app.api.routes.scrape import router as scrape_router
|
||||
from app.core.config import get_settings
|
||||
from app.core.container import ServiceContainer
|
||||
from app.core.errors import AppError
|
||||
from app.core.logging_setup import configure_logging
|
||||
from app.models.scrape import ApiResponse
|
||||
from app.services.browser_fallback import BrowserFallback
|
||||
from app.services.rakuma_client import RakumaClient
|
||||
from app.services.rakuma_session import RakumaSession
|
||||
from app.services.rakuten_client import RakutenClient
|
||||
from app.services.site_session import SiteSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def _format_validation_msg(errors: list[dict]) -> str:
|
||||
"""将校验错误整理为便于前端展示的消息。"""
|
||||
if not errors:
|
||||
return "Validation error"
|
||||
|
||||
messages: list[str] = []
|
||||
for error in errors:
|
||||
loc = ".".join(str(part) for part in error.get("loc", []) if part != "body")
|
||||
msg = str(error.get("msg", "Validation error"))
|
||||
messages.append(f"{loc}: {msg}" if loc else msg)
|
||||
|
||||
return "; ".join(messages)
|
||||
|
||||
|
||||
def build_container() -> ServiceContainer:
|
||||
"""构建服务容器，组装所有依赖"""
|
||||
settings = get_settings()
|
||||
browser_fallback = BrowserFallback(settings)
|
||||
site_session = SiteSession(settings, browser_fallback)
|
||||
rakuten_client = RakutenClient(settings, site_session)
|
||||
rakuma_session = RakumaSession(settings)
|
||||
rakuma_client = RakumaClient(settings, rakuma_session)
|
||||
return ServiceContainer(
|
||||
settings=settings,
|
||||
browser_fallback=browser_fallback,
|
||||
site_session=site_session,
|
||||
rakuten_client=rakuten_client,
|
||||
rakuma_session=rakuma_session,
|
||||
rakuma_client=rakuma_client,
|
||||
)
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""应用生命周期管理：启动时初始化各服务，关闭时释放资源"""
|
||||
container = build_container()
|
||||
app.state.container = container
|
||||
|
||||
configure_logging(container.settings)
|
||||
logger.info("应用启动：%s:%s", container.settings.app_host, container.settings.app_port)
|
||||
logger.info("日志级别：%s", container.settings.log_level)
|
||||
logger.info("当前环境：%s", container.settings.app_env)
|
||||
await container.site_session.start()
|
||||
await container.rakuma_session.start()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await container.rakuma_session.close()
|
||||
await container.site_session.close()
|
||||
await container.browser_fallback.close()
|
||||
|
||||
|
||||
def create_app() -> FastAPI:
|
||||
"""创建 FastAPI 应用实例，注册路由和异常处理器"""
|
||||
app = FastAPI(title="Rakuten Scraper Service", lifespan=lifespan)
|
||||
app.include_router(health_router)
|
||||
app.include_router(scrape_router)
|
||||
app.include_router(rakuma_router)
|
||||
|
||||
@app.exception_handler(AppError)
|
||||
async def app_error_handler(_: Request, exc: AppError) -> JSONResponse:
|
||||
"""业务异常处理器：返回结构化的错误响应"""
|
||||
return JSONResponse(
|
||||
status_code=exc.status_code,
|
||||
content=ApiResponse[None](
|
||||
success=False,
|
||||
code=exc.err_code,
|
||||
msg=exc.message,
|
||||
data=None,
|
||||
).model_dump(),
|
||||
headers=exc.headers,
|
||||
)
|
||||
|
||||
@app.exception_handler(RequestValidationError)
|
||||
async def validation_error_handler(_: Request, exc: RequestValidationError) -> JSONResponse:
|
||||
"""请求参数校验异常处理器"""
|
||||
errors = exc.errors()
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=ApiResponse[object](
|
||||
success=False,
|
||||
msg=_format_validation_msg(errors),
|
||||
data=jsonable_errors(errors),
|
||||
code=1002,
|
||||
).model_dump(),
|
||||
)
|
||||
|
||||
@app.exception_handler(ValidationError)
|
||||
async def pydantic_validation_error_handler(_: Request, exc: ValidationError) -> JSONResponse:
|
||||
"""Pydantic 模型校验异常处理器"""
|
||||
errors = exc.errors()
|
||||
return JSONResponse(
|
||||
status_code=422,
|
||||
content=ApiResponse[object](
|
||||
success=False,
|
||||
msg=_format_validation_msg(errors),
|
||||
data=jsonable_errors(errors),
|
||||
code=1002,
|
||||
).model_dump(),
|
||||
)
|
||||
|
||||
@app.exception_handler(StarletteHTTPException)
|
||||
async def http_exception_handler(_: Request, exc: StarletteHTTPException) -> JSONResponse:
|
||||
"""HTTP 异常处理器（404、500 等）"""
|
||||
status_code = int(getattr(exc, "status_code", 500) or 500)
|
||||
err_code = 1404 if status_code == 404 else 1500
|
||||
return JSONResponse(
|
||||
status_code=status_code,
|
||||
content=ApiResponse[None](
|
||||
success=False,
|
||||
msg=str(getattr(exc, "detail", "HTTP error")),
|
||||
data=None,
|
||||
code=err_code,
|
||||
).model_dump(),
|
||||
headers=getattr(exc, "headers", None),
|
||||
)
|
||||
|
||||
@app.exception_handler(Exception)
|
||||
async def unhandled_exception_handler(_: Request, exc: Exception) -> JSONResponse:
|
||||
"""兜底异常处理器：捕获所有未处理的异常"""
|
||||
logger.exception("未处理异常：%s", exc)
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content=ApiResponse[None](
|
||||
success=False,
|
||||
msg="Internal server error",
|
||||
data=None,
|
||||
code=1500,
|
||||
).model_dump(),
|
||||
)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def jsonable_errors(errors: list[dict]) -> list[dict]:
|
||||
"""剔除校验错误里不可 JSON 序列化的 ctx（如原始异常对象）"""
|
||||
return [{key: value for key, value in error.items() if key != "ctx"} for error in errors]
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
|
||||
settings = get_settings()
|
||||
configure_logging(settings)
|
||||
uvicorn.run(
|
||||
"app.main:app",
|
||||
host=settings.app_host,
|
||||
port=settings.app_port,
|
||||
log_config=None,
|
||||
timeout_keep_alive=120,
|
||||
)
|
||||
@@ -0,0 +1,701 @@
|
||||
"""API 数据模型：请求体和响应体定义
|
||||
|
||||
字段命名贴合乐天站点自身的语义（item_code / shop_code / genre_id / sku 等），
|
||||
不做跨站点的字段名归一，避免解析层与对外契约之间反复翻译。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import StrEnum
|
||||
from typing import Any, Generic, TypeVar
|
||||
|
||||
from pydantic import BaseModel, Field, HttpUrl, model_validator
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
class ApiResponse(BaseModel, Generic[T]):
|
||||
"""统一 API 响应格式"""
|
||||
|
||||
success: bool
|
||||
msg: str
|
||||
data: T | None = None
|
||||
code: int
|
||||
|
||||
|
||||
class SortOption(StrEnum):
|
||||
"""搜索排序方式，对应搜索页 `s=` 参数"""
|
||||
|
||||
STANDARD = "standard" # 站点默认相关度排序
|
||||
PRICE_ASC = "price_asc"
|
||||
PRICE_DESC = "price_desc"
|
||||
NEWEST = "newest"
|
||||
REVIEW_COUNT = "review_count"
|
||||
REVIEW_SCORE = "review_score"
|
||||
PRICE_WITH_SHIPPING_ASC = "price_with_shipping_asc"
|
||||
PRICE_WITH_SHIPPING_DESC = "price_with_shipping_desc"
|
||||
|
||||
|
||||
class ItemCondition(StrEnum):
|
||||
"""商品成色筛选"""
|
||||
|
||||
NEW = "new"
|
||||
USED = "used"
|
||||
RENTAL = "rental"
|
||||
|
||||
|
||||
class SearchRequest(BaseModel):
|
||||
"""搜索请求参数
|
||||
|
||||
三种用法（优先级从高到低）：
|
||||
1. 传 search_url：直接透传一条乐天搜索页 URL，服务端原样抓取，
|
||||
此时除 page 与 exclude_ads 外的筛选字段全部忽略；
|
||||
page 若显式指定（>1），会覆盖 URL 中的页码。
|
||||
2. 传 keyword（可叠加任意筛选字段）
|
||||
3. 只传 genre_id：抓取该分类下的商品
|
||||
"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = Field(default=1, ge=1, le=150) # 站点侧最多约 150 页（subset 6750 / 45）
|
||||
sort: SortOption = SortOption.STANDARD
|
||||
genre_id: str | None = None # 乐天分类 ID，如 565950
|
||||
min_price: int | None = Field(default=None, ge=0)
|
||||
max_price: int | None = Field(default=None, ge=0)
|
||||
shop_id: int | None = None # 限定店铺（对应 `sid` 参数，取搜索结果的 shop.shop_id）
|
||||
exclude_keyword: str | None = None # 排除词（`nitem`）
|
||||
title_only: bool = False # 仅在商品标题中匹配（`sf=1`）
|
||||
or_query: bool = False # 关键词之间用 OR 而非 AND（`st=O`）
|
||||
min_review_score: int | None = Field(default=None, ge=1, le=5) # 最低评分
|
||||
condition: ItemCondition | None = None # 新品 / 中古 / 租赁
|
||||
include_sold_out: bool = False # 包含售罄商品
|
||||
free_shipping: bool = False # 仅免运费
|
||||
has_review: bool = False # 仅有评论
|
||||
next_day_delivery: bool = False # 仅次日达
|
||||
super_deal: bool = False # 仅 SuperDEAL
|
||||
tags: list[str] = Field(default_factory=list) # 站点标签 ID（`tg`）
|
||||
search_url: HttpUrl | None = None
|
||||
exclude_ads: bool = True # 剔除搜索结果中混入的 CPC 广告位
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_search_target(self) -> SearchRequest:
|
||||
if (
|
||||
not self.search_url
|
||||
and not self.keyword.strip()
|
||||
and not self.genre_id
|
||||
and self.shop_id is None
|
||||
):
|
||||
raise ValueError("keyword、genre_id、shop_id、search_url 至少需要提供一个")
|
||||
if self.min_price is not None and self.max_price is not None and self.min_price > self.max_price:
|
||||
raise ValueError("min_price 不能大于 max_price")
|
||||
return self
|
||||
|
||||
|
||||
class ShopDetailRequest(BaseModel):
|
||||
"""乐天商家详情请求参数：传店铺代码（店铺 URL 的路径段），或直接传店铺页 URL"""
|
||||
|
||||
shop_code: str | None = None # 店铺代码，如 edion
|
||||
shop_url: HttpUrl | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> ShopDetailRequest:
|
||||
if not self.shop_url and not (self.shop_code or "").strip():
|
||||
raise ValueError("需要提供 shop_code 或 shop_url")
|
||||
return self
|
||||
|
||||
|
||||
class ShopItemsRequest(BaseModel):
|
||||
"""乐天商家商品列表请求参数
|
||||
|
||||
站点没有单独的「店铺内商品」接口，本服务转成一次限定店铺的搜索
|
||||
（搜索页的 `sid` 参数），因此支持与 /api/search 相同的排序与筛选。
|
||||
|
||||
shop_id 与 shop_code 至少提供一个；只给 shop_code 时会先取一次店铺详情
|
||||
换出 shop_id，多花一次请求，能直接给 shop_id 时优先给。
|
||||
"""
|
||||
|
||||
shop_id: int | None = None # 取自搜索结果或商家详情的 shop.shop_id
|
||||
shop_code: str | None = None # 店铺代码，如 edion
|
||||
keyword: str = "" # 在店铺内按关键词过滤
|
||||
page: int = Field(default=1, ge=1, le=150)
|
||||
sort: SortOption = SortOption.STANDARD
|
||||
genre_id: str | None = None
|
||||
min_price: int | None = Field(default=None, ge=0)
|
||||
max_price: int | None = Field(default=None, ge=0)
|
||||
condition: ItemCondition | None = None
|
||||
include_sold_out: bool = False
|
||||
free_shipping: bool = False
|
||||
exclude_ads: bool = True
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> ShopItemsRequest:
|
||||
if self.shop_id is None and not (self.shop_code or "").strip():
|
||||
raise ValueError("需要提供 shop_id 或 shop_code")
|
||||
if self.min_price is not None and self.max_price is not None and self.min_price > self.max_price:
|
||||
raise ValueError("min_price 不能大于 max_price")
|
||||
return self
|
||||
|
||||
def to_search_request(self, shop_id: int) -> SearchRequest:
|
||||
"""转成一次限定店铺的搜索请求
|
||||
|
||||
站点没有独立的「店铺内商品」页可供分页抓取，店铺商品实际就是
|
||||
`sid=` 限定后的搜索结果，因此这里复用同一条抓取链路。
|
||||
"""
|
||||
return SearchRequest(
|
||||
keyword=self.keyword,
|
||||
page=self.page,
|
||||
sort=self.sort,
|
||||
genre_id=self.genre_id,
|
||||
min_price=self.min_price,
|
||||
max_price=self.max_price,
|
||||
shop_id=shop_id,
|
||||
condition=self.condition,
|
||||
include_sold_out=self.include_sold_out,
|
||||
free_shipping=self.free_shipping,
|
||||
exclude_ads=self.exclude_ads,
|
||||
)
|
||||
|
||||
|
||||
class ShopDetailData(BaseModel):
|
||||
"""乐天商家详情数据"""
|
||||
|
||||
shop_id: int | None = None
|
||||
shop_code: str = ""
|
||||
shop_name: str = ""
|
||||
shop_url: str = ""
|
||||
introduction: str = "" # 店铺简介
|
||||
signboard_url: str = "" # 店铺招牌图
|
||||
logo_url: str = ""
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
# 站点在评价数过少时不展示评分；此时 review_score 不可信
|
||||
review_displayed: bool = False
|
||||
is_39_shop: bool = False # 39ショップ（满 3980 日元免运费）
|
||||
age_verification_required: bool = False
|
||||
status: int | None = None # 站点店铺状态码，1 = 营业中
|
||||
holidays: list[str] = Field(default_factory=list) # 店铺休息日
|
||||
|
||||
|
||||
class ItemDetailRequest(BaseModel):
|
||||
"""商品详情请求参数：传 shop_code + item_code，或直接传商品页 URL"""
|
||||
|
||||
shop_code: str | None = None # 店铺代码，如 edion（商品 URL 的第一段）
|
||||
item_code: str | None = None # 店铺内商品编号，如 4902370549263（商品 URL 的第二段）
|
||||
item_url: HttpUrl | None = None
|
||||
include_sku_variants: bool = True # SKU 组合可能多达数百条，不需要时可关闭
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_item_target(self) -> ItemDetailRequest:
|
||||
if not self.item_url and not (self.shop_code and self.item_code):
|
||||
raise ValueError("需要提供 item_url，或同时提供 shop_code 与 item_code")
|
||||
return self
|
||||
|
||||
|
||||
class GenreRequest(BaseModel):
|
||||
"""分类查询参数
|
||||
|
||||
不传 genre_id 时返回 39 个顶层分类；传入时返回该分类的信息、祖先路径与直接子分类。
|
||||
"""
|
||||
|
||||
genre_id: str | None = None
|
||||
|
||||
|
||||
class GenreNode(BaseModel):
|
||||
"""分类树上的一个节点"""
|
||||
|
||||
genre_id: str = ""
|
||||
name: str = ""
|
||||
# 该分类下的商品数。顶层列表不返回该值：站点给出的是「当前查询在该分类下的
|
||||
# 命中数」，与分类自身的商品总量不是一回事，避免误用。
|
||||
item_count: int | None = None
|
||||
shortcut: str = "" # 站点分类短代码，如 game / flower
|
||||
is_leaf: bool = False # 叶子分类，没有下级
|
||||
url: str = "" # 分类页地址
|
||||
|
||||
|
||||
class GenreData(BaseModel):
|
||||
"""分类查询结果"""
|
||||
|
||||
genre_id: str = "" # 空串表示顶层
|
||||
name: str = ""
|
||||
full_name: str = "" # 站点给出的完整分类名，仅分类页有
|
||||
description: str = "" # 站点分类描述，仅分类页有
|
||||
is_leaf: bool = False
|
||||
url: str = ""
|
||||
ancestors: list[GenreNode] = Field(default_factory=list) # 从顶层到父级，不含自身
|
||||
children: list[GenreNode] = Field(default_factory=list) # 直接子分类
|
||||
|
||||
|
||||
class ShopSummary(BaseModel):
|
||||
"""店铺信息"""
|
||||
|
||||
shop_id: int | None = None
|
||||
shop_code: str = "" # 店铺 URL 代码，如 edion；与 item_code 一起可定位商品
|
||||
shop_name: str = ""
|
||||
shop_url: str = ""
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
|
||||
|
||||
class ReviewSummary(BaseModel):
|
||||
"""评价信息"""
|
||||
|
||||
score: float = 0.0
|
||||
count: int = 0
|
||||
url: str = ""
|
||||
|
||||
|
||||
class SearchItem(BaseModel):
|
||||
"""搜索结果中的单个商品"""
|
||||
|
||||
item_id: str = "" # 乐天内部商品 ID（搜索结果的 code 字段）
|
||||
item_code: str = "" # 商品 URL 第二段，调详情接口用
|
||||
item_name: str = ""
|
||||
item_url: str = "" # 真实商品页地址；广告位已还原为 originalItemUrl
|
||||
catch_copy: str = "" # 商品副标题
|
||||
price: int = 0
|
||||
price_range: str = "" # 多 SKU 时的价格区间，如 "1000~2000"
|
||||
has_price_range: bool = False
|
||||
image_url: str = ""
|
||||
image_urls: list[str] = Field(default_factory=list)
|
||||
shop: ShopSummary = Field(default_factory=ShopSummary)
|
||||
review: ReviewSummary = Field(default_factory=ReviewSummary)
|
||||
genre_id: str = ""
|
||||
genre_path: str = "" # 形如 /0/101205/565950/566404
|
||||
genre_names: list[str] = Field(default_factory=list)
|
||||
shipping_fee: int | None = None # 站点未给出时为 null
|
||||
delivery_message: str = ""
|
||||
point_count: int = 0
|
||||
is_sold_out: bool = False
|
||||
is_ad: bool = False # CPC 广告位
|
||||
has_multi_sku: bool = False
|
||||
variant_id: str = ""
|
||||
item_options: dict[str, Any] = Field(default_factory=dict) # 站点 itemOptions 原样透出
|
||||
|
||||
|
||||
class SearchResultData(BaseModel):
|
||||
"""搜索结果数据"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = 1
|
||||
page_size: int = 0
|
||||
total_count: int = 0 # 站点声明的命中总数
|
||||
reachable_count: int = 0 # 实际可翻页取到的上限（站点 subset，随查询条件变化）
|
||||
has_more: bool = False
|
||||
# 请求页码超出 reachable_count 对应的页数。站点此时不会返回空列表，而是
|
||||
# 静默回绕到第 1 页；这里识别出来并把 items 置空，避免上游把重复数据当新数据。
|
||||
out_of_range: bool = False
|
||||
ad_count: int = 0 # 本页被识别出的广告位数量（exclude_ads=true 时已从 items 剔除）
|
||||
request_url: str = "" # 实际抓取的乐天页面地址，便于排查
|
||||
items: list[SearchItem] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkuAttribute(BaseModel):
|
||||
"""SKU 属性项"""
|
||||
|
||||
title: str = ""
|
||||
value: str = ""
|
||||
|
||||
|
||||
class SkuAxisValue(BaseModel):
|
||||
"""SKU 选择轴上的一个取值"""
|
||||
|
||||
value: str = ""
|
||||
label: str = ""
|
||||
is_sold_out: bool = False
|
||||
|
||||
|
||||
class SkuAxis(BaseModel):
|
||||
"""SKU 选择轴，如「颜色」「尺码」"""
|
||||
|
||||
key: str = ""
|
||||
label: str = ""
|
||||
values: list[SkuAxisValue] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkuVariant(BaseModel):
|
||||
"""一个具体的 SKU 组合"""
|
||||
|
||||
variant_id: str = ""
|
||||
selector_values: list[str] = Field(default_factory=list) # 与 axis 顺序对应的取值
|
||||
price: int = 0
|
||||
quantity: int = 0
|
||||
is_sold_out: bool = False
|
||||
delivery_message: str = ""
|
||||
attributes: list[SkuAttribute] = Field(default_factory=list)
|
||||
|
||||
|
||||
class SkuInfo(BaseModel):
|
||||
"""商品 SKU 信息"""
|
||||
|
||||
inventory_type: str = "" # single / multiple
|
||||
quantity: int = 0
|
||||
show_inventory: bool = False
|
||||
delivery_message: str = ""
|
||||
attributes: list[SkuAttribute] = Field(default_factory=list)
|
||||
axis: list[SkuAxis] = Field(default_factory=list)
|
||||
variants: list[SkuVariant] = Field(default_factory=list) # include_sku_variants=false 时为空
|
||||
variant_count: int = 0 # 不受 include_sku_variants 影响，始终为真实组合数
|
||||
|
||||
|
||||
class ShippingInfo(BaseModel):
|
||||
"""配送与运费信息"""
|
||||
|
||||
shipping_fee: int | None = None
|
||||
is_shipping_free: bool = False
|
||||
is_asuraku: bool = False # あす楽（次日达）
|
||||
is_next_day_delivery: bool = False
|
||||
free_shipping_threshold: int | None = None
|
||||
prefecture_id: int | None = None # 站点默认收货地（13 = 东京都）
|
||||
delivery_message: str = ""
|
||||
|
||||
|
||||
class Breadcrumb(BaseModel):
|
||||
"""分类面包屑"""
|
||||
|
||||
name: str = ""
|
||||
url: str = ""
|
||||
|
||||
|
||||
class PurchaseOptionValue(BaseModel):
|
||||
"""商品选项的一个可选值"""
|
||||
|
||||
value_id: str = ""
|
||||
name: str = ""
|
||||
|
||||
|
||||
class PurchaseOption(BaseModel):
|
||||
"""商品选项（選択肢），如「名入れ」「ラッピング」
|
||||
|
||||
加购时需要按 `名称:取值` 的形式拼进 options_field 指定的字段。
|
||||
"""
|
||||
|
||||
option_id: str = ""
|
||||
name: str = ""
|
||||
type: str = "" # select 单选 / check 多选 / text 自由文本
|
||||
is_required: bool = False
|
||||
values: list[PurchaseOptionValue] = Field(default_factory=list) # type=text 时为空
|
||||
|
||||
|
||||
class PurchaseInfo(BaseModel):
|
||||
"""构造加购请求所需的信息
|
||||
|
||||
本服务只提供数据、不执行加购——加购需要已登录的乐天账号会话，由持有登录态的
|
||||
下游负责。四个来源的加购端点与字段名各不相同，因此这里不写死字段，而是把
|
||||
「提交到哪里、固定字段是什么、数量/规格/选项各该用哪个字段名」显式描述出来：
|
||||
|
||||
payload = {**form_fields}
|
||||
payload[quantity_field] = 数量 # quantity_field 为空表示不支持指定数量
|
||||
payload[variant_field] = 选中的 sku.variants[].variant_id # variant_field 为空表示无规格
|
||||
payload[options_field] = ["选项名:取值", ...] # options_field 为空表示无选项
|
||||
|
||||
然后以 cart_method 提交到 cart_url。
|
||||
"""
|
||||
|
||||
cart_url: str = ""
|
||||
cart_method: str = "POST"
|
||||
form_fields: dict[str, str] = Field(default_factory=dict)
|
||||
quantity_field: str = ""
|
||||
variant_field: str = ""
|
||||
options_field: str = ""
|
||||
options: list[PurchaseOption] = Field(default_factory=list)
|
||||
has_required_options: bool = False
|
||||
|
||||
|
||||
class ItemDetailData(BaseModel):
|
||||
"""商品详情数据
|
||||
|
||||
部分乐天官方店的商品页会跳转到独立子站，各子站页面结构不同、可提供的字段也
|
||||
不同。source 标明这条数据由哪个站点解析而来，字段覆盖差异见 README。
|
||||
"""
|
||||
|
||||
source: str = "ichiba" # ichiba / books / brandavenue / biccamera
|
||||
source_url: str = "" # 实际解析的页面地址；跳转时与 item_url 不同
|
||||
item_id: str = ""
|
||||
item_code: str = ""
|
||||
item_name: str = ""
|
||||
catch_copy: str = ""
|
||||
description: str = "" # 店铺自填的商品说明，含 HTML
|
||||
item_url: str = ""
|
||||
price: int = 0 # 最低售价（含税，多 SKU 时为最低价）
|
||||
pre_tax_price: int = 0
|
||||
tax_flag: bool = False
|
||||
tax_rate: float = 0.0
|
||||
purchase_condition: str = "" # 站点原值，enabled 表示可购买
|
||||
is_sold_out: bool = False
|
||||
purchase_unit: int = 0 # 起订单位
|
||||
images: list[str] = Field(default_factory=list)
|
||||
shop: ShopSummary = Field(default_factory=ShopSummary)
|
||||
review: ReviewSummary = Field(default_factory=ReviewSummary)
|
||||
genre_id: str = ""
|
||||
breadcrumbs: list[Breadcrumb] = Field(default_factory=list)
|
||||
shipping: ShippingInfo = Field(default_factory=ShippingInfo)
|
||||
sku: SkuInfo = Field(default_factory=SkuInfo)
|
||||
purchase: PurchaseInfo = Field(default_factory=PurchaseInfo)
|
||||
|
||||
|
||||
class HealthData(BaseModel):
|
||||
"""健康检查响应数据"""
|
||||
|
||||
status: str
|
||||
browser_fallback_enabled: bool
|
||||
browser_fallback_ready: bool
|
||||
browser_fallback_error: str | None = None
|
||||
sessions: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
# ==========================================================================
|
||||
# ラクマ（fril.jp）
|
||||
#
|
||||
# 乐天市场是 B2C 商城（店铺 × 商品 × SKU），ラクマ 是 C2C 二手集市：
|
||||
# 每件商品都是独一无二的一件，没有 SKU、没有库存数量、没有店铺代码，
|
||||
# 卖家用一串 hash 标识。字段因此单独建模，不与市场侧强行合并。
|
||||
# ==========================================================================
|
||||
|
||||
|
||||
class RakumaSortOption(StrEnum):
|
||||
"""ラクマ 搜索排序方式，对应搜索页 `sort=` + `order=` 两个参数"""
|
||||
|
||||
STANDARD = "standard" # おすすめ順（站点默认）
|
||||
NEWEST = "newest" # 新着順
|
||||
PRICE_ASC = "price_asc"
|
||||
PRICE_DESC = "price_desc"
|
||||
LIKE_COUNT = "like_count" # いいね数順
|
||||
|
||||
|
||||
class RakumaCondition(StrEnum):
|
||||
"""ラクマ 商品状态（出品者自己申告的成色，6 档）"""
|
||||
|
||||
NEW = "new" # 新品、未使用
|
||||
ALMOST_NEW = "almost_new" # 未使用に近い
|
||||
NO_DAMAGE = "no_damage" # 目立った傷や汚れなし
|
||||
SLIGHT_DAMAGE = "slight_damage" # やや傷や汚れあり
|
||||
DAMAGED = "damaged" # 傷や汚れあり
|
||||
POOR = "poor" # 全体的に状態が悪い
|
||||
|
||||
|
||||
class RakumaTransaction(StrEnum):
|
||||
"""ラクマ 售卖状态筛选"""
|
||||
|
||||
ON_SALE = "on_sale" # 販売中のみ
|
||||
SOLD_OUT = "sold_out" # 売切れのみ
|
||||
|
||||
|
||||
class RakumaAuthenticity(StrEnum):
|
||||
"""ラクマ 正品鉴定服务类型"""
|
||||
|
||||
BEFORE_DELIVERY = "before_delivery" # お届け前鑑定
|
||||
AFTER_DELIVERY = "after_delivery" # 後から鑑定
|
||||
|
||||
|
||||
class RakumaSearchRequest(BaseModel):
|
||||
"""ラクマ 搜索请求参数
|
||||
|
||||
两种用法（优先级从高到低）：
|
||||
1. 传 search_url：透传一条 fril.jp 搜索页 URL，此时除 page 外的筛选字段全部忽略
|
||||
2. 传 keyword / category_id / brand_id（可叠加任意筛选字段）
|
||||
|
||||
keyword、category_id、brand_id、search_url 四者至少提供一个。
|
||||
"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = Field(default=1, ge=1, le=100) # 站点侧 page>100 直接 404
|
||||
sort: RakumaSortOption = RakumaSortOption.STANDARD
|
||||
category_id: str | None = None # ラクマ 分类 ID，如 788
|
||||
brand_id: str | None = None # ラクマ 品牌 ID，如 5296
|
||||
min_price: int | None = Field(default=None, ge=0)
|
||||
max_price: int | None = Field(default=None, ge=0)
|
||||
exclude_keyword: str | None = None # 排除词（站点 `excluded_query`，需与 keyword 同时使用）
|
||||
conditions: list[RakumaCondition] = Field(default_factory=list) # 可多选
|
||||
transaction: RakumaTransaction | None = None # 不传表示不限
|
||||
free_shipping: bool = False # 仅「送料込み」（卖家承担运费）
|
||||
anonymous_shipping: bool = False # 仅匿名配送
|
||||
except_for_no_brand: bool = False # 排除无品牌商品；与 brand_id 互斥
|
||||
authenticity_types: list[RakumaAuthenticity] = Field(default_factory=list)
|
||||
search_url: HttpUrl | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_search_target(self) -> RakumaSearchRequest:
|
||||
if not self.search_url and not self.keyword.strip() and not self.category_id and not self.brand_id:
|
||||
raise ValueError("keyword、category_id、brand_id、search_url 至少需要提供一个")
|
||||
if self.min_price is not None and self.max_price is not None and self.min_price > self.max_price:
|
||||
raise ValueError("min_price 不能大于 max_price")
|
||||
# 站点前端在无关键词时会拒绝下发 excluded_query，服务端也不认，这里提前拦下
|
||||
if self.exclude_keyword and not self.keyword.strip():
|
||||
raise ValueError("exclude_keyword 必须与 keyword 同时使用")
|
||||
return self
|
||||
|
||||
|
||||
class RakumaItemDetailRequest(BaseModel):
|
||||
"""ラクマ 商品详情请求参数：传 item_id（商品 URL 的最后一段），或直接传商品页 URL"""
|
||||
|
||||
item_id: str | None = None # 商品页 hash，如 4aca1d6db3e422f3a251a8a8b61e1eff
|
||||
item_url: HttpUrl | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_item_target(self) -> RakumaItemDetailRequest:
|
||||
if not self.item_url and not (self.item_id or "").strip():
|
||||
raise ValueError("需要提供 item_id 或 item_url")
|
||||
return self
|
||||
|
||||
|
||||
class RakumaShopDetailRequest(BaseModel):
|
||||
"""ラクマ 卖家详情请求参数：传 shop_id（店铺 URL 的最后一段），或直接传店铺页 URL"""
|
||||
|
||||
shop_id: str | None = None # 店铺页 hash，如 422750cb7921557bc8dba2416915d968
|
||||
shop_url: HttpUrl | None = None
|
||||
# 评价明细在单独的 /review 页上，需要多打一次请求，默认不取
|
||||
include_reviews: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> RakumaShopDetailRequest:
|
||||
if not self.shop_url and not (self.shop_id or "").strip():
|
||||
raise ValueError("需要提供 shop_id 或 shop_url")
|
||||
return self
|
||||
|
||||
|
||||
class RakumaShopItemsRequest(BaseModel):
|
||||
"""ラクマ 卖家商品列表请求参数
|
||||
|
||||
店铺页按上架顺序分页展示该卖家的全部商品（含已售出），站点不提供
|
||||
排序与筛选参数，因此这里只有页码。
|
||||
"""
|
||||
|
||||
shop_id: str | None = None
|
||||
shop_url: HttpUrl | None = None
|
||||
page: int = Field(default=1, ge=1)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def check_shop_target(self) -> RakumaShopItemsRequest:
|
||||
if not self.shop_url and not (self.shop_id or "").strip():
|
||||
raise ValueError("需要提供 shop_id 或 shop_url")
|
||||
return self
|
||||
|
||||
|
||||
class RakumaSeller(BaseModel):
|
||||
"""ラクマ 卖家（出品者）摘要"""
|
||||
|
||||
shop_id: str = "" # 店铺页 hash，可直接用于 /api/rakuma/shop_detail
|
||||
user_id: str = "" # 站点内部数值用户 ID
|
||||
shop_name: str = "" # 店铺名，卖家可自定义
|
||||
user_name: str = "" # 用户昵称
|
||||
shop_url: str = ""
|
||||
icon_url: str = ""
|
||||
seller_type: str = "" # 站点原值，如 一般 / 事業者
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
is_verified: bool = False # 本人確認済
|
||||
|
||||
|
||||
class RakumaSearchItem(BaseModel):
|
||||
"""ラクマ 搜索结果中的单个商品"""
|
||||
|
||||
item_id: str = "" # 商品页 hash，调详情接口用
|
||||
item_number: str = "" # 站点内部数值商品 ID
|
||||
item_name: str = ""
|
||||
item_url: str = ""
|
||||
price: int = 0
|
||||
image_url: str = ""
|
||||
is_sold_out: bool = False
|
||||
brand_id: str = ""
|
||||
brand_name: str = ""
|
||||
category_id: str = ""
|
||||
category_names: list[str] = Field(default_factory=list)
|
||||
seller_user_id: str = "" # 卖家数值 ID；店铺 hash 需从详情页取
|
||||
seller_type: str = ""
|
||||
|
||||
|
||||
class RakumaSearchResultData(BaseModel):
|
||||
"""ラクマ 搜索结果数据"""
|
||||
|
||||
keyword: str = ""
|
||||
page: int = 1
|
||||
page_size: int = 0
|
||||
# 站点声明的命中总数。页面上展示为「約1,190,000件」的四舍五入值，
|
||||
# 这里取的是埋点属性里的精确值。
|
||||
total_count: int = 0
|
||||
has_more: bool = False
|
||||
request_url: str = ""
|
||||
items: list[RakumaSearchItem] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RakumaItemDetailData(BaseModel):
|
||||
"""ラクマ 商品详情数据
|
||||
|
||||
C2C 集市的商品是单件的：没有 SKU 组合，没有库存数量，
|
||||
「規格」在站点上只体现为一个可选的尺码字段。
|
||||
"""
|
||||
|
||||
item_id: str = ""
|
||||
item_number: str = ""
|
||||
item_name: str = ""
|
||||
description: str = ""
|
||||
item_url: str = ""
|
||||
price: int = 0
|
||||
is_sold_out: bool = False
|
||||
images: list[str] = Field(default_factory=list)
|
||||
condition: str = "" # 商品の状態，站点原文如「目立った傷や汚れなし」
|
||||
size: str = "" # サイズ，无尺码时为空
|
||||
brand_id: str = ""
|
||||
brand_name: str = ""
|
||||
category_id: str = "" # 最具体的一级分类 ID
|
||||
breadcrumbs: list[Breadcrumb] = Field(default_factory=list)
|
||||
shipping_payer: str = "" # 配送料の負担，如「送料込」
|
||||
shipping_method: str = "" # 配送方法
|
||||
shipping_date_estimate: str = "" # 発送日の目安
|
||||
shipping_from: str = "" # 発送元の地域
|
||||
is_anonymous_shipping: bool = False # 匿名配送
|
||||
like_count: int = 0 # いいね数
|
||||
comment_count: int = 0
|
||||
posted_at: str = "" # 站点展示的相对时间，如「約1時間前」
|
||||
seller: RakumaSeller = Field(default_factory=RakumaSeller)
|
||||
|
||||
|
||||
class RakumaReview(BaseModel):
|
||||
"""ラクマ 卖家的一条交易评价"""
|
||||
|
||||
rating: str = "" # good / normal / bad
|
||||
title: str = "" # 站点原文，如「よい出品者です」
|
||||
comment: str = ""
|
||||
reviewer_name: str = ""
|
||||
reviewed_at: str = "" # 站点展示的日期，如 2026/05/04
|
||||
|
||||
|
||||
class RakumaRatingBreakdown(BaseModel):
|
||||
"""评价数量分档"""
|
||||
|
||||
good: int = 0
|
||||
normal: int = 0
|
||||
bad: int = 0
|
||||
|
||||
|
||||
class RakumaShopDetailData(BaseModel):
|
||||
"""ラクマ 卖家详情数据"""
|
||||
|
||||
shop_id: str = ""
|
||||
user_id: str = ""
|
||||
shop_name: str = ""
|
||||
user_name: str = ""
|
||||
shop_url: str = ""
|
||||
icon_url: str = ""
|
||||
cover_url: str = ""
|
||||
introduction: str = "" # プロフィール文
|
||||
review_score: float = 0.0
|
||||
review_count: int = 0
|
||||
is_verified: bool = False # 本人確認済
|
||||
verification_label: str = "" # 站点原文，如「本人確認済」/「本人確認未完了」
|
||||
item_count: int = 0 # 该卖家在售 + 已售商品总数
|
||||
# 以下三项需 include_reviews=true 才会填充
|
||||
rating_breakdown: RakumaRatingBreakdown = Field(default_factory=RakumaRatingBreakdown)
|
||||
seller_rating_breakdown: RakumaRatingBreakdown = Field(default_factory=RakumaRatingBreakdown)
|
||||
reviews: list[RakumaReview] = Field(default_factory=list)
|
||||
|
||||
|
||||
class RakumaShopItemsData(BaseModel):
|
||||
"""ラクマ 卖家商品列表数据"""
|
||||
|
||||
shop_id: str = ""
|
||||
shop_name: str = ""
|
||||
page: int = 1
|
||||
total_count: int = 0 # 该卖家的商品总数
|
||||
has_more: bool = False
|
||||
request_url: str = ""
|
||||
items: list[RakumaSearchItem] = Field(default_factory=list)
|
||||
@@ -0,0 +1,103 @@
|
||||
"""分类页/搜索页 __INITIAL_STATE__ → GenreData
|
||||
|
||||
分类数据来自 state.data.genreTree.parent_category。它不是完整分类树，而是一条
|
||||
「从根一路展开到目标分类」的链：每层只保留通往目标的那一个子节点，目标分类自身
|
||||
则挂着它的全部直接子分类。
|
||||
|
||||
不带分类查询时，根节点直接挂着 39 个顶层分类。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import GenreData, GenreNode
|
||||
from app.utils.coerce import as_dict, as_int, as_list, as_str
|
||||
|
||||
# 站点用 id=0 表示分类树的虚拟根，它不是一个真实分类
|
||||
ROOT_GENRE_ID = 0
|
||||
|
||||
|
||||
def _genre_url(genre_id: str) -> str:
|
||||
return f"{site.CATEGORY_BASE_URL}{genre_id}/" if genre_id else ""
|
||||
|
||||
|
||||
def _to_node(raw: dict[str, Any], *, with_count: bool) -> GenreNode:
|
||||
genre_id = str(as_int(raw.get("id")))
|
||||
count = raw.get("count")
|
||||
return GenreNode(
|
||||
genre_id=genre_id,
|
||||
name=as_str(raw.get("name")),
|
||||
item_count=as_int(count) if with_count and count is not None else None,
|
||||
shortcut=as_str(raw.get("shortcut")),
|
||||
is_leaf=bool(raw.get("leaf")),
|
||||
url=_genre_url(genre_id),
|
||||
)
|
||||
|
||||
|
||||
def _find_path(node: dict[str, Any], genre_id: str) -> list[dict[str, Any]] | None:
|
||||
"""在分类链中定位目标分类，返回从根到它的节点路径（含自身）"""
|
||||
if str(as_int(node.get("id"))) == genre_id:
|
||||
return [node]
|
||||
for child in as_list(node.get("children")):
|
||||
if not isinstance(child, dict):
|
||||
continue
|
||||
found = _find_path(child, genre_id)
|
||||
if found is not None:
|
||||
return [node, *found]
|
||||
return None
|
||||
|
||||
|
||||
def parse_genres(state: dict[str, Any], *, genre_id: str | None) -> GenreData:
|
||||
"""解析分类树
|
||||
|
||||
Args:
|
||||
genre_id: 目标分类；None 表示取顶层分类列表
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面里没有分类树，或目标分类不在返回的链上
|
||||
"""
|
||||
data = as_dict(as_dict(state.get("state")).get("data"))
|
||||
root = as_dict(as_dict(data.get("genreTree")).get("parent_category"))
|
||||
if not root:
|
||||
raise ScrapeParseError("页面中缺少 genreTree.parent_category 节点")
|
||||
|
||||
if genre_id is None:
|
||||
children = [
|
||||
_to_node(child, with_count=False)
|
||||
for child in as_list(root.get("children"))
|
||||
if isinstance(child, dict)
|
||||
]
|
||||
if not children:
|
||||
raise ScrapeParseError("未能取到顶层分类列表")
|
||||
return GenreData(children=children)
|
||||
|
||||
path = _find_path(root, genre_id)
|
||||
if path is None:
|
||||
raise ScrapeParseError(f"分类树中未找到分类 {genre_id}")
|
||||
|
||||
target = path[-1]
|
||||
ancestors = [
|
||||
_to_node(node, with_count=False)
|
||||
for node in path[:-1]
|
||||
if as_int(node.get("id")) != ROOT_GENRE_ID
|
||||
]
|
||||
children = [
|
||||
_to_node(child, with_count=True)
|
||||
for child in as_list(target.get("children"))
|
||||
if isinstance(child, dict)
|
||||
]
|
||||
|
||||
genre_info = as_dict(data.get("genreInfo"))
|
||||
resolved_id = str(as_int(target.get("id")))
|
||||
return GenreData(
|
||||
genre_id=resolved_id,
|
||||
name=as_str(target.get("name")),
|
||||
full_name=as_str(genre_info.get("fullGenreName")),
|
||||
description=as_str(genre_info.get("description")),
|
||||
is_leaf=bool(target.get("leaf")),
|
||||
url=_genre_url(resolved_id),
|
||||
ancestors=ancestors,
|
||||
children=children,
|
||||
)
|
||||
@@ -0,0 +1,281 @@
|
||||
"""商品详情页 __INITIAL_STATE__ → ItemDetailData
|
||||
|
||||
与搜索页不同，详情页的状态没有 `state` 包裹层，各业务块直接挂在顶层：
|
||||
item / purchase / shop / review / shipping / breadcrumbs。
|
||||
|
||||
注意：只有手机 UA 才会拿到这套统一模板；PC UA 返回的是各店铺自定义的 EUC-JP
|
||||
老页面，里面没有 __INITIAL_STATE__。UA 的选择由 RakutenClient 负责。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
PurchaseOption,
|
||||
PurchaseOptionValue,
|
||||
ReviewSummary,
|
||||
ShippingInfo,
|
||||
ShopSummary,
|
||||
SkuAttribute,
|
||||
SkuAxis,
|
||||
SkuAxisValue,
|
||||
SkuInfo,
|
||||
SkuVariant,
|
||||
)
|
||||
from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str
|
||||
|
||||
# purchase.sellType 下表示「可正常购买」的状态值
|
||||
_PURCHASABLE_CONDITION = "enabled"
|
||||
|
||||
# 普通购买的事件标识，站点前端构造加购表单时固定带上
|
||||
_NORMAL_PURCHASE_EVENT = "ES01_003_001"
|
||||
|
||||
# 库存类型 → 加购表单里的 inventory_flag
|
||||
_INVENTORY_FLAG = {"multiple": 2}
|
||||
_DEFAULT_INVENTORY_FLAG = 1
|
||||
|
||||
|
||||
def _parse_attributes(raw: Any) -> list[SkuAttribute]:
|
||||
return [
|
||||
SkuAttribute(title=as_str(attr.get("title")), value=as_str(attr.get("value")))
|
||||
for attr in as_list(raw)
|
||||
if isinstance(attr, dict)
|
||||
]
|
||||
|
||||
|
||||
def _parse_axis(raw: Any) -> list[SkuAxis]:
|
||||
axes: list[SkuAxis] = []
|
||||
for axis in as_list(raw):
|
||||
if not isinstance(axis, dict):
|
||||
continue
|
||||
axes.append(
|
||||
SkuAxis(
|
||||
key=as_str(axis.get("key")),
|
||||
label=as_str(axis.get("label")),
|
||||
values=[
|
||||
SkuAxisValue(
|
||||
value=as_str(value.get("value")),
|
||||
label=as_str(value.get("label")),
|
||||
is_sold_out=bool(value.get("isSoldOut")),
|
||||
)
|
||||
for value in as_list(axis.get("values"))
|
||||
if isinstance(value, dict)
|
||||
],
|
||||
)
|
||||
)
|
||||
return axes
|
||||
|
||||
|
||||
def _parse_variants(raw: Any) -> list[SkuVariant]:
|
||||
variants: list[SkuVariant] = []
|
||||
for variant in as_list(raw):
|
||||
if not isinstance(variant, dict):
|
||||
continue
|
||||
quantity = as_int(variant.get("quantity"))
|
||||
variants.append(
|
||||
SkuVariant(
|
||||
variant_id=as_str(variant.get("variantId")),
|
||||
selector_values=[as_str(value) for value in as_list(variant.get("selectorValues"))],
|
||||
price=as_int(variant.get("price")),
|
||||
quantity=quantity,
|
||||
# 站点未给 SKU 级的售罄标记，库存为 0 即视为该组合不可购买
|
||||
is_sold_out=quantity <= 0,
|
||||
delivery_message=as_str(variant.get("deliveryMessageRMS")),
|
||||
attributes=_parse_attributes(variant.get("attributes")),
|
||||
)
|
||||
)
|
||||
return variants
|
||||
|
||||
|
||||
def parse_purchase_options(raw: Any) -> list[PurchaseOption]:
|
||||
"""解析商品选项（選択肢）
|
||||
|
||||
结构为 {id, name, type: select|check|text, isRequired, values:[{id, name}]}；
|
||||
type=text 的选项没有候选值，由买家自由填写（如刻字内容）。
|
||||
"""
|
||||
options: list[PurchaseOption] = []
|
||||
for option in as_list(raw):
|
||||
if not isinstance(option, dict):
|
||||
continue
|
||||
options.append(
|
||||
PurchaseOption(
|
||||
option_id=str(as_int(option.get("id"))) if option.get("id") is not None else "",
|
||||
name=as_str(option.get("name")),
|
||||
type=as_str(option.get("type")),
|
||||
is_required=bool(option.get("isRequired")),
|
||||
values=[
|
||||
PurchaseOptionValue(
|
||||
value_id=str(as_int(value.get("id"))) if value.get("id") is not None else "",
|
||||
name=as_str(value.get("name")),
|
||||
)
|
||||
for value in as_list(option.get("values"))
|
||||
if isinstance(value, dict)
|
||||
],
|
||||
)
|
||||
)
|
||||
return options
|
||||
|
||||
|
||||
def _purchase_info(
|
||||
*,
|
||||
sell_type: dict[str, Any],
|
||||
raw_sku: dict[str, Any],
|
||||
information: dict[str, Any],
|
||||
shop_id: int,
|
||||
item_id: str,
|
||||
item_variant_id: str,
|
||||
) -> PurchaseInfo:
|
||||
"""组装加购所需的端点与字段
|
||||
|
||||
字段名与取值来自站点前端构造加购表单的逻辑（getPurchaseFormData）。
|
||||
basketDomain 逐商品不同（不同店铺落在不同的 basket 集群），不能写死。
|
||||
"""
|
||||
inventory_flag = _INVENTORY_FLAG.get(as_str(raw_sku.get("inventoryType")), _DEFAULT_INVENTORY_FLAG)
|
||||
form_fields = {
|
||||
"shop_bid": str(shop_id),
|
||||
"item_id": item_id,
|
||||
"inventory_flag": str(inventory_flag),
|
||||
"__event": _NORMAL_PURCHASE_EVENT,
|
||||
}
|
||||
# 单一库存商品的规格是固定的，直接填好，调用方无需再选
|
||||
if inventory_flag == _DEFAULT_INVENTORY_FLAG and item_variant_id:
|
||||
form_fields["variant_id"] = item_variant_id
|
||||
|
||||
options = parse_purchase_options(information.get("options"))
|
||||
return PurchaseInfo(
|
||||
cart_url=as_str(sell_type.get("basketDomain")),
|
||||
form_fields=form_fields,
|
||||
quantity_field="units",
|
||||
variant_field="variant_id",
|
||||
options_field="choice" if options else "",
|
||||
options=options,
|
||||
has_required_options=any(option.is_required for option in options),
|
||||
)
|
||||
|
||||
|
||||
def _pick_sell_type(sell_type: dict[str, Any]) -> dict[str, Any]:
|
||||
"""取售卖方式信息，优先普通购买，其次任意一种带价格的方式（如定期购）"""
|
||||
normal = as_dict(sell_type.get("normalPurchase"))
|
||||
if normal:
|
||||
return normal
|
||||
for value in sell_type.values():
|
||||
candidate = as_dict(value)
|
||||
if "minPrice" in candidate:
|
||||
return candidate
|
||||
return {}
|
||||
|
||||
|
||||
def parse_item_detail(
|
||||
state: dict[str, Any],
|
||||
*,
|
||||
item_url: str,
|
||||
shop_code: str,
|
||||
include_sku_variants: bool,
|
||||
) -> ItemDetailData:
|
||||
"""把商品详情页状态解析为商品详情
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 状态中不存在 item 节点
|
||||
"""
|
||||
item = state.get("item")
|
||||
if not isinstance(item, dict) or not item:
|
||||
raise ScrapeParseError("商品详情缺少 item 节点")
|
||||
|
||||
purchase = as_dict(state.get("purchase"))
|
||||
sell_type = _pick_sell_type(as_dict(purchase.get("sellType")))
|
||||
purchase_condition = as_str(sell_type.get("purchaseCondition"))
|
||||
|
||||
raw_sku = as_dict(purchase.get("sku"))
|
||||
variants = _parse_variants(raw_sku.get("variants"))
|
||||
sku = SkuInfo(
|
||||
inventory_type=as_str(raw_sku.get("inventoryType")),
|
||||
quantity=as_int(raw_sku.get("quantity")),
|
||||
show_inventory=bool(raw_sku.get("showInventory")),
|
||||
delivery_message=as_str(raw_sku.get("deliveryMessageRMS")),
|
||||
attributes=_parse_attributes(raw_sku.get("attributes")),
|
||||
axis=_parse_axis(raw_sku.get("axis")),
|
||||
variants=variants if include_sku_variants else [],
|
||||
variant_count=len(variants),
|
||||
)
|
||||
|
||||
shop_information = as_dict(as_dict(state.get("shop")).get("information"))
|
||||
shop_review = as_dict(shop_information.get("shopReview"))
|
||||
resolved_shop_code = as_str(shop_information.get("shopUrl")) or shop_code
|
||||
shop = ShopSummary(
|
||||
shop_id=as_int(shop_information.get("shopId")) or None,
|
||||
shop_code=resolved_shop_code,
|
||||
shop_name=as_str(shop_information.get("shopName")),
|
||||
shop_url=f"https://www.rakuten.co.jp/{resolved_shop_code}/" if resolved_shop_code else "",
|
||||
review_score=as_float(shop_review.get("rating")),
|
||||
review_count=as_int(shop_review.get("total")),
|
||||
)
|
||||
|
||||
item_review = as_dict(as_dict(state.get("review")).get("item"))
|
||||
review = ReviewSummary(
|
||||
score=as_float(item_review.get("totalRating")),
|
||||
count=as_int(item_review.get("count")),
|
||||
)
|
||||
|
||||
shipping_raw = as_dict(as_dict(state.get("shipping")).get("informationFromServer"))
|
||||
shipping_fee = shipping_raw.get("shippingFee")
|
||||
threshold = shipping_raw.get("freeShippingThreshold")
|
||||
shipping = ShippingInfo(
|
||||
shipping_fee=as_int(shipping_fee) if shipping_fee is not None else None,
|
||||
is_shipping_free=bool(shipping_raw.get("isShippingFree")),
|
||||
is_asuraku=bool(shipping_raw.get("isAsuraku")),
|
||||
is_next_day_delivery=bool(shipping_raw.get("isNextDayDelivery")),
|
||||
free_shipping_threshold=as_int(threshold) if threshold is not None else None,
|
||||
prefecture_id=as_int(shipping_raw.get("prefectureId")) or None,
|
||||
delivery_message=as_str(raw_sku.get("deliveryMessageRMS")),
|
||||
)
|
||||
|
||||
breadcrumbs = [
|
||||
Breadcrumb(name=as_str(crumb.get("name")), url=as_str(crumb.get("url")))
|
||||
for crumb in as_list(as_dict(state.get("breadcrumbs")).get("genreBreadcrumbs"))
|
||||
if isinstance(crumb, dict)
|
||||
]
|
||||
|
||||
images = [
|
||||
as_str(image.get("imageUrl"))
|
||||
for image in as_list(as_dict(item.get("media")).get("images"))
|
||||
if isinstance(image, dict) and as_str(image.get("imageUrl"))
|
||||
]
|
||||
|
||||
item_id = str(as_int(item.get("itemId"))) if item.get("itemId") is not None else ""
|
||||
return ItemDetailData(
|
||||
source="ichiba",
|
||||
source_url=item_url, # 市场页无跳转，解析地址即商品地址
|
||||
item_id=item_id,
|
||||
item_code=as_str(item.get("itemNumber")),
|
||||
item_name=as_str(item.get("itemName")),
|
||||
catch_copy=as_str(item.get("catchCopy")),
|
||||
description=as_str(item.get("description")),
|
||||
item_url=item_url,
|
||||
price=as_int(sell_type.get("minPrice")),
|
||||
pre_tax_price=as_int(sell_type.get("preTaxPrice")),
|
||||
tax_flag=bool(item.get("taxFlag")),
|
||||
tax_rate=as_float(shop_information.get("taxRate")),
|
||||
purchase_condition=purchase_condition,
|
||||
# purchaseCondition 是站点判定能否下单的直接依据；缺失时不臆断为售罄
|
||||
is_sold_out=bool(purchase_condition) and purchase_condition != _PURCHASABLE_CONDITION,
|
||||
purchase_unit=as_int(as_dict(purchase.get("information")).get("unit")),
|
||||
images=images,
|
||||
shop=shop,
|
||||
review=review,
|
||||
genre_id=str(as_int(item.get("genreId"))) if item.get("genreId") is not None else "",
|
||||
breadcrumbs=breadcrumbs,
|
||||
shipping=shipping,
|
||||
sku=sku,
|
||||
purchase=_purchase_info(
|
||||
sell_type=sell_type,
|
||||
raw_sku=raw_sku,
|
||||
information=as_dict(purchase.get("information")),
|
||||
shop_id=as_int(shop_information.get("shopId")),
|
||||
item_id=item_id,
|
||||
item_variant_id=as_str(item.get("variantId")),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""ラクマ（fril.jp）页面解析器
|
||||
|
||||
站点是服务端渲染的 HTML，没有内联状态 JSON，因此各模块都走 DOM 解析：
|
||||
- base — 埋点属性与文本取值的公共工具
|
||||
- search — 搜索页（商品卡片解析同时被店铺页复用）
|
||||
- item — 商品详情页
|
||||
- shop — 店铺页与评价页
|
||||
"""
|
||||
@@ -0,0 +1,144 @@
|
||||
"""ラクマ（fril.jp）页面解析的公共工具
|
||||
|
||||
站点是服务端渲染的 HTML，没有 `window.__INITIAL_STATE__` 之类的内联状态，
|
||||
因此全部走 DOM 解析。好在页面上挂了成套的埋点属性（`data-rat-*` 与
|
||||
`onclick` 里的 dataLayer JSON），它们比可见文案稳定得多，也带有可见 DOM
|
||||
上没有的字段（商品数值 ID、卖家 ID、分类 ID、品牌 ID），所以优先取这些。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from selectolax.parser import Node
|
||||
|
||||
# 「約1,190,000件中 1 - 40件」里的总数与区间
|
||||
_COUNT_RE = re.compile(r"([\d,]+)\s*件中\s*([\d,]+)\s*[-−–]\s*([\d,]+)\s*件")
|
||||
_DIGITS_RE = re.compile(r"-?\d+")
|
||||
# 页面级埋点属性 data-rat-cp-{key}="{value}"
|
||||
_RAT_PARAM_RE = re.compile(r'data-rat-cp-([\w]+)="([^"]*)"')
|
||||
|
||||
|
||||
def parse_int(text: str | int | float | None) -> int:
|
||||
"""从 "¥6,299" / "6399" / 6399 这类值里取出整数金额或计数"""
|
||||
if isinstance(text, bool) or text is None:
|
||||
return 0
|
||||
if isinstance(text, (int, float)):
|
||||
return int(text)
|
||||
digits = _DIGITS_RE.findall(text.replace(",", ""))
|
||||
return int(digits[0]) if digits else 0
|
||||
|
||||
|
||||
def parse_float(text: str | int | float | None) -> float:
|
||||
"""从 "5.0" 这类文本里取出评分"""
|
||||
if isinstance(text, bool) or text is None:
|
||||
return 0.0
|
||||
if isinstance(text, (int, float)):
|
||||
return float(text)
|
||||
match = re.search(r"\d+(?:\.\d+)?", text.replace(",", ""))
|
||||
return float(match.group()) if match else 0.0
|
||||
|
||||
|
||||
def node_text(node: Node | None) -> str:
|
||||
"""取节点的可见文本，压掉多余空白；节点不存在时返回空串"""
|
||||
if node is None:
|
||||
return ""
|
||||
return re.sub(r"\s+", " ", node.text(strip=True)).strip()
|
||||
|
||||
|
||||
def attr(node: Node | None, name: str) -> str:
|
||||
"""取节点属性，缺失时返回空串"""
|
||||
if node is None:
|
||||
return ""
|
||||
return (node.attributes.get(name) or "").strip()
|
||||
|
||||
|
||||
def image_url(node: Node | None) -> str:
|
||||
"""取图片地址：站点用 lazy load，真实地址在 data-original 上，src 是占位图"""
|
||||
if node is None:
|
||||
return ""
|
||||
return attr(node, "data-original") or attr(node, "src")
|
||||
|
||||
|
||||
def parse_total_count(text: str) -> tuple[int, int, int]:
|
||||
"""解析「N件中 X - Y件」，返回 (总数, 起, 止)；解析不出时全为 0
|
||||
|
||||
注意搜索页这里的总数是四舍五入后的展示值（約1,190,000件），
|
||||
精确值要从埋点属性 data-rat-cp-totalresults 取；店铺页则是精确值。
|
||||
"""
|
||||
match = _COUNT_RE.search(text.replace("\xa0", " "))
|
||||
if match is None:
|
||||
return 0, 0, 0
|
||||
return (
|
||||
parse_int(match.group(1)),
|
||||
parse_int(match.group(2)),
|
||||
parse_int(match.group(3)),
|
||||
)
|
||||
|
||||
|
||||
def event_payload(node: Node | None) -> dict[str, Any]:
|
||||
"""从埋点里取出商品参数字典
|
||||
|
||||
商品链接的 onclick / data-gtm-click 上挂着一段 dataLayer JSON，形如：
|
||||
{"event":"fireEvent","eventData":{"event_parameter":{
|
||||
"item_id":"844649627","seller_user_id":"12073120",
|
||||
"category_id":"788","brand_id":"5296","price":6299, ...}}}
|
||||
这里面有可见 DOM 上没有的数值 ID，是搜索结果里最可靠的数据来源。
|
||||
"""
|
||||
if node is None:
|
||||
return {}
|
||||
|
||||
for source in (node.attributes.get("data-gtm-click"), node.attributes.get("onclick")):
|
||||
if not source:
|
||||
continue
|
||||
for raw in _iter_json_objects(html.unescape(source)):
|
||||
parameter = (
|
||||
raw.get("eventData", {}).get("event_parameter")
|
||||
if isinstance(raw.get("eventData"), dict)
|
||||
else None
|
||||
)
|
||||
if isinstance(parameter, dict) and "item_id" in parameter:
|
||||
return parameter
|
||||
return {}
|
||||
|
||||
|
||||
def find_item_payload(tree: Any) -> dict[str, Any]:
|
||||
"""在整页里找出第一段带 item_id 的埋点参数
|
||||
|
||||
详情页的这段 JSON 挂在哪个元素上并不固定（在售商品挂在品牌链接上，
|
||||
已售商品的页面结构不同），因此按属性扫描而不是写死选择器。
|
||||
"""
|
||||
for node in tree.css("[data-gtm-click], [onclick]"):
|
||||
payload = event_payload(node)
|
||||
if payload:
|
||||
return payload
|
||||
return {}
|
||||
|
||||
|
||||
def rat_params(html_text: str) -> dict[str, str]:
|
||||
"""取出页面级埋点属性 `data-rat-cp-*`
|
||||
|
||||
详情页把成色、运费负担、发货地等信息也写在这组属性里。已售出商品的
|
||||
页面会换成另一套布局、规格表消失，但这组属性仍在，可用作兜底。
|
||||
"""
|
||||
return {
|
||||
match.group(1): html.unescape(match.group(2))
|
||||
for match in _RAT_PARAM_RE.finditer(html_text)
|
||||
}
|
||||
|
||||
|
||||
def _iter_json_objects(text: str):
|
||||
"""从一段掺杂着 JS 代码的文本里增量解析出所有顶层 JSON 对象"""
|
||||
decoder = json.JSONDecoder()
|
||||
index = text.find("{")
|
||||
while index >= 0:
|
||||
try:
|
||||
value, end = decoder.raw_decode(text, index)
|
||||
except ValueError:
|
||||
index = text.find("{", index + 1)
|
||||
continue
|
||||
if isinstance(value, dict):
|
||||
yield value
|
||||
index = text.find("{", max(end, index + 1))
|
||||
@@ -0,0 +1,211 @@
|
||||
"""ラクマ 商品详情页 HTML → RakumaItemDetailData
|
||||
|
||||
页面数据分三处，各取所长：
|
||||
- `<script type="application/ld+json">` 的 Product 微数据：名称、价格、描述、图片
|
||||
- `.item__details` 规格表：成色、尺码、配送方式与地区（每行 th 上的
|
||||
`item-status-{key}` class 是稳定键，比日文标签文案可靠）
|
||||
- 埋点属性 `data-rat-cp-*` 与 dataLayer JSON：数值 ID、分类 ID、品牌 ID
|
||||
|
||||
售罄判定用页面上的 SOLD OUT 标记，而不是 ld+json 的 availability——
|
||||
实测已售出商品的 ld+json 仍写 InStock，不可信。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.core import rakuma_site as site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import Breadcrumb, RakumaItemDetailData, RakumaSeller
|
||||
from app.parsers.rakuma.base import (
|
||||
attr,
|
||||
find_item_payload,
|
||||
image_url,
|
||||
node_text,
|
||||
parse_float,
|
||||
parse_int,
|
||||
rat_params,
|
||||
)
|
||||
from app.utils.rakuma_urls import split_shop_url
|
||||
|
||||
_LD_JSON_RE = re.compile(
|
||||
r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
|
||||
)
|
||||
# 商品主图：站点用 slider 展示，主图挂在 .sp-image 上（推荐位的图不在其中）
|
||||
_MAIN_IMAGE_SELECTOR = ".sp-slide img.sp-image, .item-photos img, .slider img.sp-image"
|
||||
|
||||
# 规格表里 th 图标 class 上的稳定键 → 模型字段
|
||||
_SPEC_KEYS = {
|
||||
"item-status-status": "condition",
|
||||
"item-status-size": "size",
|
||||
"item-status-carriage": "shipping_payer",
|
||||
"item-status-delivery_method": "shipping_method",
|
||||
"item-status-delivery_date": "shipping_date_estimate",
|
||||
"item-status-delivery_area": "shipping_from",
|
||||
}
|
||||
|
||||
# 站点上表示「没有填写该项」的占位文案
|
||||
_SPEC_EMPTY_VALUES = ("なし", "未定", "指定なし", "-", "―")
|
||||
|
||||
# 规格表缺失时的兜底：页面级埋点属性 data-rat-cp-{key} → 模型字段。
|
||||
# 注意这组值的措辞与规格表不完全一致（如运费负担规格表写「送料込」，
|
||||
# 埋点写「出品者」），原样透出，不做归一。
|
||||
_RAT_SPEC_KEYS = {
|
||||
"condition": "item_condition",
|
||||
"shipping_payer": "shipping_cost_payer",
|
||||
"shipping_date_estimate": "shipping_date_estimate",
|
||||
"shipping_from": "shipping_from",
|
||||
}
|
||||
|
||||
|
||||
def _parse_ld_product(html: str) -> dict[str, Any]:
|
||||
"""取出 ld+json 里的 Product 节点"""
|
||||
for match in _LD_JSON_RE.finditer(html):
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(data, dict) and data.get("@type") == "Product":
|
||||
return data
|
||||
return {}
|
||||
|
||||
|
||||
def _parse_specs(tree: HTMLParser, rat: dict[str, str]) -> dict[str, str]:
|
||||
"""解析商品情報规格表
|
||||
|
||||
每行的 th 里有个 `<i class="icon-status ... item-status-{key}">`，
|
||||
这个 key 比日文标签稳定，用它做映射。
|
||||
|
||||
已售出商品的页面会换成另一套布局、规格表整体消失，此时退回页面级埋点
|
||||
属性——它给的项少一些（没有配送方法与尺码），但成色、运费负担与发货地
|
||||
仍在，好过整片留空。
|
||||
"""
|
||||
specs: dict[str, str] = {}
|
||||
for row in tree.css("table.item__details tr"):
|
||||
icon = row.css_first("th i")
|
||||
value_node = row.css_first("td")
|
||||
if icon is None or value_node is None:
|
||||
continue
|
||||
classes = attr(icon, "class").split()
|
||||
field = next((_SPEC_KEYS[name] for name in classes if name in _SPEC_KEYS), None)
|
||||
if field is None:
|
||||
continue
|
||||
value = node_text(value_node)
|
||||
specs[field] = "" if value in _SPEC_EMPTY_VALUES else value
|
||||
|
||||
for field, key in _RAT_SPEC_KEYS.items():
|
||||
if not specs.get(field) and rat.get(key):
|
||||
specs[field] = rat[key]
|
||||
return specs
|
||||
|
||||
|
||||
def _parse_breadcrumbs(tree: HTMLParser) -> tuple[list[Breadcrumb], str]:
|
||||
"""解析分类面包屑，并返回最具体的一级分类 ID
|
||||
|
||||
取规格表里的分类行而非页头面包屑：页头那条会把品牌也混进来，
|
||||
规格表里的是纯分类链。
|
||||
"""
|
||||
crumbs: list[Breadcrumb] = []
|
||||
category_id = ""
|
||||
for row in tree.css("table.item__details tr"):
|
||||
icon = row.css_first("th i")
|
||||
if icon is None or "item-status-category" not in attr(icon, "class"):
|
||||
continue
|
||||
for link in row.css("td a"):
|
||||
url = attr(link, "href")
|
||||
crumbs.append(Breadcrumb(name=node_text(link), url=url))
|
||||
segments = [segment for segment in url.split("/") if segment]
|
||||
if segments:
|
||||
category_id = segments[-1]
|
||||
break
|
||||
return crumbs, category_id
|
||||
|
||||
|
||||
def _parse_seller(tree: HTMLParser, payload: dict[str, Any]) -> RakumaSeller:
|
||||
"""解析出品者信息块"""
|
||||
link = tree.css_first("a.shop_link, a[href*='/shop/']")
|
||||
shop_url = attr(link, "href")
|
||||
shop_id = ""
|
||||
if shop_url:
|
||||
try:
|
||||
shop_id = split_shop_url(shop_url)
|
||||
except Exception:
|
||||
shop_id = ""
|
||||
|
||||
seller_user_id = payload.get("seller_user_id")
|
||||
return RakumaSeller(
|
||||
shop_id=shop_id,
|
||||
user_id=str(seller_user_id) if seller_user_id is not None else "",
|
||||
shop_name=node_text(tree.css_first(".header-shopinfo__shop-name")),
|
||||
user_name=node_text(tree.css_first(".header-shopinfo__user-name")),
|
||||
shop_url=shop_url,
|
||||
icon_url=image_url(tree.css_first(".header-shopinfo__user-icon img")),
|
||||
seller_type=str(payload.get("seller_user_type") or ""),
|
||||
review_score=parse_float(node_text(tree.css_first(".shop_score__score"))),
|
||||
# 商品页只给评分不给评价数，需要评价数请调 /api/rakuma/shop_detail
|
||||
is_verified=tree.css_first(".header-shopinfo__verified-badge-item") is not None,
|
||||
)
|
||||
|
||||
|
||||
def parse_item_detail(html: str, *, item_id: str, item_url: str) -> RakumaItemDetailData:
|
||||
"""把商品详情页 HTML 解析为商品详情
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是商品详情页
|
||||
"""
|
||||
tree = HTMLParser(html)
|
||||
info = tree.css_first(f".{site.ITEM_PAGE_MARKER}")
|
||||
if info is None:
|
||||
raise ScrapeParseError("页面不是商品详情页（缺少商品信息区块）")
|
||||
|
||||
product = _parse_ld_product(html)
|
||||
# 这段埋点挂在哪个元素上因页面状态而异，按属性全页扫描
|
||||
payload = find_item_payload(tree)
|
||||
rat = rat_params(html)
|
||||
specs = _parse_specs(tree, rat)
|
||||
breadcrumbs, category_id = _parse_breadcrumbs(tree)
|
||||
|
||||
images = [
|
||||
url
|
||||
for url in dict.fromkeys(image_url(node) for node in tree.css(_MAIN_IMAGE_SELECTOR))
|
||||
if url and "img.fril.jp" in url
|
||||
]
|
||||
if not images and isinstance(product.get("image"), str):
|
||||
images = [product["image"]]
|
||||
|
||||
# ld+json 的 availability 对已售商品仍写 InStock，只能按页面标记判断
|
||||
page_text = info.text()
|
||||
is_sold_out = any(marker in page_text for marker in site.SOLD_OUT_MARKERS)
|
||||
|
||||
brand = product.get("brand") if isinstance(product.get("brand"), dict) else {}
|
||||
offers = product.get("offers") if isinstance(product.get("offers"), dict) else {}
|
||||
|
||||
return RakumaItemDetailData(
|
||||
item_id=item_id,
|
||||
item_number=str(payload.get("item_id") or ""),
|
||||
item_name=str(product.get("name") or "") or node_text(tree.css_first("h1.item__name")),
|
||||
description=str(product.get("description") or "")
|
||||
or node_text(tree.css_first(".item__description__line-limited")),
|
||||
item_url=item_url,
|
||||
price=parse_int(offers.get("price")) or parse_int(node_text(tree.css_first(".item__price"))),
|
||||
is_sold_out=is_sold_out,
|
||||
images=images,
|
||||
condition=specs.get("condition", ""),
|
||||
size=specs.get("size", ""),
|
||||
brand_id=str(payload.get("brand_id") or "") or rat.get("brand_id", ""),
|
||||
brand_name=str(brand.get("name") or "") or str(payload.get("brand_name") or ""),
|
||||
category_id=category_id or str(payload.get("category_id") or ""),
|
||||
breadcrumbs=breadcrumbs,
|
||||
shipping_payer=specs.get("shipping_payer", ""),
|
||||
shipping_method=specs.get("shipping_method", ""),
|
||||
shipping_date_estimate=specs.get("shipping_date_estimate", ""),
|
||||
shipping_from=specs.get("shipping_from", ""),
|
||||
is_anonymous_shipping=tree.css_first(".item__icon.anonymous") is not None,
|
||||
like_count=parse_int(node_text(tree.css_first(".like_button_set"))),
|
||||
comment_count=parse_int(node_text(tree.css_first(".go-to-comment-button"))),
|
||||
posted_at=node_text(tree.css_first(".time_ago")),
|
||||
seller=_parse_seller(tree, payload),
|
||||
)
|
||||
@@ -0,0 +1,135 @@
|
||||
"""ラクマ 搜索页 / 店铺商品列表 HTML → 商品列表
|
||||
|
||||
搜索页与店铺页的商品卡片是同一套 `.item-box` 结构（只有链接的 class 前缀
|
||||
不同：搜索页 link_search_image、店铺页 link_shop_image），因此共用一个卡片
|
||||
解析函数。
|
||||
|
||||
页面上的可见总数是四舍五入的展示值（約1,190,000件），精确值在埋点属性
|
||||
`data-rat-cp-totalresults` 上，优先取后者。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.core import rakuma_site as site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import RakumaSearchItem, RakumaSearchResultData
|
||||
from app.parsers.rakuma.base import (
|
||||
attr,
|
||||
event_payload,
|
||||
image_url,
|
||||
node_text,
|
||||
parse_int,
|
||||
parse_total_count,
|
||||
)
|
||||
from app.utils.rakuma_urls import item_id_from_url
|
||||
|
||||
# 埋点属性里的精确命中总数
|
||||
_TOTAL_RESULTS_RE = re.compile(r'data-rat-cp-totalresults="(\d+)"')
|
||||
|
||||
|
||||
def _as_str(value: object) -> str:
|
||||
"""埋点 JSON 里的值可能是数字、字符串或 null，统一收敛为字符串"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return ""
|
||||
if isinstance(value, (int, float)):
|
||||
return str(int(value))
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def parse_item_card(card: Node) -> RakumaSearchItem:
|
||||
"""解析一张商品卡片
|
||||
|
||||
优先从埋点 JSON 取结构化字段（数值 ID、分类、品牌、价格），
|
||||
可见 DOM 只用于取图片与售罄标记。
|
||||
"""
|
||||
link = (
|
||||
card.css_first("a.link_search_image")
|
||||
or card.css_first("a.link_shop_image")
|
||||
or card.css_first("a[href*='item.fril.jp']")
|
||||
)
|
||||
payload = event_payload(link)
|
||||
|
||||
item_url = attr(link, "href")
|
||||
category_names = [
|
||||
name
|
||||
for name in (
|
||||
_as_str(payload.get("first_category")),
|
||||
_as_str(payload.get("second_category")),
|
||||
_as_str(payload.get("third_category")),
|
||||
)
|
||||
if name
|
||||
]
|
||||
|
||||
# 价格优先取埋点里的数值，回退到卡片上的展示价
|
||||
price = parse_int(payload.get("price")) or parse_int(
|
||||
node_text(card.css_first(".item-box__item-price"))
|
||||
)
|
||||
|
||||
return RakumaSearchItem(
|
||||
item_id=item_id_from_url(item_url),
|
||||
item_number=_as_str(payload.get("item_id")),
|
||||
item_name=_as_str(payload.get("item_name"))
|
||||
or node_text(card.css_first(".item-box__item-name, .item-box__item-name__limited-three-lines")),
|
||||
item_url=item_url,
|
||||
price=price,
|
||||
image_url=image_url(card.css_first("img")),
|
||||
is_sold_out=card.css_first(".item-box__soldout_ribbon") is not None,
|
||||
brand_id=_as_str(payload.get("brand_id")),
|
||||
brand_name=_as_str(payload.get("brand_name"))
|
||||
or node_text(card.css_first(".item-box__item-sub-name")),
|
||||
category_id=_as_str(payload.get("category_id")),
|
||||
category_names=category_names,
|
||||
seller_user_id=_as_str(payload.get("seller_user_id")),
|
||||
seller_type=_as_str(payload.get("seller_user_type")),
|
||||
)
|
||||
|
||||
|
||||
def parse_item_cards(tree: HTMLParser) -> list[RakumaSearchItem]:
|
||||
"""解析页面上的全部商品卡片
|
||||
|
||||
只取有真实商品链接的卡片：页面上还有一批用于占位的骨架卡片
|
||||
（懒加载的推荐位），它们没有 item.fril.jp 链接。
|
||||
"""
|
||||
items: list[RakumaSearchItem] = []
|
||||
for card in tree.css(".item-box"):
|
||||
link = card.css_first("a[href*='item.fril.jp']")
|
||||
if link is None:
|
||||
continue
|
||||
items.append(parse_item_card(card))
|
||||
return items
|
||||
|
||||
|
||||
def parse_search(html: str, *, request_url: str, page: int, keyword: str) -> RakumaSearchResultData:
|
||||
"""把搜索页 HTML 解析为搜索结果
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是搜索结果页（站点对无法识别的参数值会静默返回首页）
|
||||
"""
|
||||
tree = HTMLParser(html)
|
||||
count_node = tree.css_first(f".{site.SEARCH_PAGE_MARKER}")
|
||||
if count_node is None:
|
||||
raise ScrapeParseError(
|
||||
"页面不是搜索结果页（缺少命中数区块）；"
|
||||
"站点对无法识别的筛选取值会静默返回首页，请检查筛选参数"
|
||||
)
|
||||
|
||||
items = parse_item_cards(tree)
|
||||
|
||||
# 展示值是四舍五入过的（約1,190,000件），埋点里才是精确命中数
|
||||
display_total, start, end = parse_total_count(node_text(count_node))
|
||||
match = _TOTAL_RESULTS_RE.search(html)
|
||||
total_count = int(match.group(1)) if match else display_total
|
||||
|
||||
return RakumaSearchResultData(
|
||||
keyword=keyword,
|
||||
page=page,
|
||||
page_size=len(items),
|
||||
total_count=total_count,
|
||||
# 站点 page>100 直接 404，超出可达窗口时没有下一页
|
||||
has_more=bool(items) and page < site.MAX_PAGE and (end or start + len(items) - 1) < total_count,
|
||||
request_url=request_url,
|
||||
items=items,
|
||||
)
|
||||
@@ -0,0 +1,223 @@
|
||||
"""ラクマ 店铺页 HTML → 卖家详情与卖家商品列表
|
||||
|
||||
C2C 集市里的「商家」就是个人卖家，页面在 fril.jp/shop/{hash}：
|
||||
- 店铺页本身：卖家资料 + 该卖家的商品分页列表（含已售出）
|
||||
- /review 子页：评价明细与好评/普通/差评分档计数
|
||||
|
||||
商品卡片与搜索页共用 `.item-box` 结构，直接复用 search 里的解析。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.core import rakuma_site as site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
RakumaRatingBreakdown,
|
||||
RakumaReview,
|
||||
RakumaShopDetailData,
|
||||
RakumaShopItemsData,
|
||||
)
|
||||
from app.parsers.rakuma.base import (
|
||||
attr,
|
||||
event_payload,
|
||||
image_url,
|
||||
node_text,
|
||||
parse_float,
|
||||
parse_int,
|
||||
parse_total_count,
|
||||
)
|
||||
from app.parsers.rakuma.search import parse_item_cards
|
||||
|
||||
# 评价条目标题左侧的图标 class → 评价档位
|
||||
_RATING_ICONS = {
|
||||
"icon_review_sun": "good", # よい
|
||||
"icon_review_cloud": "normal", # ふつう
|
||||
"icon_review_rain": "bad", # わるい
|
||||
}
|
||||
|
||||
# /review 页上三组分档计数的容器 id 前缀
|
||||
_ALL_RATINGS_PREFIX = "all"
|
||||
_SELLER_RATINGS_PREFIX = "seller"
|
||||
|
||||
_LD_JSON_RE = re.compile(
|
||||
r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
|
||||
)
|
||||
|
||||
|
||||
def _require_shop_page(html: str) -> HTMLParser:
|
||||
tree = HTMLParser(html)
|
||||
if tree.css_first(f".{site.SHOP_PAGE_MARKER}") is None:
|
||||
raise ScrapeParseError("页面不是店铺页（缺少店铺资料区块）")
|
||||
return tree
|
||||
|
||||
|
||||
def _parse_rating_breakdown(tree: HTMLParser, prefix: str) -> RakumaRatingBreakdown:
|
||||
"""解析一组好评/普通/差评计数
|
||||
|
||||
页面用 `<ul class="nav-pills">` 里的三个链接展示，锚点形如
|
||||
`#all-good` / `#seller-normal`，按锚点前缀区分「全部」与「出品」两组。
|
||||
"""
|
||||
counts = {"good": 0, "normal": 0, "bad": 0}
|
||||
for link in tree.css("ul.nav-pills a"):
|
||||
href = attr(link, "href")
|
||||
for rating in counts:
|
||||
if href == f"#{prefix}-{rating}":
|
||||
counts[rating] = parse_int(node_text(link))
|
||||
return RakumaRatingBreakdown(**counts)
|
||||
|
||||
|
||||
def _parse_reviews(tree: HTMLParser) -> list[RakumaReview]:
|
||||
"""解析评价列表
|
||||
|
||||
站点在「すべての評価」标签页里最多展示最新 100 条，且三个标签页
|
||||
（全部/出品/购入）的条目在 DOM 里重复出现，这里只取第一个激活面板。
|
||||
"""
|
||||
panel = tree.css_first("#all-all") or tree.css_first(".tab-pane.active")
|
||||
if panel is None:
|
||||
return []
|
||||
|
||||
reviews: list[RakumaReview] = []
|
||||
for article in panel.css("article.review-item"):
|
||||
title_node = article.css_first(".review-item-title")
|
||||
icon = title_node.css_first("i") if title_node else None
|
||||
classes = attr(icon, "class").split() if icon else []
|
||||
rating = next((_RATING_ICONS[name] for name in classes if name in _RATING_ICONS), "")
|
||||
|
||||
reviews.append(
|
||||
RakumaReview(
|
||||
rating=rating,
|
||||
title=node_text(title_node),
|
||||
comment=node_text(article.css_first(".review-item-text")),
|
||||
reviewer_name=node_text(article.css_first(".review-item-name")),
|
||||
reviewed_at=node_text(article.css_first(".review-item-date")),
|
||||
)
|
||||
)
|
||||
return reviews
|
||||
|
||||
|
||||
def _parse_store_rating(html: str) -> tuple[float, int]:
|
||||
"""从店铺页的 ld+json Store 节点取评分与评价数
|
||||
|
||||
评价数只有这里给得出来——可见 DOM 上只有星级和分数，没有条数。
|
||||
"""
|
||||
for match in _LD_JSON_RE.finditer(html):
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(data, dict) or data.get("@type") != "Store":
|
||||
continue
|
||||
rating = data.get("aggregateRating")
|
||||
if isinstance(rating, dict):
|
||||
return parse_float(rating.get("ratingValue")), parse_int(rating.get("ratingCount"))
|
||||
return 0.0, 0
|
||||
|
||||
|
||||
def parse_shop_detail(
|
||||
html: str, *, shop_id: str, shop_url: str, review_html: str | None = None
|
||||
) -> RakumaShopDetailData:
|
||||
"""把店铺页 HTML 解析为卖家详情
|
||||
|
||||
Args:
|
||||
review_html: /review 子页的 HTML；给出时才填充评价明细与分档计数
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是店铺页
|
||||
"""
|
||||
tree = _require_shop_page(html)
|
||||
|
||||
total_count, _, _ = parse_total_count(node_text(tree.css_first(".page-count")))
|
||||
badge = tree.css_first(".badge-status")
|
||||
verification_label = node_text(badge)
|
||||
|
||||
# 简介在侧栏「プロフィール」区块；卖家未填写时站点会写一句占位文案
|
||||
introduction = node_text(tree.css_first("[data-test=profile-text-top]"))
|
||||
if "設定されていません" in introduction:
|
||||
introduction = ""
|
||||
|
||||
score, review_count = _parse_store_rating(html)
|
||||
detail = RakumaShopDetailData(
|
||||
shop_id=shop_id,
|
||||
shop_name=node_text(tree.css_first(".profile-area__shop-name")),
|
||||
user_name=node_text(tree.css_first("[data-test=profile_user_name], .profile-area__user-name")),
|
||||
shop_url=shop_url,
|
||||
icon_url=image_url(tree.css_first(".profile-area__user-icon img")),
|
||||
cover_url=_cover_url(tree),
|
||||
introduction=introduction,
|
||||
review_score=score or parse_float(node_text(tree.css_first(".shop_score__score"))),
|
||||
review_count=review_count,
|
||||
is_verified="未完了" not in verification_label and bool(verification_label),
|
||||
verification_label=verification_label,
|
||||
item_count=total_count,
|
||||
)
|
||||
|
||||
# 用户数值 ID：优先取商品卡片埋点里的 seller_user_id（店铺页所有商品都属于
|
||||
# 该卖家），卖家未设头像时头像地址是站点默认图，取不到 ID。
|
||||
detail.user_id = _seller_user_id(tree) or _user_id_from_icon(detail.icon_url)
|
||||
|
||||
if review_html is not None:
|
||||
review_tree = HTMLParser(review_html)
|
||||
detail.rating_breakdown = _parse_rating_breakdown(review_tree, _ALL_RATINGS_PREFIX)
|
||||
detail.seller_rating_breakdown = _parse_rating_breakdown(review_tree, _SELLER_RATINGS_PREFIX)
|
||||
detail.reviews = _parse_reviews(review_tree)
|
||||
|
||||
return detail
|
||||
|
||||
|
||||
def _cover_url(tree: HTMLParser) -> str:
|
||||
"""封面图挂在 inline style 的 background url() 里"""
|
||||
cover = tree.css_first(".profile-area__shop-cover")
|
||||
style = attr(cover, "style")
|
||||
start = style.find("url(")
|
||||
if start < 0:
|
||||
return ""
|
||||
end = style.find(")", start)
|
||||
return style[start + 4 : end].strip("'\" ") if end > start else ""
|
||||
|
||||
|
||||
def _seller_user_id(tree: HTMLParser) -> str:
|
||||
"""从店铺页商品卡片的埋点里取卖家数值 ID"""
|
||||
for node in tree.css("[data-gtm-click], [onclick]"):
|
||||
payload = event_payload(node)
|
||||
user_id = payload.get("seller_user_id")
|
||||
if user_id:
|
||||
return str(user_id)
|
||||
return ""
|
||||
|
||||
|
||||
def _user_id_from_icon(icon_url: str) -> str:
|
||||
"""从头像地址 https://img.fril.jp/user/{id}/s/{id}.jpg 里取用户数值 ID"""
|
||||
marker = "/user/"
|
||||
start = icon_url.find(marker)
|
||||
if start < 0:
|
||||
return ""
|
||||
rest = icon_url[start + len(marker) :]
|
||||
user_id = rest.split("/", 1)[0]
|
||||
return user_id if user_id.isdigit() else ""
|
||||
|
||||
|
||||
def parse_shop_items(
|
||||
html: str, *, shop_id: str, request_url: str, page: int
|
||||
) -> RakumaShopItemsData:
|
||||
"""把店铺页 HTML 解析为该卖家的商品列表
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面不是店铺页
|
||||
"""
|
||||
tree = _require_shop_page(html)
|
||||
items = parse_item_cards(tree)
|
||||
total_count, _, end = parse_total_count(node_text(tree.css_first(".page-count")))
|
||||
|
||||
return RakumaShopItemsData(
|
||||
shop_id=shop_id,
|
||||
shop_name=node_text(tree.css_first(".profile-area__shop-name")),
|
||||
page=page,
|
||||
total_count=total_count,
|
||||
has_more=bool(items) and bool(end) and end < total_count,
|
||||
request_url=request_url,
|
||||
items=items,
|
||||
)
|
||||
@@ -0,0 +1,164 @@
|
||||
"""搜索页 __INITIAL_STATE__ → SearchResultData
|
||||
|
||||
数据位于 state.data.ichibaSearch，含 pagination 与 items 两部分。
|
||||
|
||||
需要注意的两个站点行为：
|
||||
- items 里会混入 CPC 广告位，其 url 指向 grp07.ias.rakuten.co.jp 跳转域，
|
||||
真实商品地址在 originalItemUrl；判定依据是 itemOptions.cpc 非空。
|
||||
- pagination.numFound 是命中总数，但实际只能翻到 pagination.subset 条为止。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import ReviewSummary, SearchItem, SearchResultData, ShopSummary
|
||||
from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str
|
||||
from app.utils.urls import item_url_parts
|
||||
|
||||
|
||||
def parse_search_item(raw: dict[str, Any]) -> SearchItem:
|
||||
"""解析单个搜索结果条目"""
|
||||
item_options = as_dict(raw.get("itemOptions"))
|
||||
is_ad = bool(item_options.get("cpc"))
|
||||
|
||||
# 广告位的 url 是跳转链接，真实商品地址只在 originalItemUrl 里
|
||||
original_url = as_str(raw.get("originalItemUrl"))
|
||||
fallback_url = as_str(raw.get("url"))
|
||||
item_url = original_url or fallback_url
|
||||
shop_code, item_code = item_url_parts(item_url)
|
||||
if not item_code and fallback_url and fallback_url != item_url:
|
||||
shop_code, item_code = item_url_parts(fallback_url)
|
||||
|
||||
images = [
|
||||
as_str(image.get("url"))
|
||||
for image in as_list(raw.get("images"))
|
||||
if isinstance(image, dict) and as_str(image.get("url"))
|
||||
]
|
||||
|
||||
raw_shop = as_dict(raw.get("shop"))
|
||||
shop_review = as_dict(raw_shop.get("review"))
|
||||
shop = ShopSummary(
|
||||
shop_id=as_int(raw_shop.get("id")) or None,
|
||||
shop_code=as_str(raw_shop.get("urlCode")) or shop_code,
|
||||
shop_name=as_str(raw_shop.get("name")),
|
||||
shop_url=as_str(raw_shop.get("url")),
|
||||
review_score=as_float(shop_review.get("score")),
|
||||
review_count=as_int(shop_review.get("count")),
|
||||
)
|
||||
|
||||
raw_review = as_dict(raw.get("review"))
|
||||
review = ReviewSummary(
|
||||
score=as_float(raw_review.get("score")),
|
||||
count=as_int(raw_review.get("numReviews")),
|
||||
url=as_str(raw_review.get("url")),
|
||||
)
|
||||
|
||||
genre_path = as_str(raw.get("genreIdList"))
|
||||
genre_names = [
|
||||
as_str(genre.get("name"))
|
||||
for genre in as_list(raw.get("genres"))
|
||||
if isinstance(genre, dict) and as_str(genre.get("name"))
|
||||
]
|
||||
# genreIdList 形如 /0/101205/565950/566404，末段为最具体的分类；根节点 0 不算
|
||||
genre_id = next(
|
||||
(segment for segment in reversed(genre_path.split("/")) if segment and segment != "0"),
|
||||
"",
|
||||
)
|
||||
|
||||
raw_shipping = as_dict(raw.get("shipping"))
|
||||
shipping_price = raw_shipping.get("price")
|
||||
sku_info = as_dict(raw.get("skuInfo"))
|
||||
|
||||
return SearchItem(
|
||||
item_id=as_str(raw.get("code")),
|
||||
item_code=item_code,
|
||||
item_name=as_str(raw.get("name")),
|
||||
item_url=item_url,
|
||||
catch_copy=as_str(raw.get("subtitle")),
|
||||
price=as_int(raw.get("price")),
|
||||
price_range=as_str(sku_info.get("priceRange")),
|
||||
has_price_range=bool(raw.get("hasPriceRange")),
|
||||
image_url=images[0] if images else "",
|
||||
image_urls=images,
|
||||
shop=shop,
|
||||
review=review,
|
||||
genre_id=genre_id,
|
||||
genre_path=genre_path,
|
||||
genre_names=genre_names,
|
||||
shipping_fee=as_int(shipping_price) if shipping_price is not None else None,
|
||||
delivery_message=as_str(raw_shipping.get("estimateDeliveryDay")),
|
||||
point_count=as_int(as_dict(raw.get("point")).get("count")),
|
||||
is_sold_out=bool(raw.get("isSoldOut")),
|
||||
is_ad=is_ad,
|
||||
has_multi_sku=bool(sku_info.get("hasMultiSku")),
|
||||
variant_id=as_str(raw.get("variantId")),
|
||||
item_options=item_options,
|
||||
)
|
||||
|
||||
|
||||
def parse_search(
|
||||
state: dict[str, Any],
|
||||
*,
|
||||
request_url: str,
|
||||
page: int,
|
||||
exclude_ads: bool,
|
||||
) -> SearchResultData:
|
||||
"""把搜索页状态解析为搜索结果
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 状态中不存在 ichibaSearch 节点，或站点返回了错误
|
||||
"""
|
||||
data = as_dict(as_dict(state.get("state")).get("data"))
|
||||
search = data.get("ichibaSearch")
|
||||
if not isinstance(search, dict):
|
||||
raise ScrapeParseError("搜索结果缺少 ichibaSearch 节点")
|
||||
|
||||
error = search.get("error")
|
||||
if error:
|
||||
raise ScrapeParseError(f"站点返回搜索错误：{error}")
|
||||
|
||||
pagination = as_dict(search.get("pagination"))
|
||||
ui_question = as_dict(data.get("effectiveUiQuestion"))
|
||||
|
||||
# 站点声明的每页条数；items 实际长度会因为混入广告位而略大于它
|
||||
page_size = as_int(pagination.get("pageSize"))
|
||||
total_count = as_int(pagination.get("numFound"))
|
||||
subset = as_int(pagination.get("subset")) or site.DEFAULT_SUBSET_LIMIT
|
||||
reachable_count = min(total_count, subset) if total_count else 0
|
||||
start = as_int(pagination.get("start"))
|
||||
|
||||
# 请求页超出可达窗口时，站点会静默回绕到第 1 页并把 start/page 重置为 0/1。
|
||||
# 拿站点回报的页码与请求页码对账即可识别，比自行按 subset 推算更可靠。
|
||||
site_page = as_int(ui_question.get("page"), page)
|
||||
out_of_range = page > 1 and site_page != page
|
||||
|
||||
raw_items = [item for item in as_list(search.get("items")) if isinstance(item, dict)]
|
||||
items = [parse_search_item(item) for item in raw_items]
|
||||
ad_count = sum(1 for item in items if item.is_ad)
|
||||
if exclude_ads:
|
||||
items = [item for item in items if not item.is_ad]
|
||||
if out_of_range:
|
||||
# 这一页的内容是第 1 页的副本，交给上游只会造成重复入库
|
||||
items = []
|
||||
ad_count = 0
|
||||
|
||||
has_more = (
|
||||
not out_of_range
|
||||
and bool(raw_items)
|
||||
and (start + (page_size or len(raw_items))) < reachable_count
|
||||
)
|
||||
|
||||
return SearchResultData(
|
||||
keyword=as_str(ui_question.get("keywords")),
|
||||
page=page,
|
||||
page_size=page_size,
|
||||
total_count=total_count,
|
||||
reachable_count=reachable_count,
|
||||
has_more=has_more,
|
||||
out_of_range=out_of_range,
|
||||
ad_count=ad_count,
|
||||
request_url=request_url,
|
||||
items=items,
|
||||
)
|
||||
@@ -0,0 +1,79 @@
|
||||
"""乐天店铺页 __INITIAL_STATE__ → ShopDetailData
|
||||
|
||||
店铺首页（www.rakuten.co.jp/{店铺代码}/）与搜索页一样把状态内联在
|
||||
`window.__INITIAL_STATE__` 里，店铺信息在顶层的 `shop` 节点下，结构比商品
|
||||
详情页里的 shop.information 更完整（多出招牌图、39ショップ标记、休息日）。
|
||||
|
||||
店铺 logo 不在 state 里，只出现在页面的 ld+json AggregateRating 微数据中。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import ShopDetailData
|
||||
from app.utils.coerce import as_dict, as_float, as_int, as_list, as_str
|
||||
|
||||
_LD_JSON_RE = re.compile(
|
||||
r'<script[^>]*type="application/ld\+json"[^>]*>(.*?)</script>', re.S
|
||||
)
|
||||
|
||||
|
||||
def _parse_logo_url(html: str) -> str:
|
||||
"""从 ld+json 的 AggregateRating.itemReviewed 里取店铺 logo"""
|
||||
for match in _LD_JSON_RE.finditer(html):
|
||||
try:
|
||||
data = json.loads(match.group(1))
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(data, dict) or data.get("@type") != "AggregateRating":
|
||||
continue
|
||||
reviewed = data.get("itemReviewed")
|
||||
if isinstance(reviewed, dict):
|
||||
return as_str(reviewed.get("logo"))
|
||||
return ""
|
||||
|
||||
|
||||
def parse_shop_detail(
|
||||
state: dict[str, Any], *, shop_code: str, html: str = ""
|
||||
) -> ShopDetailData:
|
||||
"""把店铺页状态解析为商家详情
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 状态中不存在 shop 节点
|
||||
"""
|
||||
shop = state.get("shop")
|
||||
if not isinstance(shop, dict) or not shop:
|
||||
raise ScrapeParseError("店铺页缺少 shop 节点")
|
||||
|
||||
review = as_dict(shop.get("shopReview"))
|
||||
resolved_code = as_str(shop.get("shopCode")) or shop_code
|
||||
# state 里的 shopUrl 是 http 的且不带尾斜杠，统一按规范形式给出
|
||||
shop_url = f"{site.WWW_BASE_URL}{resolved_code}/" if resolved_code else ""
|
||||
|
||||
holidays = [
|
||||
as_str(day)
|
||||
for day in as_list(as_dict(as_dict(shop.get("shopCalendar")).get("eventDates")).get("holiday"))
|
||||
if as_str(day)
|
||||
]
|
||||
|
||||
return ShopDetailData(
|
||||
shop_id=as_int(shop.get("shopId")) or None,
|
||||
shop_code=resolved_code,
|
||||
shop_name=as_str(shop.get("shopName")),
|
||||
shop_url=shop_url,
|
||||
introduction=as_str(shop.get("shopIntroduction")),
|
||||
signboard_url=as_str(shop.get("signboardUrl")),
|
||||
logo_url=_parse_logo_url(html),
|
||||
review_score=as_float(review.get("average")),
|
||||
review_count=as_int(review.get("total")),
|
||||
# 评价数过少时站点不展示评分，此时 average 不可信
|
||||
review_displayed=bool(review.get("didMeetDisplayRequirements")),
|
||||
is_39_shop=bool(shop.get("is39Shop")),
|
||||
age_verification_required=bool(shop.get("ageVerificationFlag")),
|
||||
status=as_int(shop.get("status")) if shop.get("status") is not None else None,
|
||||
holidays=holidays,
|
||||
)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""从页面 HTML 中抽取服务端渲染的 `window.__INITIAL_STATE__`
|
||||
|
||||
搜索页与（手机版）商品详情页都会把整页数据以 JSON 形式内联到这个全局变量里，
|
||||
因此不需要解析 DOM——直接把 JSON 取出来即可拿到结构化数据。
|
||||
|
||||
该赋值语句后面紧跟着其他脚本代码，不能按行或按分号切分，只能用增量 JSON
|
||||
解码器从 `=` 之后开始解析，读到一个完整对象为止。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
|
||||
# 页面必须包含的服务端渲染数据标记；缺失说明拿到的不是正常页面
|
||||
STATE_MARKER = "window.__INITIAL_STATE__"
|
||||
|
||||
# 页面校验器：接收 (页面 HTML, 最终落地 URL)，返回 None 表示通过，返回字符串表示
|
||||
# 失败原因（会触发抓取层换 cookie / 浏览器兜底重试）。遇到重试也无济于事的情况
|
||||
# （例如落到了不支持的站点），校验器可直接抛 AppError 快速失败。
|
||||
PageValidator = Callable[[str, str], str | None]
|
||||
|
||||
_ASSIGNMENT_RE = re.compile(r"window\.__INITIAL_STATE__\s*=\s*")
|
||||
_DECODER = json.JSONDecoder()
|
||||
|
||||
|
||||
def require_state_marker(html: str, _final_url: str) -> str | None:
|
||||
"""默认页面校验：必须含服务端渲染数据"""
|
||||
if STATE_MARKER in html:
|
||||
return None
|
||||
return f"missing {STATE_MARKER} (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def extract_initial_state(html: str) -> dict[str, Any]:
|
||||
"""抽取并解析 `window.__INITIAL_STATE__`
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 页面中没有该变量，或其内容不是合法 JSON 对象
|
||||
"""
|
||||
match = _ASSIGNMENT_RE.search(html)
|
||||
if match is None:
|
||||
raise ScrapeParseError("页面中未找到 window.__INITIAL_STATE__")
|
||||
|
||||
try:
|
||||
state, _ = _DECODER.raw_decode(html, match.end())
|
||||
except ValueError as exc:
|
||||
raise ScrapeParseError(f"window.__INITIAL_STATE__ 解析失败：{exc}") from exc
|
||||
|
||||
if not isinstance(state, dict):
|
||||
raise ScrapeParseError("window.__INITIAL_STATE__ 不是 JSON 对象")
|
||||
|
||||
return state
|
||||
@@ -0,0 +1,57 @@
|
||||
"""子站解析器注册表与商品页分派
|
||||
|
||||
乐天部分官方旗舰店的商品页会 302 跳出 item.rakuten.co.jp，落到各自独立的站点。
|
||||
这里按落地域名把页面分派给对应解析器；落到未登记的站点时抛 OffIchibaRedirectError，
|
||||
让上游能明确区分「站点不支持」与「被反爬拦截」，而不是白白重试。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import OffIchibaRedirectError, ScrapeParseError
|
||||
from app.models.scrape import ItemDetailData
|
||||
from app.parsers.state import PageValidator, require_state_marker
|
||||
from app.parsers.subsites import biccamera, books, brandavenue
|
||||
from app.parsers.subsites.base import SubsitePage, SubsiteParser
|
||||
|
||||
SUBSITE_PARSERS: dict[str, SubsiteParser] = {
|
||||
module.HOST: SubsiteParser(
|
||||
host=module.HOST,
|
||||
source=module.SOURCE,
|
||||
shop_name=module.SHOP_NAME,
|
||||
validate=module.validate,
|
||||
parse=module.parse,
|
||||
)
|
||||
for module in (books, brandavenue, biccamera)
|
||||
}
|
||||
|
||||
|
||||
def host_of(url: str) -> str:
|
||||
return urlsplit(url).hostname or ""
|
||||
|
||||
|
||||
def build_item_page_validator(requested_url: str) -> PageValidator:
|
||||
"""构造商品页校验器：按落地域名选用对应的页面校验规则
|
||||
|
||||
落到未登记的站点时直接抛错——换 cookie 或上浏览器都改变不了页面归属。
|
||||
"""
|
||||
|
||||
def validate(html: str, final_url: str) -> str | None:
|
||||
host = host_of(final_url)
|
||||
if host == site.ITEM_HOST:
|
||||
return require_state_marker(html, final_url)
|
||||
parser = SUBSITE_PARSERS.get(host)
|
||||
if parser is None:
|
||||
raise OffIchibaRedirectError(requested_url, final_url)
|
||||
return parser.validate(html)
|
||||
|
||||
return validate
|
||||
|
||||
|
||||
def parse_subsite_item(page: SubsitePage) -> ItemDetailData:
|
||||
"""把子站页面交给对应解析器"""
|
||||
parser = SUBSITE_PARSERS.get(host_of(page.final_url))
|
||||
if parser is None:
|
||||
raise ScrapeParseError(f"没有匹配的子站解析器：{page.final_url}")
|
||||
return parser.parse(page)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""子站解析器的公共契约与工具
|
||||
|
||||
乐天部分官方旗舰店的商品页会跳出市场域名，落到各自独立的站点上。这些站点
|
||||
技术栈各不相同（微数据 / Vue SSR state / Nuxt state），但对外要收敛成同一个
|
||||
ItemDetailData，因此这里定义统一的输入结构与注册契约。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from app.models.scrape import ItemDetailData
|
||||
|
||||
# 售罄判定关键词：日文页面上表示不可购买的常见措辞
|
||||
SOLD_OUT_MARKERS = ("在庫なし", "在庫切れ", "品切れ", "入荷未定", "販売終了", "取扱終了")
|
||||
|
||||
_DIGITS_RE = re.compile(r"\d+")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SubsitePage:
|
||||
"""一个待解析的子站页面及其上下文"""
|
||||
|
||||
html: str
|
||||
requested_url: str # 请求时的 item.rakuten.co.jp 地址
|
||||
final_url: str # 跳转后的实际地址
|
||||
shop_code: str # 市场侧店铺代码，如 book / stylife / biccamera
|
||||
item_code: str # 市场侧商品编号
|
||||
include_sku_variants: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SubsiteParser:
|
||||
"""一个子站的解析能力"""
|
||||
|
||||
host: str # 落地域名
|
||||
source: str # 写入 ItemDetailData.source 的标识
|
||||
shop_name: str
|
||||
validate: Callable[[str], str | None] # 返回 None 表示页面正常
|
||||
parse: Callable[[SubsitePage], ItemDetailData]
|
||||
|
||||
|
||||
def parse_price(text: str | int | float | None) -> int:
|
||||
"""从 "3,091円" / "1760" / 1760 这类值里取出整数日元金额"""
|
||||
if isinstance(text, bool) or text is None:
|
||||
return 0
|
||||
if isinstance(text, (int, float)):
|
||||
return int(text)
|
||||
digits = _DIGITS_RE.findall(text.replace(",", ""))
|
||||
return int(digits[0]) if digits else 0
|
||||
|
||||
|
||||
def looks_sold_out(status_text: str) -> bool:
|
||||
"""按页面上的库存措辞判断是否不可购买"""
|
||||
return any(marker in status_text for marker in SOLD_OUT_MARKERS)
|
||||
@@ -0,0 +1,132 @@
|
||||
"""ビックカメラ楽天市場店（biccamera.rakuten.co.jp）商品页解析
|
||||
|
||||
Nuxt 应用，整页数据内联在 `window.__NUXT__`（纯 JSON 对象，可直接增量解析）。
|
||||
商品主体在 `state.item`，字段命名已经很接近市场侧语义，且直接给出市场的
|
||||
shop_id / genre_id / 分类路径与库存数。
|
||||
|
||||
站点不提供：SKU 组合（该店商品都是单一规格）、商品评分（异步加载）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
ShippingInfo,
|
||||
ShopSummary,
|
||||
SkuInfo,
|
||||
)
|
||||
from app.parsers.subsites.base import SubsitePage
|
||||
from app.utils.coerce import as_dict, as_int, as_list, as_str
|
||||
|
||||
HOST = "biccamera.rakuten.co.jp"
|
||||
SOURCE = "biccamera"
|
||||
SHOP_NAME = "ビックカメラ楽天市場店"
|
||||
|
||||
_NUXT_RE = re.compile(r"window\.__NUXT__\s*=\s*")
|
||||
_CATEGORY_URL = "https://www.rakuten.co.jp/category/{}/"
|
||||
|
||||
|
||||
def validate(html: str) -> str | None:
|
||||
if _NUXT_RE.search(html):
|
||||
return None
|
||||
return f"biccamera nuxt state not found (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def _extract_nuxt(html: str) -> dict:
|
||||
match = _NUXT_RE.search(html)
|
||||
if match is None:
|
||||
raise ScrapeParseError("ビックカメラ页面未找到 window.__NUXT__")
|
||||
try:
|
||||
state, _ = json.JSONDecoder().raw_decode(html, match.end())
|
||||
except ValueError as exc:
|
||||
raise ScrapeParseError(f"window.__NUXT__ 解析失败：{exc}") from exc
|
||||
if not isinstance(state, dict):
|
||||
raise ScrapeParseError("window.__NUXT__ 不是 JSON 对象")
|
||||
return state
|
||||
|
||||
|
||||
def _purchase_info(state: dict, item: dict) -> PurchaseInfo:
|
||||
"""该站加购走自己的 JSON 接口，字段与市场完全不同（没有 shop_bid）
|
||||
|
||||
选项（choices）的取值结构未取到样本验证，因此只如实回报「有没有选项」，
|
||||
不给出可能不准确的选项定义——带选项的商品需要调用方另行处理。
|
||||
"""
|
||||
choices = as_list(state.get("choices"))
|
||||
return PurchaseInfo(
|
||||
cart_url=as_str(item.get("add_cart_api_url")),
|
||||
form_fields={"item_id": str(as_int(item.get("item_id")))},
|
||||
quantity_field="units",
|
||||
options_field="choice" if choices else "",
|
||||
has_required_options=bool(choices),
|
||||
)
|
||||
|
||||
|
||||
def parse(page: SubsitePage) -> ItemDetailData:
|
||||
"""解析ビックカメラ商品页"""
|
||||
nuxt = _extract_nuxt(page.html)
|
||||
item = as_dict(as_dict(nuxt.get("state")).get("item"))
|
||||
if not item or not as_str(item.get("item_name")):
|
||||
raise ScrapeParseError("ビックカメラ页面缺少 state.item")
|
||||
|
||||
sold_out = bool(item.get("sold_out_flag"))
|
||||
inventory = as_int(item.get("inventory"))
|
||||
delivery = as_str(item.get("delivery_schedule_text"))
|
||||
|
||||
breadcrumbs = [
|
||||
Breadcrumb(
|
||||
name=as_str(genre.get("genre_name")),
|
||||
url=_CATEGORY_URL.format(as_str(genre.get("genre_id"))),
|
||||
)
|
||||
for genre in as_list(item.get("genres"))
|
||||
if isinstance(genre, dict) and as_str(genre.get("genre_name"))
|
||||
]
|
||||
|
||||
images = [
|
||||
as_str(image.get("url"))
|
||||
for image in as_list(item.get("images"))
|
||||
if isinstance(image, dict) and as_str(image.get("url"))
|
||||
]
|
||||
|
||||
shop_code = as_str(item.get("shop_url")) or page.shop_code
|
||||
return ItemDetailData(
|
||||
source=SOURCE,
|
||||
source_url=page.final_url,
|
||||
item_id=str(as_int(item.get("item_id"))) if item.get("item_id") is not None else "",
|
||||
item_code=as_str(item.get("item_number")) or page.item_code,
|
||||
item_name=as_str(item.get("item_name")),
|
||||
catch_copy=as_str(item.get("catch_copy")),
|
||||
description=as_str(item.get("caption")),
|
||||
item_url=page.requested_url,
|
||||
price=as_int(item.get("price_with_tax")),
|
||||
pre_tax_price=as_int(item.get("original_price")),
|
||||
tax_flag=bool(item.get("included_tax_flag")),
|
||||
purchase_condition="soldOut" if sold_out else "enabled",
|
||||
is_sold_out=sold_out,
|
||||
purchase_unit=as_int(item.get("units")),
|
||||
images=images,
|
||||
shop=ShopSummary(
|
||||
shop_id=as_int(item.get("shop_id")) or None,
|
||||
shop_code=shop_code,
|
||||
shop_name=SHOP_NAME,
|
||||
shop_url=f"https://www.rakuten.co.jp/{shop_code}/",
|
||||
),
|
||||
genre_id=str(as_int(item.get("genre_id"))) if item.get("genre_id") is not None else "",
|
||||
breadcrumbs=breadcrumbs,
|
||||
shipping=ShippingInfo(
|
||||
# 该店商品价格含运费时站点会置位此标记，不再单独给运费金额
|
||||
is_shipping_free=bool(item.get("included_shipping_fee_flag")),
|
||||
delivery_message=delivery,
|
||||
),
|
||||
sku=SkuInfo(
|
||||
inventory_type="single",
|
||||
quantity=inventory,
|
||||
show_inventory=inventory > 0,
|
||||
delivery_message=delivery,
|
||||
),
|
||||
purchase=_purchase_info(as_dict(nuxt.get("state")), item),
|
||||
)
|
||||
@@ -0,0 +1,187 @@
|
||||
"""楽天ブックス（books.rakuten.co.jp）商品页解析
|
||||
|
||||
该站是传统服务端渲染页面，商品数据以 schema.org 微数据标注（Product / Offer /
|
||||
AggregateRating），规格与简介在 `.sec-item` 分节里，分类路径则通过页面内联的
|
||||
`var data_genres` 给出——其中的 rmsGenreId 就是市场侧的 genre_id。
|
||||
|
||||
站点不提供：SKU 组合（图书没有规格轴）、运费明细、店铺评分。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
ReviewSummary,
|
||||
ShippingInfo,
|
||||
ShopSummary,
|
||||
SkuAttribute,
|
||||
SkuInfo,
|
||||
)
|
||||
from app.parsers.subsites.base import SubsitePage, looks_sold_out, parse_price
|
||||
|
||||
HOST = "books.rakuten.co.jp"
|
||||
SOURCE = "books"
|
||||
SHOP_NAME = "楽天ブックス"
|
||||
|
||||
_GENRES_RE = re.compile(r"var\s+data_genres\s*=\s*")
|
||||
# 折叠正文里由展开控件（checkbox/label）留下的连续空行
|
||||
_BLANK_LINES_RE = re.compile(r"\n{2,}")
|
||||
_IMAGE_RE = re.compile(r"//tshop\.r10s\.jp/book/cabinet/[^\"'\s?]+\.(?:jpg|jpeg|png)", re.I)
|
||||
_CATEGORY_URL = "https://www.rakuten.co.jp/category/{}/"
|
||||
|
||||
|
||||
def validate(html: str) -> str | None:
|
||||
"""页面须带 Product 微数据，否则不是正常的商品页"""
|
||||
if 'itemprop="price"' in html and "schema.org/Product" in html:
|
||||
return None
|
||||
return f"books item page markup not found (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def _attr(tree: HTMLParser, selector: str, name: str) -> str:
|
||||
node = tree.css_first(selector)
|
||||
return (node.attributes.get(name) or "") if node else ""
|
||||
|
||||
|
||||
def _text(tree: HTMLParser, selector: str) -> str:
|
||||
node = tree.css_first(selector)
|
||||
return node.text(strip=True) if node else ""
|
||||
|
||||
|
||||
def _parse_spec(tree: HTMLParser) -> list[SkuAttribute]:
|
||||
"""商品情報分节：每个 ul 内首个 li.product-title 是字段名，其后是取值"""
|
||||
attributes: list[SkuAttribute] = []
|
||||
for row in tree.css(".sec-item__identifier__list ul"):
|
||||
cells = row.css("li")
|
||||
if len(cells) < 2:
|
||||
continue
|
||||
title = cells[0]
|
||||
if "product-title" not in (title.attributes.get("class") or ""):
|
||||
continue
|
||||
value = " ".join(cell.text(strip=True) for cell in cells[1:] if cell.text(strip=True))
|
||||
if value:
|
||||
attributes.append(SkuAttribute(title=title.text(strip=True), value=value))
|
||||
return attributes
|
||||
|
||||
|
||||
def _parse_description(tree: HTMLParser) -> str:
|
||||
"""商品説明分节：把「内容紹介」「目次」等若干小节拼起来
|
||||
|
||||
这些小节在 DOM 里是平铺的——标题 h3 与正文 div 互为兄弟节点而非父子，
|
||||
所以要从标题往后找同级的正文块，不能直接按容器取。
|
||||
"""
|
||||
parts: list[str] = []
|
||||
for title in tree.css(".sec-item__extra__title"):
|
||||
node = title.next
|
||||
while node is not None:
|
||||
classes = node.attributes.get("class") or "" if node.tag != "-text" else ""
|
||||
if "sec-item__extra__content" in classes:
|
||||
body = _BLANK_LINES_RE.sub("\n", node.text(separator="\n", strip=True)).strip()
|
||||
if body:
|
||||
parts.append(f"{title.text(strip=True)}\n{body}")
|
||||
break
|
||||
# 只在紧邻的兄弟里找，遇到下一个标题就说明这一节没有正文
|
||||
if node.tag != "-text" and "sec-item__extra__title" in classes:
|
||||
break
|
||||
node = node.next
|
||||
return "\n\n".join(parts)
|
||||
|
||||
|
||||
def _parse_genres(html: str) -> tuple[str, list[Breadcrumb]]:
|
||||
"""内联的 data_genres 给出分类路径，其中 rmsGenreId 对应市场侧 genre_id"""
|
||||
match = _GENRES_RE.search(html)
|
||||
if match is None:
|
||||
return "", []
|
||||
try:
|
||||
raw, _ = json.JSONDecoder().raw_decode(html, match.end())
|
||||
except ValueError:
|
||||
return "", []
|
||||
|
||||
# 结构是 [[{...}, {...}]]，取第一条路径
|
||||
path = raw[0] if isinstance(raw, list) and raw and isinstance(raw[0], list) else raw
|
||||
crumbs: list[Breadcrumb] = []
|
||||
genre_id = ""
|
||||
for node in path if isinstance(path, list) else []:
|
||||
if not isinstance(node, dict):
|
||||
continue
|
||||
rms_id = str(node.get("rmsGenreId") or "")
|
||||
name = str(node.get("genreName") or "")
|
||||
if not name:
|
||||
continue
|
||||
crumbs.append(Breadcrumb(name=name, url=_CATEGORY_URL.format(rms_id) if rms_id else ""))
|
||||
if rms_id:
|
||||
genre_id = rms_id
|
||||
return genre_id, crumbs
|
||||
|
||||
|
||||
def _parse_purchase(tree: HTMLParser) -> PurchaseInfo:
|
||||
"""加购信息直接取页面上的购物车表单
|
||||
|
||||
注意表单里的 item_id 与商品 URL 上的编号不是一回事，加购必须用表单里的值。
|
||||
该表单没有数量字段，无法在加购时指定件数。
|
||||
"""
|
||||
for form in tree.css("form"):
|
||||
action = form.attributes.get("action") or ""
|
||||
if "/bs/Cart" not in action:
|
||||
continue
|
||||
fields = {
|
||||
name: inp.attributes.get("value") or ""
|
||||
for inp in form.css("input")
|
||||
if (name := inp.attributes.get("name"))
|
||||
}
|
||||
return PurchaseInfo(
|
||||
cart_url=action,
|
||||
cart_method=(form.attributes.get("method") or "POST").upper(),
|
||||
form_fields=fields,
|
||||
)
|
||||
return PurchaseInfo()
|
||||
|
||||
|
||||
def parse(page: SubsitePage) -> ItemDetailData:
|
||||
"""解析楽天ブックス商品页"""
|
||||
tree = HTMLParser(page.html)
|
||||
|
||||
name = _text(tree, "#productTitle") or _text(tree, '[itemprop="name"]')
|
||||
if not name:
|
||||
raise ScrapeParseError("楽天ブックス页面未找到商品名")
|
||||
|
||||
images = ["https:" + url if url.startswith("//") else url for url in _IMAGE_RE.findall(page.html)]
|
||||
# 同一张图可能带不同裁剪参数重复出现，去重但保留出现顺序
|
||||
images = list(dict.fromkeys(images))
|
||||
|
||||
status = _text(tree, ".status")
|
||||
genre_id, breadcrumbs = _parse_genres(page.html)
|
||||
review_count = _text(tree, '[itemprop="reviewCount"]')
|
||||
purchase = _parse_purchase(tree)
|
||||
|
||||
return ItemDetailData(
|
||||
source=SOURCE,
|
||||
source_url=page.final_url,
|
||||
# 站内商品 ID 与 URL 上的编号不同，以购物车表单里的为准（下单要用它）
|
||||
item_id=purchase.form_fields.get("item_id", "") or page.item_code,
|
||||
item_code=page.item_code,
|
||||
item_name=name,
|
||||
description=_parse_description(tree),
|
||||
item_url=page.requested_url,
|
||||
price=parse_price(_attr(tree, '[itemprop="price"]', "content")),
|
||||
purchase_condition=status,
|
||||
is_sold_out=looks_sold_out(status),
|
||||
images=images,
|
||||
shop=ShopSummary(shop_code=page.shop_code, shop_name=SHOP_NAME),
|
||||
review=ReviewSummary(
|
||||
score=float(_attr(tree, '[itemprop="ratingValue"]', "content") or 0) or 0.0,
|
||||
count=parse_price(review_count),
|
||||
),
|
||||
genre_id=genre_id,
|
||||
breadcrumbs=breadcrumbs,
|
||||
# 图书统一由楽天ブックス发货，页面只给库存措辞，不给运费明细
|
||||
shipping=ShippingInfo(delivery_message=status),
|
||||
sku=SkuInfo(inventory_type="single", attributes=_parse_spec(tree), delivery_message=status),
|
||||
purchase=purchase,
|
||||
)
|
||||
@@ -0,0 +1,210 @@
|
||||
"""Rakuten Fashion / BRAND AVENUE（brandavenue.rakuten.co.jp）商品页解析
|
||||
|
||||
该站同样把整页数据内联在 `window.__INITIAL_STATE__`，但结构与市场页完全不同：
|
||||
商品主体在 `itemDetail.data.product`，店铺信息在 `env`，市场侧的 genre_id 与
|
||||
商品 ID 则藏在 `product.rms_info` 里。
|
||||
|
||||
SKU 有两个轴（颜色 / 尺码）：product_sku 给出可售组合与售价，rms_info.inventory_list
|
||||
给出各组合库存，两者按「尺码 + 颜色名」对齐。
|
||||
|
||||
站点不提供：商品评分（异步加载）、运费明细。面包屑用的是站内分类编码而非市场
|
||||
genre_id，因此只给分类名不给链接。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
Breadcrumb,
|
||||
ItemDetailData,
|
||||
PurchaseInfo,
|
||||
ShopSummary,
|
||||
SkuAttribute,
|
||||
SkuAxis,
|
||||
SkuAxisValue,
|
||||
SkuInfo,
|
||||
SkuVariant,
|
||||
)
|
||||
from app.parsers.state import extract_initial_state
|
||||
from app.parsers.subsites.base import SubsitePage, parse_price
|
||||
from app.utils.coerce import as_dict, as_int, as_list, as_str
|
||||
|
||||
HOST = "brandavenue.rakuten.co.jp"
|
||||
SOURCE = "brandavenue"
|
||||
SHOP_NAME = "Rakuten Fashion"
|
||||
|
||||
_COLOR_AXIS = "カラー"
|
||||
_SIZE_AXIS = "サイズ"
|
||||
|
||||
# cart_info.cart_url_type → 加购端点。站点前端用同名映射表（resolveCartUrl）解析；
|
||||
# 若该字段本身已经是一个 URL，则直接使用。
|
||||
_CART_URL_BY_TYPE = {
|
||||
"1": "https://ts.basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set",
|
||||
"2": "https://ts.sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set",
|
||||
"3": "https://t2.basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set",
|
||||
"4": "https://t2.sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set",
|
||||
"5": "https://basket.step.rakuten.co.jp/rms/mall/bs/cartadd/set",
|
||||
"6": "https://sp.basket.step.rakuten.co.jp/rms/mall/bss/cartadd/set",
|
||||
}
|
||||
_DEFAULT_PURCHASE_EVENT = "ES01_003_001"
|
||||
|
||||
|
||||
def _resolve_cart_url(cart_url_type: str) -> str:
|
||||
if cart_url_type.startswith("http"):
|
||||
return cart_url_type
|
||||
return _CART_URL_BY_TYPE.get(cart_url_type, "")
|
||||
|
||||
|
||||
def _purchase_info(product: dict) -> PurchaseInfo:
|
||||
"""加购字段与市场是同一套契约，差别只在端点由 cart_url_type 映射得到"""
|
||||
cart_info = as_dict(as_dict(product.get("rms_info")).get("cart_info"))
|
||||
if not cart_info:
|
||||
return PurchaseInfo()
|
||||
return PurchaseInfo(
|
||||
cart_url=_resolve_cart_url(as_str(cart_info.get("cart_url_type"))),
|
||||
form_fields={
|
||||
"shop_bid": as_str(cart_info.get("shop_bid")),
|
||||
"item_id": as_str(cart_info.get("item_id")),
|
||||
"inventory_flag": as_str(cart_info.get("inventory_type")),
|
||||
"__event": as_str(cart_info.get("event")) or _DEFAULT_PURCHASE_EVENT,
|
||||
"encode": "utf8",
|
||||
},
|
||||
quantity_field="units",
|
||||
variant_field="variant_id",
|
||||
)
|
||||
|
||||
|
||||
def validate(html: str) -> str | None:
|
||||
if "window.__INITIAL_STATE__" in html:
|
||||
return None
|
||||
return f"brandavenue state not found (body {len(html)} bytes)"
|
||||
|
||||
|
||||
def _images(html: str, image_folder: str, main_filename: str) -> list[str]:
|
||||
"""从页面直出的图片地址中收集商品图
|
||||
|
||||
图片按商品编号做了目录分片，分片规则不对外暴露，所以直接取页面里已经渲染好的
|
||||
地址，而不是自行拼接，避免规则变化导致图片全错。
|
||||
"""
|
||||
if not image_folder:
|
||||
return []
|
||||
pattern = re.compile(
|
||||
rf"https://[a-z0-9.\-]+/{re.escape(image_folder)}/[^\"'\s]+\.(?:jpg|jpeg|png)", re.I
|
||||
)
|
||||
urls = list(dict.fromkeys(pattern.findall(html)))
|
||||
if not main_filename:
|
||||
return urls
|
||||
# 主图排在最前，便于调用方直接取 images[0] 当封面
|
||||
main = main_filename.lower()
|
||||
urls.sort(key=lambda url: 0 if url.lower().endswith("/" + main) else 1)
|
||||
return urls
|
||||
|
||||
|
||||
def _breadcrumbs(product: dict) -> list[Breadcrumb]:
|
||||
"""category_l_m_cd_name 是 [大类ID, 大类名, 中类ID, 中类名] 的扁平数组"""
|
||||
flat = [as_str(value) for value in as_list(product.get("category_l_m_cd_name"))]
|
||||
crumbs: list[Breadcrumb] = []
|
||||
for index in range(0, len(flat) - 1, 2):
|
||||
name = flat[index + 1]
|
||||
if name:
|
||||
crumbs.append(Breadcrumb(name=name))
|
||||
return crumbs
|
||||
|
||||
|
||||
def _sku(product: dict, *, include_variants: bool) -> SkuInfo:
|
||||
entries = [entry for entry in as_list(product.get("product_sku")) if isinstance(entry, dict)]
|
||||
inventory = {
|
||||
(as_str(row.get("size")), as_str(row.get("color_name"))): row
|
||||
for row in as_list(as_dict(product.get("rms_info")).get("inventory_list"))
|
||||
if isinstance(row, dict)
|
||||
}
|
||||
|
||||
variants: list[SkuVariant] = []
|
||||
colors: dict[str, bool] = {}
|
||||
sizes: dict[str, bool] = {}
|
||||
for entry in entries:
|
||||
color = as_str(entry.get("product_color_name"))
|
||||
size = as_str(entry.get("product_size_name"))
|
||||
in_stock = as_str(entry.get("inventory_exist_flg")) == "1"
|
||||
row = as_dict(inventory.get((size, color)))
|
||||
attributes = [
|
||||
SkuAttribute(title=title, value=as_str(entry.get(key)))
|
||||
for title, key in (("素材", "material"), ("お手入れ", "cleaning"), ("お届け目安", "inventory_status_message"))
|
||||
if as_str(entry.get(key))
|
||||
]
|
||||
variants.append(
|
||||
SkuVariant(
|
||||
variant_id=as_str(row.get("variant_id")),
|
||||
selector_values=[color, size],
|
||||
price=parse_price(entry.get("selling_price")),
|
||||
quantity=as_int(row.get("stock")),
|
||||
is_sold_out=not in_stock,
|
||||
delivery_message=as_str(entry.get("inventory_status_message")),
|
||||
attributes=attributes,
|
||||
)
|
||||
)
|
||||
# 任一组合可售即认为该取值可选
|
||||
colors[color] = colors.get(color, False) or in_stock
|
||||
sizes[size] = sizes.get(size, False) or in_stock
|
||||
|
||||
axis = [
|
||||
SkuAxis(
|
||||
key=key,
|
||||
label=key,
|
||||
values=[
|
||||
SkuAxisValue(value=value, label=value, is_sold_out=not available)
|
||||
for value, available in mapping.items()
|
||||
if value
|
||||
],
|
||||
)
|
||||
for key, mapping in ((_COLOR_AXIS, colors), (_SIZE_AXIS, sizes))
|
||||
if any(mapping)
|
||||
]
|
||||
|
||||
return SkuInfo(
|
||||
inventory_type="multiple" if len(variants) > 1 else "single",
|
||||
quantity=sum(variant.quantity for variant in variants),
|
||||
delivery_message=as_str(entries[0].get("inventory_status_message")) if entries else "",
|
||||
axis=axis,
|
||||
variants=variants if include_variants else [],
|
||||
variant_count=len(variants),
|
||||
)
|
||||
|
||||
|
||||
def parse(page: SubsitePage) -> ItemDetailData:
|
||||
"""解析 Rakuten Fashion 商品页"""
|
||||
state = extract_initial_state(page.html)
|
||||
product = as_dict(as_dict(as_dict(state.get("itemDetail")).get("data")).get("product"))
|
||||
if not product:
|
||||
raise ScrapeParseError("Rakuten Fashion 页面缺少 itemDetail.data.product")
|
||||
|
||||
env = as_dict(state.get("env"))
|
||||
rms = as_dict(product.get("rms_info"))
|
||||
sold_out = as_int(product.get("soldout_flg")) == 1
|
||||
|
||||
return ItemDetailData(
|
||||
source=SOURCE,
|
||||
source_url=page.final_url,
|
||||
item_id=as_str(rms.get("rms_item_id")),
|
||||
item_code=page.item_code,
|
||||
item_name=as_str(product.get("product_name")),
|
||||
catch_copy=as_str(product.get("brand_name")),
|
||||
description=as_str(product.get("product_exp")),
|
||||
item_url=page.requested_url,
|
||||
price=parse_price(product.get("selling_price_no_format")),
|
||||
pre_tax_price=parse_price(product.get("fixed_price_no_format")),
|
||||
purchase_condition="soldOut" if sold_out else "enabled",
|
||||
is_sold_out=sold_out,
|
||||
images=_images(page.html, as_str(env.get("product_image_folder")), as_str(product.get("product_img_path"))),
|
||||
shop=ShopSummary(
|
||||
shop_id=as_int(env.get("shop_id")) or None,
|
||||
shop_code=as_str(env.get("shop_url")) or page.shop_code,
|
||||
shop_name=as_str(env.get("shop_name")) or SHOP_NAME,
|
||||
shop_url=f"https://www.rakuten.co.jp/{as_str(env.get('shop_url')) or page.shop_code}/",
|
||||
),
|
||||
genre_id=as_str(rms.get("genre_id")),
|
||||
breadcrumbs=_breadcrumbs(product),
|
||||
sku=_sku(product, include_variants=page.include_sku_variants),
|
||||
purchase=_purchase_info(product),
|
||||
)
|
||||
@@ -0,0 +1,185 @@
|
||||
"""浏览器兜底：纯 HTTP 被 Akamai 拦截时，用 Playwright 取回一套可用 cookie
|
||||
|
||||
为什么只是「兜底」而不是主路径：乐天的反爬是 Akamai Bot Manager，正常携带
|
||||
完整浏览器请求头 + 复用其下发的 cookie 就能通过，不像骏河屋的 Cloudflare
|
||||
Turnstile 必须真浏览器交互。因此这里的浏览器是冷路径——按需惰性启动，
|
||||
长时间不用会被回收，未安装 playwright 时整体降级为「不兜底」而非报错。
|
||||
|
||||
导航结果里的 HTML 会一并返回：既然浏览器已经把页面取到了，就没必要让调用方
|
||||
再发一次 HTTP 请求。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core import site
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BrowserVisit:
|
||||
"""一次浏览器导航的产出：cookie 快照 + 页面 HTML"""
|
||||
|
||||
cookies: list[dict[str, Any]] = field(default_factory=list)
|
||||
html: str = ""
|
||||
|
||||
|
||||
class BrowserFallback:
|
||||
"""按需启动的 Playwright 实例，用于取回通过反爬校验的 cookie
|
||||
|
||||
仅在 HTTP 抓取被判定为拦截后才会被调用；调用之间浏览器保持存活，
|
||||
由 close() 在应用关闭时释放。
|
||||
"""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self._settings = settings
|
||||
self._playwright: Any | None = None
|
||||
self._browser: Any | None = None
|
||||
self._lock = asyncio.Lock()
|
||||
self._unavailable_reason: str | None = None
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
"""配置是否允许使用浏览器兜底"""
|
||||
return self._settings.browser_fallback_enabled
|
||||
|
||||
@property
|
||||
def unavailable_reason(self) -> str | None:
|
||||
"""浏览器不可用的原因（未启用 / 未安装 / 启动失败）"""
|
||||
if not self.enabled:
|
||||
return "browser fallback is disabled"
|
||||
return self._unavailable_reason
|
||||
|
||||
@property
|
||||
def ready(self) -> bool:
|
||||
"""浏览器当前是否已启动且连接正常"""
|
||||
browser = self._browser
|
||||
if browser is None:
|
||||
return False
|
||||
is_connected = getattr(browser, "is_connected", None)
|
||||
if callable(is_connected):
|
||||
try:
|
||||
return bool(is_connected())
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
async def _ensure_browser(self) -> Any | None:
|
||||
"""惰性启动浏览器；不可用时返回 None 并记录原因，不抛异常。"""
|
||||
if not self.enabled:
|
||||
return None
|
||||
if self.ready:
|
||||
return self._browser
|
||||
|
||||
async with self._lock:
|
||||
if self.ready:
|
||||
return self._browser
|
||||
|
||||
# 上一轮已经启动失败过，直接沿用结论，避免每次请求都吃一次启动超时
|
||||
if self._unavailable_reason is not None and self._playwright is None:
|
||||
return None
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
except ImportError as exc:
|
||||
self._unavailable_reason = (
|
||||
f"playwright is not installed: {exc}; "
|
||||
'install it with pip install ".[browser]" && python -m playwright install chromium'
|
||||
)
|
||||
logger.warning("浏览器兜底不可用：%s", self._unavailable_reason)
|
||||
return None
|
||||
|
||||
await self._close_locked()
|
||||
try:
|
||||
self._playwright = await async_playwright().start()
|
||||
self._browser = await self._playwright.chromium.launch(
|
||||
headless=self._settings.browser_headless_effective,
|
||||
channel=self._settings.browser_channel or None,
|
||||
proxy=self._settings.playwright_proxy,
|
||||
timeout=self._settings.browser_launch_timeout_seconds * 1000,
|
||||
args=["--no-first-run", "--disable-blink-features=AutomationControlled"],
|
||||
)
|
||||
self._unavailable_reason = None
|
||||
logger.info(
|
||||
"浏览器兜底已启动：headless=%s channel=%s proxy=%s",
|
||||
self._settings.browser_headless_effective,
|
||||
self._settings.browser_channel,
|
||||
bool(self._settings.proxy_server),
|
||||
)
|
||||
except Exception as exc:
|
||||
self._unavailable_reason = str(exc)
|
||||
logger.exception("浏览器兜底启动失败：%s", self._unavailable_reason)
|
||||
await self._close_locked()
|
||||
return None
|
||||
|
||||
return self._browser
|
||||
|
||||
async def visit(self, url: str, *, mobile: bool) -> BrowserVisit | None:
|
||||
"""用浏览器访问 url，返回 cookie 与页面 HTML；不可用时返回 None。
|
||||
|
||||
UA 与请求头必须和后续 HTTP 客户端使用的那一套保持一致——Akamai 会把
|
||||
cookie 与指纹绑定，用 PC 指纹拿到的 cookie 拿去发手机 UA 请求可能失效。
|
||||
"""
|
||||
browser = await self._ensure_browser()
|
||||
if browser is None:
|
||||
return None
|
||||
|
||||
headers = site.default_headers(mobile=mobile)
|
||||
context = None
|
||||
try:
|
||||
context = await browser.new_context(
|
||||
user_agent=headers["User-Agent"],
|
||||
locale="ja-JP",
|
||||
timezone_id="Asia/Tokyo",
|
||||
viewport={"width": 390, "height": 844} if mobile else {"width": 1440, "height": 900},
|
||||
is_mobile=mobile,
|
||||
has_touch=mobile,
|
||||
extra_http_headers={"Accept-Language": site.ACCEPT_LANGUAGE},
|
||||
)
|
||||
page = await context.new_page()
|
||||
await page.goto(
|
||||
url,
|
||||
wait_until="domcontentloaded",
|
||||
timeout=self._settings.browser_nav_timeout_seconds * 1000,
|
||||
)
|
||||
html = await page.content()
|
||||
cookies = await context.cookies()
|
||||
logger.info("浏览器兜底导航完成：url=%s cookies=%s html_len=%s", url, len(cookies), len(html))
|
||||
return BrowserVisit(cookies=cookies, html=html)
|
||||
except Exception as exc:
|
||||
logger.warning("浏览器兜底导航失败：url=%s err=%s", url, exc)
|
||||
# 导航失败常常意味着浏览器已掉线，标记后由下次调用重建
|
||||
if not self.ready:
|
||||
self._unavailable_reason = str(exc)
|
||||
return None
|
||||
finally:
|
||||
if context is not None:
|
||||
try:
|
||||
await context.close()
|
||||
except Exception:
|
||||
logger.debug("关闭兜底浏览器上下文失败", exc_info=True)
|
||||
|
||||
async def _close_locked(self) -> None:
|
||||
"""释放浏览器与 Playwright 运行时（调用方需已持锁或处于关闭流程）"""
|
||||
if self._browser is not None:
|
||||
try:
|
||||
await self._browser.close()
|
||||
except Exception:
|
||||
logger.debug("关闭兜底浏览器失败", exc_info=True)
|
||||
self._browser = None
|
||||
if self._playwright is not None:
|
||||
try:
|
||||
await self._playwright.stop()
|
||||
except Exception:
|
||||
logger.debug("停止兜底 Playwright 运行时失败", exc_info=True)
|
||||
self._playwright = None
|
||||
|
||||
async def close(self) -> None:
|
||||
"""应用关闭时释放浏览器资源"""
|
||||
async with self._lock:
|
||||
await self._close_locked()
|
||||
@@ -0,0 +1,130 @@
|
||||
"""ラクマ 抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据
|
||||
|
||||
四个接口都走同一条 HTTP 通道（ラクマ 无 Akamai 限速，不需要分指纹通道）：
|
||||
搜索、商品详情、卖家详情、卖家商品列表。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models.scrape import (
|
||||
RakumaItemDetailData,
|
||||
RakumaItemDetailRequest,
|
||||
RakumaSearchRequest,
|
||||
RakumaSearchResultData,
|
||||
RakumaShopDetailData,
|
||||
RakumaShopDetailRequest,
|
||||
RakumaShopItemsData,
|
||||
RakumaShopItemsRequest,
|
||||
)
|
||||
from app.parsers.rakuma.item import parse_item_detail
|
||||
from app.parsers.rakuma.search import parse_search
|
||||
from app.parsers.rakuma.shop import parse_shop_detail, parse_shop_items
|
||||
from app.services.rakuma_session import RakumaSession
|
||||
from app.utils.rakuma_urls import (
|
||||
build_item_url,
|
||||
build_search_url,
|
||||
build_shop_url,
|
||||
normalize_search_url,
|
||||
split_item_url,
|
||||
split_shop_url,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RakumaClient:
|
||||
"""ラクマ（fril.jp）抓取客户端"""
|
||||
|
||||
def __init__(self, settings: Settings, session: RakumaSession):
|
||||
self._settings = settings
|
||||
self._session = session
|
||||
|
||||
async def search(self, payload: RakumaSearchRequest) -> RakumaSearchResultData:
|
||||
"""抓取搜索结果列表"""
|
||||
if payload.search_url is not None:
|
||||
url = normalize_search_url(str(payload.search_url), payload.page)
|
||||
else:
|
||||
url = build_search_url(payload)
|
||||
|
||||
logger.info("抓取 ラクマ 搜索页：url=%s", url)
|
||||
html = await self._session.fetch_html(url)
|
||||
result = parse_search(
|
||||
html,
|
||||
request_url=url,
|
||||
page=payload.page,
|
||||
keyword=payload.keyword.strip(),
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 搜索完成：url=%s items=%s total=%s",
|
||||
url, len(result.items), result.total_count,
|
||||
)
|
||||
return result
|
||||
|
||||
async def item_detail(self, payload: RakumaItemDetailRequest) -> RakumaItemDetailData:
|
||||
"""抓取商品详情"""
|
||||
if payload.item_url is not None:
|
||||
item_id = split_item_url(str(payload.item_url))
|
||||
else:
|
||||
item_id = (payload.item_id or "").strip()
|
||||
url = build_item_url(item_id)
|
||||
|
||||
logger.info("抓取 ラクマ 商品详情：url=%s", url)
|
||||
html = await self._session.fetch_html(url)
|
||||
detail = parse_item_detail(html, item_id=item_id, item_url=url)
|
||||
logger.info(
|
||||
"ラクマ 详情完成：url=%s name=%s price=%s sold_out=%s",
|
||||
url, detail.item_name[:40], detail.price, detail.is_sold_out,
|
||||
)
|
||||
return detail
|
||||
|
||||
async def shop_detail(self, payload: RakumaShopDetailRequest) -> RakumaShopDetailData:
|
||||
"""抓取卖家详情
|
||||
|
||||
评价明细在单独的 /review 页上，仅在 include_reviews=true 时并发多取一次。
|
||||
"""
|
||||
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
||||
url = build_shop_url(shop_id)
|
||||
|
||||
logger.info("抓取 ラクマ 卖家详情：url=%s reviews=%s", url, payload.include_reviews)
|
||||
if payload.include_reviews:
|
||||
html, review_html = await asyncio.gather(
|
||||
self._session.fetch_html(url),
|
||||
self._session.fetch_html(build_shop_url(shop_id, review=True)),
|
||||
)
|
||||
else:
|
||||
html, review_html = await self._session.fetch_html(url), None
|
||||
|
||||
detail = parse_shop_detail(
|
||||
html, shop_id=shop_id, shop_url=url, review_html=review_html
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 卖家详情完成：shop_id=%s name=%s items=%s reviews=%s",
|
||||
shop_id, detail.shop_name, detail.item_count, detail.review_count,
|
||||
)
|
||||
return detail
|
||||
|
||||
async def shop_items(self, payload: RakumaShopItemsRequest) -> RakumaShopItemsData:
|
||||
"""抓取卖家的商品列表"""
|
||||
shop_id = self._resolve_shop_id(payload.shop_url, payload.shop_id)
|
||||
url = build_shop_url(shop_id, page=payload.page)
|
||||
|
||||
logger.info("抓取 ラクマ 卖家商品：url=%s", url)
|
||||
html = await self._session.fetch_html(url)
|
||||
result = parse_shop_items(
|
||||
html, shop_id=shop_id, request_url=url, page=payload.page
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 卖家商品完成：shop_id=%s items=%s total=%s",
|
||||
shop_id, len(result.items), result.total_count,
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _resolve_shop_id(shop_url: object, shop_id: str | None) -> str:
|
||||
"""从 shop_url 或 shop_id 里取出店铺 hash（模型校验已保证两者不同时为空）"""
|
||||
if shop_url is not None:
|
||||
return split_shop_url(str(shop_url))
|
||||
return (shop_id or "").strip()
|
||||
@@ -0,0 +1,135 @@
|
||||
"""ラクマ（fril.jp）站点会话：单通道 HTTP 抓取
|
||||
|
||||
与乐天市场那条链路（app/services/site_session.py）分开维护，因为两站的
|
||||
抓取前提完全不同：
|
||||
|
||||
- 乐天前置 Akamai Bot Manager，无 cookie 时每个响应被拖到 ~11s，必须先访问
|
||||
首页预热再复用 cookie；且搜索页与详情页要用不同 UA，需要两条指纹通道。
|
||||
- ラクマ 实测没有这类限速：冷请求（无 cookie、无预热）即 0.5-1.1s，与预热后
|
||||
持平；PC UA 在搜索页、详情页、店铺页上都能拿到完整模板。
|
||||
|
||||
因此这里只有一条通道、不做预热，也不接浏览器兜底——没有需要兜底的拦截行为。
|
||||
若将来站点加了防护，再按乐天那套补预热与升级链路。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core import rakuma_site as site
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import (
|
||||
ItemNotFoundError,
|
||||
ResourceBusyError,
|
||||
UpstreamBlockedError,
|
||||
UpstreamRequestError,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RakumaSession:
|
||||
"""ラクマ 站点抓取会话，管理并发限流与失败重试"""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self._settings = settings
|
||||
self._semaphore = asyncio.Semaphore(settings.max_site_concurrency)
|
||||
self._client: httpx.AsyncClient | None = None
|
||||
|
||||
# ---- 生命周期 ----
|
||||
|
||||
async def start(self) -> None:
|
||||
"""创建 HTTP 客户端"""
|
||||
if self._client is not None:
|
||||
return
|
||||
self._client = httpx.AsyncClient(
|
||||
headers=site.default_headers(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
proxy=self._settings.httpx_proxy,
|
||||
http2=True,
|
||||
)
|
||||
logger.info(
|
||||
"ラクマ 会话已就绪：concurrency=%s proxy=%s",
|
||||
self._settings.max_site_concurrency,
|
||||
bool(self._settings.proxy_server),
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭 HTTP 客户端"""
|
||||
if self._client is None:
|
||||
return
|
||||
try:
|
||||
await self._client.aclose()
|
||||
except Exception:
|
||||
logger.debug("关闭 ラクマ HTTP 客户端失败", exc_info=True)
|
||||
self._client = None
|
||||
|
||||
# ---- 状态 ----
|
||||
|
||||
def status(self) -> dict[str, Any]:
|
||||
"""会话状态，供健康检查展示"""
|
||||
return {"ready": self._client is not None}
|
||||
|
||||
# ---- 抓取 ----
|
||||
|
||||
async def fetch_html(self, url: str) -> str:
|
||||
"""抓取页面 HTML
|
||||
|
||||
Raises:
|
||||
ItemNotFoundError: 目标页面 404（商品已下架或 ID 不存在）
|
||||
UpstreamRequestError: 网络异常或上游 5xx
|
||||
UpstreamBlockedError: 反复取不到正常页面
|
||||
ResourceBusyError: 等待并发槽位超时
|
||||
"""
|
||||
client = self._client
|
||||
if client is None:
|
||||
raise UpstreamRequestError("ラクマ 会话尚未初始化")
|
||||
|
||||
max_attempts = max(1, self._settings.http_max_attempts)
|
||||
last_error = "unknown error"
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise ResourceBusyError() from exc
|
||||
|
||||
try:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
response = await client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
logger.warning(
|
||||
"ラクマ 抓取请求异常：url=%s attempt=%s/%s err=%s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
raise ItemNotFoundError(f"Page not found: {url}")
|
||||
|
||||
if response.status_code < 400:
|
||||
return response.text
|
||||
|
||||
last_error = (
|
||||
f"upstream status {response.status_code}"
|
||||
if response.status_code >= 500
|
||||
else f"status {response.status_code}"
|
||||
)
|
||||
logger.warning(
|
||||
"ラクマ 抓取结果异常：url=%s attempt=%s/%s %s",
|
||||
url, attempt, max_attempts, last_error,
|
||||
)
|
||||
|
||||
if last_error.startswith("upstream status") or ":" in last_error:
|
||||
raise UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
raise UpstreamBlockedError(f"Failed to fetch {url}: {last_error}")
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
@@ -0,0 +1,169 @@
|
||||
"""乐天抓取客户端：把请求参数翻译成站点 URL，抓取后解析为结构化数据
|
||||
|
||||
两个接口分别走不同的指纹通道：
|
||||
- 搜索：PC 通道（search.rakuten.co.jp 的搜索页）
|
||||
- 详情：手机通道（item.rakuten.co.jp 只有手机 UA 才返回统一模板）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.core import site
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import ScrapeParseError
|
||||
from app.models.scrape import (
|
||||
GenreData,
|
||||
GenreRequest,
|
||||
ItemDetailData,
|
||||
ItemDetailRequest,
|
||||
SearchRequest,
|
||||
SearchResultData,
|
||||
ShopDetailData,
|
||||
ShopDetailRequest,
|
||||
ShopItemsRequest,
|
||||
)
|
||||
from app.parsers.genre import parse_genres
|
||||
from app.parsers.item import parse_item_detail
|
||||
from app.parsers.search import parse_search
|
||||
from app.parsers.shop import parse_shop_detail
|
||||
from app.parsers.state import extract_initial_state
|
||||
from app.parsers.subsites import (
|
||||
SubsitePage,
|
||||
build_item_page_validator,
|
||||
host_of,
|
||||
parse_subsite_item,
|
||||
)
|
||||
from app.services.site_session import SiteSession
|
||||
from app.utils.urls import (
|
||||
build_genre_url,
|
||||
build_item_url,
|
||||
build_search_url,
|
||||
build_shop_url,
|
||||
normalize_search_url,
|
||||
split_item_url,
|
||||
split_shop_url,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class RakutenClient:
|
||||
"""乐天市场抓取客户端"""
|
||||
|
||||
def __init__(self, settings: Settings, session: SiteSession):
|
||||
self._settings = settings
|
||||
self._session = session
|
||||
|
||||
async def search(self, payload: SearchRequest) -> SearchResultData:
|
||||
"""抓取搜索结果列表"""
|
||||
if payload.search_url is not None:
|
||||
url = normalize_search_url(str(payload.search_url), payload.page)
|
||||
else:
|
||||
url = build_search_url(payload)
|
||||
|
||||
logger.info("抓取搜索页：url=%s", url)
|
||||
html = await self._session.fetch_html(url, mobile=False)
|
||||
state = extract_initial_state(html)
|
||||
result = parse_search(
|
||||
state,
|
||||
request_url=url,
|
||||
page=payload.page,
|
||||
exclude_ads=payload.exclude_ads,
|
||||
)
|
||||
logger.info(
|
||||
"搜索完成：url=%s items=%s ads=%s total=%s",
|
||||
url, len(result.items), result.ad_count, result.total_count,
|
||||
)
|
||||
return result
|
||||
|
||||
async def genres(self, payload: GenreRequest) -> GenreData:
|
||||
"""抓取分类树：顶层分类列表，或指定分类的信息与直接子分类"""
|
||||
genre_id = (payload.genre_id or "").strip() or None
|
||||
url = build_genre_url(genre_id)
|
||||
|
||||
logger.info("抓取分类树：url=%s genre_id=%s", url, genre_id)
|
||||
html = await self._session.fetch_html(url, mobile=False)
|
||||
state = extract_initial_state(html)
|
||||
result = parse_genres(state, genre_id=genre_id)
|
||||
logger.info(
|
||||
"分类树完成：genre_id=%s name=%s children=%s",
|
||||
result.genre_id or "root", result.name, len(result.children),
|
||||
)
|
||||
return result
|
||||
|
||||
async def shop_detail(self, payload: ShopDetailRequest) -> ShopDetailData:
|
||||
"""抓取商家详情"""
|
||||
if payload.shop_url is not None:
|
||||
shop_code = split_shop_url(str(payload.shop_url))
|
||||
else:
|
||||
shop_code = (payload.shop_code or "").strip()
|
||||
url = build_shop_url(shop_code)
|
||||
|
||||
logger.info("抓取店铺页：url=%s", url)
|
||||
html = await self._session.fetch_html(url, mobile=False)
|
||||
state = extract_initial_state(html)
|
||||
result = parse_shop_detail(state, shop_code=shop_code, html=html)
|
||||
logger.info(
|
||||
"店铺详情完成：shop_code=%s shop_id=%s name=%s reviews=%s",
|
||||
result.shop_code, result.shop_id, result.shop_name, result.review_count,
|
||||
)
|
||||
return result
|
||||
|
||||
async def shop_items(self, payload: ShopItemsRequest) -> SearchResultData:
|
||||
"""抓取商家名下的商品列表
|
||||
|
||||
站点没有可分页抓取的「店铺内商品」页，店铺商品实际就是搜索结果按
|
||||
`sid=` 限定店铺后的产物，因此这里转成一次搜索。只给 shop_code 时
|
||||
需要先取一次店铺详情换出 shop_id。
|
||||
"""
|
||||
shop_id = payload.shop_id
|
||||
if shop_id is None:
|
||||
detail = await self.shop_detail(ShopDetailRequest(shop_code=payload.shop_code))
|
||||
if detail.shop_id is None:
|
||||
raise ScrapeParseError(f"未能取得店铺 {payload.shop_code} 的 shop_id")
|
||||
shop_id = detail.shop_id
|
||||
|
||||
return await self.search(payload.to_search_request(shop_id))
|
||||
|
||||
async def item_detail(self, payload: ItemDetailRequest) -> ItemDetailData:
|
||||
"""抓取商品详情"""
|
||||
if payload.item_url is not None:
|
||||
shop_code, item_code = split_item_url(str(payload.item_url))
|
||||
else:
|
||||
# 模型校验已保证此分支下两者都非空
|
||||
shop_code, item_code = payload.shop_code or "", payload.item_code or ""
|
||||
|
||||
# 统一收敛到规范地址，去掉 variantId 等查询参数——所有 SKU 组合都会随详情返回
|
||||
url = build_item_url(shop_code, item_code)
|
||||
|
||||
logger.info("抓取商品详情：url=%s", url)
|
||||
# 部分官方旗舰店的商品页会跳出市场域名，落到各自的独立站点；
|
||||
# 校验与解析都按最终落地域名分派，落到未登记站点时由校验器抛错。
|
||||
page = await self._session.fetch(
|
||||
url, mobile=True, validator=build_item_page_validator(url)
|
||||
)
|
||||
|
||||
if host_of(page.url) == site.ITEM_HOST:
|
||||
detail = parse_item_detail(
|
||||
extract_initial_state(page.html),
|
||||
item_url=url,
|
||||
shop_code=shop_code,
|
||||
include_sku_variants=payload.include_sku_variants,
|
||||
)
|
||||
else:
|
||||
detail = parse_subsite_item(
|
||||
SubsitePage(
|
||||
html=page.html,
|
||||
requested_url=url,
|
||||
final_url=page.url,
|
||||
shop_code=shop_code,
|
||||
item_code=item_code,
|
||||
include_sku_variants=payload.include_sku_variants,
|
||||
)
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"详情完成：url=%s source=%s name=%s price=%s skus=%s",
|
||||
url, detail.source, detail.item_name[:40], detail.price, detail.sku.variant_count,
|
||||
)
|
||||
return detail
|
||||
@@ -0,0 +1,310 @@
|
||||
"""站点会话：带 Akamai cookie 复用的 HTTP 抓取通道，附带浏览器兜底
|
||||
|
||||
乐天前置 Akamai Bot Manager，行为特征（实测）：
|
||||
- 请求头不完整、无 cookie 时不封禁，而是把每个响应拖到 ~11s（与响应体大小无关）
|
||||
- 补齐浏览器导航请求头并复用 Akamai 下发的 cookie 后，稳定在 ~0.6-0.9s
|
||||
|
||||
因此这里按「指纹画像」维护两条独立通道：搜索页用 PC 画像，商品详情页用手机
|
||||
画像（详情页只有手机 UA 才返回带 __INITIAL_STATE__ 的统一模板）。两条通道
|
||||
各自持有独立 cookie 罐，避免把 PC 指纹拿到的 cookie 混用到手机请求上。
|
||||
|
||||
抓取失败时的升级路径：重新预热 → 浏览器兜底取 cookie → 放弃。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from app.core import site
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import (
|
||||
ItemNotFoundError,
|
||||
ResourceBusyError,
|
||||
UpstreamBlockedError,
|
||||
UpstreamRequestError,
|
||||
)
|
||||
from app.parsers.state import PageValidator, require_state_marker
|
||||
from app.services.browser_fallback import BrowserFallback
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FetchedPage:
|
||||
"""一次成功抓取的产物"""
|
||||
|
||||
html: str
|
||||
url: str # 最终落地 URL；发生跳转时与请求地址不同
|
||||
|
||||
# Akamai 拦截页/挑战页的特征串
|
||||
_BLOCK_MARKERS = (
|
||||
"access denied",
|
||||
"pardon our interruption",
|
||||
"reference #",
|
||||
"errors.edgesuite.net",
|
||||
"/_sec/cp_challenge/",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _Profile:
|
||||
"""一条指纹通道：独立的 httpx 客户端、cookie 罐与预热状态"""
|
||||
|
||||
name: str
|
||||
mobile: bool
|
||||
client: httpx.AsyncClient
|
||||
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
warmed_at: float = 0.0
|
||||
|
||||
@property
|
||||
def cookie_names(self) -> set[str]:
|
||||
return {cookie.name for cookie in self.client.cookies.jar}
|
||||
|
||||
|
||||
class SiteSession:
|
||||
"""乐天站点抓取会话，管理 cookie 预热、并发限流与失败升级"""
|
||||
|
||||
def __init__(self, settings: Settings, browser_fallback: BrowserFallback):
|
||||
self._settings = settings
|
||||
self._browser = browser_fallback
|
||||
self._semaphore = asyncio.Semaphore(settings.max_site_concurrency)
|
||||
self._profiles: dict[str, _Profile] = {}
|
||||
|
||||
# ---- 生命周期 ----
|
||||
|
||||
async def start(self) -> None:
|
||||
"""创建两条指纹通道的 HTTP 客户端"""
|
||||
for name, mobile in (("pc", False), ("sp", True)):
|
||||
if name in self._profiles:
|
||||
continue
|
||||
self._profiles[name] = _Profile(
|
||||
name=name,
|
||||
mobile=mobile,
|
||||
client=httpx.AsyncClient(
|
||||
headers=site.default_headers(mobile=mobile),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
follow_redirects=True,
|
||||
proxy=self._settings.httpx_proxy,
|
||||
http2=True,
|
||||
),
|
||||
)
|
||||
logger.info(
|
||||
"站点会话已就绪：profiles=%s concurrency=%s proxy=%s",
|
||||
list(self._profiles),
|
||||
self._settings.max_site_concurrency,
|
||||
bool(self._settings.proxy_server),
|
||||
)
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭所有 HTTP 客户端"""
|
||||
for profile in self._profiles.values():
|
||||
try:
|
||||
await profile.client.aclose()
|
||||
except Exception:
|
||||
logger.debug("关闭 HTTP 客户端失败：profile=%s", profile.name, exc_info=True)
|
||||
self._profiles.clear()
|
||||
|
||||
# ---- 状态 ----
|
||||
|
||||
def profile_status(self) -> dict[str, dict[str, Any]]:
|
||||
"""各通道的预热状态，供健康检查展示"""
|
||||
now = time.monotonic()
|
||||
return {
|
||||
name: {
|
||||
"warmed": profile.warmed_at > 0,
|
||||
"age_seconds": round(now - profile.warmed_at, 1) if profile.warmed_at else None,
|
||||
"cookies": sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES)),
|
||||
}
|
||||
for name, profile in self._profiles.items()
|
||||
}
|
||||
|
||||
# ---- 抓取 ----
|
||||
|
||||
async def fetch_html(self, url: str, *, mobile: bool) -> str:
|
||||
"""抓取要求含 __INITIAL_STATE__ 的页面，只取 HTML"""
|
||||
page = await self.fetch(url, mobile=mobile)
|
||||
return page.html
|
||||
|
||||
async def fetch(
|
||||
self,
|
||||
url: str,
|
||||
*,
|
||||
mobile: bool,
|
||||
validator: PageValidator | None = None,
|
||||
) -> FetchedPage:
|
||||
"""抓取页面，返回 HTML 与最终落地地址
|
||||
|
||||
失败时按 重新预热 → 浏览器兜底 的顺序逐级升级重试。
|
||||
|
||||
Args:
|
||||
validator: 页面校验器，默认要求页面含 __INITIAL_STATE__。跨站抓取时
|
||||
由调用方注入按落地域名分派的校验逻辑。
|
||||
|
||||
Raises:
|
||||
ItemNotFoundError: 目标页面 404
|
||||
UpstreamBlockedError: 反复被反爬阻断
|
||||
UpstreamRequestError: 网络异常或上游 5xx
|
||||
ResourceBusyError: 等待并发槽位超时
|
||||
AppError: 校验器判定为不可重试的失败（如落到不支持的站点）
|
||||
"""
|
||||
profile = self._profiles["sp" if mobile else "pc"]
|
||||
max_attempts = max(1, self._settings.http_max_attempts)
|
||||
validate = validator or require_state_marker
|
||||
last_error: str = "unknown error"
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(
|
||||
self._semaphore.acquire(),
|
||||
timeout=self._settings.request_timeout_seconds,
|
||||
)
|
||||
except TimeoutError as exc:
|
||||
raise ResourceBusyError() from exc
|
||||
|
||||
try:
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
await self._ensure_warm(profile)
|
||||
try:
|
||||
response = await profile.client.get(url)
|
||||
except httpx.HTTPError as exc:
|
||||
last_error = f"{type(exc).__name__}: {exc}"
|
||||
logger.warning(
|
||||
"抓取请求异常：url=%s profile=%s attempt=%s/%s err=%s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
)
|
||||
continue
|
||||
|
||||
if response.status_code == 404:
|
||||
raise ItemNotFoundError(f"Page not found: {url}")
|
||||
|
||||
text = response.text
|
||||
final_url = str(response.url)
|
||||
if response.status_code < 400:
|
||||
# 校验器可能直接抛 AppError 表示「重试也没用」，此处不拦截
|
||||
reason = validate(text, final_url)
|
||||
if reason is None:
|
||||
return FetchedPage(html=text, url=final_url)
|
||||
last_error = self._refine_failure(reason, text)
|
||||
else:
|
||||
last_error = self._describe_error_status(response.status_code)
|
||||
logger.warning(
|
||||
"抓取结果异常：url=%s profile=%s attempt=%s/%s %s",
|
||||
url, profile.name, attempt, max_attempts, last_error,
|
||||
)
|
||||
|
||||
if attempt >= max_attempts:
|
||||
break
|
||||
|
||||
# 第一次失败先便宜地换一套 cookie；仍失败才动用浏览器
|
||||
if attempt == 1:
|
||||
await self._invalidate(profile)
|
||||
else:
|
||||
page = await self._escalate_to_browser(profile, url, validate)
|
||||
if page is not None:
|
||||
return page
|
||||
|
||||
if self._is_server_error(last_error):
|
||||
raise UpstreamRequestError(f"Upstream request failed: {last_error}")
|
||||
raise UpstreamBlockedError(f"Blocked while fetching {url}: {last_error}")
|
||||
finally:
|
||||
self._semaphore.release()
|
||||
|
||||
# ---- 内部 ----
|
||||
|
||||
@staticmethod
|
||||
def _describe_error_status(status_code: int) -> str:
|
||||
return (
|
||||
f"upstream status {status_code}"
|
||||
if status_code >= 500
|
||||
else f"status {status_code}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _refine_failure(reason: str, text: str) -> str:
|
||||
"""页面内容校验失败时，优先报出更具体的反爬挑战页特征"""
|
||||
lowered = text[:4000].lower()
|
||||
matched = next((marker for marker in _BLOCK_MARKERS if marker in lowered), None)
|
||||
return f"challenge page detected ({matched})" if matched else reason
|
||||
|
||||
@staticmethod
|
||||
def _is_server_error(reason: str) -> bool:
|
||||
return reason.startswith("upstream status") or reason.startswith("httpx") or "Error:" in reason
|
||||
|
||||
async def _ensure_warm(self, profile: _Profile) -> None:
|
||||
"""确保通道持有新鲜的 Akamai cookie；过期或缺失时访问首页预热"""
|
||||
if self._is_warm(profile):
|
||||
return
|
||||
|
||||
async with profile.lock:
|
||||
if self._is_warm(profile):
|
||||
return
|
||||
try:
|
||||
response = await profile.client.get(self._settings.home_url)
|
||||
profile.warmed_at = time.monotonic()
|
||||
logger.info(
|
||||
"会话预热完成：profile=%s status=%s cookies=%s",
|
||||
profile.name,
|
||||
response.status_code,
|
||||
sorted(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES)),
|
||||
)
|
||||
except httpx.HTTPError as exc:
|
||||
# 预热失败不阻断本次抓取：直连目标页仍可能成功，只是慢
|
||||
logger.warning("会话预热失败：profile=%s err=%s", profile.name, exc)
|
||||
profile.warmed_at = time.monotonic()
|
||||
|
||||
def _is_warm(self, profile: _Profile) -> bool:
|
||||
if not profile.warmed_at:
|
||||
return False
|
||||
if time.monotonic() - profile.warmed_at > self._settings.session_ttl_seconds:
|
||||
return False
|
||||
return bool(profile.cookie_names & set(site.AKAMAI_COOKIE_NAMES))
|
||||
|
||||
async def _invalidate(self, profile: _Profile) -> None:
|
||||
"""清空通道 cookie 并强制下次重新预热"""
|
||||
async with profile.lock:
|
||||
profile.client.cookies.clear()
|
||||
profile.warmed_at = 0.0
|
||||
logger.info("已清空会话 cookie，将重新预热：profile=%s", profile.name)
|
||||
|
||||
async def _escalate_to_browser(
|
||||
self, profile: _Profile, url: str, validate: PageValidator
|
||||
) -> FetchedPage | None:
|
||||
"""用浏览器访问目标页，把 cookie 回灌给 HTTP 客户端
|
||||
|
||||
浏览器已经拿到合格页面时直接返回，省掉一次重复请求。
|
||||
"""
|
||||
visit = await self._browser.visit(url, mobile=profile.mobile)
|
||||
if visit is None:
|
||||
logger.warning(
|
||||
"浏览器兜底不可用，放弃升级：profile=%s reason=%s",
|
||||
profile.name,
|
||||
self._browser.unavailable_reason,
|
||||
)
|
||||
return None
|
||||
|
||||
async with profile.lock:
|
||||
for cookie in visit.cookies:
|
||||
name = cookie.get("name")
|
||||
value = cookie.get("value")
|
||||
if not name or value is None:
|
||||
continue
|
||||
profile.client.cookies.set(
|
||||
name,
|
||||
value,
|
||||
domain=cookie.get("domain") or "",
|
||||
path=cookie.get("path") or "/",
|
||||
)
|
||||
profile.warmed_at = time.monotonic()
|
||||
|
||||
# 浏览器不回报最终 URL，这里以请求地址为准；跨站跳转场景下 HTTP 通道已先行
|
||||
# 报错，走不到这一步。
|
||||
if validate(visit.html, url) is None:
|
||||
logger.info("浏览器兜底直接取回页面：url=%s profile=%s", url, profile.name)
|
||||
return FetchedPage(html=visit.html, url=url)
|
||||
|
||||
logger.warning("浏览器兜底页面仍未通过校验：url=%s profile=%s", url, profile.name)
|
||||
return None
|
||||
@@ -0,0 +1,44 @@
|
||||
"""站点 JSON 取值的类型收敛工具
|
||||
|
||||
`__INITIAL_STATE__` 里同一字段在不同商品/店铺上可能是 null、数字或字符串
|
||||
（例如运费既可能是 0 也可能是 null，评分既可能是 4.5 也可能是 "4.5"），
|
||||
统一在这里做容错转换，避免解析器里到处写 isinstance 判断。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
|
||||
def as_dict(value: Any) -> dict[str, Any]:
|
||||
"""非 dict（含 None）一律收敛为空 dict"""
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def as_list(value: Any) -> list[Any]:
|
||||
"""非 list（含 None）一律收敛为空 list"""
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
|
||||
def as_str(value: Any) -> str:
|
||||
"""非字符串一律收敛为空串"""
|
||||
return value if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def as_int(value: Any, default: int = 0) -> int:
|
||||
"""尽力转成 int；布尔值不视为数字，避免 True 被当成 1"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return default
|
||||
try:
|
||||
return int(float(value))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def as_float(value: Any, default: float = 0.0) -> float:
|
||||
"""尽力转成 float；布尔值不视为数字"""
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float, str)):
|
||||
return default
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
@@ -0,0 +1,157 @@
|
||||
"""ラクマ（fril.jp）页面 URL 的构建与解析
|
||||
|
||||
搜索页 URL 形如：
|
||||
https://fril.jp/s?query=switch&category_id=788&statuses=5,4&min=1000&transaction=selling
|
||||
|
||||
参数名与取值取自站点前端 bundle 中 SearchPanel 组件的 `_url()` 方法，
|
||||
不是猜测。注意站点对无法识别的参数值不会报错，而是**静默返回首页**
|
||||
（如 statuses=99 返回 75KB 的首页 HTML），因此所有枚举值都必须来自码表。
|
||||
|
||||
商品页 URL 形如：https://item.fril.jp/{商品hash}
|
||||
店铺页 URL 形如：https://fril.jp/shop/{店铺hash}
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote, urlencode, urlsplit, urlunsplit, parse_qsl
|
||||
|
||||
from app.core import rakuma_site as site
|
||||
from app.core.errors import InvalidRequestError
|
||||
from app.models.scrape import RakumaSearchRequest
|
||||
|
||||
|
||||
def build_search_url(payload: RakumaSearchRequest) -> str:
|
||||
"""根据搜索请求参数构建 ラクマ 搜索页 URL"""
|
||||
params: list[tuple[str, str]] = []
|
||||
|
||||
keyword = payload.keyword.strip()
|
||||
if keyword:
|
||||
params.append(("query", keyword))
|
||||
if payload.exclude_keyword:
|
||||
params.append(("excluded_query", payload.exclude_keyword))
|
||||
if payload.category_id:
|
||||
params.append(("category_id", str(payload.category_id).strip()))
|
||||
# 站点前端把这两项做成互斥分支：勾选「排除无品牌」时不下发 brand_id
|
||||
if payload.except_for_no_brand:
|
||||
params.append(("except_for_no_brand", "true"))
|
||||
elif payload.brand_id:
|
||||
params.append(("brand_id", str(payload.brand_id).strip()))
|
||||
|
||||
if payload.min_price is not None:
|
||||
params.append(("min", str(payload.min_price)))
|
||||
if payload.max_price is not None:
|
||||
params.append(("max", str(payload.max_price)))
|
||||
|
||||
if payload.authenticity_types:
|
||||
codes = [site.AUTHENTICITY_CODES[value.value] for value in payload.authenticity_types]
|
||||
params.append(("authenticity_types", ",".join(codes)))
|
||||
if payload.conditions:
|
||||
codes = [site.CONDITION_CODES[value.value] for value in payload.conditions]
|
||||
params.append(("statuses", ",".join(codes)))
|
||||
if payload.free_shipping:
|
||||
params.append(("carriage", site.CARRIAGE_INCLUDED))
|
||||
if payload.transaction is not None:
|
||||
params.append(("transaction", site.TRANSACTION_CODES[payload.transaction.value]))
|
||||
if payload.anonymous_shipping:
|
||||
params.append(("anonymous_shipping", "true"))
|
||||
|
||||
sort_code, order_code = site.SORT_CODES[payload.sort.value]
|
||||
params.append(("sort", sort_code))
|
||||
params.append(("order", order_code))
|
||||
|
||||
if payload.page > 1:
|
||||
params.append(("page", str(payload.page)))
|
||||
|
||||
return f"{site.SEARCH_BASE_URL}?{urlencode(params)}"
|
||||
|
||||
|
||||
def normalize_search_url(raw_url: str, page: int) -> str:
|
||||
"""校验透传的搜索 URL，并在需要时覆盖其中的页码
|
||||
|
||||
page 大于 1 时以入参为准覆盖 URL 中的 `page`，这样上游可以复用同一条 URL 翻页。
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.SEARCH_HOST:
|
||||
raise InvalidRequestError(f"search_url 必须是 {site.SEARCH_HOST} 下的搜索页地址")
|
||||
|
||||
if page <= 1:
|
||||
return raw_url
|
||||
|
||||
query = [
|
||||
(key, value)
|
||||
for key, value in parse_qsl(parsed.query, keep_blank_values=True)
|
||||
if key != "page"
|
||||
]
|
||||
query.append(("page", str(page)))
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query), parsed.fragment))
|
||||
|
||||
|
||||
def build_item_url(item_id: str) -> str:
|
||||
"""由商品 hash 拼出商品详情页 URL"""
|
||||
item = quote(item_id.strip().strip("/"), safe="")
|
||||
if not item:
|
||||
raise InvalidRequestError("item_id 不能为空")
|
||||
return f"{site.ITEM_BASE_URL}{item}"
|
||||
|
||||
|
||||
def split_item_url(raw_url: str) -> str:
|
||||
"""从商品页 URL 中解析出商品 hash
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 item.fril.jp 下的商品页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
raise InvalidRequestError(f"item_url 必须是 {site.ITEM_HOST} 下的商品页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if not segments:
|
||||
raise InvalidRequestError(f"无法从 item_url 中解析商品编号：{raw_url}")
|
||||
return segments[0]
|
||||
|
||||
|
||||
def build_shop_url(shop_id: str, *, page: int = 1, review: bool = False) -> str:
|
||||
"""由店铺 hash 拼出店铺页 URL
|
||||
|
||||
Args:
|
||||
page: 商品列表页码，1 时不带参数
|
||||
review: 取评价页（/review）而非商品列表页
|
||||
"""
|
||||
shop = quote(shop_id.strip().strip("/"), safe="")
|
||||
if not shop:
|
||||
raise InvalidRequestError("shop_id 不能为空")
|
||||
|
||||
url = f"{site.SHOP_BASE_URL}{shop}"
|
||||
if review:
|
||||
return f"{url}/review"
|
||||
return f"{url}?page={page}" if page > 1 else url
|
||||
|
||||
|
||||
def split_shop_url(raw_url: str) -> str:
|
||||
"""从店铺页 URL 中解析出店铺 hash
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 fril.jp/shop/ 下的店铺页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.SEARCH_HOST:
|
||||
raise InvalidRequestError(f"shop_url 必须是 {site.SEARCH_HOST} 下的店铺页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if len(segments) < 2 or segments[0] != "shop":
|
||||
raise InvalidRequestError(f"无法从 shop_url 中解析店铺编号：{raw_url}")
|
||||
return segments[1]
|
||||
|
||||
|
||||
def item_id_from_url(url: str) -> str:
|
||||
"""尽力从任意商品链接中解析商品 hash，失败时返回空串
|
||||
|
||||
用于列表页解析——单条链接解析不出来不应中断整页解析。
|
||||
"""
|
||||
try:
|
||||
parsed = urlsplit(url)
|
||||
except ValueError:
|
||||
return ""
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
return ""
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
return segments[0] if segments else ""
|
||||
@@ -0,0 +1,164 @@
|
||||
"""乐天页面 URL 的构建与解析
|
||||
|
||||
搜索页 URL 形如：
|
||||
https://search.rakuten.co.jp/search/mall/{关键词}/{分类ID}/?p=2&s=2&min=1000&f=2&f=101
|
||||
其中关键词段在只按分类检索时用 `-` 占位；`f` 可重复出现，每个值代表一项筛选。
|
||||
|
||||
商品页 URL 形如：
|
||||
https://item.rakuten.co.jp/{店铺代码}/{商品编号}/
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from urllib.parse import quote, urlencode, urlparse, urlsplit, urlunsplit, parse_qsl
|
||||
|
||||
from app.core import site
|
||||
from app.core.errors import InvalidRequestError
|
||||
from app.models.scrape import SearchRequest
|
||||
|
||||
# 只按分类检索（无关键词）时，关键词段的占位符
|
||||
_KEYWORD_PLACEHOLDER = "-"
|
||||
|
||||
# www.rakuten.co.jp 下已被站点自身占用、不可能是店铺代码的一级路径
|
||||
_RESERVED_WWW_PATHS = frozenset({"category", "event", "search", "rms", "info", "help"})
|
||||
|
||||
|
||||
def build_search_url(payload: SearchRequest) -> str:
|
||||
"""根据搜索请求参数构建乐天搜索页 URL"""
|
||||
keyword = payload.keyword.strip()
|
||||
path = quote(keyword, safe="") if keyword else _KEYWORD_PLACEHOLDER
|
||||
url = f"{site.SEARCH_BASE_URL}{path}/"
|
||||
if payload.genre_id:
|
||||
url = f"{url}{quote(str(payload.genre_id), safe='')}/"
|
||||
|
||||
params: list[tuple[str, str]] = []
|
||||
if payload.page > 1:
|
||||
params.append(("p", str(payload.page)))
|
||||
|
||||
sort_code = site.SORT_CODES.get(payload.sort.value)
|
||||
if sort_code:
|
||||
params.append(("s", sort_code))
|
||||
|
||||
if payload.min_price is not None:
|
||||
params.append(("min", str(payload.min_price)))
|
||||
if payload.max_price is not None:
|
||||
params.append(("max", str(payload.max_price)))
|
||||
if payload.shop_id is not None:
|
||||
params.append(("sid", str(payload.shop_id)))
|
||||
if payload.exclude_keyword:
|
||||
params.append(("nitem", payload.exclude_keyword))
|
||||
if payload.title_only:
|
||||
params.append(("sf", "1"))
|
||||
if payload.or_query:
|
||||
params.append(("st", "O"))
|
||||
if payload.min_review_score is not None:
|
||||
params.append(("review", str(payload.min_review_score)))
|
||||
if payload.tags:
|
||||
params.append(("tg", ",".join(str(tag) for tag in payload.tags)))
|
||||
|
||||
# 成色与各布尔筛选共用可重复的 `f` 参数
|
||||
if payload.condition is not None:
|
||||
params.append(("f", site.CONDITION_CODES[payload.condition.value]))
|
||||
for field_name, code in site.BOOL_FILTER_CODES.items():
|
||||
if getattr(payload, field_name):
|
||||
params.append(("f", code))
|
||||
|
||||
if not params:
|
||||
return url
|
||||
return f"{url}?{urlencode(params)}"
|
||||
|
||||
|
||||
def normalize_search_url(raw_url: str, page: int) -> str:
|
||||
"""校验透传的搜索 URL，并在需要时覆盖其中的页码
|
||||
|
||||
page 大于 1 时以入参为准覆盖 URL 中的 `p`，这样上游可以复用同一条 URL 翻页。
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.SEARCH_HOST:
|
||||
raise InvalidRequestError(f"search_url 必须是 {site.SEARCH_HOST} 下的搜索页地址")
|
||||
|
||||
if page <= 1:
|
||||
return raw_url
|
||||
|
||||
query = [(key, value) for key, value in parse_qsl(parsed.query, keep_blank_values=True) if key != "p"]
|
||||
query.append(("p", str(page)))
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query), parsed.fragment))
|
||||
|
||||
|
||||
def build_genre_url(genre_id: str | None) -> str:
|
||||
"""构建取分类树用的页面地址
|
||||
|
||||
指定分类时用分类页——它比搜索页多给 genreInfo（完整分类名与描述）。
|
||||
不指定时退回搜索页：分类分面与查询内容无关，用哨兵关键词能拿到干净的顶层列表。
|
||||
"""
|
||||
if genre_id:
|
||||
return f"{site.CATEGORY_BASE_URL}{quote(str(genre_id).strip().strip('/'), safe='')}/"
|
||||
return f"{site.SEARCH_BASE_URL}{site.GENRE_FACET_PROBE_KEYWORD}/"
|
||||
|
||||
|
||||
def build_item_url(shop_code: str, item_code: str) -> str:
|
||||
"""由店铺代码与商品编号拼出商品详情页 URL"""
|
||||
shop = quote(shop_code.strip().strip("/"), safe="")
|
||||
item = quote(item_code.strip().strip("/"), safe="")
|
||||
if not shop or not item:
|
||||
raise InvalidRequestError("shop_code 与 item_code 均不能为空")
|
||||
return f"{site.ITEM_BASE_URL}{shop}/{item}/"
|
||||
|
||||
|
||||
def split_item_url(raw_url: str) -> tuple[str, str]:
|
||||
"""从商品页 URL 中解析出 (店铺代码, 商品编号)
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 item.rakuten.co.jp 下的商品页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
raise InvalidRequestError(f"item_url 必须是 {site.ITEM_HOST} 下的商品页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if len(segments) < 2:
|
||||
raise InvalidRequestError(f"无法从 item_url 中解析店铺代码与商品编号：{raw_url}")
|
||||
return segments[0], segments[1]
|
||||
|
||||
|
||||
def build_shop_url(shop_code: str) -> str:
|
||||
"""由店铺代码拼出店铺首页 URL"""
|
||||
shop = quote(shop_code.strip().strip("/"), safe="")
|
||||
if not shop:
|
||||
raise InvalidRequestError("shop_code 不能为空")
|
||||
return f"{site.WWW_BASE_URL}{shop}/"
|
||||
|
||||
|
||||
def split_shop_url(raw_url: str) -> str:
|
||||
"""从店铺页 URL 中解析出店铺代码
|
||||
|
||||
Raises:
|
||||
InvalidRequestError: 不是 www.rakuten.co.jp 下的店铺页地址
|
||||
"""
|
||||
parsed = urlsplit(raw_url)
|
||||
if parsed.hostname != site.WWW_HOST:
|
||||
raise InvalidRequestError(f"shop_url 必须是 {site.WWW_HOST} 下的店铺页地址")
|
||||
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if not segments:
|
||||
raise InvalidRequestError(f"无法从 shop_url 中解析店铺代码：{raw_url}")
|
||||
# 排除分类页等非店铺路径，避免把 /category/101205/ 当成店铺代码 category
|
||||
if segments[0] in _RESERVED_WWW_PATHS:
|
||||
raise InvalidRequestError(f"shop_url 不是店铺首页地址：{raw_url}")
|
||||
return segments[0]
|
||||
|
||||
|
||||
def item_url_parts(url: str) -> tuple[str, str]:
|
||||
"""尽力从任意商品链接中解析 (店铺代码, 商品编号)，失败时返回空串
|
||||
|
||||
用于搜索结果——广告位链接可能指向跳转域名，解析不出来不应中断整页解析。
|
||||
"""
|
||||
try:
|
||||
parsed = urlparse(url)
|
||||
except ValueError:
|
||||
return "", ""
|
||||
if parsed.hostname != site.ITEM_HOST:
|
||||
return "", ""
|
||||
segments = [segment for segment in parsed.path.split("/") if segment]
|
||||
if len(segments) < 2:
|
||||
return "", ""
|
||||
return segments[0], segments[1]
|
||||
Reference in New Issue
Block a user