jp/surugaya
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Init

Browse Source
This commit is contained in:
Jerry Yan
2026-06-08 16:09:13 +08:00
commit 518c6b3c7a
32 changed files with 4156 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/services/surugaya_client.py
+615
View File
@@ -0,0 +1,615 @@
"""骏河屋（suruga-ya.jp）抓取客户端
负责搜索列表和商品详情的抓取与解析。
通过 Cloudflare 会话复用机制，避免每次请求都触发验证挑战。
"""
from __future__ import annotations
import asyncio
import json
import logging
import re
from datetime import datetime
from pathlib import Path
from urllib.parse import parse_qs, urlencode, urlparse
from uuid import uuid4
from typing import Any
from selectolax.parser import HTMLParser
from app.core.config import Settings
from app.core.errors import ScrapeParseError, UpstreamBlockedError
from app.models.scrape import CategoryData, DetailRequest, ProductDetailData, ProductSummary, SearchRequest, \
SearchResultData
from app.services.browser_pool import BrowserPool
from app.services.cloudflare_session import CloudflareSessionManager
from app.services.session_store import SessionStore
from app.utils.app_utils import is_empty_str
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
logger = logging.getLogger(__name__)
class SurugayaClient:
"""骏河屋抓取客户端，提供搜索和商品详情两个核心能力
工作流程：
1. 获取已通过 Cloudflare 验证的会话（优先复用，必要时新建）
2. 使用浏览器池中的页面，携带会话 Cookie 访问目标页面
3. 解析页面 HTML 提取结构化数据
"""
def __init__(
self,
settings: Settings,
browser_pool: BrowserPool,
session_manager: CloudflareSessionManager,
session_store: SessionStore,
):
self._settings = settings
self._browser_pool = browser_pool
self._session_manager = session_manager
self._session_store = session_store
self._categories_cache_key = "surugaya:categories:cache"
async def _raise_upstream_error(self, *, page: object, response: object | None, url: str) -> None:
status = getattr(response, "status", None)
try:
await self._session_manager.invalidate(url)
except Exception as exc:
logger.warning("清理上游异常会话失败：url=%s err=%s", url, exc)
screenshot_path: str | None = None
try:
screenshot_dir = Path(__file__).resolve().parents[2] / self._settings.log_dir / "screenshots"
screenshot_dir.mkdir(parents=True, exist_ok=True)
screenshot_file = screenshot_dir / f"{uuid4().hex}.png"
await page.screenshot(path=str(screenshot_file), full_page=True)
screenshot_path = str(screenshot_file)
except Exception:
screenshot_path = None
raise UpstreamBlockedError(f"Upstream status：{status}, 截图地址：{screenshot_path}")
async def search(self, payload: SearchRequest) -> SearchResultData:
"""抓取搜索列表页
流程：
1. 构建/使用搜索 URL
2. 获取已通过 Cloudflare 验证的会话
3. 用浏览器页面访问并获取 HTML
4. 解析商品总数和商品列表
Args:
payload: 搜索请求参数，包含 search_word、page 等
Returns:
SearchResultData: 包含搜索结果列表、总数、分页信息等
"""
logger.debug("开始抓取搜索页：payload=%s", json.dumps(payload.model_dump(), ensure_ascii=False))
url = str(payload.search_url) if payload.search_url else self._build_search_url(payload)
session = await self._session_manager.get_verified_session(url)
if session.html is not None:
# 新创建的会话已携带页面 HTML，直接使用，避免二次加载
html = session.html
else:
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
response = await page.goto(
url,
wait_until="domcontentloaded",
timeout=int(self._settings.request_timeout_seconds * 1000),
)
status = response.status if response is not None else None
if status != 200:
await self._raise_upstream_error(page=page, response=response, url=url)
html = await page.content()
tree = HTMLParser(html)
# 解析商品总数量，格式示例：該当件数:10,428件中 1-24件
current_page = self._extract_page_value(payload)
total_count = 0
page_size = 24
has_more = 0
hit_node = tree.css_first("#search_header .search_option .hit")
if hit_node:
raw_text = hit_node.text()
match = re.search(r'該当件数:([\d,]+)件', raw_text)
if match:
total_count = int(match.group(1).replace(',', ''))
if total_count > 0:
has_more = int(current_page * page_size < total_count)
# 解析商品列表
items = self._parse_search_items(tree)
logger.info("搜索页解析完成 ===> 关键词: %s 抓取商品数=%s", payload.search_word, len(items))
return SearchResultData(
query=payload.search_word,
page=current_page,
page_size=page_size,
total_count=total_count,
has_more=has_more,
items=items,
session_id=session.session_id,
)
async def fetch_detail(self, payload: DetailRequest) -> ProductDetailData:
"""抓取商品详情页
流程：
1. 根据请求参数确定商品 URL 和 ID（支持直接传 ID 或完整 URL）
2. 获取已通过 Cloudflare 验证的会话
3. 用浏览器页面访问并获取 HTML
4. 从 DOM 中提取商品名称、图片、价格、售卖状态、描述等
Args:
payload: 详情请求参数，包含 id 或 product_url
Returns:
ProductDetailData: 商品详情数据
Raises:
ScrapeParseError: 当 id 和 product_url 都未提供时
"""
goods_id = payload.id or ""
if is_empty_str(goods_id):
raise ScrapeParseError("id is empty")
url = self._build_product_url(goods_id)
logger.info("开始抓取详情页：goods_id=%s", goods_id)
logger.debug("详情页URL：url=%s", url)
# 抓取页面 HTML
session = await self._session_manager.get_verified_session(url)
if session.html is not None:
html = session.html
else:
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
response = await page.goto(
url,
wait_until="domcontentloaded",
timeout=int(self._settings.request_timeout_seconds * 1000),
)
status = response.status if response is not None else None
if status != 200:
await self._raise_upstream_error(page=page, response=response, url=url)
html = await page.content()
tree = HTMLParser(html)
# 提取商品标题
title_node = tree.css_first("#item_title")
goods_name = title_node.text().strip() if title_node else None
if not goods_name:
raise ScrapeParseError("商品名称解析失败")
# 提取商品主图
img_node = tree.css_first("div.easyzoom--with-thumbnails img")
goods_image: str = (img_node.attributes.get("src") or "") if img_node else ""
if goods_image and "database/images/no_photo.jpg" in goods_image:
goods_image = "https://oss.daimatech.com/suruga-ya/no_photo.jpg"
# 提取商品价格：优先从 .price_choise 中取，其次从 span.purchase-price 中取
price_text = ""
price_choise = tree.css_first("#cart .price_choise")
if price_choise:
buy_node = price_choise.css_first(".text-price-detail.price-buy")
if buy_node:
price_text = buy_node.text().strip()
if not price_text:
purchase_node = tree.css_first("span.purchase-price")
if purchase_node:
price_text = purchase_node.text().strip()
goods_price: int = format_price(price_text) or 0 if price_text else 0
# 判断售卖状态：
out_of_stock_dom = tree.css_first("div.out-of-stock-text")
on_sold = 0 if out_of_stock_dom else 1
# 解析规格列表
sku = ""
sku_list = []
pdom = tree.css_first("#item_title")
if pdom and pdom.parent:
direct_children = pdom.parent.css(".price_group > .item-price")
sku_index = 0
for node in direct_children:
sku_index += 1
vo: dict[str, Any] = {"sku_name": "", "price": 0, "stock": 0}
label_node = node.css_first("label")
if label_node:
vo["sku_name"] = label_node.attributes.get("data-label", "").strip()
input_node = node.css_first("input.form-check-input")
if input_node:
zaiko_str = input_node.attributes.get("data-zaiko", "")
if zaiko_str and zaiko_str != "null":
try:
zaiko = json.loads(zaiko_str)
if isinstance(zaiko, dict):
vo.update(zaiko)
vo["price"] = int(zaiko.get("baika", 0))
vo["stock"] = int(zaiko.get("zaiko", 0))
vo["sku_id"] = str(sku_index)
if sku_index == 1:
sku = vo["sku_id"]
except json.JSONDecodeError:
vo["zaiko_raw"] = zaiko_str
sku_list.append(vo)
# 提取库存
stock = 0
if sku_list:
# 从 SKU 列表中找到匹配当前 SKU 的库存
stock = next((item["stock"] for item in sku_list if item.get("sku_id") == sku), 0)
else:
quantity_selection = tree.css_first("#quantity_selection")
if quantity_selection:
stock = 1
last_option = quantity_selection.css_first("option:last-child")
if last_option:
option_val = last_option.attributes.get("value")
if option_val and option_val.isdigit():
stock = int(option_val)
# 提取商品描述
desc_note = tree.css_first("p.note.text-break")
description = desc_note.text().strip() if desc_note else ""
# 提取店铺来源
supplier = "" # 默认表示官方自营
# 利用与目标 div.mb-2 平级的 #naviplus-review-list-5 缩小查找范围以提升效率
review_node = tree.css_first("#naviplus-review-list-5")
if review_node and review_node.parent:
context_node = review_node.parent
for node in context_node.css("div.mb-2"):
text = node.text()
if "この商品は" in text and "販売" in text and "発送" in text:
# 提取 mb-2 下的全部文本，并清理多余的换行和空白字符
supplier = " ".join(text.split())
break
# 提取价格备注
price_note = None
price_note_node = tree.css_first(".item-price-note-wrap")
if price_note_node:
price_note = price_note_node.text().strip()
# 提取运费相关
shipping_comission_fee = get_shipping_fee(goods_price)
# 提取面包屑
breadcrumb_list = self.get_breadcrumb_list(tree)
detail = ProductDetailData(
goods_id=goods_id,
goods_name=goods_name,
goods_image=goods_image,
goods_price=goods_price,
stock=stock,
on_sold=on_sold,
sku=sku,
sku_list=sku_list,
description=description,
supplier=supplier,
price_note=price_note,
shipping_comission_fee=shipping_comission_fee,
breadcrumb_list=breadcrumb_list,
)
logger.debug("详情页解析完成：goods_id=%s goods_name=%s ", detail.goods_id, detail.goods_name)
return detail
async def fetch_categories(self) -> list[CategoryData]:
"""获取商品分类（带缓存）
解析骏河屋的商品分类，由于分类不易变动，
优先从 redis 获取缓存，若 redis 未配置或缓存失效，则重新抓取。
默认缓存 24 小时。
"""
# 如果配置了 Redis，尝试从 Redis 获取缓存
if self._session_store._redis is not None:
try:
cached_data = await self._session_store._redis.get(self._categories_cache_key)
if cached_data:
logger.debug("命中商品分类 Redis 缓存")
data_list = json.loads(cached_data)
return [CategoryData(**item) for item in data_list]
except Exception as e:
logger.warning("读取 Redis 缓存失败: %s", e)
logger.info("开始获取并解析商品分类...")
top_categories: list[CategoryData] = []
target_url = "https://www.suruga-ya.jp/"
session = await self._session_manager.get_verified_session(target_url, True)
html_content = ""
if session.html is not None:
html_content = session.html
else:
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
try:
response = await page.goto(
target_url,
wait_until="domcontentloaded",
timeout=int(self._settings.request_timeout_seconds * 1000),
)
status = response.status if response is not None else None
if status != 200:
raise ScrapeParseError(f"抓取首页分类失败，状态码 {status}")
html_content = await page.content()
except Exception as e:
logger.error("抓取首页分类失败：%s", e)
raise ScrapeParseError(f"抓取首页分类失败: {e!s}") from e
tree = HTMLParser(html_content)
# 解析一级分类列表
cate_nodes = tree.css(".catemenu_group .cate_menu .cate_item")
for cate_item in cate_nodes:
href = cate_item.css_first("a").attributes.get("href")
if not href:
continue
# 提取 href 的路径名（不含 .html 和前置的 /，例如：/avsoft.html -> avsoft）
match = re.search(r'/([^/]+)(?:\.html|/)', href)
if not match:
continue
cate_id = match.group(1)
# 过滤掉指定的分类
if cate_id in ["boyslove"]:
continue
# 提取包含的文字内容（拼接多个 span，例如 おもちゃ・ + ホビー）
h4_node = cate_item.css_first("a h4")
if not h4_node:
continue
cate_name = h4_node.text().strip()
# 清理多余空格与换行，防止拼接文本里出现多余空白
cate_name = "".join(cate_name.split())
icon_node = cate_item.css_first("a img")
icon = icon_node.attributes.get("src") if icon_node else None
pid = "0"
top_categories.append(CategoryData(id=cate_id, name=cate_name, icon=icon, pid=pid, href=href))
# 解析二级分类列表
seen_pairs: set[tuple[str, str]] = set()
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
for top_category in top_categories:
try:
category_url = f"https://www.suruga-ya.jp{top_category.href}"
response = await page.goto(
category_url,
wait_until="domcontentloaded",
timeout=int(self._settings.request_timeout_seconds * 1000),
)
status = response.status if response is not None else None
if status != 200:
await self._raise_upstream_error(page=page, response=response, url=category_url)
category_html = await page.content()
category_tree = HTMLParser(category_html)
menu_node = category_tree.css_first("#menu")
if not menu_node:
continue
related_block = None
for block in menu_node.css(".block"):
h2_node = block.css_first("h2")
if h2_node and "関連ジャンルで絞り込む" in h2_node.text():
related_block = block
break
if not related_block:
continue
for a_node in related_block.css("ul.border_bottom li a"):
sub_name = "".join(a_node.text().split())
if not sub_name:
continue
sub_href = a_node.attributes.get("href") or ""
parsed = urlparse(sub_href)
sub_id = ""
if parsed.path == "/search":
query = parse_qs(parsed.query)
sub_id = (query.get("category") or [""])[0]
if not sub_id:
match = re.search(r"/([^/]+)\.html$", parsed.path)
if match:
sub_id = match.group(1)
if not sub_id:
continue
pair = (top_category.id, sub_id)
if pair in seen_pairs:
continue
seen_pairs.add(pair)
top_category.children.append(
CategoryData(id=sub_id, name=sub_name, pid=top_category.id, href=sub_href))
# 休眠1秒，避免对服务器造成过大压力
await asyncio.sleep(1)
except Exception as e:
logger.warning("抓取二级分类失败：top_id=%s err=%s", top_category.id, e)
continue
# 更新 Redis 缓存
if top_categories and self._session_store._redis is not None:
try:
logger.debug("更新商品分类 Redis 缓存")
data_list = [item.model_dump() for item in top_categories]
# 缓存有效期 24 小时 (86400秒)
await self._session_store._redis.setex(
self._categories_cache_key,
86400 * 30,
json.dumps(data_list, ensure_ascii=False)
)
except Exception as e:
logger.warning("写入 Redis 缓存失败: %s", e)
return top_categories
# ------------------------------------------------------------------
# 内部辅助方法
# ------------------------------------------------------------------
def _build_search_url(self, payload: SearchRequest) -> str:
"""根据搜索参数构建骏河屋搜索 URL
将 payload 中的 search_word、page 等参数拼接为完整的搜索地址。
search_url 字段本身不参与拼接，它用于直接指定完整 URL。
"""
params = payload.model_dump(exclude_none=True)
params.pop("search_url", None)
# if not str(params.get("search_word", "")).strip():
# raise ScrapeParseError("search_word is required")
query_string = urlencode(params, doseq=True)
base_url = self._settings.base_url.rstrip("/")
return f"{base_url}/search?{query_string}"
@staticmethod
def _extract_page_value(payload: SearchRequest) -> int:
"""安全地提取页码，确保返回 >= 1 的整数"""
page_raw = payload.model_dump(exclude_none=True).get("page", 1)
try:
current_page = int(page_raw)
except (TypeError, ValueError):
return 1
return current_page if current_page >= 1 else 1
def _build_product_url(self, product_id: str | None) -> str:
"""根据商品 ID 构建详情页 URL"""
if not product_id:
raise ScrapeParseError("Either product_id or product_url must be provided")
base_url = self._settings.base_url.rstrip("/")
return f"{base_url}/product/detail/{product_id}"
def _parse_search_items(self, tree: HTMLParser) -> list[ProductSummary]:
"""解析搜索结果列表
从搜索页 DOM 中提取每个商品的 ID、名称、图片、价格、市场价、售卖状态。
自动去重（按 goods_id），最多返回 50 条结果。
"""
item_nodes = tree.css("div.item_box div.item")
if not item_nodes:
item_nodes = tree.css("div.item")
items: list[ProductSummary] = []
seen_goods_ids: set[str] = set()
for node in item_nodes:
link_node = node.css_first("div.title a") or node.css_first("div.photo_box a")
href = link_node.attributes.get("href") if link_node else None
if not href:
continue
goods_link = self._normalize_url(href)
goods_id = self._extract_product_id(goods_link) or self._extract_product_id(href)
if not goods_id:
continue
if goods_id in seen_goods_ids:
continue
seen_goods_ids.add(goods_id)
name_node = node.css_first("h3.product-name")
goods_name = name_node.text().strip() if name_node else f"product-{goods_id}"
img_node = node.css_first("div.photo_box img")
goods_image_str = (img_node.attributes.get("src") or "").strip() if img_node else ""
goods_image = surugaya_photo_url_to_cdn(goods_image_str, goods_id)
# 解析商品售价（自营价格）
goods_price = 0
price_teika = node.css_first(".item_price .price_teika")
if price_teika is not None:
strong = price_teika.css_first(".text-red strong")
price_text = strong.text().strip() if strong is not None else price_teika.text().strip()
parsed_goods_price = format_price(price_text)
if parsed_goods_price is not None:
goods_price = parsed_goods_price
# 解析第三方市场价
third_shop_price = 0
makepla_strong = node.css_first(".item_price .makeplaTit .text-red strong")
if makepla_strong is not None:
price_text = makepla_strong.text().strip()
parsed_third_shop_price = format_price(price_text)
if parsed_third_shop_price is not None:
third_shop_price = parsed_third_shop_price
# 判断售卖状态：品切れ = 售罄
on_sold = 1
sold_out = node.css_first(".item_price p.price")
if sold_out is not None and "品切れ" in sold_out.text().strip():
on_sold = 0
# 售罄时，售价回退到市场价
goods_price = third_shop_price
now = str(int(datetime.now().timestamp()))
items.append(
ProductSummary(
goods_id=goods_id,
goods_name=goods_name,
goods_image=goods_image,
goods_price=goods_price,
third_shop_price=third_shop_price,
goods_link=goods_link,
on_sold=on_sold,
created_at=now,
updated_at=now,
)
)
return items
@staticmethod
def _extract_product_id(text: str) -> str | None:
"""从 URL 或文本中提取商品 ID
匹配 /product/detail/{id} 或 /product/other/{id} 格式
"""
match = re.search(r"/product/(?:detail|other)/([A-Za-z0-9]+)", text)
return match.group(1) if match else None
@staticmethod
def _normalize_url(url: str) -> str:
"""将相对 URL 补全为完整的 https URL"""
if url.startswith("http://") or url.startswith("https://"):
return url
return f"https://www.suruga-ya.jp{url}"
def get_breadcrumb_list(self, tree):
dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")
breadcrumb_list = []
for index, dom in enumerate(dom_list):
a_node = dom.css_first("a")
if not a_node:
continue
breadcrumb_list.append({
"position": index + 1,
"is_active": index == len(dom_list) - 1,
"name": a_node.text().strip(),
"href": a_node.attributes.get("href") or "",
})
return breadcrumb_list