jp/surugaya
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

支持其他店铺

Browse Source
This commit is contained in:
Jerry Yan
2026-07-08 21:53:20 +08:00
parent af8d3a859f
commit 7ddb41cea1
10 changed files with 932 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/api/routes/scrape.py
+24
View File
@@ -18,6 +18,7 @@ from app.models.scrape import (
DetailRequest,
OrderShippingFeeData,
OrderShippingFeeRequest,
OtherShopListData,
ProductDetailData,
PurchaseTaskRequest,
SearchRequest,
@@ -122,6 +123,29 @@ async def item_detail(
)
@router.post(
"/item_other_shops",
response_model=ApiResponse[OtherShopListData],
dependencies=[Depends(require_bearer_token)],
)
async def item_other_shops(
payload: DetailRequest,
container: ServiceContainer = Depends(get_container),
) -> ApiResponse[OtherShopListData]:
"""获取商品「他のショップ」全部店铺报价列表
根据商品 ID 抓取骏河屋 /product/other/{id} 页面，返回全部店铺（含官方自营+加盟店/市场店铺）报价列表；
列表中每项的 detail_url 已带上对应 tenpo_cd/branch_number，可直接用于 /api/item_detail 跳转查看该店铺报价。
"""
data = await container.surugaya_client.fetch_other_shops(payload)
return ApiResponse[OtherShopListData](
success=True,
msg="success",
data=data,
code=0,
)
@router.get(
"/categories",
response_model=ApiResponse[list[CategoryData]],
app/models/scrape.py
+26
View File
@@ -40,6 +40,8 @@ class DetailRequest(BaseModel):
"""商品详情请求参数，支持 id 或 product_url"""
id: str | None = Field(default=None, min_length=3, max_length=40)
product_url: HttpUrl | None = None
tenpo_cd: str | None = None # 指定店铺代码；跳转查看某个加盟店/市场店铺的报价时传入
branch_number: str | None = None # 指定分支编号，与 tenpo_cd 搭配使用
class ProductSummary(BaseModel):
@@ -67,6 +69,24 @@ class SearchResultData(BaseModel):
items: list[ProductSummary]
class OtherShopItem(BaseModel):
"""商品「他のショップ」列表中的单个店铺报价
来自 /product/other/{goods_id} 页面，含官方自营与加盟店/市场店铺。
"""
tenpo_cd: str = "" # 店铺代码；空串 = 官方自营
shop_name: str = "" # 店铺名称；官方自营固定为"駿河屋"
shop_url: str = "" # 店铺主页链接
is_official: bool = False # 是否为官方自营（tenpo_cd 为空）
condition: str = "" # 成色/规格描述，如"中古"
price: int = 0 # 该店铺售价（日元）
branch_number: str = "" # 分支编号
rating_score: str = "" # 店铺评分，如"5.0"
rating_count: int = 0 # 评价数量
shipping_note: str = "" # 发货时效描述，如"1日〜3日以内に発送します"
detail_url: str = "" # 该报价对应的商品详情页链接（含 tenpo_cd/branch_number）
class ProductDetailData(BaseModel):
"""商品详情数据"""
goods_id: str
@@ -86,6 +106,12 @@ class ProductDetailData(BaseModel):
breadcrumb_list: list[dict[str, Any]] = [] # 面包屑
class OtherShopListData(BaseModel):
"""商品「他のショップ」全部店铺（含官方自营+加盟店/市场店铺）报价列表"""
goods_id: str
other_shop_list: list[OtherShopItem] = []
class HealthData(BaseModel):
"""健康检查响应数据"""
status: str
app/services/surugaya_client.py
+135 -7
View File
@@ -19,14 +19,14 @@ from selectolax.parser import HTMLParser
from app.core.config import Settings
from app.core.errors import ScrapeParseError, UpstreamBlockedError
from app.models.scrape import CategoryData, DetailRequest, ProductDetailData, ProductSummary, SearchRequest, \
SearchResultData
from app.models.scrape import CategoryData, DetailRequest, OtherShopItem, OtherShopListData, ProductDetailData, \
ProductSummary, SearchRequest, SearchResultData
from app.services.browser_pool import BrowserPool
from app.services.cloudflare_session import CloudflareSessionManager
from app.services.session_store import SessionStore
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
from surugaya_common.app_utils import is_empty_str
from surugaya_common.urls import BASE_URL
from surugaya_common.urls import BASE_URL, product_other_url
logger = logging.getLogger(__name__)
@@ -147,7 +147,7 @@ class SurugayaClient:
if is_empty_str(goods_id):
raise ScrapeParseError("id is empty")
url = self._build_product_url(goods_id)
url = self._build_product_url(goods_id, tenpo_cd=payload.tenpo_cd, branch_number=payload.branch_number)
logger.info("开始抓取详情页：goods_id=%s", goods_id)
logger.debug("详情页URL：url=%s", url)
@@ -283,6 +283,121 @@ class SurugayaClient:
logger.debug("详情页解析完成：goods_id=%s goods_name=%s ", detail.goods_id, detail.goods_name)
return detail
async def fetch_other_shops(self, payload: DetailRequest) -> OtherShopListData:
"""获取商品「他のショップ」全部店铺报价列表
独立于 fetch_detail，仅在需要展示/跳转其他店铺时按需调用，
避免每次详情请求都多付出一次 Cloudflare 保护页面抓取的成本。
Args:
payload: 详情请求参数，仅使用其中的 id
Returns:
OtherShopListData: 全部店铺（含官方自营+加盟店/市场店铺）报价列表
Raises:
ScrapeParseError: 当 id 未提供时
"""
goods_id = payload.id or ""
if is_empty_str(goods_id):
raise ScrapeParseError("id is empty")
other_shop_list = await self._fetch_other_shop_list(goods_id)
return OtherShopListData(goods_id=goods_id, other_shop_list=other_shop_list)
async def _fetch_other_shop_list(self, goods_id: str) -> list[OtherShopItem]:
"""抓取并解析商品「他のショップ」页面，返回全部店铺（含官方自营）报价列表
该页面独立于详情页，需额外一次 Cloudflare 保护页面请求；
解析失败或抓取异常时降级为空列表，不影响详情页主流程。
"""
other_url = product_other_url(goods_id)
try:
other_html, _ = await self._session_manager.fetch_html(other_url)
except Exception as exc:
logger.warning("抓取他のショップ页面失败：goods_id=%s err=%s", goods_id, exc)
return []
try:
return self._parse_other_shop_list(HTMLParser(other_html))
except Exception:
logger.exception("解析他のショップ页面失败：goods_id=%s", goods_id)
return []
@staticmethod
def _parse_other_shop_list(tree: HTMLParser) -> list[OtherShopItem]:
"""解析「他のショップ」页面的全部店铺（tbl_all 标签页）报价列表
每行对应一个店铺报价；tenpo_cd 为空表示官方自营（駿河屋本店）。
"""
items: list[OtherShopItem] = []
for row in tree.css("#tbl_all tr.item"):
tds = row.css("td")
if len(tds) < 3:
continue
price_node = tds[0].css_first("strong.text-red")
price = format_price(price_node.text().strip()) or 0 if price_node else 0
condition_node = tds[1].css_first("h2.title_product")
condition = condition_node.text().strip() if condition_node else ""
detail_link_node = tds[1].css_first("a[href*='/product/detail/']")
detail_href = detail_link_node.attributes.get("href") if detail_link_node else ""
detail_url = f"{BASE_URL}{detail_href}" if detail_href else ""
shop_col = tds[2]
shop_link_node = shop_col.css_first("a[href^='/shop/']")
tenpo_cd = ""
shop_name = "駿河屋"
shop_url = f"{BASE_URL}/"
if shop_link_node:
shop_href = shop_link_node.attributes.get("href") or ""
tenpo_cd = shop_href.rsplit("/", 1)[-1]
shop_name = shop_link_node.text().strip()
shop_url = f"{BASE_URL}{shop_href}"
rating_score = ""
rating_count = 0
rating_node = shop_col.css_first("div:not([class])")
if rating_node:
rating_text = rating_node.text().strip()
rating_match = re.match(r"([\d.]+)\((\d+)", rating_text)
if rating_match:
rating_score = rating_match.group(1)
rating_count = int(rating_match.group(2))
shipping_note_node = row.css_first("ul.shipping_campaing_info li.padT5")
shipping_note = shipping_note_node.text().strip() if shipping_note_node else ""
branch_number = ""
campaign_node = row.css_first("li.ajax-campaign-placeholder")
if campaign_node:
zaiko_raw = campaign_node.attributes.get("data-zaiko_data") or ""
if zaiko_raw:
try:
zaiko = json.loads(zaiko_raw)
branch_number = str(zaiko.get("branch_number", ""))
except json.JSONDecodeError:
pass
items.append(
OtherShopItem(
tenpo_cd=tenpo_cd,
shop_name=shop_name,
shop_url=shop_url,
is_official=tenpo_cd == "",
condition=condition,
price=price,
branch_number=branch_number,
rating_score=rating_score,
rating_count=rating_count,
shipping_note=shipping_note,
detail_url=detail_url,
)
)
return items
async def fetch_categories(self) -> list[CategoryData]:
"""获取商品分类（带缓存）
@@ -470,12 +585,25 @@ class SurugayaClient:
return 1
return current_page if current_page >= 1 else 1
def _build_product_url(self, product_id: str | None) -> str:
"""根据商品 ID 构建详情页 URL"""
def _build_product_url(
self,
product_id: str | None,
tenpo_cd: str | None = None,
branch_number: str | None = None,
) -> str:
"""根据商品 ID 构建详情页 URL；tenpo_cd/branch_number 用于跳转查看指定店铺的报价"""
if not product_id:
raise ScrapeParseError("Either product_id or product_url must be provided")
base_url = self._settings.base_url.rstrip("/")
return f"{base_url}/product/detail/{product_id}"
url = f"{base_url}/product/detail/{product_id}"
params = []
if tenpo_cd:
params.append(f"tenpo_cd={tenpo_cd}")
if branch_number:
params.append(f"branch_number={branch_number}")
if params:
url += "?" + "&".join(params)
return url
def _parse_search_items(self, tree: HTMLParser) -> list[ProductSummary]:
"""解析搜索结果列表