From 9a1f2f40fc7b4573c2cd8ab3471e1562b17d6007 Mon Sep 17 00:00:00 2001
From: Jerry Yan <792602257@qq.com>
Date: Mon, 27 Jul 2026 10:57:02 +0800
Subject: [PATCH] =?UTF-8?q?=E9=97=A8=E5=BA=97=E4=BF=A1=E6=81=AF?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
README.md | 3 +
app/api/routes/scrape.py | 53 +++-
app/models/scrape.py | 84 ++++++
app/services/surugaya_client.py | 271 +++++++++++++++++-
.../src/surugaya_common/urls.py | 10 +
tests/test_common_contracts.py | 7 +
6 files changed, 414 insertions(+), 14 deletions(-)
diff --git a/README.md b/README.md
index 270c486..12873d2 100644
--- a/README.md
+++ b/README.md
@@ -15,6 +15,9 @@
- `GET /health`
- `POST /api/search`
- `POST /api/item_detail`
+- `POST /api/item_other_shops`
+- `POST /api/shop_info`、`POST /api/shop_items`（加盟店店铺资料与在售商品，见 [docs/api_shop.md](docs/api_shop.md)）
+- `GET /api/news`（见 [docs/api_news.md](docs/api_news.md)）
## 部署
diff --git a/app/api/routes/scrape.py b/app/api/routes/scrape.py
index e54fe48..5f1f860 100644
--- a/app/api/routes/scrape.py
+++ b/app/api/routes/scrape.py
@@ -22,7 +22,11 @@ from app.models.scrape import (
ProductDetailData,
PurchaseTaskRequest,
SearchRequest,
- SearchResultData, TradeMonitorRequest,
+ SearchResultData,
+ ShopInfoData,
+ ShopInfoRequest,
+ ShopItemsData,
+ ShopItemsRequest, TradeMonitorRequest,
)
from surugaya_common.http_utils import generate_signed_headers
from app.utils.parse_util import (
@@ -146,6 +150,53 @@ async def item_other_shops(
)
+@router.post(
+ "/shop_info",
+ response_model=ApiResponse[ShopInfoData],
+ dependencies=[Depends(require_bearer_token)],
+)
+async def shop_info(
+ payload: ShopInfoRequest,
+ container: ServiceContainer = Depends(get_container),
+) -> ApiResponse[ShopInfoData]:
+ """获取加盟店/市场店铺信息
+
+ 抓取骏河屋 /shop/{tenpo_cd} 页面，返回店名、logo、评分、评价数与店铺公告；
+ include_details=true 时额外抓取 配送（含都道府县运费表）/ポリシー（特商法表记）/
+ 返品保証/連絡（地址）四个子页，多付出四次页面请求。
+ """
+ data = await container.surugaya_client.fetch_shop_info(payload)
+ return ApiResponse[ShopInfoData](
+ success=True,
+ msg="success",
+ data=data,
+ code=0,
+ )
+
+
+@router.post(
+ "/shop_items",
+ response_model=ApiResponse[ShopItemsData],
+ dependencies=[Depends(require_bearer_token)],
+)
+async def shop_items(
+ payload: ShopItemsRequest,
+ container: ServiceContainer = Depends(get_container),
+) -> ApiResponse[ShopItemsData]:
+ """获取加盟店/市场店铺商品列表
+
+ 按 tenpo_cd 抓取该店铺在售商品，支持店内关键词检索与分页（24 条/页）；
+ rankBy、category 等搜索参数可原样透传。返回项的 goods_link 已带 tenpo_cd。
+ """
+ data = await container.surugaya_client.fetch_shop_items(payload)
+ return ApiResponse[ShopItemsData](
+ success=True,
+ msg="success",
+ data=data,
+ code=0,
+ )
+
+
@router.get(
"/categories",
response_model=ApiResponse[list[CategoryData]],
diff --git a/app/models/scrape.py b/app/models/scrape.py
index 6f91260..97df306 100644
--- a/app/models/scrape.py
+++ b/app/models/scrape.py
@@ -114,6 +114,90 @@ class OtherShopListData(BaseModel):
other_shop_list: list[OtherShopItem] = []
+class ShopInfoRequest(BaseModel):
+ """加盟店/市场店铺信息请求参数"""
+ tenpo_cd: str = Field(min_length=1, max_length=20, pattern=r"^[A-Za-z0-9_-]+$")
+ include_details: bool = False # 为 True 时额外抓取 配送/ポリシー/返品保証/連絡 四个子页（多付出四次页面请求）
+
+
+class ShopShippingFeeItem(BaseModel):
+ """店铺「配送」页中按都道府县列出的一条运费"""
+ prefecture: str = "" # 都道府县名，如"東京都"
+ fee: int = 0 # 该都道府县运费（日元）
+
+
+class ShopDeliveryInfo(BaseModel):
+ """店铺「配送」子页信息"""
+ description: str = "" # 发货说明原文（日文）
+ shipping_fee_list: list[ShopShippingFeeItem] = [] # 按都道府县的运费表
+
+
+class ShopPolicyInfo(BaseModel):
+ """店铺「ポリシー」子页信息（含特定商取引法表记）"""
+ description: str = "" # 政策全文（日文）
+
+
+class ShopServiceInfo(BaseModel):
+ """店铺「返品、保証、払い戻し」子页信息"""
+ return_policy: str = "" # 返品、払い戻し 段落
+ warranty: str = "" # 保証 段落
+
+
+class ShopAddress(BaseModel):
+ """店铺地址（联系地址或返品地址）"""
+ label: str = "" # 地址类型，如"住所"/"返品先"
+ postal_code: str = "" # 邮编
+ prefecture: str = "" # 都道府县
+ city: str = "" # 市区町村
+ street: str = "" # 番地
+ building: str = "" # 大楼/公寓名（可能为空）
+
+
+class ShopContactInfo(BaseModel):
+ """店铺「連絡」子页信息"""
+ address_list: list[ShopAddress] = []
+
+
+class ShopInfoData(BaseModel):
+ """加盟店/市场店铺信息
+
+ 基础字段来自 /shop/{tenpo_cd} 主页；delivery/policy/service/contact
+ 仅当请求 include_details=True 时才会填充，否则为 null。
+ """
+ tenpo_cd: str
+ shop_name: str = "" # 店铺名称
+ shop_url: str = "" # 店铺主页链接
+ logo_url: str = "" # 店铺 logo 图片链接
+ rating_score: str = "" # 店铺评分，如"5.0"
+ rating_count: int = 0 # 评价数量
+ notice: str = "" # 店铺公告（ショップ情報）原文
+ delivery: ShopDeliveryInfo | None = None
+ policy: ShopPolicyInfo | None = None
+ service: ShopServiceInfo | None = None
+ contact: ShopContactInfo | None = None
+
+
+class ShopItemsRequest(BaseModel):
+ """加盟店/市场店铺商品列表请求参数"""
+ model_config = ConfigDict(extra="allow")
+
+ tenpo_cd: str = Field(min_length=1, max_length=20, pattern=r"^[A-Za-z0-9_-]+$")
+ search_word: str = "" # 店内关键词检索，留空为全部商品
+ page: int = Field(default=1, ge=1, le=100)
+
+
+class ShopItemsData(BaseModel):
+ """加盟店/市场店铺商品列表数据"""
+ tenpo_cd: str
+ shop_name: str = "" # 店铺名称，从列表页标题解析
+ query: str = ""
+ page: int = 1
+ page_size: int = 0
+ total_count: int = 0
+ has_more: int = 0
+ items: list[ProductSummary] = []
+
+
class HealthData(BaseModel):
"""健康检查响应数据"""
status: str
diff --git a/app/services/surugaya_client.py b/app/services/surugaya_client.py
index 8c5e79c..fe450f4 100644
--- a/app/services/surugaya_client.py
+++ b/app/services/surugaya_client.py
@@ -20,13 +20,14 @@ from selectolax.parser import HTMLParser
from app.core.config import Settings
from app.core.errors import ScrapeParseError, UpstreamBlockedError
from app.models.scrape import CategoryData, DetailRequest, OtherShopItem, OtherShopListData, ProductDetailData, \
- ProductSummary, SearchRequest, SearchResultData
+ ProductSummary, SearchRequest, SearchResultData, ShopAddress, ShopContactInfo, ShopDeliveryInfo, ShopInfoData, \
+ ShopInfoRequest, ShopItemsData, ShopItemsRequest, ShopPolicyInfo, ShopServiceInfo, ShopShippingFeeItem
from app.services.browser_pool import BrowserPool
from app.services.cloudflare_session import CloudflareSessionManager
from app.services.session_store import SessionStore
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
from surugaya_common.app_utils import is_empty_str
-from surugaya_common.urls import BASE_URL, product_other_url
+from surugaya_common.urls import BASE_URL, product_other_url, shop_section_url, shop_url
logger = logging.getLogger(__name__)
@@ -98,18 +99,9 @@ class SurugayaClient:
# 解析商品总数量，格式示例：該当件数:10,428件中 1-24件
current_page = self._extract_page_value(payload)
- total_count = 0
page_size = 24
- has_more = 0
- hit_node = tree.css_first("#search_header .search_option .hit")
- if hit_node:
- raw_text = hit_node.text()
- match = re.search(r'該当件数:([\d,]+)件', raw_text)
- if match:
- total_count = int(match.group(1).replace(',', ''))
-
- if total_count > 0:
- has_more = int(current_page * page_size < total_count)
+ total_count = self._extract_total_count(tree)
+ has_more = int(current_page * page_size < total_count) if total_count > 0 else 0
# 解析商品列表
items = self._parse_search_items(tree)
@@ -125,6 +117,246 @@ class SurugayaClient:
session_id=session.session_id,
)
+ async def fetch_shop_items(self, payload: ShopItemsRequest) -> ShopItemsData:
+ """抓取加盟店/市场店铺的商品列表
+
+ 店铺商品列表复用站点的搜索页（/search?tenpo_code=xxx），
+ DOM 结构与普通搜索完全一致，因此解析逻辑与 search 共用。
+
+ Args:
+ payload: 店铺商品请求参数，包含 tenpo_cd、search_word、page 等
+
+ Returns:
+ ShopItemsData: 店铺名称、商品列表、总数和分页信息
+ """
+ url = self._build_shop_items_url(payload)
+ logger.info("开始抓取店铺商品列表：tenpo_cd=%s page=%s", payload.tenpo_cd, payload.page)
+ logger.debug("店铺商品列表URL：url=%s", url)
+
+ html, _ = await self._session_manager.fetch_html(url)
+ tree = HTMLParser(html)
+
+ page_size = 24
+ total_count = self._extract_total_count(tree)
+ has_more = int(payload.page * page_size < total_count) if total_count > 0 else 0
+
+ # 列表页标题形如「駿河屋 山口大学前店の商品一覧」，去掉后缀即为店名
+ shop_name = ""
+ title_node = tree.css_first("#search_header .search_option .hit h2")
+ if title_node:
+ shop_name = re.sub(r"の商品一覧$", "", title_node.text().strip())
+
+ items = self._parse_search_items(tree)
+ logger.info("店铺商品列表解析完成 ===> tenpo_cd=%s 抓取商品数=%s", payload.tenpo_cd, len(items))
+
+ return ShopItemsData(
+ tenpo_cd=payload.tenpo_cd,
+ shop_name=shop_name,
+ query=payload.search_word,
+ page=payload.page,
+ page_size=page_size,
+ total_count=total_count,
+ has_more=has_more,
+ items=items,
+ )
+
+ async def fetch_shop_info(self, payload: ShopInfoRequest) -> ShopInfoData:
+ """抓取加盟店/市场店铺信息
+
+ 基础信息来自店铺主页 /shop/{tenpo_cd}；当 include_details=True 时，
+ 额外并发抓取 配送/ポリシー/返品保証/連絡 四个子页（多付出四次页面请求），
+ 单个子页抓取或解析失败降级为 None，不影响主页信息返回。
+
+ Args:
+ payload: 店铺信息请求参数，包含 tenpo_cd、include_details
+
+ Returns:
+ ShopInfoData: 店铺信息
+
+ Raises:
+ ScrapeParseError: 店铺主页解析失败（如店铺不存在）
+ """
+ tenpo_cd = payload.tenpo_cd
+ target_url = shop_url(tenpo_cd)
+ logger.info("开始抓取店铺信息：tenpo_cd=%s include_details=%s", tenpo_cd, payload.include_details)
+
+ html, _ = await self._session_manager.fetch_html(target_url)
+ info = self._parse_shop_info(HTMLParser(html), tenpo_cd)
+
+ if payload.include_details:
+ delivery, policy, service, contact = await asyncio.gather(
+ self._fetch_shop_section(tenpo_cd, "delivery", self._parse_shop_delivery),
+ self._fetch_shop_section(tenpo_cd, "policy", self._parse_shop_policy),
+ self._fetch_shop_section(tenpo_cd, "service", self._parse_shop_service),
+ self._fetch_shop_section(tenpo_cd, "contact", self._parse_shop_contact),
+ )
+ info.delivery = delivery
+ info.policy = policy
+ info.service = service
+ info.contact = contact
+
+ logger.debug("店铺信息解析完成：tenpo_cd=%s shop_name=%s", info.tenpo_cd, info.shop_name)
+ return info
+
+ async def _fetch_shop_section(self, tenpo_cd: str, section: str, parse: Any) -> Any:
+ """抓取并解析单个店铺子页；抓取或解析失败时降级为 None，不影响主流程。"""
+ section_url = shop_section_url(section, tenpo_cd)
+ try:
+ html, _ = await self._session_manager.fetch_html(section_url)
+ except Exception as exc:
+ logger.warning("抓取店铺子页失败：tenpo_cd=%s section=%s err=%s", tenpo_cd, section, exc)
+ return None
+
+ try:
+ return parse(HTMLParser(html))
+ except Exception:
+ logger.exception("解析店铺子页失败：tenpo_cd=%s section=%s", tenpo_cd, section)
+ return None
+
+ @staticmethod
+ def _parse_shop_info(tree: HTMLParser, tenpo_cd: str) -> ShopInfoData:
+ """解析店铺主页 /shop/{tenpo_cd} 的基础信息（店名、logo、评分、公告）。"""
+ brand = tree.css_first(".shop_brand")
+ if brand is None:
+ raise ScrapeParseError(f"店铺信息解析失败：tenpo_cd={tenpo_cd}")
+
+ name_node = brand.css_first(".shop_info h1")
+ shop_name = name_node.text().strip() if name_node else ""
+ if not shop_name:
+ raise ScrapeParseError(f"店铺名称解析失败：tenpo_cd={tenpo_cd}")
+
+ logo_node = brand.css_first(".shop_logo img")
+ logo_src = (logo_node.attributes.get("src") or "") if logo_node else ""
+ logo_url = SurugayaClient._normalize_url(logo_src) if logo_src else ""
+
+ # 评分区形如：5.0
(1345件)
+ rating_score = ""
+ rating_count = 0
+ point_node = brand.css_first(".shop_info .point")
+ if point_node:
+ score_node = point_node.css_first(".pull-left")
+ if score_node:
+ score_match = re.search(r"[\d.]+", score_node.text())
+ if score_match:
+ rating_score = score_match.group(0)
+ count_node = point_node.css_first(".pull-right")
+ if count_node:
+ count_match = re.search(r"(\d+)", count_node.text().replace(",", ""))
+ if count_match:
+ rating_count = int(count_match.group(1))
+
+ notice_node = tree.css_first("#search_result .contact-top .content_text")
+ notice = SurugayaClient._clean_block_text(notice_node)
+
+ return ShopInfoData(
+ tenpo_cd=tenpo_cd,
+ shop_name=shop_name,
+ shop_url=shop_url(tenpo_cd),
+ logo_url=logo_url,
+ rating_score=rating_score,
+ rating_count=rating_count,
+ notice=notice,
+ )
+
+ @staticmethod
+ def _parse_shop_delivery(tree: HTMLParser) -> ShopDeliveryInfo:
+ """解析店铺「配送」子页：发货说明 + 按都道府县的运费表。"""
+ description = SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text"))
+
+ shipping_fee_list: list[ShopShippingFeeItem] = []
+ for row in tree.css("#main2 table.table-bordered tr"):
+ tds = row.css("td")
+ if len(tds) < 2:
+ # 表头行（th）无 td，跳过
+ continue
+ prefecture = tds[0].text().strip()
+ fee = format_price(tds[1].text().strip()) or 0
+ if prefecture:
+ shipping_fee_list.append(ShopShippingFeeItem(prefecture=prefecture, fee=fee))
+
+ return ShopDeliveryInfo(description=description, shipping_fee_list=shipping_fee_list)
+
+ @staticmethod
+ def _parse_shop_policy(tree: HTMLParser) -> ShopPolicyInfo:
+ """解析店铺「ポリシー」子页：政策全文（含特定商取引法表记）。"""
+ return ShopPolicyInfo(description=SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text")))
+
+ @staticmethod
+ def _parse_shop_service(tree: HTMLParser) -> ShopServiceInfo:
+ """解析店铺「返品、保証、払い戻し」子页。
+
+ 页面结构为 h4 标题与 content_text 正文交替排列，按出现顺序配对，
+ 再依据标题文案分派到 return_policy / warranty。
+ """
+ info = ShopServiceInfo()
+ container = tree.css_first("#main2 #search_result > div")
+ if container is None:
+ return info
+
+ current_title = ""
+ for node in container.iter():
+ if node.tag == "h4":
+ current_title = node.text().strip()
+ continue
+ if "content_text" not in (node.attributes.get("class") or ""):
+ continue
+
+ text = SurugayaClient._clean_block_text(node)
+ if "保証" in current_title:
+ info.warranty = text
+ elif "返品" in current_title or "払い戻し" in current_title:
+ info.return_policy = text
+
+ return info
+
+ @staticmethod
+ def _parse_shop_contact(tree: HTMLParser) -> ShopContactInfo:
+ """解析店铺「連絡」子页：联系地址与返品地址。"""
+ field_map = {
+ "郵便番号": "postal_code",
+ "都道府県": "prefecture",
+ "市区町村": "city",
+ "番地": "street",
+ "ビル・マンション名": "building",
+ }
+
+ address_list: list[ShopAddress] = []
+ for block in tree.css("#main2 .contact-top"):
+ label_node = block.css_first("h4")
+ if label_node is None:
+ continue
+
+ address = ShopAddress(label=label_node.text().strip())
+ for p in block.css("p"):
+ key, sep, value = p.text().partition("：")
+ if not sep:
+ continue
+ field = field_map.get(key.strip())
+ if field:
+ setattr(address, field, value.strip())
+
+ if address.postal_code or address.prefecture:
+ address_list.append(address)
+
+ return ShopContactInfo(address_list=address_list)
+
+ @staticmethod
+ def _clean_block_text(node: object | None) -> str:
+ """提取节点文本并归一化空白：去掉行首尾空白与连续空行，保留段落换行。"""
+ if node is None:
+ return ""
+ lines = [line.strip() for line in node.text().splitlines()]
+ return "\n".join(line for line in lines if line)
+
+ @staticmethod
+ def _extract_total_count(tree: HTMLParser) -> int:
+ """从搜索/店铺列表页头部解析命中总数，格式示例：該当件数:10,428件中 1-24件"""
+ hit_node = tree.css_first("#search_header .search_option .hit")
+ if hit_node is None:
+ return 0
+ match = re.search(r"該当件数:([\d,]+)件", hit_node.text())
+ return int(match.group(1).replace(",", "")) if match else 0
+
async def fetch_detail(self, payload: DetailRequest) -> ProductDetailData:
"""抓取商品详情页
@@ -597,6 +829,19 @@ class SurugayaClient:
return 1
return current_page if current_page >= 1 else 1
+ def _build_shop_items_url(self, payload: ShopItemsRequest) -> str:
+ """构建店铺商品列表 URL
+
+ 店内商品列表即站点搜索页按 tenpo_code 过滤的结果；
+ tenpo_cd 转成站点侧的 tenpo_code 参数，其余字段（rankBy、category 等）原样透传。
+ """
+ params = payload.model_dump(exclude_none=True)
+ params["tenpo_code"] = params.pop("tenpo_cd")
+
+ query_string = urlencode(params, doseq=True)
+ base_url = self._settings.base_url.rstrip("/")
+ return f"{base_url}/search?{query_string}"
+
def _build_product_url(
self,
product_id: str | None,
diff --git a/packages/surugaya_common/src/surugaya_common/urls.py b/packages/surugaya_common/src/surugaya_common/urls.py
index 8e7dd94..11d28f9 100644
--- a/packages/surugaya_common/src/surugaya_common/urls.py
+++ b/packages/surugaya_common/src/surugaya_common/urls.py
@@ -27,6 +27,16 @@ def product_other_url(product_id: str) -> str:
return f"{BASE_URL}/product/other/{product_id}"
+def shop_url(tenpo_cd: str) -> str:
+ """加盟店/市场店铺主页地址：店名、logo、评分与店铺公告。"""
+ return f"{BASE_URL}/shop/{tenpo_cd}"
+
+
+def shop_section_url(section: str, tenpo_cd: str) -> str:
+ """店铺子页地址，section 取 delivery/policy/service/contact/order/rating。"""
+ return f"{BASE_URL}/shop/{section}/{tenpo_cd}"
+
+
def trade_detail_url(trade_code: str) -> str:
"""交易（订单）详情页地址。"""
return f"{BASE_URL}/pcmypage/action_sell_search/detail?trade_code={trade_code}"
diff --git a/tests/test_common_contracts.py b/tests/test_common_contracts.py
index 6321b7a..72dfa2f 100644
--- a/tests/test_common_contracts.py
+++ b/tests/test_common_contracts.py
@@ -7,6 +7,8 @@ from surugaya_common.urls import (
CARGO_DETAIL_URL,
MYPAGE_URL,
product_detail_url,
+ shop_section_url,
+ shop_url,
trade_detail_url,
)
@@ -40,6 +42,11 @@ class TestUrls:
== "https://www.suruga-ya.jp/pcmypage/action_sell_search/detail?trade_code=T123456"
)
+ def test_shop_urls(self):
+ assert shop_url("400451") == "https://www.suruga-ya.jp/shop/400451"
+ assert shop_section_url("delivery", "400451") == "https://www.suruga-ya.jp/shop/delivery/400451"
+ assert shop_section_url("contact", "410118") == "https://www.suruga-ya.jp/shop/contact/410118"
+
class TestFormatJapanesePrice:
def test_supported_formats(self):