jp/surugaya
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

门店信息

Browse Source
This commit is contained in:
q792602257
2026-07-27 10:57:02 +08:00
parent ef7eb9dbb1
commit 9a1f2f40fc
6 changed files with 414 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+3
View File
@@ -15,6 +15,9 @@
- `GET /health`
- `POST /api/search`
- `POST /api/item_detail`
- `POST /api/item_other_shops`
- `POST /api/shop_info`、`POST /api/shop_items`（加盟店店铺资料与在售商品，见 [docs/api_shop.md](docs/api_shop.md)）
- `GET /api/news`（见 [docs/api_news.md](docs/api_news.md)）
## 部署
app/api/routes/scrape.py
+52 -1
View File
@@ -22,7 +22,11 @@ from app.models.scrape import (
ProductDetailData,
PurchaseTaskRequest,
SearchRequest,
SearchResultData, TradeMonitorRequest,
SearchResultData,
ShopInfoData,
ShopInfoRequest,
ShopItemsData,
ShopItemsRequest, TradeMonitorRequest,
)
from surugaya_common.http_utils import generate_signed_headers
from app.utils.parse_util import (
@@ -146,6 +150,53 @@ async def item_other_shops(
)
@router.post(
"/shop_info",
response_model=ApiResponse[ShopInfoData],
dependencies=[Depends(require_bearer_token)],
)
async def shop_info(
payload: ShopInfoRequest,
container: ServiceContainer = Depends(get_container),
) -> ApiResponse[ShopInfoData]:
"""获取加盟店/市场店铺信息
抓取骏河屋 /shop/{tenpo_cd} 页面，返回店名、logo、评分、评价数与店铺公告；
include_details=true 时额外抓取 配送（含都道府县运费表）/ポリシー（特商法表记）/
返品保証/連絡（地址）四个子页，多付出四次页面请求。
"""
data = await container.surugaya_client.fetch_shop_info(payload)
return ApiResponse[ShopInfoData](
success=True,
msg="success",
data=data,
code=0,
)
@router.post(
"/shop_items",
response_model=ApiResponse[ShopItemsData],
dependencies=[Depends(require_bearer_token)],
)
async def shop_items(
payload: ShopItemsRequest,
container: ServiceContainer = Depends(get_container),
) -> ApiResponse[ShopItemsData]:
"""获取加盟店/市场店铺商品列表
按 tenpo_cd 抓取该店铺在售商品，支持店内关键词检索与分页（24 条/页）；
rankBy、category 等搜索参数可原样透传。返回项的 goods_link 已带 tenpo_cd。
"""
data = await container.surugaya_client.fetch_shop_items(payload)
return ApiResponse[ShopItemsData](
success=True,
msg="success",
data=data,
code=0,
)
@router.get(
"/categories",
response_model=ApiResponse[list[CategoryData]],
app/models/scrape.py
+84
View File
@@ -114,6 +114,90 @@ class OtherShopListData(BaseModel):
other_shop_list: list[OtherShopItem] = []
class ShopInfoRequest(BaseModel):
"""加盟店/市场店铺信息请求参数"""
tenpo_cd: str = Field(min_length=1, max_length=20, pattern=r"^[A-Za-z0-9_-]+$")
include_details: bool = False # 为 True 时额外抓取 配送/ポリシー/返品保証/連絡 四个子页（多付出四次页面请求）
class ShopShippingFeeItem(BaseModel):
"""店铺「配送」页中按都道府县列出的一条运费"""
prefecture: str = "" # 都道府县名，如"東京都"
fee: int = 0 # 该都道府县运费（日元）
class ShopDeliveryInfo(BaseModel):
"""店铺「配送」子页信息"""
description: str = "" # 发货说明原文（日文）
shipping_fee_list: list[ShopShippingFeeItem] = [] # 按都道府县的运费表
class ShopPolicyInfo(BaseModel):
"""店铺「ポリシー」子页信息（含特定商取引法表记）"""
description: str = "" # 政策全文（日文）
class ShopServiceInfo(BaseModel):
"""店铺「返品、保証、払い戻し」子页信息"""
return_policy: str = "" # 返品、払い戻し 段落
warranty: str = "" # 保証 段落
class ShopAddress(BaseModel):
"""店铺地址（联系地址或返品地址）"""
label: str = "" # 地址类型，如"住所"/"返品先"
postal_code: str = "" # 邮编
prefecture: str = "" # 都道府县
city: str = "" # 市区町村
street: str = "" # 番地
building: str = "" # 大楼/公寓名（可能为空）
class ShopContactInfo(BaseModel):
"""店铺「連絡」子页信息"""
address_list: list[ShopAddress] = []
class ShopInfoData(BaseModel):
"""加盟店/市场店铺信息
基础字段来自 /shop/{tenpo_cd} 主页；delivery/policy/service/contact
仅当请求 include_details=True 时才会填充，否则为 null。
"""
tenpo_cd: str
shop_name: str = "" # 店铺名称
shop_url: str = "" # 店铺主页链接
logo_url: str = "" # 店铺 logo 图片链接
rating_score: str = "" # 店铺评分，如"5.0"
rating_count: int = 0 # 评价数量
notice: str = "" # 店铺公告（ショップ情報）原文
delivery: ShopDeliveryInfo | None = None
policy: ShopPolicyInfo | None = None
service: ShopServiceInfo | None = None
contact: ShopContactInfo | None = None
class ShopItemsRequest(BaseModel):
"""加盟店/市场店铺商品列表请求参数"""
model_config = ConfigDict(extra="allow")
tenpo_cd: str = Field(min_length=1, max_length=20, pattern=r"^[A-Za-z0-9_-]+$")
search_word: str = "" # 店内关键词检索，留空为全部商品
page: int = Field(default=1, ge=1, le=100)
class ShopItemsData(BaseModel):
"""加盟店/市场店铺商品列表数据"""
tenpo_cd: str
shop_name: str = "" # 店铺名称，从列表页标题解析
query: str = ""
page: int = 1
page_size: int = 0
total_count: int = 0
has_more: int = 0
items: list[ProductSummary] = []
class HealthData(BaseModel):
"""健康检查响应数据"""
status: str
app/services/surugaya_client.py
+258 -13
View File
@@ -20,13 +20,14 @@ from selectolax.parser import HTMLParser
from app.core.config import Settings
from app.core.errors import ScrapeParseError, UpstreamBlockedError
from app.models.scrape import CategoryData, DetailRequest, OtherShopItem, OtherShopListData, ProductDetailData, \
ProductSummary, SearchRequest, SearchResultData
ProductSummary, SearchRequest, SearchResultData, ShopAddress, ShopContactInfo, ShopDeliveryInfo, ShopInfoData, \
ShopInfoRequest, ShopItemsData, ShopItemsRequest, ShopPolicyInfo, ShopServiceInfo, ShopShippingFeeItem
from app.services.browser_pool import BrowserPool
from app.services.cloudflare_session import CloudflareSessionManager
from app.services.session_store import SessionStore
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
from surugaya_common.app_utils import is_empty_str
from surugaya_common.urls import BASE_URL, product_other_url
from surugaya_common.urls import BASE_URL, product_other_url, shop_section_url, shop_url
logger = logging.getLogger(__name__)
@@ -98,18 +99,9 @@ class SurugayaClient:
# 解析商品总数量，格式示例：該当件数:10,428件中 1-24件
current_page = self._extract_page_value(payload)
total_count = 0
page_size = 24
has_more = 0
hit_node = tree.css_first("#search_header .search_option .hit")
if hit_node:
raw_text = hit_node.text()
match = re.search(r'該当件数:([\d,]+)件', raw_text)
if match:
total_count = int(match.group(1).replace(',', ''))
if total_count > 0:
has_more = int(current_page * page_size < total_count)
total_count = self._extract_total_count(tree)
has_more = int(current_page * page_size < total_count) if total_count > 0 else 0
# 解析商品列表
items = self._parse_search_items(tree)
@@ -125,6 +117,246 @@ class SurugayaClient:
session_id=session.session_id,
)
async def fetch_shop_items(self, payload: ShopItemsRequest) -> ShopItemsData:
"""抓取加盟店/市场店铺的商品列表
店铺商品列表复用站点的搜索页（/search?tenpo_code=xxx），
DOM 结构与普通搜索完全一致，因此解析逻辑与 search 共用。
Args:
payload: 店铺商品请求参数，包含 tenpo_cd、search_word、page 等
Returns:
ShopItemsData: 店铺名称、商品列表、总数和分页信息
"""
url = self._build_shop_items_url(payload)
logger.info("开始抓取店铺商品列表：tenpo_cd=%s page=%s", payload.tenpo_cd, payload.page)
logger.debug("店铺商品列表URL：url=%s", url)
html, _ = await self._session_manager.fetch_html(url)
tree = HTMLParser(html)
page_size = 24
total_count = self._extract_total_count(tree)
has_more = int(payload.page * page_size < total_count) if total_count > 0 else 0
# 列表页标题形如「駿河屋　山口大学前店の商品一覧」，去掉后缀即为店名
shop_name = ""
title_node = tree.css_first("#search_header .search_option .hit h2")
if title_node:
shop_name = re.sub(r"の商品一覧$", "", title_node.text().strip())
items = self._parse_search_items(tree)
logger.info("店铺商品列表解析完成 ===> tenpo_cd=%s 抓取商品数=%s", payload.tenpo_cd, len(items))
return ShopItemsData(
tenpo_cd=payload.tenpo_cd,
shop_name=shop_name,
query=payload.search_word,
page=payload.page,
page_size=page_size,
total_count=total_count,
has_more=has_more,
items=items,
)
async def fetch_shop_info(self, payload: ShopInfoRequest) -> ShopInfoData:
"""抓取加盟店/市场店铺信息
基础信息来自店铺主页 /shop/{tenpo_cd}；当 include_details=True 时，
额外并发抓取 配送/ポリシー/返品保証/連絡 四个子页（多付出四次页面请求），
单个子页抓取或解析失败降级为 None，不影响主页信息返回。
Args:
payload: 店铺信息请求参数，包含 tenpo_cd、include_details
Returns:
ShopInfoData: 店铺信息
Raises:
ScrapeParseError: 店铺主页解析失败（如店铺不存在）
"""
tenpo_cd = payload.tenpo_cd
target_url = shop_url(tenpo_cd)
logger.info("开始抓取店铺信息：tenpo_cd=%s include_details=%s", tenpo_cd, payload.include_details)
html, _ = await self._session_manager.fetch_html(target_url)
info = self._parse_shop_info(HTMLParser(html), tenpo_cd)
if payload.include_details:
delivery, policy, service, contact = await asyncio.gather(
self._fetch_shop_section(tenpo_cd, "delivery", self._parse_shop_delivery),
self._fetch_shop_section(tenpo_cd, "policy", self._parse_shop_policy),
self._fetch_shop_section(tenpo_cd, "service", self._parse_shop_service),
self._fetch_shop_section(tenpo_cd, "contact", self._parse_shop_contact),
)
info.delivery = delivery
info.policy = policy
info.service = service
info.contact = contact
logger.debug("店铺信息解析完成：tenpo_cd=%s shop_name=%s", info.tenpo_cd, info.shop_name)
return info
async def _fetch_shop_section(self, tenpo_cd: str, section: str, parse: Any) -> Any:
"""抓取并解析单个店铺子页；抓取或解析失败时降级为 None，不影响主流程。"""
section_url = shop_section_url(section, tenpo_cd)
try:
html, _ = await self._session_manager.fetch_html(section_url)
except Exception as exc:
logger.warning("抓取店铺子页失败：tenpo_cd=%s section=%s err=%s", tenpo_cd, section, exc)
return None
try:
return parse(HTMLParser(html))
except Exception:
logger.exception("解析店铺子页失败：tenpo_cd=%s section=%s", tenpo_cd, section)
return None
@staticmethod
def _parse_shop_info(tree: HTMLParser, tenpo_cd: str) -> ShopInfoData:
"""解析店铺主页 /shop/{tenpo_cd} 的基础信息（店名、logo、评分、公告）。"""
brand = tree.css_first(".shop_brand")
if brand is None:
raise ScrapeParseError(f"店铺信息解析失败：tenpo_cd={tenpo_cd}")
name_node = brand.css_first(".shop_info h1")
shop_name = name_node.text().strip() if name_node else ""
if not shop_name:
raise ScrapeParseError(f"店铺名称解析失败：tenpo_cd={tenpo_cd}")
logo_node = brand.css_first(".shop_logo img")
logo_src = (logo_node.attributes.get("src") or "") if logo_node else ""
logo_url = SurugayaClient._normalize_url(logo_src) if logo_src else ""
# 评分区形如：<div class="pull-left padR20">5.0</div><div class="pull-right">(1345件)</div>
rating_score = ""
rating_count = 0
point_node = brand.css_first(".shop_info .point")
if point_node:
score_node = point_node.css_first(".pull-left")
if score_node:
score_match = re.search(r"[\d.]+", score_node.text())
if score_match:
rating_score = score_match.group(0)
count_node = point_node.css_first(".pull-right")
if count_node:
count_match = re.search(r"(\d+)", count_node.text().replace(",", ""))
if count_match:
rating_count = int(count_match.group(1))
notice_node = tree.css_first("#search_result .contact-top .content_text")
notice = SurugayaClient._clean_block_text(notice_node)
return ShopInfoData(
tenpo_cd=tenpo_cd,
shop_name=shop_name,
shop_url=shop_url(tenpo_cd),
logo_url=logo_url,
rating_score=rating_score,
rating_count=rating_count,
notice=notice,
)
@staticmethod
def _parse_shop_delivery(tree: HTMLParser) -> ShopDeliveryInfo:
"""解析店铺「配送」子页：发货说明 + 按都道府县的运费表。"""
description = SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text"))
shipping_fee_list: list[ShopShippingFeeItem] = []
for row in tree.css("#main2 table.table-bordered tr"):
tds = row.css("td")
if len(tds) < 2:
# 表头行（th）无 td，跳过
continue
prefecture = tds[0].text().strip()
fee = format_price(tds[1].text().strip()) or 0
if prefecture:
shipping_fee_list.append(ShopShippingFeeItem(prefecture=prefecture, fee=fee))
return ShopDeliveryInfo(description=description, shipping_fee_list=shipping_fee_list)
@staticmethod
def _parse_shop_policy(tree: HTMLParser) -> ShopPolicyInfo:
"""解析店铺「ポリシー」子页：政策全文（含特定商取引法表记）。"""
return ShopPolicyInfo(description=SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text")))
@staticmethod
def _parse_shop_service(tree: HTMLParser) -> ShopServiceInfo:
"""解析店铺「返品、保証、払い戻し」子页。
页面结构为 h4 标题与 content_text 正文交替排列，按出现顺序配对，
再依据标题文案分派到 return_policy / warranty。
"""
info = ShopServiceInfo()
container = tree.css_first("#main2 #search_result > div")
if container is None:
return info
current_title = ""
for node in container.iter():
if node.tag == "h4":
current_title = node.text().strip()
continue
if "content_text" not in (node.attributes.get("class") or ""):
continue
text = SurugayaClient._clean_block_text(node)
if "保証" in current_title:
info.warranty = text
elif "返品" in current_title or "払い戻し" in current_title:
info.return_policy = text
return info
@staticmethod
def _parse_shop_contact(tree: HTMLParser) -> ShopContactInfo:
"""解析店铺「連絡」子页：联系地址与返品地址。"""
field_map = {
"郵便番号": "postal_code",
"都道府県": "prefecture",
"市区町村": "city",
"番地": "street",
"ビル・マンション名": "building",
}
address_list: list[ShopAddress] = []
for block in tree.css("#main2 .contact-top"):
label_node = block.css_first("h4")
if label_node is None:
continue
address = ShopAddress(label=label_node.text().strip())
for p in block.css("p"):
key, sep, value = p.text().partition("：")
if not sep:
continue
field = field_map.get(key.strip())
if field:
setattr(address, field, value.strip())
if address.postal_code or address.prefecture:
address_list.append(address)
return ShopContactInfo(address_list=address_list)
@staticmethod
def _clean_block_text(node: object | None) -> str:
"""提取节点文本并归一化空白：去掉行首尾空白与连续空行，保留段落换行。"""
if node is None:
return ""
lines = [line.strip() for line in node.text().splitlines()]
return "\n".join(line for line in lines if line)
@staticmethod
def _extract_total_count(tree: HTMLParser) -> int:
"""从搜索/店铺列表页头部解析命中总数，格式示例：該当件数:10,428件中 1-24件"""
hit_node = tree.css_first("#search_header .search_option .hit")
if hit_node is None:
return 0
match = re.search(r"該当件数:([\d,]+)件", hit_node.text())
return int(match.group(1).replace(",", "")) if match else 0
async def fetch_detail(self, payload: DetailRequest) -> ProductDetailData:
"""抓取商品详情页
@@ -597,6 +829,19 @@ class SurugayaClient:
return 1
return current_page if current_page >= 1 else 1
def _build_shop_items_url(self, payload: ShopItemsRequest) -> str:
"""构建店铺商品列表 URL
店内商品列表即站点搜索页按 tenpo_code 过滤的结果；
tenpo_cd 转成站点侧的 tenpo_code 参数，其余字段（rankBy、category 等）原样透传。
"""
params = payload.model_dump(exclude_none=True)
params["tenpo_code"] = params.pop("tenpo_cd")
query_string = urlencode(params, doseq=True)
base_url = self._settings.base_url.rstrip("/")
return f"{base_url}/search?{query_string}"
def _build_product_url(
self,
product_id: str | None,
packages/surugaya_common/src/surugaya_common/urls.py
+10
View File
@@ -27,6 +27,16 @@ def product_other_url(product_id: str) -> str:
return f"{BASE_URL}/product/other/{product_id}"
def shop_url(tenpo_cd: str) -> str:
"""加盟店/市场店铺主页地址：店名、logo、评分与店铺公告。"""
return f"{BASE_URL}/shop/{tenpo_cd}"
def shop_section_url(section: str, tenpo_cd: str) -> str:
"""店铺子页地址，section 取 delivery/policy/service/contact/order/rating。"""
return f"{BASE_URL}/shop/{section}/{tenpo_cd}"
def trade_detail_url(trade_code: str) -> str:
"""交易（订单）详情页地址。"""
return f"{BASE_URL}/pcmypage/action_sell_search/detail?trade_code={trade_code}"
tests/test_common_contracts.py
+7
View File
@@ -7,6 +7,8 @@ from surugaya_common.urls import (
CARGO_DETAIL_URL,
MYPAGE_URL,
product_detail_url,
shop_section_url,
shop_url,
trade_detail_url,
)
@@ -40,6 +42,11 @@ class TestUrls:
== "https://www.suruga-ya.jp/pcmypage/action_sell_search/detail?trade_code=T123456"
)
def test_shop_urls(self):
assert shop_url("400451") == "https://www.suruga-ya.jp/shop/400451"
assert shop_section_url("delivery", "400451") == "https://www.suruga-ya.jp/shop/delivery/400451"
assert shop_section_url("contact", "410118") == "https://www.suruga-ya.jp/shop/contact/410118"
class TestFormatJapanesePrice:
def test_supported_formats(self):