门店信息
This commit is contained in:
+258
-13
@@ -20,13 +20,14 @@ from selectolax.parser import HTMLParser
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import ScrapeParseError, UpstreamBlockedError
|
||||
from app.models.scrape import CategoryData, DetailRequest, OtherShopItem, OtherShopListData, ProductDetailData, \
|
||||
ProductSummary, SearchRequest, SearchResultData
|
||||
ProductSummary, SearchRequest, SearchResultData, ShopAddress, ShopContactInfo, ShopDeliveryInfo, ShopInfoData, \
|
||||
ShopInfoRequest, ShopItemsData, ShopItemsRequest, ShopPolicyInfo, ShopServiceInfo, ShopShippingFeeItem
|
||||
from app.services.browser_pool import BrowserPool
|
||||
from app.services.cloudflare_session import CloudflareSessionManager
|
||||
from app.services.session_store import SessionStore
|
||||
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
|
||||
from surugaya_common.app_utils import is_empty_str
|
||||
from surugaya_common.urls import BASE_URL, product_other_url
|
||||
from surugaya_common.urls import BASE_URL, product_other_url, shop_section_url, shop_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -98,18 +99,9 @@ class SurugayaClient:
|
||||
|
||||
# 解析商品总数量，格式示例：該当件数:10,428件中 1-24件
|
||||
current_page = self._extract_page_value(payload)
|
||||
total_count = 0
|
||||
page_size = 24
|
||||
has_more = 0
|
||||
hit_node = tree.css_first("#search_header .search_option .hit")
|
||||
if hit_node:
|
||||
raw_text = hit_node.text()
|
||||
match = re.search(r'該当件数:([\d,]+)件', raw_text)
|
||||
if match:
|
||||
total_count = int(match.group(1).replace(',', ''))
|
||||
|
||||
if total_count > 0:
|
||||
has_more = int(current_page * page_size < total_count)
|
||||
total_count = self._extract_total_count(tree)
|
||||
has_more = int(current_page * page_size < total_count) if total_count > 0 else 0
|
||||
|
||||
# 解析商品列表
|
||||
items = self._parse_search_items(tree)
|
||||
@@ -125,6 +117,246 @@ class SurugayaClient:
|
||||
session_id=session.session_id,
|
||||
)
|
||||
|
||||
async def fetch_shop_items(self, payload: ShopItemsRequest) -> ShopItemsData:
|
||||
"""抓取加盟店/市场店铺的商品列表
|
||||
|
||||
店铺商品列表复用站点的搜索页（/search?tenpo_code=xxx），
|
||||
DOM 结构与普通搜索完全一致，因此解析逻辑与 search 共用。
|
||||
|
||||
Args:
|
||||
payload: 店铺商品请求参数，包含 tenpo_cd、search_word、page 等
|
||||
|
||||
Returns:
|
||||
ShopItemsData: 店铺名称、商品列表、总数和分页信息
|
||||
"""
|
||||
url = self._build_shop_items_url(payload)
|
||||
logger.info("开始抓取店铺商品列表：tenpo_cd=%s page=%s", payload.tenpo_cd, payload.page)
|
||||
logger.debug("店铺商品列表URL：url=%s", url)
|
||||
|
||||
html, _ = await self._session_manager.fetch_html(url)
|
||||
tree = HTMLParser(html)
|
||||
|
||||
page_size = 24
|
||||
total_count = self._extract_total_count(tree)
|
||||
has_more = int(payload.page * page_size < total_count) if total_count > 0 else 0
|
||||
|
||||
# 列表页标题形如「駿河屋 山口大学前店の商品一覧」，去掉后缀即为店名
|
||||
shop_name = ""
|
||||
title_node = tree.css_first("#search_header .search_option .hit h2")
|
||||
if title_node:
|
||||
shop_name = re.sub(r"の商品一覧$", "", title_node.text().strip())
|
||||
|
||||
items = self._parse_search_items(tree)
|
||||
logger.info("店铺商品列表解析完成 ===> tenpo_cd=%s 抓取商品数=%s", payload.tenpo_cd, len(items))
|
||||
|
||||
return ShopItemsData(
|
||||
tenpo_cd=payload.tenpo_cd,
|
||||
shop_name=shop_name,
|
||||
query=payload.search_word,
|
||||
page=payload.page,
|
||||
page_size=page_size,
|
||||
total_count=total_count,
|
||||
has_more=has_more,
|
||||
items=items,
|
||||
)
|
||||
|
||||
async def fetch_shop_info(self, payload: ShopInfoRequest) -> ShopInfoData:
|
||||
"""抓取加盟店/市场店铺信息
|
||||
|
||||
基础信息来自店铺主页 /shop/{tenpo_cd}；当 include_details=True 时，
|
||||
额外并发抓取 配送/ポリシー/返品保証/連絡 四个子页（多付出四次页面请求），
|
||||
单个子页抓取或解析失败降级为 None，不影响主页信息返回。
|
||||
|
||||
Args:
|
||||
payload: 店铺信息请求参数，包含 tenpo_cd、include_details
|
||||
|
||||
Returns:
|
||||
ShopInfoData: 店铺信息
|
||||
|
||||
Raises:
|
||||
ScrapeParseError: 店铺主页解析失败（如店铺不存在）
|
||||
"""
|
||||
tenpo_cd = payload.tenpo_cd
|
||||
target_url = shop_url(tenpo_cd)
|
||||
logger.info("开始抓取店铺信息：tenpo_cd=%s include_details=%s", tenpo_cd, payload.include_details)
|
||||
|
||||
html, _ = await self._session_manager.fetch_html(target_url)
|
||||
info = self._parse_shop_info(HTMLParser(html), tenpo_cd)
|
||||
|
||||
if payload.include_details:
|
||||
delivery, policy, service, contact = await asyncio.gather(
|
||||
self._fetch_shop_section(tenpo_cd, "delivery", self._parse_shop_delivery),
|
||||
self._fetch_shop_section(tenpo_cd, "policy", self._parse_shop_policy),
|
||||
self._fetch_shop_section(tenpo_cd, "service", self._parse_shop_service),
|
||||
self._fetch_shop_section(tenpo_cd, "contact", self._parse_shop_contact),
|
||||
)
|
||||
info.delivery = delivery
|
||||
info.policy = policy
|
||||
info.service = service
|
||||
info.contact = contact
|
||||
|
||||
logger.debug("店铺信息解析完成：tenpo_cd=%s shop_name=%s", info.tenpo_cd, info.shop_name)
|
||||
return info
|
||||
|
||||
async def _fetch_shop_section(self, tenpo_cd: str, section: str, parse: Any) -> Any:
|
||||
"""抓取并解析单个店铺子页；抓取或解析失败时降级为 None，不影响主流程。"""
|
||||
section_url = shop_section_url(section, tenpo_cd)
|
||||
try:
|
||||
html, _ = await self._session_manager.fetch_html(section_url)
|
||||
except Exception as exc:
|
||||
logger.warning("抓取店铺子页失败：tenpo_cd=%s section=%s err=%s", tenpo_cd, section, exc)
|
||||
return None
|
||||
|
||||
try:
|
||||
return parse(HTMLParser(html))
|
||||
except Exception:
|
||||
logger.exception("解析店铺子页失败：tenpo_cd=%s section=%s", tenpo_cd, section)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_shop_info(tree: HTMLParser, tenpo_cd: str) -> ShopInfoData:
|
||||
"""解析店铺主页 /shop/{tenpo_cd} 的基础信息（店名、logo、评分、公告）。"""
|
||||
brand = tree.css_first(".shop_brand")
|
||||
if brand is None:
|
||||
raise ScrapeParseError(f"店铺信息解析失败：tenpo_cd={tenpo_cd}")
|
||||
|
||||
name_node = brand.css_first(".shop_info h1")
|
||||
shop_name = name_node.text().strip() if name_node else ""
|
||||
if not shop_name:
|
||||
raise ScrapeParseError(f"店铺名称解析失败：tenpo_cd={tenpo_cd}")
|
||||
|
||||
logo_node = brand.css_first(".shop_logo img")
|
||||
logo_src = (logo_node.attributes.get("src") or "") if logo_node else ""
|
||||
logo_url = SurugayaClient._normalize_url(logo_src) if logo_src else ""
|
||||
|
||||
# 评分区形如：<div class="pull-left padR20">5.0</div><div class="pull-right">(1345件)</div>
|
||||
rating_score = ""
|
||||
rating_count = 0
|
||||
point_node = brand.css_first(".shop_info .point")
|
||||
if point_node:
|
||||
score_node = point_node.css_first(".pull-left")
|
||||
if score_node:
|
||||
score_match = re.search(r"[\d.]+", score_node.text())
|
||||
if score_match:
|
||||
rating_score = score_match.group(0)
|
||||
count_node = point_node.css_first(".pull-right")
|
||||
if count_node:
|
||||
count_match = re.search(r"(\d+)", count_node.text().replace(",", ""))
|
||||
if count_match:
|
||||
rating_count = int(count_match.group(1))
|
||||
|
||||
notice_node = tree.css_first("#search_result .contact-top .content_text")
|
||||
notice = SurugayaClient._clean_block_text(notice_node)
|
||||
|
||||
return ShopInfoData(
|
||||
tenpo_cd=tenpo_cd,
|
||||
shop_name=shop_name,
|
||||
shop_url=shop_url(tenpo_cd),
|
||||
logo_url=logo_url,
|
||||
rating_score=rating_score,
|
||||
rating_count=rating_count,
|
||||
notice=notice,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_shop_delivery(tree: HTMLParser) -> ShopDeliveryInfo:
|
||||
"""解析店铺「配送」子页：发货说明 + 按都道府县的运费表。"""
|
||||
description = SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text"))
|
||||
|
||||
shipping_fee_list: list[ShopShippingFeeItem] = []
|
||||
for row in tree.css("#main2 table.table-bordered tr"):
|
||||
tds = row.css("td")
|
||||
if len(tds) < 2:
|
||||
# 表头行（th）无 td，跳过
|
||||
continue
|
||||
prefecture = tds[0].text().strip()
|
||||
fee = format_price(tds[1].text().strip()) or 0
|
||||
if prefecture:
|
||||
shipping_fee_list.append(ShopShippingFeeItem(prefecture=prefecture, fee=fee))
|
||||
|
||||
return ShopDeliveryInfo(description=description, shipping_fee_list=shipping_fee_list)
|
||||
|
||||
@staticmethod
|
||||
def _parse_shop_policy(tree: HTMLParser) -> ShopPolicyInfo:
|
||||
"""解析店铺「ポリシー」子页：政策全文（含特定商取引法表记）。"""
|
||||
return ShopPolicyInfo(description=SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text")))
|
||||
|
||||
@staticmethod
|
||||
def _parse_shop_service(tree: HTMLParser) -> ShopServiceInfo:
|
||||
"""解析店铺「返品、保証、払い戻し」子页。
|
||||
|
||||
页面结构为 h4 标题与 content_text 正文交替排列，按出现顺序配对，
|
||||
再依据标题文案分派到 return_policy / warranty。
|
||||
"""
|
||||
info = ShopServiceInfo()
|
||||
container = tree.css_first("#main2 #search_result > div")
|
||||
if container is None:
|
||||
return info
|
||||
|
||||
current_title = ""
|
||||
for node in container.iter():
|
||||
if node.tag == "h4":
|
||||
current_title = node.text().strip()
|
||||
continue
|
||||
if "content_text" not in (node.attributes.get("class") or ""):
|
||||
continue
|
||||
|
||||
text = SurugayaClient._clean_block_text(node)
|
||||
if "保証" in current_title:
|
||||
info.warranty = text
|
||||
elif "返品" in current_title or "払い戻し" in current_title:
|
||||
info.return_policy = text
|
||||
|
||||
return info
|
||||
|
||||
@staticmethod
|
||||
def _parse_shop_contact(tree: HTMLParser) -> ShopContactInfo:
|
||||
"""解析店铺「連絡」子页：联系地址与返品地址。"""
|
||||
field_map = {
|
||||
"郵便番号": "postal_code",
|
||||
"都道府県": "prefecture",
|
||||
"市区町村": "city",
|
||||
"番地": "street",
|
||||
"ビル・マンション名": "building",
|
||||
}
|
||||
|
||||
address_list: list[ShopAddress] = []
|
||||
for block in tree.css("#main2 .contact-top"):
|
||||
label_node = block.css_first("h4")
|
||||
if label_node is None:
|
||||
continue
|
||||
|
||||
address = ShopAddress(label=label_node.text().strip())
|
||||
for p in block.css("p"):
|
||||
key, sep, value = p.text().partition("：")
|
||||
if not sep:
|
||||
continue
|
||||
field = field_map.get(key.strip())
|
||||
if field:
|
||||
setattr(address, field, value.strip())
|
||||
|
||||
if address.postal_code or address.prefecture:
|
||||
address_list.append(address)
|
||||
|
||||
return ShopContactInfo(address_list=address_list)
|
||||
|
||||
@staticmethod
|
||||
def _clean_block_text(node: object | None) -> str:
|
||||
"""提取节点文本并归一化空白：去掉行首尾空白与连续空行，保留段落换行。"""
|
||||
if node is None:
|
||||
return ""
|
||||
lines = [line.strip() for line in node.text().splitlines()]
|
||||
return "\n".join(line for line in lines if line)
|
||||
|
||||
@staticmethod
|
||||
def _extract_total_count(tree: HTMLParser) -> int:
|
||||
"""从搜索/店铺列表页头部解析命中总数，格式示例：該当件数:10,428件中 1-24件"""
|
||||
hit_node = tree.css_first("#search_header .search_option .hit")
|
||||
if hit_node is None:
|
||||
return 0
|
||||
match = re.search(r"該当件数:([\d,]+)件", hit_node.text())
|
||||
return int(match.group(1).replace(",", "")) if match else 0
|
||||
|
||||
async def fetch_detail(self, payload: DetailRequest) -> ProductDetailData:
|
||||
"""抓取商品详情页
|
||||
|
||||
@@ -597,6 +829,19 @@ class SurugayaClient:
|
||||
return 1
|
||||
return current_page if current_page >= 1 else 1
|
||||
|
||||
def _build_shop_items_url(self, payload: ShopItemsRequest) -> str:
|
||||
"""构建店铺商品列表 URL
|
||||
|
||||
店内商品列表即站点搜索页按 tenpo_code 过滤的结果；
|
||||
tenpo_cd 转成站点侧的 tenpo_code 参数，其余字段（rankBy、category 等）原样透传。
|
||||
"""
|
||||
params = payload.model_dump(exclude_none=True)
|
||||
params["tenpo_code"] = params.pop("tenpo_cd")
|
||||
|
||||
query_string = urlencode(params, doseq=True)
|
||||
base_url = self._settings.base_url.rstrip("/")
|
||||
return f"{base_url}/search?{query_string}"
|
||||
|
||||
def _build_product_url(
|
||||
self,
|
||||
product_id: str | None,
|
||||
|
||||
Reference in New Issue
Block a user