diff --git a/README.md b/README.md index 270c486..12873d2 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,9 @@ - `GET /health` - `POST /api/search` - `POST /api/item_detail` +- `POST /api/item_other_shops` +- `POST /api/shop_info`、`POST /api/shop_items`(加盟店店铺资料与在售商品,见 [docs/api_shop.md](docs/api_shop.md)) +- `GET /api/news`(见 [docs/api_news.md](docs/api_news.md)) ## 部署 diff --git a/app/api/routes/scrape.py b/app/api/routes/scrape.py index e54fe48..5f1f860 100644 --- a/app/api/routes/scrape.py +++ b/app/api/routes/scrape.py @@ -22,7 +22,11 @@ from app.models.scrape import ( ProductDetailData, PurchaseTaskRequest, SearchRequest, - SearchResultData, TradeMonitorRequest, + SearchResultData, + ShopInfoData, + ShopInfoRequest, + ShopItemsData, + ShopItemsRequest, TradeMonitorRequest, ) from surugaya_common.http_utils import generate_signed_headers from app.utils.parse_util import ( @@ -146,6 +150,53 @@ async def item_other_shops( ) +@router.post( + "/shop_info", + response_model=ApiResponse[ShopInfoData], + dependencies=[Depends(require_bearer_token)], +) +async def shop_info( + payload: ShopInfoRequest, + container: ServiceContainer = Depends(get_container), +) -> ApiResponse[ShopInfoData]: + """获取加盟店/市场店铺信息 + + 抓取骏河屋 /shop/{tenpo_cd} 页面,返回店名、logo、评分、评价数与店铺公告; + include_details=true 时额外抓取 配送(含都道府县运费表)/ポリシー(特商法表记)/ + 返品保証/連絡(地址)四个子页,多付出四次页面请求。 + """ + data = await container.surugaya_client.fetch_shop_info(payload) + return ApiResponse[ShopInfoData]( + success=True, + msg="success", + data=data, + code=0, + ) + + +@router.post( + "/shop_items", + response_model=ApiResponse[ShopItemsData], + dependencies=[Depends(require_bearer_token)], +) +async def shop_items( + payload: ShopItemsRequest, + container: ServiceContainer = Depends(get_container), +) -> ApiResponse[ShopItemsData]: + """获取加盟店/市场店铺商品列表 + + 按 tenpo_cd 抓取该店铺在售商品,支持店内关键词检索与分页(24 条/页); + rankBy、category 等搜索参数可原样透传。返回项的 goods_link 已带 tenpo_cd。 + """ + data = await container.surugaya_client.fetch_shop_items(payload) + return ApiResponse[ShopItemsData]( + success=True, + msg="success", + data=data, + code=0, + ) + + @router.get( "/categories", response_model=ApiResponse[list[CategoryData]], diff --git a/app/models/scrape.py b/app/models/scrape.py index 6f91260..97df306 100644 --- a/app/models/scrape.py +++ b/app/models/scrape.py @@ -114,6 +114,90 @@ class OtherShopListData(BaseModel): other_shop_list: list[OtherShopItem] = [] +class ShopInfoRequest(BaseModel): + """加盟店/市场店铺信息请求参数""" + tenpo_cd: str = Field(min_length=1, max_length=20, pattern=r"^[A-Za-z0-9_-]+$") + include_details: bool = False # 为 True 时额外抓取 配送/ポリシー/返品保証/連絡 四个子页(多付出四次页面请求) + + +class ShopShippingFeeItem(BaseModel): + """店铺「配送」页中按都道府县列出的一条运费""" + prefecture: str = "" # 都道府县名,如"東京都" + fee: int = 0 # 该都道府县运费(日元) + + +class ShopDeliveryInfo(BaseModel): + """店铺「配送」子页信息""" + description: str = "" # 发货说明原文(日文) + shipping_fee_list: list[ShopShippingFeeItem] = [] # 按都道府县的运费表 + + +class ShopPolicyInfo(BaseModel): + """店铺「ポリシー」子页信息(含特定商取引法表记)""" + description: str = "" # 政策全文(日文) + + +class ShopServiceInfo(BaseModel): + """店铺「返品、保証、払い戻し」子页信息""" + return_policy: str = "" # 返品、払い戻し 段落 + warranty: str = "" # 保証 段落 + + +class ShopAddress(BaseModel): + """店铺地址(联系地址或返品地址)""" + label: str = "" # 地址类型,如"住所"/"返品先" + postal_code: str = "" # 邮编 + prefecture: str = "" # 都道府县 + city: str = "" # 市区町村 + street: str = "" # 番地 + building: str = "" # 大楼/公寓名(可能为空) + + +class ShopContactInfo(BaseModel): + """店铺「連絡」子页信息""" + address_list: list[ShopAddress] = [] + + +class ShopInfoData(BaseModel): + """加盟店/市场店铺信息 + + 基础字段来自 /shop/{tenpo_cd} 主页;delivery/policy/service/contact + 仅当请求 include_details=True 时才会填充,否则为 null。 + """ + tenpo_cd: str + shop_name: str = "" # 店铺名称 + shop_url: str = "" # 店铺主页链接 + logo_url: str = "" # 店铺 logo 图片链接 + rating_score: str = "" # 店铺评分,如"5.0" + rating_count: int = 0 # 评价数量 + notice: str = "" # 店铺公告(ショップ情報)原文 + delivery: ShopDeliveryInfo | None = None + policy: ShopPolicyInfo | None = None + service: ShopServiceInfo | None = None + contact: ShopContactInfo | None = None + + +class ShopItemsRequest(BaseModel): + """加盟店/市场店铺商品列表请求参数""" + model_config = ConfigDict(extra="allow") + + tenpo_cd: str = Field(min_length=1, max_length=20, pattern=r"^[A-Za-z0-9_-]+$") + search_word: str = "" # 店内关键词检索,留空为全部商品 + page: int = Field(default=1, ge=1, le=100) + + +class ShopItemsData(BaseModel): + """加盟店/市场店铺商品列表数据""" + tenpo_cd: str + shop_name: str = "" # 店铺名称,从列表页标题解析 + query: str = "" + page: int = 1 + page_size: int = 0 + total_count: int = 0 + has_more: int = 0 + items: list[ProductSummary] = [] + + class HealthData(BaseModel): """健康检查响应数据""" status: str diff --git a/app/services/surugaya_client.py b/app/services/surugaya_client.py index 8c5e79c..fe450f4 100644 --- a/app/services/surugaya_client.py +++ b/app/services/surugaya_client.py @@ -20,13 +20,14 @@ from selectolax.parser import HTMLParser from app.core.config import Settings from app.core.errors import ScrapeParseError, UpstreamBlockedError from app.models.scrape import CategoryData, DetailRequest, OtherShopItem, OtherShopListData, ProductDetailData, \ - ProductSummary, SearchRequest, SearchResultData + ProductSummary, SearchRequest, SearchResultData, ShopAddress, ShopContactInfo, ShopDeliveryInfo, ShopInfoData, \ + ShopInfoRequest, ShopItemsData, ShopItemsRequest, ShopPolicyInfo, ShopServiceInfo, ShopShippingFeeItem from app.services.browser_pool import BrowserPool from app.services.cloudflare_session import CloudflareSessionManager from app.services.session_store import SessionStore from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn from surugaya_common.app_utils import is_empty_str -from surugaya_common.urls import BASE_URL, product_other_url +from surugaya_common.urls import BASE_URL, product_other_url, shop_section_url, shop_url logger = logging.getLogger(__name__) @@ -98,18 +99,9 @@ class SurugayaClient: # 解析商品总数量,格式示例:該当件数:10,428件中 1-24件 current_page = self._extract_page_value(payload) - total_count = 0 page_size = 24 - has_more = 0 - hit_node = tree.css_first("#search_header .search_option .hit") - if hit_node: - raw_text = hit_node.text() - match = re.search(r'該当件数:([\d,]+)件', raw_text) - if match: - total_count = int(match.group(1).replace(',', '')) - - if total_count > 0: - has_more = int(current_page * page_size < total_count) + total_count = self._extract_total_count(tree) + has_more = int(current_page * page_size < total_count) if total_count > 0 else 0 # 解析商品列表 items = self._parse_search_items(tree) @@ -125,6 +117,246 @@ class SurugayaClient: session_id=session.session_id, ) + async def fetch_shop_items(self, payload: ShopItemsRequest) -> ShopItemsData: + """抓取加盟店/市场店铺的商品列表 + + 店铺商品列表复用站点的搜索页(/search?tenpo_code=xxx), + DOM 结构与普通搜索完全一致,因此解析逻辑与 search 共用。 + + Args: + payload: 店铺商品请求参数,包含 tenpo_cd、search_word、page 等 + + Returns: + ShopItemsData: 店铺名称、商品列表、总数和分页信息 + """ + url = self._build_shop_items_url(payload) + logger.info("开始抓取店铺商品列表:tenpo_cd=%s page=%s", payload.tenpo_cd, payload.page) + logger.debug("店铺商品列表URL:url=%s", url) + + html, _ = await self._session_manager.fetch_html(url) + tree = HTMLParser(html) + + page_size = 24 + total_count = self._extract_total_count(tree) + has_more = int(payload.page * page_size < total_count) if total_count > 0 else 0 + + # 列表页标题形如「駿河屋 山口大学前店の商品一覧」,去掉后缀即为店名 + shop_name = "" + title_node = tree.css_first("#search_header .search_option .hit h2") + if title_node: + shop_name = re.sub(r"の商品一覧$", "", title_node.text().strip()) + + items = self._parse_search_items(tree) + logger.info("店铺商品列表解析完成 ===> tenpo_cd=%s 抓取商品数=%s", payload.tenpo_cd, len(items)) + + return ShopItemsData( + tenpo_cd=payload.tenpo_cd, + shop_name=shop_name, + query=payload.search_word, + page=payload.page, + page_size=page_size, + total_count=total_count, + has_more=has_more, + items=items, + ) + + async def fetch_shop_info(self, payload: ShopInfoRequest) -> ShopInfoData: + """抓取加盟店/市场店铺信息 + + 基础信息来自店铺主页 /shop/{tenpo_cd};当 include_details=True 时, + 额外并发抓取 配送/ポリシー/返品保証/連絡 四个子页(多付出四次页面请求), + 单个子页抓取或解析失败降级为 None,不影响主页信息返回。 + + Args: + payload: 店铺信息请求参数,包含 tenpo_cd、include_details + + Returns: + ShopInfoData: 店铺信息 + + Raises: + ScrapeParseError: 店铺主页解析失败(如店铺不存在) + """ + tenpo_cd = payload.tenpo_cd + target_url = shop_url(tenpo_cd) + logger.info("开始抓取店铺信息:tenpo_cd=%s include_details=%s", tenpo_cd, payload.include_details) + + html, _ = await self._session_manager.fetch_html(target_url) + info = self._parse_shop_info(HTMLParser(html), tenpo_cd) + + if payload.include_details: + delivery, policy, service, contact = await asyncio.gather( + self._fetch_shop_section(tenpo_cd, "delivery", self._parse_shop_delivery), + self._fetch_shop_section(tenpo_cd, "policy", self._parse_shop_policy), + self._fetch_shop_section(tenpo_cd, "service", self._parse_shop_service), + self._fetch_shop_section(tenpo_cd, "contact", self._parse_shop_contact), + ) + info.delivery = delivery + info.policy = policy + info.service = service + info.contact = contact + + logger.debug("店铺信息解析完成:tenpo_cd=%s shop_name=%s", info.tenpo_cd, info.shop_name) + return info + + async def _fetch_shop_section(self, tenpo_cd: str, section: str, parse: Any) -> Any: + """抓取并解析单个店铺子页;抓取或解析失败时降级为 None,不影响主流程。""" + section_url = shop_section_url(section, tenpo_cd) + try: + html, _ = await self._session_manager.fetch_html(section_url) + except Exception as exc: + logger.warning("抓取店铺子页失败:tenpo_cd=%s section=%s err=%s", tenpo_cd, section, exc) + return None + + try: + return parse(HTMLParser(html)) + except Exception: + logger.exception("解析店铺子页失败:tenpo_cd=%s section=%s", tenpo_cd, section) + return None + + @staticmethod + def _parse_shop_info(tree: HTMLParser, tenpo_cd: str) -> ShopInfoData: + """解析店铺主页 /shop/{tenpo_cd} 的基础信息(店名、logo、评分、公告)。""" + brand = tree.css_first(".shop_brand") + if brand is None: + raise ScrapeParseError(f"店铺信息解析失败:tenpo_cd={tenpo_cd}") + + name_node = brand.css_first(".shop_info h1") + shop_name = name_node.text().strip() if name_node else "" + if not shop_name: + raise ScrapeParseError(f"店铺名称解析失败:tenpo_cd={tenpo_cd}") + + logo_node = brand.css_first(".shop_logo img") + logo_src = (logo_node.attributes.get("src") or "") if logo_node else "" + logo_url = SurugayaClient._normalize_url(logo_src) if logo_src else "" + + # 评分区形如:
5.0
(1345件)
+ rating_score = "" + rating_count = 0 + point_node = brand.css_first(".shop_info .point") + if point_node: + score_node = point_node.css_first(".pull-left") + if score_node: + score_match = re.search(r"[\d.]+", score_node.text()) + if score_match: + rating_score = score_match.group(0) + count_node = point_node.css_first(".pull-right") + if count_node: + count_match = re.search(r"(\d+)", count_node.text().replace(",", "")) + if count_match: + rating_count = int(count_match.group(1)) + + notice_node = tree.css_first("#search_result .contact-top .content_text") + notice = SurugayaClient._clean_block_text(notice_node) + + return ShopInfoData( + tenpo_cd=tenpo_cd, + shop_name=shop_name, + shop_url=shop_url(tenpo_cd), + logo_url=logo_url, + rating_score=rating_score, + rating_count=rating_count, + notice=notice, + ) + + @staticmethod + def _parse_shop_delivery(tree: HTMLParser) -> ShopDeliveryInfo: + """解析店铺「配送」子页:发货说明 + 按都道府县的运费表。""" + description = SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text")) + + shipping_fee_list: list[ShopShippingFeeItem] = [] + for row in tree.css("#main2 table.table-bordered tr"): + tds = row.css("td") + if len(tds) < 2: + # 表头行(th)无 td,跳过 + continue + prefecture = tds[0].text().strip() + fee = format_price(tds[1].text().strip()) or 0 + if prefecture: + shipping_fee_list.append(ShopShippingFeeItem(prefecture=prefecture, fee=fee)) + + return ShopDeliveryInfo(description=description, shipping_fee_list=shipping_fee_list) + + @staticmethod + def _parse_shop_policy(tree: HTMLParser) -> ShopPolicyInfo: + """解析店铺「ポリシー」子页:政策全文(含特定商取引法表记)。""" + return ShopPolicyInfo(description=SurugayaClient._clean_block_text(tree.css_first("#main2 .content_text"))) + + @staticmethod + def _parse_shop_service(tree: HTMLParser) -> ShopServiceInfo: + """解析店铺「返品、保証、払い戻し」子页。 + + 页面结构为 h4 标题与 content_text 正文交替排列,按出现顺序配对, + 再依据标题文案分派到 return_policy / warranty。 + """ + info = ShopServiceInfo() + container = tree.css_first("#main2 #search_result > div") + if container is None: + return info + + current_title = "" + for node in container.iter(): + if node.tag == "h4": + current_title = node.text().strip() + continue + if "content_text" not in (node.attributes.get("class") or ""): + continue + + text = SurugayaClient._clean_block_text(node) + if "保証" in current_title: + info.warranty = text + elif "返品" in current_title or "払い戻し" in current_title: + info.return_policy = text + + return info + + @staticmethod + def _parse_shop_contact(tree: HTMLParser) -> ShopContactInfo: + """解析店铺「連絡」子页:联系地址与返品地址。""" + field_map = { + "郵便番号": "postal_code", + "都道府県": "prefecture", + "市区町村": "city", + "番地": "street", + "ビル・マンション名": "building", + } + + address_list: list[ShopAddress] = [] + for block in tree.css("#main2 .contact-top"): + label_node = block.css_first("h4") + if label_node is None: + continue + + address = ShopAddress(label=label_node.text().strip()) + for p in block.css("p"): + key, sep, value = p.text().partition(":") + if not sep: + continue + field = field_map.get(key.strip()) + if field: + setattr(address, field, value.strip()) + + if address.postal_code or address.prefecture: + address_list.append(address) + + return ShopContactInfo(address_list=address_list) + + @staticmethod + def _clean_block_text(node: object | None) -> str: + """提取节点文本并归一化空白:去掉行首尾空白与连续空行,保留段落换行。""" + if node is None: + return "" + lines = [line.strip() for line in node.text().splitlines()] + return "\n".join(line for line in lines if line) + + @staticmethod + def _extract_total_count(tree: HTMLParser) -> int: + """从搜索/店铺列表页头部解析命中总数,格式示例:該当件数:10,428件中 1-24件""" + hit_node = tree.css_first("#search_header .search_option .hit") + if hit_node is None: + return 0 + match = re.search(r"該当件数:([\d,]+)件", hit_node.text()) + return int(match.group(1).replace(",", "")) if match else 0 + async def fetch_detail(self, payload: DetailRequest) -> ProductDetailData: """抓取商品详情页 @@ -597,6 +829,19 @@ class SurugayaClient: return 1 return current_page if current_page >= 1 else 1 + def _build_shop_items_url(self, payload: ShopItemsRequest) -> str: + """构建店铺商品列表 URL + + 店内商品列表即站点搜索页按 tenpo_code 过滤的结果; + tenpo_cd 转成站点侧的 tenpo_code 参数,其余字段(rankBy、category 等)原样透传。 + """ + params = payload.model_dump(exclude_none=True) + params["tenpo_code"] = params.pop("tenpo_cd") + + query_string = urlencode(params, doseq=True) + base_url = self._settings.base_url.rstrip("/") + return f"{base_url}/search?{query_string}" + def _build_product_url( self, product_id: str | None, diff --git a/packages/surugaya_common/src/surugaya_common/urls.py b/packages/surugaya_common/src/surugaya_common/urls.py index 8e7dd94..11d28f9 100644 --- a/packages/surugaya_common/src/surugaya_common/urls.py +++ b/packages/surugaya_common/src/surugaya_common/urls.py @@ -27,6 +27,16 @@ def product_other_url(product_id: str) -> str: return f"{BASE_URL}/product/other/{product_id}" +def shop_url(tenpo_cd: str) -> str: + """加盟店/市场店铺主页地址:店名、logo、评分与店铺公告。""" + return f"{BASE_URL}/shop/{tenpo_cd}" + + +def shop_section_url(section: str, tenpo_cd: str) -> str: + """店铺子页地址,section 取 delivery/policy/service/contact/order/rating。""" + return f"{BASE_URL}/shop/{section}/{tenpo_cd}" + + def trade_detail_url(trade_code: str) -> str: """交易(订单)详情页地址。""" return f"{BASE_URL}/pcmypage/action_sell_search/detail?trade_code={trade_code}" diff --git a/tests/test_common_contracts.py b/tests/test_common_contracts.py index 6321b7a..72dfa2f 100644 --- a/tests/test_common_contracts.py +++ b/tests/test_common_contracts.py @@ -7,6 +7,8 @@ from surugaya_common.urls import ( CARGO_DETAIL_URL, MYPAGE_URL, product_detail_url, + shop_section_url, + shop_url, trade_detail_url, ) @@ -40,6 +42,11 @@ class TestUrls: == "https://www.suruga-ya.jp/pcmypage/action_sell_search/detail?trade_code=T123456" ) + def test_shop_urls(self): + assert shop_url("400451") == "https://www.suruga-ya.jp/shop/400451" + assert shop_section_url("delivery", "400451") == "https://www.suruga-ya.jp/shop/delivery/400451" + assert shop_section_url("contact", "410118") == "https://www.suruga-ya.jp/shop/contact/410118" + class TestFormatJapanesePrice: def test_supported_formats(self):