Init

app/services/__init__.py

@@ -0,0 +1 @@
+"""Service layer implementations."""

app/services/browser_pool.py

@@ -0,0 +1,299 @@
+"""浏览器池：管理 Playwright 浏览器实例的生命周期和并发访问
+
+核心能力：
+- 启动/关闭 Chromium 浏览器实例（进程级单例）
+- 通过信号量控制并发页面数量，避免 API 并发把浏览器资源打爆
+- 支持本地调试模式下保留浏览器窗口（出错或成功时），方便人工排查
+"""
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+from app.core.config import Settings
+from app.core.errors import BrowserUnavailableError, ResourceBusyError
+
+logger = logging.getLogger(__name__)
+
+
+class BrowserPool:
+    """浏览器实例池，提供并发安全的页面会话获取"""
+
+    def __init__(self, settings: Settings):
+        self._settings = settings
+        self._playwright: Any | None = None
+        self._browser: Any | None = None
+        self._semaphore = asyncio.Semaphore(settings.max_site_concurrency)
+        self._restart_lock = asyncio.Lock()
+        self._started = False
+        self._startup_error: str | None = None
+        self._retained_contexts: list[Any] = []
+        self._retained_pages: list[Any] = []
+
+    @property
+    def enabled(self) -> bool:
+        """浏览器功能是否启用"""
+        return self._settings.browser_enabled
+
+    @property
+    def ready(self) -> bool:
+        """浏览器是否已就绪（未启用或已启动成功均视为就绪）"""
+        return not self.enabled or self._started
+
+    @property
+    def startup_error(self) -> str | None:
+        """浏览器启动失败时的错误信息"""
+        return self._startup_error
+
+    async def start(self) -> None:
+        """启动 Playwright 浏览器实例
+
+        仅在 browser_enabled=True 且尚未启动时执行。
+        启动失败会记录错误信息到 startup_error，但不会抛异常（服务仍可启动）。
+        """
+        if not self.enabled or self._started:
+            return
+
+        try:
+            from playwright.async_api import async_playwright
+
+            headless = self._settings.browser_headless_effective
+            logger.info(
+                "启动浏览器：enabled=%s headless=%s keep_open_on_error=%s channel=%s proxy=%s env=%s",
+                self._settings.browser_enabled,
+                headless,
+                self._settings.browser_keep_open_on_error_effective,
+                self._settings.browser_channel,
+                bool(self._settings.proxy_server),
+                self._settings.app_env,
+            )
+            self._playwright = await async_playwright().start()
+            chromium = self._playwright.chromium
+            launch_args = [ 
+                "--disable-blink-features=AutomationControlled",
+                "--disable-dev-shm-usage",
+                "--disable-web-security",
+                "--no-first-run",
+                "--disable-infobars",
+                "--no-sandbox",
+                "--disable-setuid-sandbox",
+                "--disable-gpu",
+                "--use-mock-keychain",
+                "--password-store=basic",
+            ]
+            # 本地可视化调试时强制窗口化，避免浏览器启动后不可见
+            if not headless:
+                launch_args.extend(["--start-maximized", "--window-position=120,120"])
+            self._browser = await chromium.launch(
+                headless=headless,
+                channel=self._settings.browser_channel,
+                proxy=self._settings.proxy,
+                args=launch_args,
+            )
+            self._started = True
+            self._startup_error = None
+            logger.info("浏览器启动完成")
+        except Exception as exc:
+            self._playwright = None
+            self._browser = None
+            self._started = False
+            self._startup_error = str(exc)
+            logger.exception("浏览器启动失败：%s", self._startup_error)
+
+    def _is_browser_connected(self) -> bool:
+        """检查当前浏览器对象是否仍与 Playwright driver 保持连接。"""
+        browser = self._browser
+        if not self._started or browser is None:
+            return False
+
+        is_connected = getattr(browser, "is_connected", None)
+        if callable(is_connected):
+            try:
+                return bool(is_connected())
+            except Exception:
+                return False
+
+        return True
+
+    async def close(self) -> None:
+        """关闭浏览器与 Playwright 运行时，通常发生在服务退出时"""
+        self._retained_pages.clear()
+        for context in self._retained_contexts:
+            try:
+                await context.close()
+            except Exception:
+                logger.exception("关闭保留的调试上下文失败")
+        self._retained_contexts.clear()
+        if self._browser is not None:
+            try:
+                await self._browser.close()
+            except Exception:
+                logger.exception("关闭浏览器实例失败")
+        if self._playwright is not None:
+            try:
+                await self._playwright.stop()
+            except Exception:
+                logger.exception("停止 Playwright 运行时失败")
+        self._browser = None
+        self._playwright = None
+        self._started = False
+
+    async def _restart_browser(self, reason: str) -> None:
+        """在浏览器掉线后串行重建 Playwright 运行时。"""
+        async with self._restart_lock:
+            if self._is_browser_connected():
+                return
+
+            logger.warning("检测到浏览器连接不可用，准备重建浏览器实例：reason=%s", reason)
+            await self.close()
+            await self.start()
+            if not self._is_browser_connected():
+                raise BrowserUnavailableError(self._startup_error or f"Browser restart failed: {reason}")
+
+    async def _ensure_browser_ready(self) -> None:
+        """确保浏览器池处于可用状态；若连接已断开则自动重建。"""
+        if not self.enabled:
+            raise BrowserUnavailableError("Browser support is disabled")
+
+        if self._is_browser_connected():
+            return
+
+        await self._restart_browser("browser is disconnected before acquiring page session")
+
+    @staticmethod
+    def _should_retry_context_creation(exc: Exception) -> bool:
+        """识别 Playwright driver 断链类错误，允许执行一次重建后重试。"""
+        message = str(exc).lower()
+        retry_markers = (
+            "connection closed while reading from the driver",
+            "browser has been closed",
+            "target page, context or browser has been closed",
+            "connection closed",
+        )
+        return any(marker in message for marker in retry_markers)
+
+    async def _retain_context(self, context: Any, page: Any | None, reason: str) -> None:
+        """保留浏览器上下文用于本地调试排查
+
+        当本地调试模式开启时，出错或成功后不关闭浏览器窗口，
+        方便人工查看页面状态。保留数量超过池大小时自动关闭最早的。
+        """
+        if page is not None:
+            try:
+                await page.bring_to_front()
+                logger.debug("%s：已将页面置顶：url=%s", reason, page.url)
+            except Exception:
+                logger.exception("%s：页面置顶失败", reason)
+            self._retained_pages.append(page)
+        self._retained_contexts.append(context)
+
+        # 超出保留上限时关闭最早的上下文
+        limit = max(1, int(self._settings.browser_pool_size))
+        while len(self._retained_contexts) > limit:
+            old = self._retained_contexts.pop(0)
+            try:
+                await old.close()
+            except Exception:
+                logger.exception("关闭旧的保留调试上下文失败")
+
+    @asynccontextmanager
+    async def page_session(
+        self,
+        storage_state: dict[str, Any] | None = None,
+        *,
+        keep_open_on_success: bool | None = None,
+    ) -> AsyncIterator[tuple[Any, Any]]:
+        """获取一个浏览器页面会话（上下文管理器）
+
+        内部通过信号量做并发控制，超时未获取到槽位会抛出 ResourceBusyError。
+        页面创建时会设置日语环境、User-Agent 等反检测参数。
+
+        Args:
+            storage_state: 浏览器存储状态（含 Cookie），用于会话复用
+            keep_open_on_success: 请求成功后是否保留浏览器窗口（用于本地调试）
+
+        Yields:
+            (context, page) 元组，context 为浏览器上下文，page 为页面实例
+
+        Raises:
+            BrowserUnavailableError: 浏览器未启用或未启动
+            ResourceBusyError: 等待可用槽位超时
+        """
+        await self._ensure_browser_ready()
+
+        should_keep_open_on_success = (
+            self._settings.browser_keep_open_effective
+            if keep_open_on_success is None
+            else keep_open_on_success
+        )
+
+        try:
+            await asyncio.wait_for(
+                self._semaphore.acquire(),
+                timeout=self._settings.page_acquire_timeout_seconds,
+            )
+        except TimeoutError as exc:
+            raise ResourceBusyError() from exc
+
+        context = None
+        page = None
+        try:
+            for attempt in range(2):
+                try:
+                    context = await self._browser.new_context(
+                        locale=self._settings.browser_locale,
+                        timezone_id=self._settings.browser_timezone_id,
+                        user_agent=self._settings.browser_user_agent,
+                        viewport=None if not self._settings.browser_headless_effective else {"width": 1440, "height": 900},
+                        storage_state=storage_state,
+                    )
+                    break
+                except Exception as exc:
+                    if attempt == 0 and self._should_retry_context_creation(exc):
+                        logger.warning("创建浏览器上下文失败，尝试重建浏览器后重试一次：err=%s", exc)
+                        await self._restart_browser(f"new_context failed: {exc}")
+                        continue
+                    raise
+
+            if self._settings.browser_stealth_enabled:
+                try:
+                    from playwright_stealth import Stealth
+
+                    await Stealth().apply_stealth_async(context)
+                    # logger.debug("已为浏览器上下文应用 Stealth 模式")
+                except Exception as e:
+                    logger.warning("应用 playwright-stealth 失败，隐身模式未生效, err=%s", e)
+
+            page = await context.new_page()
+            await page.set_extra_http_headers(
+                {
+                    "Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
+                    "Cache-Control": "no-cache",
+                    "Pragma": "no-cache",
+                }
+            )
+            yield context, page
+        except Exception:
+            if self._settings.browser_keep_open_on_error_effective and context is not None:
+                await self._retain_context(context, page, reason="抓取失败，已保留页面用于排查")
+                logger.warning(
+                    "本地调试模式已保留浏览器页面用于排查，窗口不会自动关闭；请手动查看页面。当前保留上下文数=%s",
+                    len(self._retained_contexts),
+                )
+                page = None
+                context = None
+            raise
+        finally:
+            # 本地调试模式下保留窗口
+            if context is not None and should_keep_open_on_success:
+                await self._retain_context(context, page, reason="本地调试：已保留页面（请求成功）")
+                page = None
+                context = None
+            if page is not None:
+                await page.close()
+            if context is not None:
+                await context.close()
+            self._semaphore.release()

app/services/cargo_service.py

@@ -0,0 +1,383 @@
+import asyncio
+import logging
+import random
+import re
+
+from playwright.async_api import async_playwright, Page
+from playwright.async_api import TimeoutError as PlaywrightTimeoutError
+from selectolax.parser import HTMLParser
+
+from app.core.config import Settings
+from app.models.scrape import CargoRequest
+from app.utils.parse_util import extract_product_id, format_price
+
+logger = logging.getLogger(__name__)
+
+
+class CargoService:
+    """购物车服务"""
+
+    def __init__(self, settings: Settings):
+        self._settings = settings
+
+    async def get_cargo_info(self, request: CargoRequest) -> dict:
+        """获取购物车信息
+        
+        每次访问都会启动一个全新的 Playwright 浏览器实例，
+        创建全新的上下文，避免任何状态残留。
+        先将指定的商品加入购物车，然后再进入购物车页面解析信息。
+        """
+        async with async_playwright() as p:
+            headless = self._settings.browser_headless_effective
+            launch_args = [
+                "--disable-blink-features=AutomationControlled",
+                "--disable-infobars",
+                "--no-sandbox",
+                "--disable-setuid-sandbox",
+                "--disable-dev-shm-usage",
+                "--disable-gpu",
+                "--use-mock-keychain",
+                "--password-store=basic",
+            ]
+            if not headless:
+                launch_args.extend(["--start-maximized", "--window-position=0,0"])
+
+            # 启动全新的浏览器进程
+            browser = await p.chromium.launch(
+                headless=headless,
+                channel=self._settings.browser_channel,
+                proxy=self._settings.proxy,
+                args=launch_args,
+            )
+
+            try:
+                # 创建全新上下文
+                context = await browser.new_context(
+                    locale=self._settings.browser_locale,
+                    timezone_id=self._settings.browser_timezone_id,
+                    user_agent=self._settings.browser_user_agent,
+                    viewport=None if not headless else {"width": 1440, "height": 900},
+                )
+
+                if self._settings.browser_stealth_enabled:
+                    try:
+                        from playwright_stealth import Stealth
+                        await Stealth().apply_stealth_async(context)
+                    except Exception as e:
+                        logger.warning("应用 playwright-stealth 失败: %s", e)
+
+                page = await context.new_page()
+                await page.set_extra_http_headers(
+                    {
+                        "Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
+                        "Cache-Control": "no-cache",
+                        "Pragma": "no-cache",
+                    }
+                )
+
+                target_url = "https://www.suruga-ya.jp/cargo/detail"
+                # 先进入购物车页面
+                await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
+
+                # 判断购物车是否有商品
+                cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
+
+                # 清空购物车
+                await asyncio.sleep(random.uniform(1, 3))  # 加入随机延迟，模拟人工操作间隔
+                if cart_goods_num > 0:
+                    await self.clear_cart(page)
+
+                # 将待采购的商品加入购物车
+                item_list = request.payload.item_list
+                for index, item in enumerate(item_list):
+                    product_id = item.id
+                    if not product_id:
+                        logger.warning("索引 %s 的商品未提供 id，跳过。", index)
+                        continue
+
+                    number = item.number or 1
+                    if number <= 0:
+                        logger.warning("索引 %s 的商品数量为 0，跳过。", index)
+                        continue
+
+                    logger.debug("进入商品 %s 详情页，准备购买数量: %s...", product_id, number)
+                    detail_url = f"https://www.suruga-ya.jp/product/detail/{product_id}"
+                    await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
+
+                    # 判断是否缺货
+                    out_of_stock_dom = page.locator("div.out-of-stock-text")
+                    is_out_of_stock = await out_of_stock_dom.is_visible()
+                    if is_out_of_stock:
+                        logger.warning("商品 %s 缺货，跳过", product_id)
+                        continue
+
+                    # 提取商品最大购买数量
+                    try:
+                        quantity_select = page.locator("#quantity_selection")
+                        await quantity_select.wait_for(state="visible", timeout=5000)
+                        last_option_locator = quantity_select.locator("option").last
+                        buy_number_limit_str = await last_option_locator.get_attribute("value")
+                        buy_number_limit = int(buy_number_limit_str) if buy_number_limit_str else 0
+                        logger.debug("商品 %s 最大购买数量: %s", product_id, buy_number_limit)
+                        if 0 < buy_number_limit < number:
+                            number = buy_number_limit
+                    except PlaywrightTimeoutError:
+                        logger.debug("商品 %s 没有数量选择框，默认购买数量: %s", product_id, number)
+
+                    try:
+                        btn_buy = page.locator("button.btn_buy").first
+                        await btn_buy.wait_for(state="visible", timeout=5000)
+
+                        if await btn_buy.is_disabled():
+                            logger.warning("商品 %s 已售罄或无法购买（按钮被禁用），跳过", product_id)
+                            continue
+
+                        # 根据 number 循环点击加入购物车
+                        for i in range(number):
+                            logger.debug("点击加入购物车按钮，第 %s 次", i + 1)
+                            await btn_buy.click()
+                            if i < number - 1:
+                                # 多次点击之间加入随机延迟，防止被识别为机器人或触发频率限制
+                                await asyncio.sleep(random.uniform(0.8, 1.5))
+                    except PlaywrightTimeoutError:
+                        logger.warning("商品 %s 找不到购买按钮(可能是已下架/缺货)，跳过", product_id)
+                        continue
+
+                logger.info("再次访问购物车页面: %s", target_url)
+                await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
+
+                # 解析购物车
+                try:
+                    primary = page.locator("#container #main3 #primary")
+                    if not await primary.is_visible(timeout=10000):
+                        logger.warning("购物车商品元素未找到，跳过解析")
+                        raise Exception("购物车商品获取失败")
+
+                    result = await self.parse_cargo_info(page)
+                except Exception as e:
+                    logger.warning("购物车价格元素未找到，跳过解析: %s", e)
+                    raise Exception("购物车价格获取失败")
+
+                return result
+            finally:
+                await browser.close()
+
+    async def parse_cargo_info(self, page: Page):
+        """
+        解析商品总额，运费和手续费金额等信息
+        """
+        logger.debug("解析购物车信息")
+        total_goods_amount = 0
+        total_shipping_fee = 0
+        total_commission_fee = 0
+        items: list[dict] = []
+
+        total_price_dom = page.locator("#total_box div.total_price")
+        try:
+            await total_price_dom.first.wait_for(state="attached", timeout=20000)
+
+            total_price_str = await total_price_dom.locator("p.price_name .total_price").first.text_content()
+            if total_price_str:
+                total_goods_amount = format_price(total_price_str.strip()) or 0
+                logger.debug("商品总额: %s", total_goods_amount)
+
+            delivery_charge_html = await total_price_dom.locator("p.delivery_charge").first.inner_html()
+            if delivery_charge_html:
+                shipping_match = re.search(r"([\d,]+)(?=円の送料)", delivery_charge_html)
+                if shipping_match:
+                    total_shipping_fee = int(shipping_match.group(1).replace(",", ""))
+                    logger.debug("运费: %s", total_shipping_fee)
+
+                handling_match = re.search(r"通信販売手数料.*?:([\d,]+)(?=円)", delivery_charge_html)
+                if handling_match:
+                    total_commission_fee = int(handling_match.group(1).replace(",", ""))
+                    logger.debug("手续费: %s", total_commission_fee)
+        except PlaywrightTimeoutError:
+            pass
+
+        table = page.locator("#container #main3 #primary table.item_box")
+        await table.wait_for(state="visible", timeout=10000)
+
+        rows = table.locator("tr.item_row")
+        for i in range(await rows.count()):
+            row = rows.nth(i)
+            logger.debug("解析商品 %s", i)
+
+            link_node = row.locator("td.photo_box a").first
+            href = await link_node.get_attribute("href") or ""
+
+            img_node = row.locator("td.photo_box img").first
+            goods_image = await img_node.get_attribute("src") or ""
+
+            name_node = row.locator("td.item_detail .item_name a").first
+            goods_name = (await name_node.text_content() or "").strip()
+
+            price_node = row.locator("td.price").first
+            price_text = (await price_node.text_content() or "").strip()
+            goods_price = format_price(price_text) or 0
+
+            quantity = 1
+            qty_input = row.locator("input[name*='qty'], input[name*='quantity']").first
+            if await qty_input.count():
+                qty_val = (await qty_input.get_attribute("value") or "").strip()
+                if qty_val.isdigit():
+                    quantity = int(qty_val)
+            else:
+                qty_select = row.locator("select[name*='qty'], select[name*='quantity'], select.quantity").first
+                if await qty_select.count():
+                    selected = qty_select.locator("option[selected]").first
+                    if await selected.count():
+                        qty_val = (await selected.get_attribute("value") or "").strip()
+                        if qty_val.isdigit():
+                            quantity = int(qty_val)
+
+            items.append(
+                {
+                    "goods_id": extract_product_id(href),
+                    "goods_name": goods_name,
+                    "goods_price": goods_price,
+                    "goods_image": goods_image,
+                    "quantity": quantity,
+                }
+            )
+
+        if total_goods_amount <= 0:
+            total_goods_amount = sum(item["goods_price"] * item["quantity"] for item in items)
+
+        return {
+            "total_goods_amount": total_goods_amount,
+            "total_shipping_fee": total_shipping_fee,
+            "total_commission_fee": total_commission_fee,
+            "items": items,
+        }
+
+    def _parse_cargo_html(self, html: str) -> dict:
+        tree = HTMLParser(html)
+
+        items = []
+        # 根据常见的骏河屋购物车结构进行解析
+        item_nodes = tree.css("table.cart_table tr.item_row, .cart-item")
+        if not item_nodes:
+            item_nodes = tree.css("div.item_box")  # Fallback
+
+        for node in item_nodes:
+            # 解析商品ID
+            id_node = node.css_first("input[name='goods_id'], .goods_id")
+            goods_id = id_node.attributes.get("value", "") if id_node else ""
+            if not goods_id:
+                # 尝试从链接提取
+                link = node.css_first("a[href*='/product/detail/']")
+                if link:
+                    href = link.attributes.get("href", "")
+                    import re
+                    match = re.search(r'/product/detail/([A-Za-z0-9]+)', href)
+                    if match:
+                        goods_id = match.group(1)
+
+            if not goods_id:
+                continue
+
+            # 解析商品名称
+            name_node = node.css_first(".item_name, .title, p.name")
+            goods_name = name_node.text().strip() if name_node else f"product-{goods_id}"
+
+            # 解析商品价格
+            price_node = node.css_first(".price, .item_price, td.price")
+            price_text = price_node.text().strip() if price_node else "0"
+            goods_price = format_price(price_text) or 0
+
+            # 解析商品图片
+            img_node = node.css_first("img")
+            goods_image = img_node.attributes.get("src", "") if img_node else ""
+
+            items.append({
+                "goods_id": goods_id,
+                "goods_name": goods_name,
+                "goods_price": goods_price,
+                "goods_image": goods_image,
+                "number": 1
+            })
+
+        # 提取金额信息
+        total_goods_amount = 0
+        total_shipping_fee = 0
+        total_commission_fee = 0
+
+        # 寻找包含金额的总计区域
+        summary_nodes = tree.css(".summary_box, .total_box, table.total_table tr")
+        for node in summary_nodes:
+            text = node.text().replace(" ", "").replace("\n", "")
+            if "商品合計" in text or "小計" in text:
+                price_node = node.css_first(".price, td:last-child")
+                if price_node:
+                    total_goods_amount = format_price(price_node.text().strip()) or 0
+            elif "送料" in text:
+                price_node = node.css_first(".price, td:last-child")
+                if price_node:
+                    total_shipping_fee = format_price(price_node.text().strip()) or 0
+            elif "通信販売手数料" in text or "代引手数料" in text or "手数料" in text:
+                price_node = node.css_first(".price, td:last-child")
+                if price_node:
+                    total_commission_fee = format_price(price_node.text().strip()) or 0
+
+        return {
+            "items": items,
+            "total_goods_amount": total_goods_amount,
+            "total_shipping_fee": total_shipping_fee,
+            "total_commission_fee": total_commission_fee
+        }
+
+    async def clear_cart(self, page):
+        """
+        清空购物车
+        """
+        cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
+        if cart_goods_num < 1:
+            return
+
+        is_delete = 0
+        # 购物车商品数量超过3个后会出现"全削除"按钮
+        if cart_goods_num > 3:
+            try:
+                delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
+                # 3. 显式等待按钮在页面上可见（容错处理：比如给它最多 3000 毫秒的加载时间）
+                # 如果按钮是一开始就在静态 HTML 里的，这一步也可以省略，直接用下面的 is_visible()
+                await delete_button.wait_for(state="visible", timeout=3000)
+                if await delete_button.is_visible():
+                    logger.debug("检测到‘全削除’按钮存在，正在执行点击...")
+                    await delete_button.click()
+                    is_delete = 1
+            except Exception:
+                logger.debug("未检测到‘全削除’按钮。")
+
+        if is_delete == 1:
+            return
+
+        # 遍历删除
+        item_selector = "table.item_box tbody tr.item"
+        remove_selector = "table.item_box a.remove_product"
+
+        while True:
+            cart_goods_num = await page.locator(item_selector).count()
+            if cart_goods_num == 0:
+                break
+
+            remove_link = page.locator(remove_selector).first
+
+            try:
+                await remove_link.click()
+                # 删除按钮不会触发弹窗，直接等待购物车条目数减少即可。
+                await page.wait_for_function(
+                    """({selector, expectedCount}) => {
+                        return document.querySelectorAll(selector).length < expectedCount;
+                    }""",
+                    arg={"selector": item_selector, "expectedCount": cart_goods_num},
+                    timeout=15_000,
+                )
+                await page.wait_for_timeout(random.randint(800, 1500))
+            except PlaywrightTimeoutError:
+                latest_count = await page.locator(item_selector).count()
+                if latest_count < cart_goods_num:
+                    continue
+
+                await page.wait_for_timeout(1000)

app/services/cloudflare_session.py

@@ -0,0 +1,252 @@
+"""Cloudflare 会话管理器：自动获取并复用通过验证的浏览器会话
+
+骏河屋网站受 Cloudflare 保护，首次访问需通过浏览器完成 JS 挑战。
+本模块负责：
+- 检测并等待 Cloudflare 挑战完成
+- 保存通过验证后的浏览器存储状态（含 cf_clearance Cookie）
+- 后续请求复用已验证会话，避免重复触发挑战
+- 会话失效时支持主动失效和重建
+"""
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import logging
+from collections.abc import Iterable
+from dataclasses import dataclass
+from urllib.parse import urlparse
+from uuid import uuid4
+from pathlib import Path
+
+from app.core.config import Settings
+from app.core.errors import CloudflareChallengeError, UpstreamBlockedError
+from app.services.browser_pool import BrowserPool
+from app.services.session_store import SessionState, SessionStore
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass(slots=True)
+class VerifiedSession:
+    """已通过 Cloudflare 验证的会话信息"""
+    session_id: str
+    storage_state: dict
+    cf_clearance: str | None  # cf_clearance Cookie 值
+    html: str | None = None  # 新创建会话时携带的页面 HTML，复用场景为 None
+
+
+class CloudflareSessionManager:
+    """Cloudflare 会话管理器，负责会话的获取、验证和失效"""
+
+    def __init__(
+            self,
+            settings: Settings,
+            browser_pool: BrowserPool,
+            session_store: SessionStore,
+    ):
+        self._settings = settings
+        self._browser_pool = browser_pool
+        self._session_store = session_store
+
+    async def get_verified_session(self, target_url: str, force_refresh: bool = False) -> VerifiedSession:
+        """获取一个已通过 Cloudflare 验证的会话
+
+        优先复用已有会话；当 force_refresh=True 或无可用会话时，
+        启动浏览器访问目标页面，等待 Cloudflare 挑战通过后保存会话。
+
+        Args:
+            target_url: 目标页面 URL，用于确定会话名称
+            force_refresh: 是否强制创建新会话（忽略已有会话）
+
+        Returns:
+            VerifiedSession: 已验证的会话信息
+
+        Raises:
+            UpstreamBlockedError: 目标站点返回阻断响应（如 1020/403）
+            CloudflareChallengeError: 等待 Cloudflare 挑战超时
+        """
+        session_name = self._session_name_for_url(target_url)
+        if not force_refresh:
+            existing = await self._session_store.get_session(session_name)
+            if existing is not None:
+                logger.debug("复用已验证会话：session_name=%s session_id=%s", session_name, existing.session_id)
+                return self._to_verified_session(existing)
+
+        logger.debug("创建新会话：session_name=%s force_refresh=%s", session_name, force_refresh)
+        session = await self._create_verified_session(session_name=session_name, target_url=target_url)
+        return session
+
+    async def invalidate(self, target_url: str) -> None:
+        """主动让会话失效
+
+        当识别到被阻断/挑战失败时调用，下次请求将创建新会话。
+
+        Args:
+            target_url: 目标页面 URL，用于定位需要失效的会话
+        """
+        session_name = self._session_name_for_url(target_url)
+        logger.warning("会话失效：session_name=%s", session_name)
+        await self._session_store.invalidate_session(session_name)
+
+    async def _create_verified_session(self, session_name: str, target_url: str) -> VerifiedSession:
+        """通过真实浏览器访问目标页面，让 Cloudflare 挑战在浏览器侧完成
+
+        流程：
+        1. 从浏览器池获取页面会话
+        2. 访问目标页面
+        3. 等待 Cloudflare 挑战通过
+        4. 保存浏览器存储状态和 cf_clearance Cookie
+        5. 同时返回页面 HTML，避免调用方二次加载同一页面
+        """
+        async with self._browser_pool.page_session(keep_open_on_success=False) as (context, page):
+            logger.debug("开始访问（用于通过挑战）：%s", target_url)
+            response = await page.goto(
+                target_url,
+                wait_until="domcontentloaded",
+                timeout=int(self._settings.request_timeout_seconds * 1000),
+            )
+            status = response.status
+            if status != 200:
+                try:
+                    screenshot_dir = Path(__file__).resolve().parents[2] / self._settings.log_dir / "screenshots"
+                    screenshot_dir.mkdir(parents=True, exist_ok=True)
+                    screenshot_file = screenshot_dir / f"{uuid4().hex}.png"
+                    await page.screenshot(path=str(screenshot_file), full_page=True)
+                    screenshot_path = str(screenshot_file)
+                except Exception:
+                    screenshot_path = None
+
+                raise UpstreamBlockedError(f"Upstream status={status}, 页面返回非200状态码, 截图地址：{screenshot_path}")
+            await self._wait_for_clearance(page)
+            html_content = await page.content()
+            storage_state = await context.storage_state()
+            cookies = await context.cookies()
+            cf_clearance = self._get_cookie_value(cookies, "cf_clearance")
+            logger.debug("挑战通过：session_name=%s cf_clearance=%s", session_name, bool(cf_clearance))
+            saved = await self._session_store.save_session(
+                session_name=session_name,
+                session_id=str(uuid4()),
+                user_agent=self._settings.browser_user_agent,
+                proxy_key=self._settings.proxy_server or "direct",
+                storage_state=storage_state,
+                cf_clearance=cf_clearance,
+            )
+            return VerifiedSession(
+                session_id=saved.session_id,
+                storage_state=saved.storage_state,
+                cf_clearance=saved.cf_clearance,
+                html=html_content,
+            )
+
+    async def _wait_for_clearance(self, page: object) -> None:
+        """检测并等待 Cloudflare 挑战完成
+
+        循环检测页面标题和内容：
+        - 如果检测到阻断响应（如 1020/403），直接抛出异常
+        - 如果检测到挑战页面（如 "Just a moment"），继续等待
+        - 如果既非阻断也非挑战，认为已通过
+
+        Raises:
+            UpstreamBlockedError: 检测到被 Cloudflare 阻断
+            CloudflareChallengeError: 等待挑战超时
+        """
+        timeout_at = asyncio.get_running_loop().time() + self._settings.cloudflare_wait_timeout_seconds
+        while True:
+            title = (await page.title()).lower()
+            content = (await page.content()).lower()
+
+            if self._is_blocked_response(title, content):
+                logger.warning("检测到被阻断响应（可能是 1020/403）")
+                raise UpstreamBlockedError("Cloudflare returned a blocked response")
+            if not self._looks_like_challenge(title, content):
+                return
+
+            try:
+                cf_iframe = page.frame_locator('iframe[src*="turnstile"], iframe[src*="cloudflare"]')
+                if await cf_iframe.locator('input[type="checkbox"]').count() > 0:
+                    checkbox = cf_iframe.locator('input[type="checkbox"]')
+                    if await checkbox.is_visible():
+                        logger.debug("检测到 Turnstile 复选框，尝试自动点击")
+                        await checkbox.click()
+                        await page.wait_for_timeout(2000)
+            except Exception as e:
+                logger.debug("自动点击 Turnstile 失败或未找到: %s", e)
+
+            if asyncio.get_running_loop().time() >= timeout_at:
+                logger.warning("等待挑战超时（秒）：%s", self._settings.cloudflare_wait_timeout_seconds)
+                raise CloudflareChallengeError()
+
+            await page.wait_for_timeout(1500)
+
+    @staticmethod
+    def _get_cookie_value(cookies: Iterable[dict], name: str) -> str | None:
+        """从 Cookie 列表中获取指定名称的 Cookie 值"""
+        for cookie in cookies:
+            if cookie.get("name") == name:
+                return cookie.get("value")
+        return None
+
+    @staticmethod
+    def _looks_like_challenge(title: str, content: str) -> bool:
+        """判断页面是否仍处于 Cloudflare 挑战中
+
+        先排除正常页面（搜索列表/商品详情页中也可能包含 Cloudflare 脚本），
+        再检查标题和内容中是否包含挑战标志（如 "Just a moment"）。
+        """
+        # 正常页面的 DOM 特征，出现这些说明挑战已通过
+        if (
+                'class="item_box' in content
+                or "product-name" in content
+                or 'id="item_title"' in content
+                or "h1_title_product" in content
+                or 'id="item_detailinfo"' in content
+                or "tbl_product_info" in content
+                or "商品詳細情報" in content
+        ):
+            return False
+
+        challenge_markers = (
+            "just a moment",
+            "checking your browser",
+            "verify you are human",
+            "attention required",
+            "cf-challenge",
+            "cf-turnstile",
+            "cf_chl_opt",
+            "cf-please-wait",
+            "challenge-form",
+        )
+        return any(marker in title or marker in content for marker in challenge_markers)
+
+    @staticmethod
+    def _is_blocked_response(title: str, content: str) -> bool:
+        """判断页面是否为 Cloudflare 阻断响应（如 1020 错误）"""
+        blocked_markers = (
+            "access denied",
+            "error code 1020",
+            "forbidden",
+            "temporarily blocked",
+        )
+        return any(marker in title or marker in content for marker in blocked_markers)
+
+    def _session_name_for_url(self, target_url: str) -> str:
+        """根据 URL、代理和 User-Agent 生成会话名称
+
+        同一域名下仍共享会话，但会按当前代理与 UA 做进一步隔离，
+        避免不同运行身份误复用同一套 Cloudflare 验证状态。
+        """
+        parsed = urlparse(target_url)
+        proxy_key = self._settings.proxy_server or "direct"
+        user_agent = self._settings.browser_user_agent
+        user_agent_hash = hashlib.sha1(user_agent.encode("utf-8")).hexdigest()[:12]
+        return f"{parsed.netloc}|{proxy_key}|{user_agent_hash}"
+
+    @staticmethod
+    def _to_verified_session(state: SessionState) -> VerifiedSession:
+        """将 SessionState 转换为 VerifiedSession（复用场景，html 为空）"""
+        return VerifiedSession(
+            session_id=state.session_id,
+            storage_state=state.storage_state,
+            cf_clearance=state.cf_clearance,
+            html=None,
+        )

app/services/session_store.py

@@ -0,0 +1,139 @@
+"""会话存储：管理 Cloudflare 验证通过后的浏览器会话状态
+
+支持两种存储后端：
+- Redis（推荐生产环境使用，支持多实例共享）
+- 进程内存（默认回退，仅单实例可用）
+
+会话包含浏览器存储状态（Cookie、localStorage 等）和 cf_clearance 值，
+设置 TTL 自动过期，过期后需要重新通过 Cloudflare 挑战。
+"""
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any
+
+from app.core.config import Settings
+
+try:
+    from redis.asyncio import Redis
+except Exception:  # pragma: no cover - optional until dependencies are installed.
+    Redis = Any  # type: ignore[misc,assignment]
+
+
+@dataclass(slots=True)
+class SessionState:
+    """会话状态数据"""
+    session_id: str
+    user_agent: str
+    proxy_key: str
+    storage_state: dict[str, Any]
+    cf_clearance: str | None
+    created_at: str
+    expires_at: str
+
+    @property
+    def is_expired(self) -> bool:
+        """会话是否已过期"""
+        return datetime.now(UTC) >= datetime.fromisoformat(self.expires_at)
+
+
+class SessionStore:
+    """会话存储，支持 Redis 和进程内存两种后端"""
+
+    def __init__(self, settings: Settings):
+        self._settings = settings
+        self._redis: Redis | None = None
+        self._memory: dict[str, SessionState] = {}
+
+    async def start(self) -> None:
+        """初始化存储后端：配置了 Redis 时连接 Redis，否则使用进程内存"""
+        if not self._settings.redis_host:
+            return
+
+        from redis.asyncio import Redis as AsyncRedis
+
+        self._redis = AsyncRedis(
+            host=self._settings.redis_host,
+            port=self._settings.redis_port,
+            password=self._settings.redis_password,
+            db=self._settings.redis_db,
+            decode_responses=True,
+        )
+        await self._redis.ping()
+
+    async def close(self) -> None:
+        """关闭存储后端连接"""
+        if self._redis is not None:
+            await self._redis.aclose()
+            self._redis = None
+
+    @property
+    def redis_enabled(self) -> bool:
+        """Redis 是否已连接"""
+        return self._redis is not None
+
+    def _session_key(self, session_name: str) -> str:
+        """生成 Redis/内存中的存储键"""
+        return f"{self._settings.session_namespace}:session:{session_name}"
+
+    async def get_session(self, session_name: str) -> SessionState | None:
+        """获取会话状态，过期会话自动失效并返回 None"""
+        key = self._session_key(session_name)
+        if self._redis is not None:
+            raw = await self._redis.get(key)
+            if not raw:
+                return None
+            session = SessionState(**json.loads(raw))
+            if session.is_expired:
+                await self.invalidate_session(session_name)
+                return None
+            return session
+
+        session = self._memory.get(key)
+        if session and session.is_expired:
+            self._memory.pop(key, None)
+            return None
+        return session
+
+    async def save_session(
+        self,
+        session_name: str,
+        session_id: str,
+        user_agent: str,
+        proxy_key: str,
+        storage_state: dict[str, Any],
+        cf_clearance: str | None,
+    ) -> SessionState:
+        """保存会话状态
+
+        创建带 TTL 的会话记录，Redis 后端会自动在 TTL 到期后删除。
+        """
+        now = datetime.now(UTC)
+        state = SessionState(
+            session_id=session_id,
+            user_agent=user_agent,
+            proxy_key=proxy_key,
+            storage_state=storage_state,
+            cf_clearance=cf_clearance,
+            created_at=now.isoformat(),
+            expires_at=(now + timedelta(seconds=self._settings.session_ttl_seconds)).isoformat(),
+        )
+        key = self._session_key(session_name)
+
+        if self._redis is not None:
+            ttl = self._settings.session_ttl_seconds
+            await self._redis.set(key, json.dumps(asdict(state)), ex=ttl)
+            return state
+
+        self._memory[key] = state
+        return state
+
+    async def invalidate_session(self, session_name: str) -> None:
+        """使会话失效，删除存储中的会话记录"""
+        key = self._session_key(session_name)
+        if self._redis is not None:
+            await self._redis.delete(key)
+            return
+        self._memory.pop(key, None)

app/services/surugaya_client.py

@@ -0,0 +1,615 @@
+"""骏河屋（suruga-ya.jp）抓取客户端
+
+负责搜索列表和商品详情的抓取与解析。
+通过 Cloudflare 会话复用机制，避免每次请求都触发验证挑战。
+"""
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import re
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import parse_qs, urlencode, urlparse
+from uuid import uuid4
+from typing import Any
+
+from selectolax.parser import HTMLParser
+
+from app.core.config import Settings
+from app.core.errors import ScrapeParseError, UpstreamBlockedError
+from app.models.scrape import CategoryData, DetailRequest, ProductDetailData, ProductSummary, SearchRequest, \
+    SearchResultData
+from app.services.browser_pool import BrowserPool
+from app.services.cloudflare_session import CloudflareSessionManager
+from app.services.session_store import SessionStore
+from app.utils.app_utils import is_empty_str
+from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
+
+logger = logging.getLogger(__name__)
+
+
+class SurugayaClient:
+    """骏河屋抓取客户端，提供搜索和商品详情两个核心能力
+
+    工作流程：
+    1. 获取已通过 Cloudflare 验证的会话（优先复用，必要时新建）
+    2. 使用浏览器池中的页面，携带会话 Cookie 访问目标页面
+    3. 解析页面 HTML 提取结构化数据
+    """
+
+    def __init__(
+            self,
+            settings: Settings,
+            browser_pool: BrowserPool,
+            session_manager: CloudflareSessionManager,
+            session_store: SessionStore,
+    ):
+        self._settings = settings
+        self._browser_pool = browser_pool
+        self._session_manager = session_manager
+        self._session_store = session_store
+
+        self._categories_cache_key = "surugaya:categories:cache"
+
+    async def _raise_upstream_error(self, *, page: object, response: object | None, url: str) -> None:
+        status = getattr(response, "status", None)
+
+        try:
+            await self._session_manager.invalidate(url)
+        except Exception as exc:
+            logger.warning("清理上游异常会话失败：url=%s err=%s", url, exc)
+
+        screenshot_path: str | None = None
+        try:
+            screenshot_dir = Path(__file__).resolve().parents[2] / self._settings.log_dir / "screenshots"
+            screenshot_dir.mkdir(parents=True, exist_ok=True)
+            screenshot_file = screenshot_dir / f"{uuid4().hex}.png"
+            await page.screenshot(path=str(screenshot_file), full_page=True)
+            screenshot_path = str(screenshot_file)
+        except Exception:
+            screenshot_path = None
+
+        raise UpstreamBlockedError(f"Upstream status：{status}, 截图地址：{screenshot_path}")
+
+    async def search(self, payload: SearchRequest) -> SearchResultData:
+        """抓取搜索列表页
+
+        流程：
+        1. 构建/使用搜索 URL
+        2. 获取已通过 Cloudflare 验证的会话
+        3. 用浏览器页面访问并获取 HTML
+        4. 解析商品总数和商品列表
+
+        Args:
+            payload: 搜索请求参数，包含 search_word、page 等
+
+        Returns:
+            SearchResultData: 包含搜索结果列表、总数、分页信息等
+        """
+        logger.debug("开始抓取搜索页：payload=%s", json.dumps(payload.model_dump(), ensure_ascii=False))
+        url = str(payload.search_url) if payload.search_url else self._build_search_url(payload)
+
+        session = await self._session_manager.get_verified_session(url)
+        if session.html is not None:
+            # 新创建的会话已携带页面 HTML，直接使用，避免二次加载
+            html = session.html
+        else:
+            async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
+                response = await page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=int(self._settings.request_timeout_seconds * 1000),
+                )
+                status = response.status if response is not None else None
+                if status != 200:
+                    await self._raise_upstream_error(page=page, response=response, url=url)
+                html = await page.content()
+
+        tree = HTMLParser(html)
+
+        # 解析商品总数量，格式示例：該当件数:10,428件中 1-24件
+        current_page = self._extract_page_value(payload)
+        total_count = 0
+        page_size = 24
+        has_more = 0
+        hit_node = tree.css_first("#search_header .search_option .hit")
+        if hit_node:
+            raw_text = hit_node.text()
+            match = re.search(r'該当件数:([\d,]+)件', raw_text)
+            if match:
+                total_count = int(match.group(1).replace(',', ''))
+
+        if total_count > 0:
+            has_more = int(current_page * page_size < total_count)
+
+        # 解析商品列表
+        items = self._parse_search_items(tree)
+        logger.info("搜索页解析完成 ===> 关键词: %s 抓取商品数=%s", payload.search_word, len(items))
+
+        return SearchResultData(
+            query=payload.search_word,
+            page=current_page,
+            page_size=page_size,
+            total_count=total_count,
+            has_more=has_more,
+            items=items,
+            session_id=session.session_id,
+        )
+
+    async def fetch_detail(self, payload: DetailRequest) -> ProductDetailData:
+        """抓取商品详情页
+
+        流程：
+        1. 根据请求参数确定商品 URL 和 ID（支持直接传 ID 或完整 URL）
+        2. 获取已通过 Cloudflare 验证的会话
+        3. 用浏览器页面访问并获取 HTML
+        4. 从 DOM 中提取商品名称、图片、价格、售卖状态、描述等
+
+        Args:
+            payload: 详情请求参数，包含 id 或 product_url
+
+        Returns:
+            ProductDetailData: 商品详情数据
+
+        Raises:
+            ScrapeParseError: 当 id 和 product_url 都未提供时
+        """
+        goods_id = payload.id or ""
+        if is_empty_str(goods_id):
+            raise ScrapeParseError("id is empty")
+
+        url = self._build_product_url(goods_id)
+
+        logger.info("开始抓取详情页：goods_id=%s", goods_id)
+        logger.debug("详情页URL：url=%s", url)
+
+        # 抓取页面 HTML
+        session = await self._session_manager.get_verified_session(url)
+        if session.html is not None:
+            html = session.html
+        else:
+            async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
+                response = await page.goto(
+                    url,
+                    wait_until="domcontentloaded",
+                    timeout=int(self._settings.request_timeout_seconds * 1000),
+                )
+                status = response.status if response is not None else None
+                if status != 200:
+                    await self._raise_upstream_error(page=page, response=response, url=url)
+                html = await page.content()
+
+        tree = HTMLParser(html)
+
+        # 提取商品标题
+        title_node = tree.css_first("#item_title")
+        goods_name = title_node.text().strip() if title_node else None
+        if not goods_name:
+            raise ScrapeParseError("商品名称解析失败")
+
+        # 提取商品主图
+        img_node = tree.css_first("div.easyzoom--with-thumbnails img")
+        goods_image: str = (img_node.attributes.get("src") or "") if img_node else ""
+        if goods_image and "database/images/no_photo.jpg" in goods_image:
+            goods_image = "https://oss.daimatech.com/suruga-ya/no_photo.jpg"
+
+        # 提取商品价格：优先从 .price_choise 中取，其次从 span.purchase-price 中取
+        price_text = ""
+        price_choise = tree.css_first("#cart .price_choise")
+        if price_choise:
+            buy_node = price_choise.css_first(".text-price-detail.price-buy")
+            if buy_node:
+                price_text = buy_node.text().strip()
+
+        if not price_text:
+            purchase_node = tree.css_first("span.purchase-price")
+            if purchase_node:
+                price_text = purchase_node.text().strip()
+
+        goods_price: int = format_price(price_text) or 0 if price_text else 0
+
+        # 判断售卖状态：
+        out_of_stock_dom = tree.css_first("div.out-of-stock-text")
+        on_sold = 0 if out_of_stock_dom else 1
+
+        # 解析规格列表
+        sku = ""
+        sku_list = []
+        pdom = tree.css_first("#item_title")
+        if pdom and pdom.parent:
+            direct_children = pdom.parent.css(".price_group > .item-price")
+            sku_index = 0
+            for node in direct_children:
+                sku_index += 1
+                vo: dict[str, Any] = {"sku_name": "", "price": 0, "stock": 0}
+
+                label_node = node.css_first("label")
+                if label_node:
+                    vo["sku_name"] = label_node.attributes.get("data-label", "").strip()
+
+                input_node = node.css_first("input.form-check-input")
+                if input_node:
+                    zaiko_str = input_node.attributes.get("data-zaiko", "")
+                    if zaiko_str and zaiko_str != "null":
+                        try:
+                            zaiko = json.loads(zaiko_str)
+                            if isinstance(zaiko, dict):
+                                vo.update(zaiko)
+                                vo["price"] = int(zaiko.get("baika", 0))
+                                vo["stock"] = int(zaiko.get("zaiko", 0))
+                                vo["sku_id"] = str(sku_index)
+                                if sku_index == 1:
+                                    sku = vo["sku_id"]
+                        except json.JSONDecodeError:
+                            vo["zaiko_raw"] = zaiko_str
+
+                sku_list.append(vo)
+
+        # 提取库存
+        stock = 0
+        if sku_list:
+            # 从 SKU 列表中找到匹配当前 SKU 的库存
+            stock = next((item["stock"] for item in sku_list if item.get("sku_id") == sku), 0)
+        else:
+            quantity_selection = tree.css_first("#quantity_selection")
+            if quantity_selection:
+                stock = 1
+                last_option = quantity_selection.css_first("option:last-child")
+                if last_option:
+                    option_val = last_option.attributes.get("value")
+                    if option_val and option_val.isdigit():
+                        stock = int(option_val)
+
+        # 提取商品描述
+        desc_note = tree.css_first("p.note.text-break")
+        description = desc_note.text().strip() if desc_note else ""
+
+        # 提取店铺来源
+        supplier = ""  # 默认表示官方自营
+        # 利用与目标 div.mb-2 平级的 #naviplus-review-list-5 缩小查找范围以提升效率
+        review_node = tree.css_first("#naviplus-review-list-5")
+        if review_node and review_node.parent:
+            context_node = review_node.parent
+            for node in context_node.css("div.mb-2"):
+                text = node.text()
+                if "この商品は" in text and "販売" in text and "発送" in text:
+                    # 提取 mb-2 下的全部文本，并清理多余的换行和空白字符
+                    supplier = " ".join(text.split())
+                    break
+
+        # 提取价格备注
+        price_note = None
+        price_note_node = tree.css_first(".item-price-note-wrap")
+        if price_note_node:
+            price_note = price_note_node.text().strip()
+
+        # 提取运费相关
+        shipping_comission_fee = get_shipping_fee(goods_price)
+
+        # 提取面包屑
+        breadcrumb_list = self.get_breadcrumb_list(tree)
+
+        detail = ProductDetailData(
+            goods_id=goods_id,
+            goods_name=goods_name,
+            goods_image=goods_image,
+            goods_price=goods_price,
+            stock=stock,
+            on_sold=on_sold,
+            sku=sku,
+            sku_list=sku_list,
+            description=description,
+            supplier=supplier,
+            price_note=price_note,
+            shipping_comission_fee=shipping_comission_fee,
+            breadcrumb_list=breadcrumb_list,
+        )
+        logger.debug("详情页解析完成：goods_id=%s goods_name=%s ", detail.goods_id, detail.goods_name)
+        return detail
+
+    async def fetch_categories(self) -> list[CategoryData]:
+        """获取商品分类（带缓存）
+        
+        解析骏河屋的商品分类，由于分类不易变动，
+        优先从 redis 获取缓存，若 redis 未配置或缓存失效，则重新抓取。
+        默认缓存 24 小时。
+        """
+        # 如果配置了 Redis，尝试从 Redis 获取缓存
+        if self._session_store._redis is not None:
+            try:
+                cached_data = await self._session_store._redis.get(self._categories_cache_key)
+                if cached_data:
+                    logger.debug("命中商品分类 Redis 缓存")
+                    data_list = json.loads(cached_data)
+                    return [CategoryData(**item) for item in data_list]
+            except Exception as e:
+                logger.warning("读取 Redis 缓存失败: %s", e)
+
+        logger.info("开始获取并解析商品分类...")
+        top_categories: list[CategoryData] = []
+
+        target_url = "https://www.suruga-ya.jp/"
+        session = await self._session_manager.get_verified_session(target_url, True)
+
+        html_content = ""
+        if session.html is not None:
+            html_content = session.html
+        else:
+            async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
+                try:
+                    response = await page.goto(
+                        target_url,
+                        wait_until="domcontentloaded",
+                        timeout=int(self._settings.request_timeout_seconds * 1000),
+                    )
+                    status = response.status if response is not None else None
+                    if status != 200:
+                        raise ScrapeParseError(f"抓取首页分类失败，状态码 {status}")
+
+                    html_content = await page.content()
+                except Exception as e:
+                    logger.error("抓取首页分类失败：%s", e)
+                    raise ScrapeParseError(f"抓取首页分类失败: {e!s}") from e
+
+        tree = HTMLParser(html_content)
+
+        # 解析一级分类列表
+        cate_nodes = tree.css(".catemenu_group .cate_menu .cate_item")
+        for cate_item in cate_nodes:
+            href = cate_item.css_first("a").attributes.get("href")
+            if not href:
+                continue
+
+            # 提取 href 的路径名（不含 .html 和前置的 /，例如：/avsoft.html -> avsoft）
+            match = re.search(r'/([^/]+)(?:\.html|/)', href)
+            if not match:
+                continue
+
+            cate_id = match.group(1)
+            # 过滤掉指定的分类
+            if cate_id in ["boyslove"]:
+                continue
+
+            # 提取包含的文字内容（拼接多个 span，例如 おもちゃ・ + ホビー）
+            h4_node = cate_item.css_first("a h4")
+            if not h4_node:
+                continue
+
+            cate_name = h4_node.text().strip()
+            # 清理多余空格与换行，防止拼接文本里出现多余空白
+            cate_name = "".join(cate_name.split())
+
+            icon_node = cate_item.css_first("a img")
+            icon = icon_node.attributes.get("src") if icon_node else None
+
+            pid = "0"
+
+            top_categories.append(CategoryData(id=cate_id, name=cate_name, icon=icon, pid=pid, href=href))
+
+        # 解析二级分类列表
+        seen_pairs: set[tuple[str, str]] = set()
+        async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
+            for top_category in top_categories:
+                try:
+                    category_url = f"https://www.suruga-ya.jp{top_category.href}"
+                    response = await page.goto(
+                        category_url,
+                        wait_until="domcontentloaded",
+                        timeout=int(self._settings.request_timeout_seconds * 1000),
+                    )
+                    status = response.status if response is not None else None
+                    if status != 200:
+                        await self._raise_upstream_error(page=page, response=response, url=category_url)
+                    category_html = await page.content()
+
+                    category_tree = HTMLParser(category_html)
+                    menu_node = category_tree.css_first("#menu")
+                    if not menu_node:
+                        continue
+
+                    related_block = None
+                    for block in menu_node.css(".block"):
+                        h2_node = block.css_first("h2")
+                        if h2_node and "関連ジャンルで絞り込む" in h2_node.text():
+                            related_block = block
+                            break
+                    if not related_block:
+                        continue
+
+                    for a_node in related_block.css("ul.border_bottom li a"):
+                        sub_name = "".join(a_node.text().split())
+                        if not sub_name:
+                            continue
+
+                        sub_href = a_node.attributes.get("href") or ""
+                        parsed = urlparse(sub_href)
+                        sub_id = ""
+                        if parsed.path == "/search":
+                            query = parse_qs(parsed.query)
+                            sub_id = (query.get("category") or [""])[0]
+
+                        if not sub_id:
+                            match = re.search(r"/([^/]+)\.html$", parsed.path)
+                            if match:
+                                sub_id = match.group(1)
+
+                        if not sub_id:
+                            continue
+
+                        pair = (top_category.id, sub_id)
+                        if pair in seen_pairs:
+                            continue
+                        seen_pairs.add(pair)
+                        top_category.children.append(
+                            CategoryData(id=sub_id, name=sub_name, pid=top_category.id, href=sub_href))
+                        # 休眠1秒，避免对服务器造成过大压力
+                        await asyncio.sleep(1)
+
+                except Exception as e:
+                    logger.warning("抓取二级分类失败：top_id=%s err=%s", top_category.id, e)
+                    continue
+
+        # 更新 Redis 缓存
+        if top_categories and self._session_store._redis is not None:
+            try:
+                logger.debug("更新商品分类 Redis 缓存")
+                data_list = [item.model_dump() for item in top_categories]
+                # 缓存有效期 24 小时 (86400秒)
+                await self._session_store._redis.setex(
+                    self._categories_cache_key,
+                    86400 * 30,
+                    json.dumps(data_list, ensure_ascii=False)
+                )
+            except Exception as e:
+                logger.warning("写入 Redis 缓存失败: %s", e)
+
+        return top_categories
+
+    # ------------------------------------------------------------------
+    # 内部辅助方法
+    # ------------------------------------------------------------------
+
+    def _build_search_url(self, payload: SearchRequest) -> str:
+        """根据搜索参数构建骏河屋搜索 URL
+
+        将 payload 中的 search_word、page 等参数拼接为完整的搜索地址。
+        search_url 字段本身不参与拼接，它用于直接指定完整 URL。
+        """
+        params = payload.model_dump(exclude_none=True)
+        params.pop("search_url", None)
+        # if not str(params.get("search_word", "")).strip():
+        #     raise ScrapeParseError("search_word is required")
+
+        query_string = urlencode(params, doseq=True)
+        base_url = self._settings.base_url.rstrip("/")
+        return f"{base_url}/search?{query_string}"
+
+    @staticmethod
+    def _extract_page_value(payload: SearchRequest) -> int:
+        """安全地提取页码，确保返回 >= 1 的整数"""
+        page_raw = payload.model_dump(exclude_none=True).get("page", 1)
+        try:
+            current_page = int(page_raw)
+        except (TypeError, ValueError):
+            return 1
+        return current_page if current_page >= 1 else 1
+
+    def _build_product_url(self, product_id: str | None) -> str:
+        """根据商品 ID 构建详情页 URL"""
+        if not product_id:
+            raise ScrapeParseError("Either product_id or product_url must be provided")
+        base_url = self._settings.base_url.rstrip("/")
+        return f"{base_url}/product/detail/{product_id}"
+
+    def _parse_search_items(self, tree: HTMLParser) -> list[ProductSummary]:
+        """解析搜索结果列表
+
+        从搜索页 DOM 中提取每个商品的 ID、名称、图片、价格、市场价、售卖状态。
+        自动去重（按 goods_id），最多返回 50 条结果。
+        """
+        item_nodes = tree.css("div.item_box div.item")
+        if not item_nodes:
+            item_nodes = tree.css("div.item")
+
+        items: list[ProductSummary] = []
+        seen_goods_ids: set[str] = set()
+
+        for node in item_nodes:
+            link_node = node.css_first("div.title a") or node.css_first("div.photo_box a")
+            href = link_node.attributes.get("href") if link_node else None
+            if not href:
+                continue
+
+            goods_link = self._normalize_url(href)
+            goods_id = self._extract_product_id(goods_link) or self._extract_product_id(href)
+            if not goods_id:
+                continue
+            if goods_id in seen_goods_ids:
+                continue
+            seen_goods_ids.add(goods_id)
+
+            name_node = node.css_first("h3.product-name")
+            goods_name = name_node.text().strip() if name_node else f"product-{goods_id}"
+
+            img_node = node.css_first("div.photo_box img")
+            goods_image_str = (img_node.attributes.get("src") or "").strip() if img_node else ""
+            goods_image = surugaya_photo_url_to_cdn(goods_image_str, goods_id)
+
+            # 解析商品售价（自营价格）
+            goods_price = 0
+            price_teika = node.css_first(".item_price .price_teika")
+            if price_teika is not None:
+                strong = price_teika.css_first(".text-red strong")
+                price_text = strong.text().strip() if strong is not None else price_teika.text().strip()
+                parsed_goods_price = format_price(price_text)
+                if parsed_goods_price is not None:
+                    goods_price = parsed_goods_price
+
+            # 解析第三方市场价
+            third_shop_price = 0
+            makepla_strong = node.css_first(".item_price .makeplaTit .text-red strong")
+            if makepla_strong is not None:
+                price_text = makepla_strong.text().strip()
+                parsed_third_shop_price = format_price(price_text)
+                if parsed_third_shop_price is not None:
+                    third_shop_price = parsed_third_shop_price
+
+            # 判断售卖状态：品切れ = 售罄
+            on_sold = 1
+            sold_out = node.css_first(".item_price p.price")
+            if sold_out is not None and "品切れ" in sold_out.text().strip():
+                on_sold = 0
+                # 售罄时，售价回退到市场价
+                goods_price = third_shop_price
+
+            now = str(int(datetime.now().timestamp()))
+
+            items.append(
+                ProductSummary(
+                    goods_id=goods_id,
+                    goods_name=goods_name,
+                    goods_image=goods_image,
+                    goods_price=goods_price,
+                    third_shop_price=third_shop_price,
+                    goods_link=goods_link,
+                    on_sold=on_sold,
+                    created_at=now,
+                    updated_at=now,
+                )
+            )
+        return items
+
+    @staticmethod
+    def _extract_product_id(text: str) -> str | None:
+        """从 URL 或文本中提取商品 ID
+
+        匹配 /product/detail/{id} 或 /product/other/{id} 格式
+        """
+        match = re.search(r"/product/(?:detail|other)/([A-Za-z0-9]+)", text)
+        return match.group(1) if match else None
+
+    @staticmethod
+    def _normalize_url(url: str) -> str:
+        """将相对 URL 补全为完整的 https URL"""
+        if url.startswith("http://") or url.startswith("https://"):
+            return url
+        return f"https://www.suruga-ya.jp{url}"
+
+    def get_breadcrumb_list(self, tree):
+        dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")
+
+        breadcrumb_list = []
+        for index, dom in enumerate(dom_list):
+            a_node = dom.css_first("a")
+            if not a_node:
+                continue
+                
+            breadcrumb_list.append({
+                "position": index + 1,
+                "is_active": index == len(dom_list) - 1,
+                "name": a_node.text().strip(),
+                "href": a_node.attributes.get("href") or "",
+            })
+        return breadcrumb_list