workspace— step4

app/api/routes/scrape.py

@@ -6,6 +6,7 @@ from typing import Any
 from urllib import error, request
 
 from fastapi import APIRouter, Depends
+from pydantic import ValidationError
 
 from app.api.dependencies import get_container, require_bearer_token
 from app.core.container import ServiceContainer
@@ -29,6 +30,7 @@ from app.utils.parse_util import (
     get_shipping_fee,
 )
 from surugaya_common import redis_keys
+from surugaya_common.models import SurugayaOrderDetail
 
 router = APIRouter(prefix="/api", tags=["scrape"])
 
@@ -279,18 +281,18 @@ async def order_add_trade_monitor(
 
 @router.get(
     "/trade",
-    response_model=ApiResponse[dict[str, Any]],
+    response_model=ApiResponse[SurugayaOrderDetail],
     dependencies=[Depends(require_bearer_token)],
 )
 async def trade(
         trade_code: str,
         container: ServiceContainer = Depends(get_container),
-) -> ApiResponse[dict[str, Any]]:
+) -> ApiResponse[SurugayaOrderDetail]:
     """
     获取订单详情
     """
     if not trade_code or trade_code.strip() == "":
-        return ApiResponse[dict[str, Any]](
+        return ApiResponse[SurugayaOrderDetail](
             success=False,
             msg="trade_code 不能为空",
             data=None,
@@ -298,7 +300,7 @@ async def trade(
         )
 
     if container.session_store._redis is None:
-        return ApiResponse[dict[str, Any]](
+        return ApiResponse[SurugayaOrderDetail](
             success=False,
             msg="Redis 未配置",
             data=None,
@@ -323,17 +325,29 @@ async def trade(
                 approximate=True,
             )
 
-        return ApiResponse[dict[str, Any]](
+        return ApiResponse[SurugayaOrderDetail](
             success=False,
             msg="未找到该交易",
             data=None,
             code=404,
         )
 
-    return ApiResponse[dict[str, Any]](
+    # Redis 中存储的是 worker 写入的 SurugayaOrderDetail 序列化结果，
+    # 用共享契约校验并类型化；历史脏数据校验失败时降级为结构化错误而非 500。
+    try:
+        order_detail = SurugayaOrderDetail.model_validate_json(trade_data)
+    except ValidationError as exc:
+        return ApiResponse[SurugayaOrderDetail](
+            success=False,
+            msg=f"订单详情数据结构异常: {exc.error_count()} 处校验失败",
+            data=None,
+            code=500,
+        )
+
+    return ApiResponse[SurugayaOrderDetail](
         success=True,
         msg="success",
-        data=json.loads(trade_data),
+        data=order_detail,
         code=0,
     )
 

packages/surugaya_common/pyproject.toml

@@ -12,9 +12,14 @@ dependencies = [
     "redis==8.0.0",
 ]
 
+# playwright 仅作为可选 extra：只有解析器/浏览器工具需要它，核心安装保持 light。
+# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集，确保 workspace 解析到单一 playwright。
+[project.optional-dependencies]
+browser = ["playwright>=1.54.0,<2.0.0"]
+
 [tool.hatch.build.targets.wheel]
 packages = ["src/surugaya_common"]
 
-# Keep surugaya_common light: stdlib + pydantic + redis only. redis is pinned to
+# 保持 surugaya_common core 轻量：仅 stdlib + pydantic + redis(==8.0.0，与 API 和 worker 对齐)。
-# ==8.0.0 to stay in lockstep with both the API service and the worker. Do not add
+# playwright 只允许出现在上面的 [browser] extra 中，core 安装绝不引入。
-# fastapi/playwright/opencv/numpy/pyautogui/pillow here.
+# 不要在此加入 fastapi/opencv/numpy/pyautogui/pillow。

packages/surugaya_common/src/surugaya_common/browser_utils.py

@@ -0,0 +1,108 @@
+"""浏览器拟真交互工具（需 playwright，安装 surugaya_common[browser]）。
+
+从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 human_scroll，逻辑保持一致。
+注：page_utils.py 中的纯函数 format_japanese_price 未迁移——它与 API 端
+app/utils/parse_util.py 的 format_price 语义不同，合并属行为变更，单独处理。
+"""
+
+from playwright.async_api import Page
+import asyncio
+import random
+
+
+async def human_scroll(page: Page, mode: str, value: int = 0):
+    """
+    模拟人类行为的网页滚动方法
+    :param page: Playwright 的 Page 对象
+    :param mode: 滚动模式 -> 'top'(滚动到顶部), 'bottom'(滚动到底部), 'random'(随机上下滚动), 'distance'(滚动指定距离)
+    :param value: 当 mode='distance' 时，代表要滚动的像素距离（正数向下，负数向上）
+    """
+
+    async def smooth_scroll_step(pixel_distance: int):
+        """核心拟真算法：将一个大距离拆分成带有加速度和随机抖动的微小步长"""
+        if pixel_distance == 0:
+            return
+
+        direction = 1 if pixel_distance > 0 else -1
+        remaining = abs(pixel_distance)
+
+        while remaining > 0:
+            # 模拟人类滚动时越接近目标速度越慢的特征（减速缓冲）
+            if remaining > 100:
+                step = random.randint(30, 70)
+            elif remaining > 30:
+                step = random.randint(10, 25)
+            else:
+                step = remaining
+
+            remaining -= step
+            scroll_y = step * direction
+
+            # 执行单步微调滚动
+            await page.evaluate(f"window.scrollBy(0, {scroll_y});")
+
+            # 随机微小停顿，模拟手指滑动的摩擦阻力（20ms - 50ms）
+            await asyncio.sleep(random.uniform(0.02, 0.05))
+
+            # 0.5% 的极低概率触发人类的“视线停留”
+            if random.random() < 0.005:
+                await asyncio.sleep(random.uniform(0.3, 0.8))
+
+    # ==================== 模式 0：滚动到顶部 ====================
+    if mode == 'top':
+        # print("[HumanScroll] 正在模拟向上滚动至页面顶部...")
+        last_scroll = -1
+        while True:
+            # 获取当前滚动条位置
+            current_scroll = await page.evaluate("window.scrollY")
+
+            # 防死循环：如果位置不再变化，说明已经到顶或页面无法滚动
+            if current_scroll <= 0 or current_scroll == last_scroll:
+                break
+
+            last_scroll = current_scroll
+            chunk = random.randint(300, 700)
+            await smooth_scroll_step(-chunk)
+            await asyncio.sleep(random.uniform(0.5, 1.2))  # 每滑一下停顿看一眼
+
+    # ==================== 模式 1：滚动到底部 ====================
+    elif mode == 'bottom':
+        # print("[HumanScroll] 正在模拟向下滚动至页面底部...")
+        last_scroll = -1
+        while True:
+            # 合并 evaluate 请求，减少与浏览器的 IPC 通信开销
+            scroll_info = await page.evaluate("() => [window.scrollY + window.innerHeight, document.body.scrollHeight]")
+            current_scroll, total_height = scroll_info[0], scroll_info[1]
+
+            # 防死循环：如果位置不再变化，说明已经到底或页面无法滚动
+            if current_scroll >= total_height - 10 or current_scroll == last_scroll:
+                break
+
+            last_scroll = current_scroll
+            chunk = random.randint(300, 700)
+            await smooth_scroll_step(chunk)
+            await asyncio.sleep(random.uniform(0.5, 1.2))  # 每滑一下停顿看一眼
+
+    # ==================== 模式 2：随机上下滚动几下 ====================
+    elif mode == 'random':
+        # print("[HumanScroll] 正在执行人类迷惑行为：随机上下滚动...")
+        # 随机决定滚动 3 到 6 下
+        scroll_times = random.randint(3, 6)
+
+        for _ in range(scroll_times):
+            # 70% 概率向下，30% 概率向上
+            if random.random() > 0.3:
+                distance = random.randint(200, 500)  # 向下滚
+            else:
+                distance = random.randint(-400, -150)  # 向上回滚
+
+            await smooth_scroll_step(distance)
+            await asyncio.sleep(random.uniform(0.6, 1.5))  # 停顿
+
+    # ==================== 模式 3：滚动指定距离 ====================
+    elif mode == 'distance':
+        # print(f"[HumanScroll] 正在精准模拟滚动指定距离: {value} 像素")
+        await smooth_scroll_step(value)
+
+    else:
+        raise ValueError("未知的滚动模式！请选择 'top', 'bottom', 'random' 或 'distance'")

packages/surugaya_common/src/surugaya_common/parsers/__init__.py

@@ -0,0 +1 @@
+"""surugaya_common 解析器子包（部分依赖 playwright，需安装 surugaya_common[browser]）。"""

packages/surugaya_common/src/surugaya_common/parsers/surugaya/__init__.py

@@ -0,0 +1,11 @@
+"""骏河屋页面解析器（需 playwright，安装 surugaya_common[browser]）。"""
+
+from surugaya_common.parsers.surugaya.order_detail import (
+    SurugayaOrderDetailParser,
+    parse_surugaya_order_detail,
+)
+
+__all__ = [
+    "SurugayaOrderDetailParser",
+    "parse_surugaya_order_detail",
+]

packages/surugaya_common/src/surugaya_common/parsers/surugaya/order_detail.py

@@ -0,0 +1,329 @@
+"""骏河屋交易详情页解析器（需 playwright，安装 surugaya_common[browser]）。
+
+从 jp_pruchase_zdh_5/src/parsers/surugaya/order_detail.py 迁移而来，模型导入改为
+共享包 surugaya_common.models。
+
+相对 worker 原版的唯一行为差异：修正了 ITEM_FIELD_MAP 的字段映射 bug——原版将
+品番/状態/数量/金額 误映射为 product_id/status/number/product_total_amount，与
+SurugayaOrderItem 模型字段 product_code/condition/quantity/line_total 不符，被
+pydantic 默认 extra=ignore 静默丢弃，导致这 4 列恒为 None。此处已对齐模型字段。
+worker 本地副本仍为旧版，应在 repoint 到本包时一并淘汰。
+"""
+
+import re
+from typing import Any
+
+from playwright.async_api import Locator, Page
+
+from surugaya_common.models import (
+    SurugayaOrderDetail,
+    SurugayaOrderItem,
+    SurugayaShippingInfo,
+    SurugayaOrderStatus,
+    SurugayaOrderSummary,
+)
+
+
+# 订单摘要表中的日文字段到英文字段映射。
+SUMMARY_FIELD_MAP = {
+    "取引番号": "trade_number",
+    "注文日": "order_date",
+    "商品合計": "items_subtotal",
+    "商品点数": "item_count",
+    "支払方法": "payment_method",
+    "代引き手数料": "cash_on_delivery_fee",
+    "発送日": "shipping_date",
+    "お問い合わせ番号": "inquiry_number",
+    "送料・通信販売手数料": "shipping_and_handling_fee",
+    "電子商品券": "digital_gift_certificate_amount",
+    "総合計": "grand_total",
+}
+
+# 收件信息表的日文字段到英文字段映射。
+SHIPPING_FIELD_MAP = {
+    "氏名": "recipient_name",
+    "住所": "recipient_address",
+    "電話番号": "recipient_phone_number",
+    "メールアドレス": "recipient_email",
+}
+
+# 商品明细表的列名到英文字段映射。
+# 字段名已对齐 SurugayaOrderItem 模型（修正了 worker 原版的映射 bug：
+# product_code/condition/quantity/line_total 此前因键名不符被 pydantic 静默丢弃）。
+ITEM_FIELD_MAP = {
+    "品番": "product_code",
+    "状態": "condition",
+    "枝番": "branch_number",
+    "商品タイトル": "product_title",
+    "単価": "unit_price",
+    "数量": "quantity",
+    "値引額": "discount_amount",
+    "金額": "line_total",
+    "備考": "remarks",
+}
+
+# 需要按金额规则清洗的字段集合。
+MONEY_FIELDS = {
+    "items_subtotal",
+    "cash_on_delivery_fee",
+    "shipping_and_handling_fee",
+    "digital_gift_certificate_amount",
+    "grand_total",
+    "unit_price",
+    "discount_amount",
+    "line_total",
+}
+
+# 需要按整数规则清洗的字段集合。
+INT_FIELDS = {
+    "item_count",
+    "quantity",
+}
+
+class SurugayaOrderDetailParser:
+    """骏河屋交易详情页解析器。"""
+
+    @classmethod
+    async def parse_page(cls, page: Page) -> SurugayaOrderDetail:
+        """对外主入口：传入已加载详情页的 `Page`，返回结构化订单对象。"""
+        # 适配 `SurugayaOrderDetailParser.parse_page(page)` 这种类调用方式。
+        parser = cls()
+        if await page.locator("#pageCont").count() < 1:
+            raise ValueError("Surugaya order detail page not found")
+        return await parser._parse_page_content(page)
+
+    async def _parse_page_content(self, page: Page) -> SurugayaOrderDetail:
+        """解析正常交易详情页中的 `#pageCont` 核心区域。"""
+
+        page_cont = page.locator("#pageCont").first
+        tables = page_cont.locator("table")
+        table_count = await tables.count()
+        if table_count < 4:
+            raise ValueError("Expected 4 trade tables under #pageCont")
+
+        status_table = tables.nth(0)
+        summary_table = tables.nth(1)
+        shipping_table = tables.nth(2)
+        items_table = tables.nth(3)
+
+        status = await self._parse_status_table(status_table)
+        summary = await self._parse_summary_table(summary_table)
+        shipping = await self._parse_shipping_table(shipping_table)
+        items = await self._parse_items_table(items_table)
+
+        return SurugayaOrderDetail(
+            status=status,
+            summary=summary,
+            shipping=shipping,
+            items=items,
+            raw_tables={
+                "status": status.raw,
+                "summary": summary.raw,
+                "shipping": shipping.raw,
+                "items": [item.raw for item in items],
+            },
+        )
+
+    async def _parse_status_table(self, table: Locator) -> SurugayaOrderStatus:
+        """解析第 1 张交易状态表。"""
+        rows = table.locator("tr")
+        row_count = await rows.count()
+        if row_count < 1:
+            return SurugayaOrderStatus()
+
+        first_row = await self._extract_row_cells(rows.nth(0))
+        raw: dict[str, str] = {}
+        trade_status = None
+        status_message = None
+
+        if len(first_row) >= 2 and first_row[0]["tag"] == "th" and first_row[1]["tag"] == "td":
+            label = first_row[0]["text"]
+            value = first_row[1]["text"]
+            raw[label] = value
+            lines = self._split_lines(value)
+            trade_status = lines[0] if lines else None
+            status_message = " ".join(lines[1:]) or None
+
+        return SurugayaOrderStatus(
+            trade_status=trade_status,
+            status_message=status_message,
+            raw=raw,
+        )
+
+    async def _parse_summary_table(self, table: Locator) -> SurugayaOrderSummary:
+        """解析第 2 张订单摘要表。"""
+        raw_pairs = await self._parse_key_value_rows(table)
+        normalized = self._normalize_mapped_values(raw_pairs, SUMMARY_FIELD_MAP)
+        return SurugayaOrderSummary(**normalized, raw=raw_pairs)
+
+    async def _parse_shipping_table(self, table: Locator) -> SurugayaShippingInfo:
+        """解析第 3 张收件信息表。"""
+        raw_pairs = await self._parse_key_value_rows(table)
+        normalized = self._normalize_mapped_values(raw_pairs, SHIPPING_FIELD_MAP)
+        return SurugayaShippingInfo(**normalized, raw=raw_pairs)
+
+    async def _parse_items_table(self, table: Locator) -> list[SurugayaOrderItem]:
+        """解析第 4 张商品明细表。"""
+        rows = table.locator("tr")
+        row_count = await rows.count()
+        if row_count < 2:
+            return []
+
+        header_cells = await self._extract_row_cells(rows.nth(0))
+        headers = [cell["text"] for cell in header_cells if cell["tag"] == "th" and cell["text"]]
+        if not headers:
+            return []
+
+        items: list[SurugayaOrderItem] = []
+        expected_columns = len(headers)
+
+        for index in range(1, row_count):
+            cells = await self._extract_row_cells(rows.nth(index))
+            if not cells:
+                continue
+
+            td_cells = [cell for cell in cells if cell["tag"] == "td"]
+            if len(td_cells) != expected_columns:
+                continue
+
+            raw_item = {
+                header: td_cells[position]["text"]
+                for position, header in enumerate(headers)
+            }
+
+            if not any(raw_item.values()):
+                continue
+
+            normalized = self._normalize_mapped_values(raw_item, ITEM_FIELD_MAP)
+            items.append(SurugayaOrderItem(**normalized, raw=raw_item))
+
+        return items
+
+    async def _parse_key_value_rows(self, table: Locator) -> dict[str, str]:
+        """通用解析：按 `th -> td` 配对方式抽取键值。"""
+        rows = table.locator("tr")
+        row_count = await rows.count()
+        result: dict[str, str] = {}
+
+        for index in range(row_count):
+            cells = await self._extract_row_cells(rows.nth(index))
+            if not cells:
+                continue
+
+            pointer = 0
+            while pointer < len(cells):
+                current = cells[pointer]
+                if current["tag"] != "th":
+                    pointer += 1
+                    continue
+
+                label = current["text"]
+                pointer += 1
+                if not label:
+                    continue
+
+                while pointer < len(cells) and cells[pointer]["tag"] != "td":
+                    pointer += 1
+
+                if pointer >= len(cells):
+                    break
+
+                result[label] = cells[pointer]["text"]
+                pointer += 1
+
+        return result
+
+    async def _extract_row_cells(self, row: Locator) -> list[dict[str, str]]:
+        """提取一行中的所有单元格，并保留标签名与纯文本。"""
+        return await row.locator(":scope > th, :scope > td").evaluate_all(
+            """cells => cells.map((cell) => ({
+                tag: cell.tagName.toLowerCase(),
+                text: (cell.innerText || "").replace(/\\u00a0/g, " ").trim(),
+            }))"""
+        )
+
+    def _normalize_mapped_values(
+        self,
+        raw_data: dict[str, str],
+        field_map: dict[str, str],
+    ) -> dict[str, Any]:
+        """将日文键值映射并清洗为结构化字段。"""
+        normalized: dict[str, Any] = {}
+
+        for label, value in raw_data.items():
+            english_key = field_map.get(label)
+            if not english_key:
+                continue
+
+            cleaned_value = self._clean_text(value)
+            if english_key in MONEY_FIELDS:
+                normalized[english_key] = self._normalize_money(cleaned_value)
+            elif english_key in INT_FIELDS:
+                normalized[english_key] = self._normalize_int(cleaned_value)
+            elif english_key.endswith("_date"):
+                normalized[english_key] = self._normalize_date(cleaned_value)
+            else:
+                normalized[english_key] = cleaned_value
+
+        return normalized
+
+    def _split_lines(self, text: str | None) -> list[str]:
+        """按行拆分文本，并清理每一行空白。"""
+        if not text:
+            return []
+
+        lines = [self._clean_text(part) for part in text.splitlines()]
+        return [line for line in lines if line]
+
+    def _clean_text(self, text: str | None) -> str | None:
+        """清洗文本中的多余空白并统一空值。"""
+        if text is None:
+            return None
+
+        cleaned = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
+        return cleaned or None
+
+    def _normalize_money(self, text: str | None) -> int | None:
+        """将 `￥1,234` / `1,234円` 这类金额文本转成整数。"""
+        if not text:
+            return None
+
+        digits = re.findall(r"-?\d[\d,]*", text)
+        if not digits:
+            return None
+
+        value = digits[0].replace(",", "")
+        try:
+            return int(value)
+        except ValueError:
+            return None
+
+    def _normalize_int(self, text: str | None) -> int | None:
+        """提取文本中的首个整数值。"""
+        if not text:
+            return None
+
+        digits = re.findall(r"\d+", text)
+        if not digits:
+            return None
+
+        return int(digits[0])
+
+    def _normalize_date(self, text: str | None) -> str | None:
+        """将 `YYYY.MM.DD` / `YYYY/MM/DD` 统一为 `YYYY-MM-DD`。"""
+        if not text:
+            return None
+
+        match = re.search(r"(\d{4})[./-](\d{2})[./-](\d{2})", text)
+        if not match:
+            return text
+
+        year, month, day = match.groups()
+        return f"{year}-{month}-{day}"
+
+
+async def parse_surugaya_order_detail(page: Page) -> dict[str, Any]:
+    """便捷函数：直接返回可 JSON 序列化的订单字典。"""
+    parser = SurugayaOrderDetailParser()
+    result = await parser.parse_page(page)
+    return result.model_dump(mode="json")

tests/test_order_detail_parser.py

@@ -0,0 +1,50 @@
+"""SurugayaOrderDetailParser 商品明细字段映射的回归测试。
+
+锁定 ITEM_FIELD_MAP 字段名 bug 的修复：品番/状態/数量/金額 必须正确填充
+product_code/condition/quantity/line_total，而非被 pydantic 静默丢弃。
+"""
+
+from surugaya_common.models import SurugayaOrderItem
+from surugaya_common.parsers.surugaya.order_detail import (
+    ITEM_FIELD_MAP,
+    SurugayaOrderDetailParser,
+)
+
+
+def test_item_field_map_targets_are_real_model_fields():
+    """ITEM_FIELD_MAP 的每个目标字段都必须是 SurugayaOrderItem 的真实字段。"""
+    model_fields = set(SurugayaOrderItem.model_fields)
+    for jp_column, en_field in ITEM_FIELD_MAP.items():
+        assert en_field in model_fields, f"{jp_column} 映射到非法字段 {en_field}"
+
+
+def test_normalize_item_row_populates_all_corrected_fields():
+    """修复后：品番/状態/数量/金額 应正确填充，金额/整数字段完成归一化。"""
+    parser = SurugayaOrderDetailParser()
+    raw_item = {
+        "品番": "ABC-123",
+        "状態": "中古",
+        "枝番": "1",
+        "商品タイトル": "测试商品",
+        "単価": "1,250円",
+        "数量": "2",
+        "値引額": "100円",
+        "金額": "2,400円",
+        "備考": "x",
+    }
+
+    normalized = parser._normalize_mapped_values(raw_item, ITEM_FIELD_MAP)
+    item = SurugayaOrderItem(**normalized, raw=raw_item)
+
+    # 此前被静默丢弃的 4 个字段现在应被正确填充
+    assert item.product_code == "ABC-123"
+    assert item.condition == "中古"
+    assert item.quantity == 2  # INT_FIELDS 归一
+    assert item.line_total == 2400  # MONEY_FIELDS 归一，逗号去除
+
+    # 原本就正确的字段保持不变
+    assert item.branch_number == "1"
+    assert item.product_title == "测试商品"
+    assert item.unit_price == 1250
+    assert item.discount_amount == 100
+    assert item.remarks == "x"

uv.lock

@@ -524,11 +524,18 @@ dependencies = [
     { name = "redis" },
 ]
 
+[package.optional-dependencies]
+browser = [
+    { name = "playwright" },
+]
+
 [package.metadata]
 requires-dist = [
+    { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" },
     { name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
     { name = "redis", specifier = "==8.0.0" },
 ]
+provides-extras = ["browser"]
 
 [[package]]
 name = "surugaya-scraper-service"