From 66aebcf5653108450a459e0d6513d3548166e583 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Tue, 9 Jun 2026 18:22:55 +0800 Subject: [PATCH] =?UTF-8?q?workspace=E2=80=94=20step4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/routes/scrape.py | 28 +- packages/surugaya_common/pyproject.toml | 11 +- .../src/surugaya_common/browser_utils.py | 108 ++++++ .../src/surugaya_common/parsers/__init__.py | 1 + .../parsers/surugaya/__init__.py | 11 + .../parsers/surugaya/order_detail.py | 329 ++++++++++++++++++ tests/test_order_detail_parser.py | 50 +++ uv.lock | 7 + 8 files changed, 535 insertions(+), 10 deletions(-) create mode 100644 packages/surugaya_common/src/surugaya_common/browser_utils.py create mode 100644 packages/surugaya_common/src/surugaya_common/parsers/__init__.py create mode 100644 packages/surugaya_common/src/surugaya_common/parsers/surugaya/__init__.py create mode 100644 packages/surugaya_common/src/surugaya_common/parsers/surugaya/order_detail.py create mode 100644 tests/test_order_detail_parser.py diff --git a/app/api/routes/scrape.py b/app/api/routes/scrape.py index 8f24fe0..4f7148f 100644 --- a/app/api/routes/scrape.py +++ b/app/api/routes/scrape.py @@ -6,6 +6,7 @@ from typing import Any from urllib import error, request from fastapi import APIRouter, Depends +from pydantic import ValidationError from app.api.dependencies import get_container, require_bearer_token from app.core.container import ServiceContainer @@ -29,6 +30,7 @@ from app.utils.parse_util import ( get_shipping_fee, ) from surugaya_common import redis_keys +from surugaya_common.models import SurugayaOrderDetail router = APIRouter(prefix="/api", tags=["scrape"]) @@ -279,18 +281,18 @@ async def order_add_trade_monitor( @router.get( "/trade", - response_model=ApiResponse[dict[str, Any]], + response_model=ApiResponse[SurugayaOrderDetail], dependencies=[Depends(require_bearer_token)], ) async def trade( trade_code: str, container: ServiceContainer = Depends(get_container), -) -> ApiResponse[dict[str, Any]]: +) -> ApiResponse[SurugayaOrderDetail]: """ 获取订单详情 """ if not trade_code or trade_code.strip() == "": - return ApiResponse[dict[str, Any]]( + return ApiResponse[SurugayaOrderDetail]( success=False, msg="trade_code 不能为空", data=None, @@ -298,7 +300,7 @@ async def trade( ) if container.session_store._redis is None: - return ApiResponse[dict[str, Any]]( + return ApiResponse[SurugayaOrderDetail]( success=False, msg="Redis 未配置", data=None, @@ -323,17 +325,29 @@ async def trade( approximate=True, ) - return ApiResponse[dict[str, Any]]( + return ApiResponse[SurugayaOrderDetail]( success=False, msg="未找到该交易", data=None, code=404, ) - return ApiResponse[dict[str, Any]]( + # Redis 中存储的是 worker 写入的 SurugayaOrderDetail 序列化结果, + # 用共享契约校验并类型化;历史脏数据校验失败时降级为结构化错误而非 500。 + try: + order_detail = SurugayaOrderDetail.model_validate_json(trade_data) + except ValidationError as exc: + return ApiResponse[SurugayaOrderDetail]( + success=False, + msg=f"订单详情数据结构异常: {exc.error_count()} 处校验失败", + data=None, + code=500, + ) + + return ApiResponse[SurugayaOrderDetail]( success=True, msg="success", - data=json.loads(trade_data), + data=order_detail, code=0, ) diff --git a/packages/surugaya_common/pyproject.toml b/packages/surugaya_common/pyproject.toml index 7704be2..7392d42 100644 --- a/packages/surugaya_common/pyproject.toml +++ b/packages/surugaya_common/pyproject.toml @@ -12,9 +12,14 @@ dependencies = [ "redis==8.0.0", ] +# playwright 仅作为可选 extra:只有解析器/浏览器工具需要它,核心安装保持 light。 +# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集,确保 workspace 解析到单一 playwright。 +[project.optional-dependencies] +browser = ["playwright>=1.54.0,<2.0.0"] + [tool.hatch.build.targets.wheel] packages = ["src/surugaya_common"] -# Keep surugaya_common light: stdlib + pydantic + redis only. redis is pinned to -# ==8.0.0 to stay in lockstep with both the API service and the worker. Do not add -# fastapi/playwright/opencv/numpy/pyautogui/pillow here. +# 保持 surugaya_common core 轻量:仅 stdlib + pydantic + redis(==8.0.0,与 API 和 worker 对齐)。 +# playwright 只允许出现在上面的 [browser] extra 中,core 安装绝不引入。 +# 不要在此加入 fastapi/opencv/numpy/pyautogui/pillow。 diff --git a/packages/surugaya_common/src/surugaya_common/browser_utils.py b/packages/surugaya_common/src/surugaya_common/browser_utils.py new file mode 100644 index 0000000..530bbd0 --- /dev/null +++ b/packages/surugaya_common/src/surugaya_common/browser_utils.py @@ -0,0 +1,108 @@ +"""浏览器拟真交互工具(需 playwright,安装 surugaya_common[browser])。 + +从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 human_scroll,逻辑保持一致。 +注:page_utils.py 中的纯函数 format_japanese_price 未迁移——它与 API 端 +app/utils/parse_util.py 的 format_price 语义不同,合并属行为变更,单独处理。 +""" + +from playwright.async_api import Page +import asyncio +import random + + +async def human_scroll(page: Page, mode: str, value: int = 0): + """ + 模拟人类行为的网页滚动方法 + :param page: Playwright 的 Page 对象 + :param mode: 滚动模式 -> 'top'(滚动到顶部), 'bottom'(滚动到底部), 'random'(随机上下滚动), 'distance'(滚动指定距离) + :param value: 当 mode='distance' 时,代表要滚动的像素距离(正数向下,负数向上) + """ + + async def smooth_scroll_step(pixel_distance: int): + """核心拟真算法:将一个大距离拆分成带有加速度和随机抖动的微小步长""" + if pixel_distance == 0: + return + + direction = 1 if pixel_distance > 0 else -1 + remaining = abs(pixel_distance) + + while remaining > 0: + # 模拟人类滚动时越接近目标速度越慢的特征(减速缓冲) + if remaining > 100: + step = random.randint(30, 70) + elif remaining > 30: + step = random.randint(10, 25) + else: + step = remaining + + remaining -= step + scroll_y = step * direction + + # 执行单步微调滚动 + await page.evaluate(f"window.scrollBy(0, {scroll_y});") + + # 随机微小停顿,模拟手指滑动的摩擦阻力(20ms - 50ms) + await asyncio.sleep(random.uniform(0.02, 0.05)) + + # 0.5% 的极低概率触发人类的“视线停留” + if random.random() < 0.005: + await asyncio.sleep(random.uniform(0.3, 0.8)) + + # ==================== 模式 0:滚动到顶部 ==================== + if mode == 'top': + # print("[HumanScroll] 正在模拟向上滚动至页面顶部...") + last_scroll = -1 + while True: + # 获取当前滚动条位置 + current_scroll = await page.evaluate("window.scrollY") + + # 防死循环:如果位置不再变化,说明已经到顶或页面无法滚动 + if current_scroll <= 0 or current_scroll == last_scroll: + break + + last_scroll = current_scroll + chunk = random.randint(300, 700) + await smooth_scroll_step(-chunk) + await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼 + + # ==================== 模式 1:滚动到底部 ==================== + elif mode == 'bottom': + # print("[HumanScroll] 正在模拟向下滚动至页面底部...") + last_scroll = -1 + while True: + # 合并 evaluate 请求,减少与浏览器的 IPC 通信开销 + scroll_info = await page.evaluate("() => [window.scrollY + window.innerHeight, document.body.scrollHeight]") + current_scroll, total_height = scroll_info[0], scroll_info[1] + + # 防死循环:如果位置不再变化,说明已经到底或页面无法滚动 + if current_scroll >= total_height - 10 or current_scroll == last_scroll: + break + + last_scroll = current_scroll + chunk = random.randint(300, 700) + await smooth_scroll_step(chunk) + await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼 + + # ==================== 模式 2:随机上下滚动几下 ==================== + elif mode == 'random': + # print("[HumanScroll] 正在执行人类迷惑行为:随机上下滚动...") + # 随机决定滚动 3 到 6 下 + scroll_times = random.randint(3, 6) + + for _ in range(scroll_times): + # 70% 概率向下,30% 概率向上 + if random.random() > 0.3: + distance = random.randint(200, 500) # 向下滚 + else: + distance = random.randint(-400, -150) # 向上回滚 + + await smooth_scroll_step(distance) + await asyncio.sleep(random.uniform(0.6, 1.5)) # 停顿 + + # ==================== 模式 3:滚动指定距离 ==================== + elif mode == 'distance': + # print(f"[HumanScroll] 正在精准模拟滚动指定距离: {value} 像素") + await smooth_scroll_step(value) + + else: + raise ValueError("未知的滚动模式!请选择 'top', 'bottom', 'random' 或 'distance'") diff --git a/packages/surugaya_common/src/surugaya_common/parsers/__init__.py b/packages/surugaya_common/src/surugaya_common/parsers/__init__.py new file mode 100644 index 0000000..27956f0 --- /dev/null +++ b/packages/surugaya_common/src/surugaya_common/parsers/__init__.py @@ -0,0 +1 @@ +"""surugaya_common 解析器子包(部分依赖 playwright,需安装 surugaya_common[browser])。""" diff --git a/packages/surugaya_common/src/surugaya_common/parsers/surugaya/__init__.py b/packages/surugaya_common/src/surugaya_common/parsers/surugaya/__init__.py new file mode 100644 index 0000000..73dc884 --- /dev/null +++ b/packages/surugaya_common/src/surugaya_common/parsers/surugaya/__init__.py @@ -0,0 +1,11 @@ +"""骏河屋页面解析器(需 playwright,安装 surugaya_common[browser])。""" + +from surugaya_common.parsers.surugaya.order_detail import ( + SurugayaOrderDetailParser, + parse_surugaya_order_detail, +) + +__all__ = [ + "SurugayaOrderDetailParser", + "parse_surugaya_order_detail", +] diff --git a/packages/surugaya_common/src/surugaya_common/parsers/surugaya/order_detail.py b/packages/surugaya_common/src/surugaya_common/parsers/surugaya/order_detail.py new file mode 100644 index 0000000..7a82131 --- /dev/null +++ b/packages/surugaya_common/src/surugaya_common/parsers/surugaya/order_detail.py @@ -0,0 +1,329 @@ +"""骏河屋交易详情页解析器(需 playwright,安装 surugaya_common[browser])。 + +从 jp_pruchase_zdh_5/src/parsers/surugaya/order_detail.py 迁移而来,模型导入改为 +共享包 surugaya_common.models。 + +相对 worker 原版的唯一行为差异:修正了 ITEM_FIELD_MAP 的字段映射 bug——原版将 +品番/状態/数量/金額 误映射为 product_id/status/number/product_total_amount,与 +SurugayaOrderItem 模型字段 product_code/condition/quantity/line_total 不符,被 +pydantic 默认 extra=ignore 静默丢弃,导致这 4 列恒为 None。此处已对齐模型字段。 +worker 本地副本仍为旧版,应在 repoint 到本包时一并淘汰。 +""" + +import re +from typing import Any + +from playwright.async_api import Locator, Page + +from surugaya_common.models import ( + SurugayaOrderDetail, + SurugayaOrderItem, + SurugayaShippingInfo, + SurugayaOrderStatus, + SurugayaOrderSummary, +) + + +# 订单摘要表中的日文字段到英文字段映射。 +SUMMARY_FIELD_MAP = { + "取引番号": "trade_number", + "注文日": "order_date", + "商品合計": "items_subtotal", + "商品点数": "item_count", + "支払方法": "payment_method", + "代引き手数料": "cash_on_delivery_fee", + "発送日": "shipping_date", + "お問い合わせ番号": "inquiry_number", + "送料・通信販売手数料": "shipping_and_handling_fee", + "電子商品券": "digital_gift_certificate_amount", + "総合計": "grand_total", +} + +# 收件信息表的日文字段到英文字段映射。 +SHIPPING_FIELD_MAP = { + "氏名": "recipient_name", + "住所": "recipient_address", + "電話番号": "recipient_phone_number", + "メールアドレス": "recipient_email", +} + +# 商品明细表的列名到英文字段映射。 +# 字段名已对齐 SurugayaOrderItem 模型(修正了 worker 原版的映射 bug: +# product_code/condition/quantity/line_total 此前因键名不符被 pydantic 静默丢弃)。 +ITEM_FIELD_MAP = { + "品番": "product_code", + "状態": "condition", + "枝番": "branch_number", + "商品タイトル": "product_title", + "単価": "unit_price", + "数量": "quantity", + "値引額": "discount_amount", + "金額": "line_total", + "備考": "remarks", +} + +# 需要按金额规则清洗的字段集合。 +MONEY_FIELDS = { + "items_subtotal", + "cash_on_delivery_fee", + "shipping_and_handling_fee", + "digital_gift_certificate_amount", + "grand_total", + "unit_price", + "discount_amount", + "line_total", +} + +# 需要按整数规则清洗的字段集合。 +INT_FIELDS = { + "item_count", + "quantity", +} + +class SurugayaOrderDetailParser: + """骏河屋交易详情页解析器。""" + + @classmethod + async def parse_page(cls, page: Page) -> SurugayaOrderDetail: + """对外主入口:传入已加载详情页的 `Page`,返回结构化订单对象。""" + # 适配 `SurugayaOrderDetailParser.parse_page(page)` 这种类调用方式。 + parser = cls() + if await page.locator("#pageCont").count() < 1: + raise ValueError("Surugaya order detail page not found") + return await parser._parse_page_content(page) + + async def _parse_page_content(self, page: Page) -> SurugayaOrderDetail: + """解析正常交易详情页中的 `#pageCont` 核心区域。""" + + page_cont = page.locator("#pageCont").first + tables = page_cont.locator("table") + table_count = await tables.count() + if table_count < 4: + raise ValueError("Expected 4 trade tables under #pageCont") + + status_table = tables.nth(0) + summary_table = tables.nth(1) + shipping_table = tables.nth(2) + items_table = tables.nth(3) + + status = await self._parse_status_table(status_table) + summary = await self._parse_summary_table(summary_table) + shipping = await self._parse_shipping_table(shipping_table) + items = await self._parse_items_table(items_table) + + return SurugayaOrderDetail( + status=status, + summary=summary, + shipping=shipping, + items=items, + raw_tables={ + "status": status.raw, + "summary": summary.raw, + "shipping": shipping.raw, + "items": [item.raw for item in items], + }, + ) + + async def _parse_status_table(self, table: Locator) -> SurugayaOrderStatus: + """解析第 1 张交易状态表。""" + rows = table.locator("tr") + row_count = await rows.count() + if row_count < 1: + return SurugayaOrderStatus() + + first_row = await self._extract_row_cells(rows.nth(0)) + raw: dict[str, str] = {} + trade_status = None + status_message = None + + if len(first_row) >= 2 and first_row[0]["tag"] == "th" and first_row[1]["tag"] == "td": + label = first_row[0]["text"] + value = first_row[1]["text"] + raw[label] = value + lines = self._split_lines(value) + trade_status = lines[0] if lines else None + status_message = " ".join(lines[1:]) or None + + return SurugayaOrderStatus( + trade_status=trade_status, + status_message=status_message, + raw=raw, + ) + + async def _parse_summary_table(self, table: Locator) -> SurugayaOrderSummary: + """解析第 2 张订单摘要表。""" + raw_pairs = await self._parse_key_value_rows(table) + normalized = self._normalize_mapped_values(raw_pairs, SUMMARY_FIELD_MAP) + return SurugayaOrderSummary(**normalized, raw=raw_pairs) + + async def _parse_shipping_table(self, table: Locator) -> SurugayaShippingInfo: + """解析第 3 张收件信息表。""" + raw_pairs = await self._parse_key_value_rows(table) + normalized = self._normalize_mapped_values(raw_pairs, SHIPPING_FIELD_MAP) + return SurugayaShippingInfo(**normalized, raw=raw_pairs) + + async def _parse_items_table(self, table: Locator) -> list[SurugayaOrderItem]: + """解析第 4 张商品明细表。""" + rows = table.locator("tr") + row_count = await rows.count() + if row_count < 2: + return [] + + header_cells = await self._extract_row_cells(rows.nth(0)) + headers = [cell["text"] for cell in header_cells if cell["tag"] == "th" and cell["text"]] + if not headers: + return [] + + items: list[SurugayaOrderItem] = [] + expected_columns = len(headers) + + for index in range(1, row_count): + cells = await self._extract_row_cells(rows.nth(index)) + if not cells: + continue + + td_cells = [cell for cell in cells if cell["tag"] == "td"] + if len(td_cells) != expected_columns: + continue + + raw_item = { + header: td_cells[position]["text"] + for position, header in enumerate(headers) + } + + if not any(raw_item.values()): + continue + + normalized = self._normalize_mapped_values(raw_item, ITEM_FIELD_MAP) + items.append(SurugayaOrderItem(**normalized, raw=raw_item)) + + return items + + async def _parse_key_value_rows(self, table: Locator) -> dict[str, str]: + """通用解析:按 `th -> td` 配对方式抽取键值。""" + rows = table.locator("tr") + row_count = await rows.count() + result: dict[str, str] = {} + + for index in range(row_count): + cells = await self._extract_row_cells(rows.nth(index)) + if not cells: + continue + + pointer = 0 + while pointer < len(cells): + current = cells[pointer] + if current["tag"] != "th": + pointer += 1 + continue + + label = current["text"] + pointer += 1 + if not label: + continue + + while pointer < len(cells) and cells[pointer]["tag"] != "td": + pointer += 1 + + if pointer >= len(cells): + break + + result[label] = cells[pointer]["text"] + pointer += 1 + + return result + + async def _extract_row_cells(self, row: Locator) -> list[dict[str, str]]: + """提取一行中的所有单元格,并保留标签名与纯文本。""" + return await row.locator(":scope > th, :scope > td").evaluate_all( + """cells => cells.map((cell) => ({ + tag: cell.tagName.toLowerCase(), + text: (cell.innerText || "").replace(/\\u00a0/g, " ").trim(), + }))""" + ) + + def _normalize_mapped_values( + self, + raw_data: dict[str, str], + field_map: dict[str, str], + ) -> dict[str, Any]: + """将日文键值映射并清洗为结构化字段。""" + normalized: dict[str, Any] = {} + + for label, value in raw_data.items(): + english_key = field_map.get(label) + if not english_key: + continue + + cleaned_value = self._clean_text(value) + if english_key in MONEY_FIELDS: + normalized[english_key] = self._normalize_money(cleaned_value) + elif english_key in INT_FIELDS: + normalized[english_key] = self._normalize_int(cleaned_value) + elif english_key.endswith("_date"): + normalized[english_key] = self._normalize_date(cleaned_value) + else: + normalized[english_key] = cleaned_value + + return normalized + + def _split_lines(self, text: str | None) -> list[str]: + """按行拆分文本,并清理每一行空白。""" + if not text: + return [] + + lines = [self._clean_text(part) for part in text.splitlines()] + return [line for line in lines if line] + + def _clean_text(self, text: str | None) -> str | None: + """清洗文本中的多余空白并统一空值。""" + if text is None: + return None + + cleaned = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip() + return cleaned or None + + def _normalize_money(self, text: str | None) -> int | None: + """将 `¥1,234` / `1,234円` 这类金额文本转成整数。""" + if not text: + return None + + digits = re.findall(r"-?\d[\d,]*", text) + if not digits: + return None + + value = digits[0].replace(",", "") + try: + return int(value) + except ValueError: + return None + + def _normalize_int(self, text: str | None) -> int | None: + """提取文本中的首个整数值。""" + if not text: + return None + + digits = re.findall(r"\d+", text) + if not digits: + return None + + return int(digits[0]) + + def _normalize_date(self, text: str | None) -> str | None: + """将 `YYYY.MM.DD` / `YYYY/MM/DD` 统一为 `YYYY-MM-DD`。""" + if not text: + return None + + match = re.search(r"(\d{4})[./-](\d{2})[./-](\d{2})", text) + if not match: + return text + + year, month, day = match.groups() + return f"{year}-{month}-{day}" + + +async def parse_surugaya_order_detail(page: Page) -> dict[str, Any]: + """便捷函数:直接返回可 JSON 序列化的订单字典。""" + parser = SurugayaOrderDetailParser() + result = await parser.parse_page(page) + return result.model_dump(mode="json") diff --git a/tests/test_order_detail_parser.py b/tests/test_order_detail_parser.py new file mode 100644 index 0000000..e8a800c --- /dev/null +++ b/tests/test_order_detail_parser.py @@ -0,0 +1,50 @@ +"""SurugayaOrderDetailParser 商品明细字段映射的回归测试。 + +锁定 ITEM_FIELD_MAP 字段名 bug 的修复:品番/状態/数量/金額 必须正确填充 +product_code/condition/quantity/line_total,而非被 pydantic 静默丢弃。 +""" + +from surugaya_common.models import SurugayaOrderItem +from surugaya_common.parsers.surugaya.order_detail import ( + ITEM_FIELD_MAP, + SurugayaOrderDetailParser, +) + + +def test_item_field_map_targets_are_real_model_fields(): + """ITEM_FIELD_MAP 的每个目标字段都必须是 SurugayaOrderItem 的真实字段。""" + model_fields = set(SurugayaOrderItem.model_fields) + for jp_column, en_field in ITEM_FIELD_MAP.items(): + assert en_field in model_fields, f"{jp_column} 映射到非法字段 {en_field}" + + +def test_normalize_item_row_populates_all_corrected_fields(): + """修复后:品番/状態/数量/金額 应正确填充,金额/整数字段完成归一化。""" + parser = SurugayaOrderDetailParser() + raw_item = { + "品番": "ABC-123", + "状態": "中古", + "枝番": "1", + "商品タイトル": "测试商品", + "単価": "1,250円", + "数量": "2", + "値引額": "100円", + "金額": "2,400円", + "備考": "x", + } + + normalized = parser._normalize_mapped_values(raw_item, ITEM_FIELD_MAP) + item = SurugayaOrderItem(**normalized, raw=raw_item) + + # 此前被静默丢弃的 4 个字段现在应被正确填充 + assert item.product_code == "ABC-123" + assert item.condition == "中古" + assert item.quantity == 2 # INT_FIELDS 归一 + assert item.line_total == 2400 # MONEY_FIELDS 归一,逗号去除 + + # 原本就正确的字段保持不变 + assert item.branch_number == "1" + assert item.product_title == "测试商品" + assert item.unit_price == 1250 + assert item.discount_amount == 100 + assert item.remarks == "x" diff --git a/uv.lock b/uv.lock index 6ba2eb0..c01ba68 100644 --- a/uv.lock +++ b/uv.lock @@ -524,11 +524,18 @@ dependencies = [ { name = "redis" }, ] +[package.optional-dependencies] +browser = [ + { name = "playwright" }, +] + [package.metadata] requires-dist = [ + { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" }, { name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "redis", specifier = "==8.0.0" }, ] +provides-extras = ["browser"] [[package]] name = "surugaya-scraper-service"