Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 q792602257
|
11eaa7443c | ||
|
q792602257
|
66aebcf565 | ||
|
q792602257
|
5ceabe9c9a | ||
|
q792602257
|
4e591b07eb | ||
|
q792602257
|
7eb3647539 |
+28
-14
@@ -6,6 +6,7 @@ from typing import Any
|
||||
from urllib import error, request
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.api.dependencies import get_container, require_bearer_token
|
||||
from app.core.container import ServiceContainer
|
||||
@@ -22,12 +23,14 @@ from app.models.scrape import (
|
||||
SearchRequest,
|
||||
SearchResultData, TradeMonitorRequest,
|
||||
)
|
||||
from app.utils.http_utils import generate_signed_headers
|
||||
from surugaya_common.http_utils import generate_signed_headers
|
||||
from app.utils.parse_util import (
|
||||
get_franchise_shipping_fee,
|
||||
get_handle_fee,
|
||||
get_shipping_fee,
|
||||
)
|
||||
from surugaya_common import redis_keys
|
||||
from surugaya_common.models import SurugayaOrderDetail
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["scrape"])
|
||||
|
||||
@@ -202,7 +205,7 @@ async def order_add_purchase_task(
|
||||
code=500,
|
||||
)
|
||||
|
||||
stream_name = "jp_surugaya_purchase_task_dev" if payload.is_dev == "1" else "jp_surugaya_purchase_task"
|
||||
stream_name = redis_keys.purchase_task_stream(payload.is_dev)
|
||||
params_str = payload.model_dump_json(by_alias=True)
|
||||
|
||||
await container.session_store._redis.xadd(
|
||||
@@ -259,8 +262,7 @@ async def order_add_trade_monitor(
|
||||
code=400,
|
||||
)
|
||||
|
||||
queue_name = "jp_surugaya_trade_delay_dev" if payload.is_dev == "1" else "jp_surugaya_trade_delay"
|
||||
queue_name += f":{node_id}"
|
||||
queue_name = redis_keys.trade_delay_zset(node_id, payload.is_dev)
|
||||
|
||||
score = int(time.time()) + 3
|
||||
member = json.dumps(payload.payload)
|
||||
@@ -279,18 +281,18 @@ async def order_add_trade_monitor(
|
||||
|
||||
@router.get(
|
||||
"/trade",
|
||||
response_model=ApiResponse[dict[str, Any]],
|
||||
response_model=ApiResponse[SurugayaOrderDetail],
|
||||
dependencies=[Depends(require_bearer_token)],
|
||||
)
|
||||
async def trade(
|
||||
trade_code: str,
|
||||
container: ServiceContainer = Depends(get_container),
|
||||
) -> ApiResponse[dict[str, Any]]:
|
||||
) -> ApiResponse[SurugayaOrderDetail]:
|
||||
"""
|
||||
获取订单详情
|
||||
"""
|
||||
if not trade_code or trade_code.strip() == "":
|
||||
return ApiResponse[dict[str, Any]](
|
||||
return ApiResponse[SurugayaOrderDetail](
|
||||
success=False,
|
||||
msg="trade_code 不能为空",
|
||||
data=None,
|
||||
@@ -298,7 +300,7 @@ async def trade(
|
||||
)
|
||||
|
||||
if container.session_store._redis is None:
|
||||
return ApiResponse[dict[str, Any]](
|
||||
return ApiResponse[SurugayaOrderDetail](
|
||||
success=False,
|
||||
msg="Redis 未配置",
|
||||
data=None,
|
||||
@@ -306,34 +308,46 @@ async def trade(
|
||||
)
|
||||
|
||||
trade_data = await container.session_store._redis.hget(
|
||||
name="jp_surugaya_trade",
|
||||
name=redis_keys.trade_hash(),
|
||||
key=trade_code,
|
||||
)
|
||||
|
||||
if not trade_data:
|
||||
# 去重：同一 trade_code 在 3600s 内不重复下发
|
||||
dedup_key = f"jp_surugaya_trade_pending:{trade_code}"
|
||||
dedup_key = redis_keys.trade_pending_key(trade_code)
|
||||
dedup_exists = await container.session_store._redis.get(dedup_key)
|
||||
if not dedup_exists:
|
||||
await container.session_store._redis.setex(dedup_key, 3600, "1")
|
||||
await container.session_store._redis.xadd(
|
||||
name="jp_surugaya_trade_fetch",
|
||||
name=redis_keys.trade_fetch_stream(),
|
||||
fields={"payload": json.dumps({"trade_code": trade_code})},
|
||||
maxlen=100,
|
||||
approximate=True,
|
||||
)
|
||||
|
||||
return ApiResponse[dict[str, Any]](
|
||||
return ApiResponse[SurugayaOrderDetail](
|
||||
success=False,
|
||||
msg="未找到该交易",
|
||||
data=None,
|
||||
code=404,
|
||||
)
|
||||
|
||||
return ApiResponse[dict[str, Any]](
|
||||
# Redis 中存储的是 worker 写入的 SurugayaOrderDetail 序列化结果，
|
||||
# 用共享契约校验并类型化；历史脏数据校验失败时降级为结构化错误而非 500。
|
||||
try:
|
||||
order_detail = SurugayaOrderDetail.model_validate_json(trade_data)
|
||||
except ValidationError as exc:
|
||||
return ApiResponse[SurugayaOrderDetail](
|
||||
success=False,
|
||||
msg=f"订单详情数据结构异常: {exc.error_count()} 处校验失败",
|
||||
data=None,
|
||||
code=500,
|
||||
)
|
||||
|
||||
return ApiResponse[SurugayaOrderDetail](
|
||||
success=True,
|
||||
msg="success",
|
||||
data=json.loads(trade_data),
|
||||
data=order_detail,
|
||||
code=0,
|
||||
)
|
||||
|
||||
|
||||
@@ -24,8 +24,8 @@ from app.models.scrape import CategoryData, DetailRequest, ProductDetailData, Pr
|
||||
from app.services.browser_pool import BrowserPool
|
||||
from app.services.cloudflare_session import CloudflareSessionManager
|
||||
from app.services.session_store import SessionStore
|
||||
from app.utils.app_utils import is_empty_str
|
||||
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
|
||||
from surugaya_common.app_utils import is_empty_str
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
[build-system]
|
||||
requires = ["hatchling"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "surugaya-common"
|
||||
version = "0.2.0"
|
||||
description = "Shared contracts (Redis keys, future models) for jp_surugaya services"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"pydantic>=2.0.0,<3.0.0",
|
||||
"redis==8.0.0",
|
||||
]
|
||||
|
||||
# playwright 仅作为可选 extra：只有解析器/浏览器工具需要它，核心安装保持 light。
|
||||
# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集，确保 workspace 解析到单一 playwright。
|
||||
[project.optional-dependencies]
|
||||
browser = ["playwright>=1.54.0,<2.0.0"]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/surugaya_common"]
|
||||
|
||||
# 保持 surugaya_common core 轻量：仅 stdlib + pydantic + redis(==8.0.0，与 API 和 worker 对齐)。
|
||||
# playwright 只允许出现在上面的 [browser] extra 中，core 安装绝不引入。
|
||||
# 不要在此加入 fastapi/opencv/numpy/pyautogui/pillow。
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Shared contracts for jp_surugaya services (Redis keys, future models)."""
|
||||
|
||||
from surugaya_common import redis_keys
|
||||
|
||||
__all__ = ["redis_keys"]
|
||||
@@ -1,8 +1,15 @@
|
||||
"""通用字符串工具（stdlib only）。
|
||||
|
||||
从 app/utils/app_utils.py 迁移而来，作为 surugaya_common 的共享实现，
|
||||
供 API 服务与 worker 复用。
|
||||
"""
|
||||
|
||||
|
||||
def is_valid_str(s):
|
||||
"""判断字符串是否有效，即非空且非空格。"""
|
||||
return isinstance(s, str) and len(s.strip()) > 0
|
||||
|
||||
|
||||
def is_empty_str(s):
|
||||
"""判断字符串是否为空，即空或仅包含空格。"""
|
||||
return isinstance(s, str) and len(s.strip()) == 0
|
||||
@@ -0,0 +1,108 @@
|
||||
"""浏览器拟真交互工具（需 playwright，安装 surugaya_common[browser]）。
|
||||
|
||||
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 human_scroll，逻辑保持一致。
|
||||
注：page_utils.py 中的纯函数 format_japanese_price 未迁移——它与 API 端
|
||||
app/utils/parse_util.py 的 format_price 语义不同，合并属行为变更，单独处理。
|
||||
"""
|
||||
|
||||
from playwright.async_api import Page
|
||||
import asyncio
|
||||
import random
|
||||
|
||||
|
||||
async def human_scroll(page: Page, mode: str, value: int = 0):
|
||||
"""
|
||||
模拟人类行为的网页滚动方法
|
||||
:param page: Playwright 的 Page 对象
|
||||
:param mode: 滚动模式 -> 'top'(滚动到顶部), 'bottom'(滚动到底部), 'random'(随机上下滚动), 'distance'(滚动指定距离)
|
||||
:param value: 当 mode='distance' 时，代表要滚动的像素距离（正数向下，负数向上）
|
||||
"""
|
||||
|
||||
async def smooth_scroll_step(pixel_distance: int):
|
||||
"""核心拟真算法：将一个大距离拆分成带有加速度和随机抖动的微小步长"""
|
||||
if pixel_distance == 0:
|
||||
return
|
||||
|
||||
direction = 1 if pixel_distance > 0 else -1
|
||||
remaining = abs(pixel_distance)
|
||||
|
||||
while remaining > 0:
|
||||
# 模拟人类滚动时越接近目标速度越慢的特征（减速缓冲）
|
||||
if remaining > 100:
|
||||
step = random.randint(30, 70)
|
||||
elif remaining > 30:
|
||||
step = random.randint(10, 25)
|
||||
else:
|
||||
step = remaining
|
||||
|
||||
remaining -= step
|
||||
scroll_y = step * direction
|
||||
|
||||
# 执行单步微调滚动
|
||||
await page.evaluate(f"window.scrollBy(0, {scroll_y});")
|
||||
|
||||
# 随机微小停顿，模拟手指滑动的摩擦阻力（20ms - 50ms）
|
||||
await asyncio.sleep(random.uniform(0.02, 0.05))
|
||||
|
||||
# 0.5% 的极低概率触发人类的“视线停留”
|
||||
if random.random() < 0.005:
|
||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||
|
||||
# ==================== 模式 0：滚动到顶部 ====================
|
||||
if mode == 'top':
|
||||
# print("[HumanScroll] 正在模拟向上滚动至页面顶部...")
|
||||
last_scroll = -1
|
||||
while True:
|
||||
# 获取当前滚动条位置
|
||||
current_scroll = await page.evaluate("window.scrollY")
|
||||
|
||||
# 防死循环：如果位置不再变化，说明已经到顶或页面无法滚动
|
||||
if current_scroll <= 0 or current_scroll == last_scroll:
|
||||
break
|
||||
|
||||
last_scroll = current_scroll
|
||||
chunk = random.randint(300, 700)
|
||||
await smooth_scroll_step(-chunk)
|
||||
await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼
|
||||
|
||||
# ==================== 模式 1：滚动到底部 ====================
|
||||
elif mode == 'bottom':
|
||||
# print("[HumanScroll] 正在模拟向下滚动至页面底部...")
|
||||
last_scroll = -1
|
||||
while True:
|
||||
# 合并 evaluate 请求，减少与浏览器的 IPC 通信开销
|
||||
scroll_info = await page.evaluate("() => [window.scrollY + window.innerHeight, document.body.scrollHeight]")
|
||||
current_scroll, total_height = scroll_info[0], scroll_info[1]
|
||||
|
||||
# 防死循环：如果位置不再变化，说明已经到底或页面无法滚动
|
||||
if current_scroll >= total_height - 10 or current_scroll == last_scroll:
|
||||
break
|
||||
|
||||
last_scroll = current_scroll
|
||||
chunk = random.randint(300, 700)
|
||||
await smooth_scroll_step(chunk)
|
||||
await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼
|
||||
|
||||
# ==================== 模式 2：随机上下滚动几下 ====================
|
||||
elif mode == 'random':
|
||||
# print("[HumanScroll] 正在执行人类迷惑行为：随机上下滚动...")
|
||||
# 随机决定滚动 3 到 6 下
|
||||
scroll_times = random.randint(3, 6)
|
||||
|
||||
for _ in range(scroll_times):
|
||||
# 70% 概率向下，30% 概率向上
|
||||
if random.random() > 0.3:
|
||||
distance = random.randint(200, 500) # 向下滚
|
||||
else:
|
||||
distance = random.randint(-400, -150) # 向上回滚
|
||||
|
||||
await smooth_scroll_step(distance)
|
||||
await asyncio.sleep(random.uniform(0.6, 1.5)) # 停顿
|
||||
|
||||
# ==================== 模式 3：滚动指定距离 ====================
|
||||
elif mode == 'distance':
|
||||
# print(f"[HumanScroll] 正在精准模拟滚动指定距离: {value} 像素")
|
||||
await smooth_scroll_step(value)
|
||||
|
||||
else:
|
||||
raise ValueError("未知的滚动模式！请选择 'top', 'bottom', 'random' 或 'distance'")
|
||||
+10
-3
@@ -1,3 +1,9 @@
|
||||
"""HTTP 签名工具（stdlib only）。
|
||||
|
||||
从 app/utils/http_utils.py 迁移而来，逻辑与 worker 端实现一致，
|
||||
作为 surugaya_common 的共享实现统一维护。
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import hmac
|
||||
@@ -5,10 +11,11 @@ import secrets
|
||||
import hashlib
|
||||
from typing import Dict, Any
|
||||
|
||||
|
||||
def generate_signed_headers(app_id: str, app_secret: str, data: Dict[str, Any], method: str = "POST") -> Dict[str, str]:
|
||||
timestamp = str(int(time.time() * 1000))
|
||||
nonce = secrets.token_hex(16)
|
||||
|
||||
|
||||
payload_str = json.dumps(data, ensure_ascii=False)
|
||||
payload_bytes = payload_str.encode("utf-8")
|
||||
actual_body_hash = hashlib.sha256(payload_bytes).hexdigest()
|
||||
@@ -34,5 +41,5 @@ def generate_signed_headers(app_id: str, app_secret: str, data: Dict[str, Any],
|
||||
"X-Ca-Signature": signature,
|
||||
"Content-Type": "application/json; charset=utf-8"
|
||||
}
|
||||
|
||||
return headers
|
||||
|
||||
return headers
|
||||
@@ -0,0 +1,17 @@
|
||||
"""surugaya_common 共享数据模型。"""
|
||||
|
||||
from surugaya_common.models.order import (
|
||||
SurugayaOrderDetail,
|
||||
SurugayaOrderItem,
|
||||
SurugayaOrderStatus,
|
||||
SurugayaOrderSummary,
|
||||
SurugayaShippingInfo,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SurugayaOrderDetail",
|
||||
"SurugayaOrderItem",
|
||||
"SurugayaOrderStatus",
|
||||
"SurugayaOrderSummary",
|
||||
"SurugayaShippingInfo",
|
||||
]
|
||||
@@ -0,0 +1,77 @@
|
||||
"""骏河屋订单详情的结构化模型（pydantic + typing only）。
|
||||
|
||||
从 jp_pruchase_zdh_5/src/parsers/surugaya/models.py 迁移而来，作为 API 服务与
|
||||
worker 共享的订单契约统一维护。worker 端解析订单详情页后产出这些结构，写入
|
||||
Redis Hash（见 surugaya_common.redis_keys.trade_hash）；API 的 /api/trade 读取
|
||||
同一份数据。
|
||||
"""
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SurugayaOrderStatus(BaseModel):
|
||||
"""交易状态表的结构化结果。"""
|
||||
|
||||
trade_status: str | None = None
|
||||
status_message: str | None = None
|
||||
# 保留日文原始键值，便于排障和后续补字段。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaOrderSummary(BaseModel):
|
||||
"""订单摘要表的结构化结果。"""
|
||||
|
||||
trade_number: str | None = None
|
||||
order_date: str | None = None
|
||||
items_subtotal: int | None = None
|
||||
item_count: int | None = None
|
||||
payment_method: str | None = None
|
||||
cash_on_delivery_fee: int | None = None
|
||||
shipping_date: str | None = None
|
||||
inquiry_number: str | None = None
|
||||
shipping_and_handling_fee: int | None = None
|
||||
digital_gift_certificate_amount: int | None = None
|
||||
grand_total: int | None = None
|
||||
# 保留订单摘要表的原始日文字段。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaShippingInfo(BaseModel):
|
||||
"""收件信息表的结构化结果。"""
|
||||
|
||||
recipient_name: str | None = None
|
||||
recipient_address: str | None = None
|
||||
recipient_phone_number: str | None = None
|
||||
recipient_email: str | None = None
|
||||
# 保留配送信息表的原始日文字段。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaOrderItem(BaseModel):
|
||||
"""商品明细表中的单行商品记录。"""
|
||||
|
||||
product_code: str | None = None
|
||||
condition: str | None = None
|
||||
branch_number: str | None = None
|
||||
product_title: str | None = None
|
||||
unit_price: int | None = None
|
||||
quantity: int | None = None
|
||||
discount_amount: int | None = None
|
||||
line_total: int | None = None
|
||||
remarks: str | None = None
|
||||
# 每一行商品同时保留原始列名和值。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaOrderDetail(BaseModel):
|
||||
"""骏河屋交易详情页的完整结构化结果。"""
|
||||
|
||||
site: Literal["surugaya"] = "surugaya"
|
||||
status: SurugayaOrderStatus
|
||||
summary: SurugayaOrderSummary
|
||||
shipping: SurugayaShippingInfo
|
||||
items: list[SurugayaOrderItem] = Field(default_factory=list)
|
||||
# 以业务分段聚合保留原始表格结果。
|
||||
raw_tables: dict[str, Any] = Field(default_factory=dict)
|
||||
@@ -0,0 +1 @@
|
||||
"""surugaya_common 解析器子包（部分依赖 playwright，需安装 surugaya_common[browser]）。"""
|
||||
@@ -0,0 +1,11 @@
|
||||
"""骏河屋页面解析器（需 playwright，安装 surugaya_common[browser]）。"""
|
||||
|
||||
from surugaya_common.parsers.surugaya.order_detail import (
|
||||
SurugayaOrderDetailParser,
|
||||
parse_surugaya_order_detail,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SurugayaOrderDetailParser",
|
||||
"parse_surugaya_order_detail",
|
||||
]
|
||||
@@ -0,0 +1,329 @@
|
||||
"""骏河屋交易详情页解析器（需 playwright，安装 surugaya_common[browser]）。
|
||||
|
||||
从 jp_pruchase_zdh_5/src/parsers/surugaya/order_detail.py 迁移而来，模型导入改为
|
||||
共享包 surugaya_common.models。
|
||||
|
||||
相对 worker 原版的唯一行为差异：修正了 ITEM_FIELD_MAP 的字段映射 bug——原版将
|
||||
品番/状態/数量/金額 误映射为 product_id/status/number/product_total_amount，与
|
||||
SurugayaOrderItem 模型字段 product_code/condition/quantity/line_total 不符，被
|
||||
pydantic 默认 extra=ignore 静默丢弃，导致这 4 列恒为 None。此处已对齐模型字段。
|
||||
worker 本地副本仍为旧版，应在 repoint 到本包时一并淘汰。
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from playwright.async_api import Locator, Page
|
||||
|
||||
from surugaya_common.models import (
|
||||
SurugayaOrderDetail,
|
||||
SurugayaOrderItem,
|
||||
SurugayaShippingInfo,
|
||||
SurugayaOrderStatus,
|
||||
SurugayaOrderSummary,
|
||||
)
|
||||
|
||||
|
||||
# 订单摘要表中的日文字段到英文字段映射。
|
||||
SUMMARY_FIELD_MAP = {
|
||||
"取引番号": "trade_number",
|
||||
"注文日": "order_date",
|
||||
"商品合計": "items_subtotal",
|
||||
"商品点数": "item_count",
|
||||
"支払方法": "payment_method",
|
||||
"代引き手数料": "cash_on_delivery_fee",
|
||||
"発送日": "shipping_date",
|
||||
"お問い合わせ番号": "inquiry_number",
|
||||
"送料・通信販売手数料": "shipping_and_handling_fee",
|
||||
"電子商品券": "digital_gift_certificate_amount",
|
||||
"総合計": "grand_total",
|
||||
}
|
||||
|
||||
# 收件信息表的日文字段到英文字段映射。
|
||||
SHIPPING_FIELD_MAP = {
|
||||
"氏名": "recipient_name",
|
||||
"住所": "recipient_address",
|
||||
"電話番号": "recipient_phone_number",
|
||||
"メールアドレス": "recipient_email",
|
||||
}
|
||||
|
||||
# 商品明细表的列名到英文字段映射。
|
||||
# 字段名已对齐 SurugayaOrderItem 模型（修正了 worker 原版的映射 bug：
|
||||
# product_code/condition/quantity/line_total 此前因键名不符被 pydantic 静默丢弃）。
|
||||
ITEM_FIELD_MAP = {
|
||||
"品番": "product_code",
|
||||
"状態": "condition",
|
||||
"枝番": "branch_number",
|
||||
"商品タイトル": "product_title",
|
||||
"単価": "unit_price",
|
||||
"数量": "quantity",
|
||||
"値引額": "discount_amount",
|
||||
"金額": "line_total",
|
||||
"備考": "remarks",
|
||||
}
|
||||
|
||||
# 需要按金额规则清洗的字段集合。
|
||||
MONEY_FIELDS = {
|
||||
"items_subtotal",
|
||||
"cash_on_delivery_fee",
|
||||
"shipping_and_handling_fee",
|
||||
"digital_gift_certificate_amount",
|
||||
"grand_total",
|
||||
"unit_price",
|
||||
"discount_amount",
|
||||
"line_total",
|
||||
}
|
||||
|
||||
# 需要按整数规则清洗的字段集合。
|
||||
INT_FIELDS = {
|
||||
"item_count",
|
||||
"quantity",
|
||||
}
|
||||
|
||||
class SurugayaOrderDetailParser:
|
||||
"""骏河屋交易详情页解析器。"""
|
||||
|
||||
@classmethod
|
||||
async def parse_page(cls, page: Page) -> SurugayaOrderDetail:
|
||||
"""对外主入口：传入已加载详情页的 `Page`，返回结构化订单对象。"""
|
||||
# 适配 `SurugayaOrderDetailParser.parse_page(page)` 这种类调用方式。
|
||||
parser = cls()
|
||||
if await page.locator("#pageCont").count() < 1:
|
||||
raise ValueError("Surugaya order detail page not found")
|
||||
return await parser._parse_page_content(page)
|
||||
|
||||
async def _parse_page_content(self, page: Page) -> SurugayaOrderDetail:
|
||||
"""解析正常交易详情页中的 `#pageCont` 核心区域。"""
|
||||
|
||||
page_cont = page.locator("#pageCont").first
|
||||
tables = page_cont.locator("table")
|
||||
table_count = await tables.count()
|
||||
if table_count < 4:
|
||||
raise ValueError("Expected 4 trade tables under #pageCont")
|
||||
|
||||
status_table = tables.nth(0)
|
||||
summary_table = tables.nth(1)
|
||||
shipping_table = tables.nth(2)
|
||||
items_table = tables.nth(3)
|
||||
|
||||
status = await self._parse_status_table(status_table)
|
||||
summary = await self._parse_summary_table(summary_table)
|
||||
shipping = await self._parse_shipping_table(shipping_table)
|
||||
items = await self._parse_items_table(items_table)
|
||||
|
||||
return SurugayaOrderDetail(
|
||||
status=status,
|
||||
summary=summary,
|
||||
shipping=shipping,
|
||||
items=items,
|
||||
raw_tables={
|
||||
"status": status.raw,
|
||||
"summary": summary.raw,
|
||||
"shipping": shipping.raw,
|
||||
"items": [item.raw for item in items],
|
||||
},
|
||||
)
|
||||
|
||||
async def _parse_status_table(self, table: Locator) -> SurugayaOrderStatus:
|
||||
"""解析第 1 张交易状态表。"""
|
||||
rows = table.locator("tr")
|
||||
row_count = await rows.count()
|
||||
if row_count < 1:
|
||||
return SurugayaOrderStatus()
|
||||
|
||||
first_row = await self._extract_row_cells(rows.nth(0))
|
||||
raw: dict[str, str] = {}
|
||||
trade_status = None
|
||||
status_message = None
|
||||
|
||||
if len(first_row) >= 2 and first_row[0]["tag"] == "th" and first_row[1]["tag"] == "td":
|
||||
label = first_row[0]["text"]
|
||||
value = first_row[1]["text"]
|
||||
raw[label] = value
|
||||
lines = self._split_lines(value)
|
||||
trade_status = lines[0] if lines else None
|
||||
status_message = " ".join(lines[1:]) or None
|
||||
|
||||
return SurugayaOrderStatus(
|
||||
trade_status=trade_status,
|
||||
status_message=status_message,
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
async def _parse_summary_table(self, table: Locator) -> SurugayaOrderSummary:
|
||||
"""解析第 2 张订单摘要表。"""
|
||||
raw_pairs = await self._parse_key_value_rows(table)
|
||||
normalized = self._normalize_mapped_values(raw_pairs, SUMMARY_FIELD_MAP)
|
||||
return SurugayaOrderSummary(**normalized, raw=raw_pairs)
|
||||
|
||||
async def _parse_shipping_table(self, table: Locator) -> SurugayaShippingInfo:
|
||||
"""解析第 3 张收件信息表。"""
|
||||
raw_pairs = await self._parse_key_value_rows(table)
|
||||
normalized = self._normalize_mapped_values(raw_pairs, SHIPPING_FIELD_MAP)
|
||||
return SurugayaShippingInfo(**normalized, raw=raw_pairs)
|
||||
|
||||
async def _parse_items_table(self, table: Locator) -> list[SurugayaOrderItem]:
|
||||
"""解析第 4 张商品明细表。"""
|
||||
rows = table.locator("tr")
|
||||
row_count = await rows.count()
|
||||
if row_count < 2:
|
||||
return []
|
||||
|
||||
header_cells = await self._extract_row_cells(rows.nth(0))
|
||||
headers = [cell["text"] for cell in header_cells if cell["tag"] == "th" and cell["text"]]
|
||||
if not headers:
|
||||
return []
|
||||
|
||||
items: list[SurugayaOrderItem] = []
|
||||
expected_columns = len(headers)
|
||||
|
||||
for index in range(1, row_count):
|
||||
cells = await self._extract_row_cells(rows.nth(index))
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
td_cells = [cell for cell in cells if cell["tag"] == "td"]
|
||||
if len(td_cells) != expected_columns:
|
||||
continue
|
||||
|
||||
raw_item = {
|
||||
header: td_cells[position]["text"]
|
||||
for position, header in enumerate(headers)
|
||||
}
|
||||
|
||||
if not any(raw_item.values()):
|
||||
continue
|
||||
|
||||
normalized = self._normalize_mapped_values(raw_item, ITEM_FIELD_MAP)
|
||||
items.append(SurugayaOrderItem(**normalized, raw=raw_item))
|
||||
|
||||
return items
|
||||
|
||||
async def _parse_key_value_rows(self, table: Locator) -> dict[str, str]:
|
||||
"""通用解析：按 `th -> td` 配对方式抽取键值。"""
|
||||
rows = table.locator("tr")
|
||||
row_count = await rows.count()
|
||||
result: dict[str, str] = {}
|
||||
|
||||
for index in range(row_count):
|
||||
cells = await self._extract_row_cells(rows.nth(index))
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
pointer = 0
|
||||
while pointer < len(cells):
|
||||
current = cells[pointer]
|
||||
if current["tag"] != "th":
|
||||
pointer += 1
|
||||
continue
|
||||
|
||||
label = current["text"]
|
||||
pointer += 1
|
||||
if not label:
|
||||
continue
|
||||
|
||||
while pointer < len(cells) and cells[pointer]["tag"] != "td":
|
||||
pointer += 1
|
||||
|
||||
if pointer >= len(cells):
|
||||
break
|
||||
|
||||
result[label] = cells[pointer]["text"]
|
||||
pointer += 1
|
||||
|
||||
return result
|
||||
|
||||
async def _extract_row_cells(self, row: Locator) -> list[dict[str, str]]:
|
||||
"""提取一行中的所有单元格，并保留标签名与纯文本。"""
|
||||
return await row.locator(":scope > th, :scope > td").evaluate_all(
|
||||
"""cells => cells.map((cell) => ({
|
||||
tag: cell.tagName.toLowerCase(),
|
||||
text: (cell.innerText || "").replace(/\\u00a0/g, " ").trim(),
|
||||
}))"""
|
||||
)
|
||||
|
||||
def _normalize_mapped_values(
|
||||
self,
|
||||
raw_data: dict[str, str],
|
||||
field_map: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
"""将日文键值映射并清洗为结构化字段。"""
|
||||
normalized: dict[str, Any] = {}
|
||||
|
||||
for label, value in raw_data.items():
|
||||
english_key = field_map.get(label)
|
||||
if not english_key:
|
||||
continue
|
||||
|
||||
cleaned_value = self._clean_text(value)
|
||||
if english_key in MONEY_FIELDS:
|
||||
normalized[english_key] = self._normalize_money(cleaned_value)
|
||||
elif english_key in INT_FIELDS:
|
||||
normalized[english_key] = self._normalize_int(cleaned_value)
|
||||
elif english_key.endswith("_date"):
|
||||
normalized[english_key] = self._normalize_date(cleaned_value)
|
||||
else:
|
||||
normalized[english_key] = cleaned_value
|
||||
|
||||
return normalized
|
||||
|
||||
def _split_lines(self, text: str | None) -> list[str]:
|
||||
"""按行拆分文本，并清理每一行空白。"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
lines = [self._clean_text(part) for part in text.splitlines()]
|
||||
return [line for line in lines if line]
|
||||
|
||||
def _clean_text(self, text: str | None) -> str | None:
|
||||
"""清洗文本中的多余空白并统一空值。"""
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
cleaned = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
|
||||
return cleaned or None
|
||||
|
||||
def _normalize_money(self, text: str | None) -> int | None:
|
||||
"""将 `￥1,234` / `1,234円` 这类金额文本转成整数。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
digits = re.findall(r"-?\d[\d,]*", text)
|
||||
if not digits:
|
||||
return None
|
||||
|
||||
value = digits[0].replace(",", "")
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def _normalize_int(self, text: str | None) -> int | None:
|
||||
"""提取文本中的首个整数值。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
digits = re.findall(r"\d+", text)
|
||||
if not digits:
|
||||
return None
|
||||
|
||||
return int(digits[0])
|
||||
|
||||
def _normalize_date(self, text: str | None) -> str | None:
|
||||
"""将 `YYYY.MM.DD` / `YYYY/MM/DD` 统一为 `YYYY-MM-DD`。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
match = re.search(r"(\d{4})[./-](\d{2})[./-](\d{2})", text)
|
||||
if not match:
|
||||
return text
|
||||
|
||||
year, month, day = match.groups()
|
||||
return f"{year}-{month}-{day}"
|
||||
|
||||
|
||||
async def parse_surugaya_order_detail(page: Page) -> dict[str, Any]:
|
||||
"""便捷函数：直接返回可 JSON 序列化的订单字典。"""
|
||||
parser = SurugayaOrderDetailParser()
|
||||
result = await parser.parse_page(page)
|
||||
return result.model_dump(mode="json")
|
||||
@@ -0,0 +1,80 @@
|
||||
"""Centralized Redis key contract shared across jp_surugaya services.
|
||||
|
||||
Single source of truth for every Redis key schema. Helper functions reproduce
|
||||
the EXACT literals currently produced inline by the API service so behaviour is
|
||||
byte-for-byte identical after refactoring.
|
||||
|
||||
Current literals (as of this extraction):
|
||||
purchase task stream : jp_surugaya_purchase_task[_dev]
|
||||
trade delay zset : jp_surugaya_trade_delay[_dev]:{node_id}
|
||||
trade hash : jp_surugaya_trade
|
||||
trade pending dedup : jp_surugaya_trade_pending:{trade_code}
|
||||
trade fetch stream : jp_surugaya_trade_fetch
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# --- Base key constants (no suffixes applied) -------------------------------
|
||||
|
||||
PURCHASE_TASK_STREAM: str = "jp_surugaya_purchase_task"
|
||||
TRADE_DELAY_ZSET: str = "jp_surugaya_trade_delay"
|
||||
TRADE_HASH: str = "jp_surugaya_trade"
|
||||
TRADE_PENDING_PREFIX: str = "jp_surugaya_trade_pending"
|
||||
TRADE_FETCH_STREAM: str = "jp_surugaya_trade_fetch"
|
||||
|
||||
# Suffix appended to dev-mode keys.
|
||||
_DEV_SUFFIX: str = "_dev"
|
||||
|
||||
|
||||
def _is_dev(is_dev: object) -> bool:
|
||||
"""Normalize the dev flag.
|
||||
|
||||
The API passes payload.is_dev which is the string "1" for dev mode.
|
||||
Accepts the raw string flag ("1"), a bool, or anything truthy-comparable
|
||||
while preserving the exact original semantics (is_dev == "1").
|
||||
"""
|
||||
if isinstance(is_dev, str):
|
||||
return is_dev == "1"
|
||||
return bool(is_dev)
|
||||
|
||||
|
||||
def purchase_task_stream(is_dev: object = None) -> str:
|
||||
"""Redis Stream key for the purchase task queue.
|
||||
|
||||
Reproduces:
|
||||
"jp_surugaya_purchase_task_dev" if is_dev == "1" else "jp_surugaya_purchase_task"
|
||||
"""
|
||||
if _is_dev(is_dev):
|
||||
return f"{PURCHASE_TASK_STREAM}{_DEV_SUFFIX}"
|
||||
return PURCHASE_TASK_STREAM
|
||||
|
||||
|
||||
def trade_delay_zset(node_id: object, is_dev: object = None) -> str:
|
||||
"""Redis Sorted Set key for delayed trade monitoring, scoped by node_id.
|
||||
|
||||
Reproduces:
|
||||
base = "jp_surugaya_trade_delay_dev" if is_dev == "1" else "jp_surugaya_trade_delay"
|
||||
base + f":{node_id}"
|
||||
"""
|
||||
base = TRADE_DELAY_ZSET
|
||||
if _is_dev(is_dev):
|
||||
base = f"{base}{_DEV_SUFFIX}"
|
||||
return f"{base}:{node_id}"
|
||||
|
||||
|
||||
def trade_hash() -> str:
|
||||
"""Redis Hash key storing trade order details keyed by trade_code."""
|
||||
return TRADE_HASH
|
||||
|
||||
|
||||
def trade_pending_key(trade_code: object) -> str:
|
||||
"""TTL dedup key for a pending trade fetch.
|
||||
|
||||
Reproduces: f"jp_surugaya_trade_pending:{trade_code}"
|
||||
"""
|
||||
return f"{TRADE_PENDING_PREFIX}:{trade_code}"
|
||||
|
||||
|
||||
def trade_fetch_stream() -> str:
|
||||
"""Redis Stream key for trade fetch tasks."""
|
||||
return TRADE_FETCH_STREAM
|
||||
+8
-1
@@ -10,10 +10,11 @@ dependencies = [
|
||||
"playwright-stealth>=1.0.0,<3.0.0",
|
||||
"pydantic>=2.0.0,<3.0.0",
|
||||
"pydantic-settings>=2.4.0,<3.0.0",
|
||||
"redis>=5.0.0,<6.0.0",
|
||||
"redis==8.0.0",
|
||||
"selectolax>=0.3.21,<1.0.0",
|
||||
"starlette>=0.37.0,<1.0.0",
|
||||
"uvicorn[standard]>=0.30.0,<1.0.0",
|
||||
"surugaya-common",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -29,3 +30,9 @@ testpaths = ["tests"]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
|
||||
[tool.uv.workspace]
|
||||
members = ["packages/*"]
|
||||
|
||||
[tool.uv.sources]
|
||||
surugaya-common = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
"""SurugayaOrderDetailParser 商品明细字段映射的回归测试。
|
||||
|
||||
锁定 ITEM_FIELD_MAP 字段名 bug 的修复：品番/状態/数量/金額 必须正确填充
|
||||
product_code/condition/quantity/line_total，而非被 pydantic 静默丢弃。
|
||||
"""
|
||||
|
||||
from surugaya_common.models import SurugayaOrderItem
|
||||
from surugaya_common.parsers.surugaya.order_detail import (
|
||||
ITEM_FIELD_MAP,
|
||||
SurugayaOrderDetailParser,
|
||||
)
|
||||
|
||||
|
||||
def test_item_field_map_targets_are_real_model_fields():
|
||||
"""ITEM_FIELD_MAP 的每个目标字段都必须是 SurugayaOrderItem 的真实字段。"""
|
||||
model_fields = set(SurugayaOrderItem.model_fields)
|
||||
for jp_column, en_field in ITEM_FIELD_MAP.items():
|
||||
assert en_field in model_fields, f"{jp_column} 映射到非法字段 {en_field}"
|
||||
|
||||
|
||||
def test_normalize_item_row_populates_all_corrected_fields():
|
||||
"""修复后：品番/状態/数量/金額 应正确填充，金额/整数字段完成归一化。"""
|
||||
parser = SurugayaOrderDetailParser()
|
||||
raw_item = {
|
||||
"品番": "ABC-123",
|
||||
"状態": "中古",
|
||||
"枝番": "1",
|
||||
"商品タイトル": "测试商品",
|
||||
"単価": "1,250円",
|
||||
"数量": "2",
|
||||
"値引額": "100円",
|
||||
"金額": "2,400円",
|
||||
"備考": "x",
|
||||
}
|
||||
|
||||
normalized = parser._normalize_mapped_values(raw_item, ITEM_FIELD_MAP)
|
||||
item = SurugayaOrderItem(**normalized, raw=raw_item)
|
||||
|
||||
# 此前被静默丢弃的 4 个字段现在应被正确填充
|
||||
assert item.product_code == "ABC-123"
|
||||
assert item.condition == "中古"
|
||||
assert item.quantity == 2 # INT_FIELDS 归一
|
||||
assert item.line_total == 2400 # MONEY_FIELDS 归一，逗号去除
|
||||
|
||||
# 原本就正确的字段保持不变
|
||||
assert item.branch_number == "1"
|
||||
assert item.product_title == "测试商品"
|
||||
assert item.unit_price == 1250
|
||||
assert item.discount_amount == 100
|
||||
assert item.remarks == "x"
|
||||
@@ -2,6 +2,12 @@ version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.12"
|
||||
|
||||
[manifest]
|
||||
members = [
|
||||
"surugaya-common",
|
||||
"surugaya-scraper-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "annotated-doc"
|
||||
version = "0.0.4"
|
||||
@@ -360,15 +366,6 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pyjwt"
|
||||
version = "2.12.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/c2/27/a3b6e5bf6ff856d2509292e95c8f57f0df7017cf5394921fc4e4ef40308a/pyjwt-2.12.1.tar.gz", hash = "sha256:c74a7a2adf861c04d002db713dd85f84beb242228e671280bf709d765b03672b", size = 102564, upload-time = "2026-03-13T19:27:37.25Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/e5/7a/8dd906bd22e79e47397a61742927f6747fe93242ef86645ee9092e610244/pyjwt-2.12.1-py3-none-any.whl", hash = "sha256:28ca37c070cad8ba8cd9790cd940535d40274d22f80ab87f3ac6a713e6e8454c", size = 29726, upload-time = "2026-03-13T19:27:35.677Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "8.4.2"
|
||||
@@ -454,14 +451,11 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "redis"
|
||||
version = "5.3.1"
|
||||
version = "8.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "pyjwt" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/6a/cf/128b1b6d7086200c9f387bd4be9b2572a30b90745ef078bd8b235042dc9f/redis-5.3.1.tar.gz", hash = "sha256:ca49577a531ea64039b5a36db3d6cd1a0c7a60c34124d46924a45b956e8cf14c", size = 4626200, upload-time = "2025-07-25T08:06:27.778Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/53/ae/ed461cca5780b5fc8b9fe8ca0ed98d89508645fb9d880c24cc42c087678f/redis-8.0.0.tar.gz", hash = "sha256:a00c5355432051ac14e593b8b197fc76c887ee12d55a0984f69328a1115fdc49", size = 5101591, upload-time = "2026-05-28T12:45:13.5Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/7f/26/5c5fa0e83c3621db835cfc1f1d789b37e7fa99ed54423b5f519beb931aa7/redis-5.3.1-py3-none-any.whl", hash = "sha256:dc1909bd24669cc31b5f67a039700b16ec30571096c5f1f0d9d2324bff31af97", size = 272833, upload-time = "2025-07-25T08:06:26.317Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/27/e3/b519734372d305bd547534a9f32e4ce9f98552af753dce72cf3483a0ff0b/redis-8.0.0-py3-none-any.whl", hash = "sha256:c938c18338585009f0bc310f4c7e4e4b4d37639356c4ac072cedf3af570c8dc7", size = 499870, upload-time = "2026-05-28T12:45:11.697Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -521,6 +515,28 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "surugaya-common"
|
||||
version = "0.2.0"
|
||||
source = { editable = "packages/surugaya_common" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "redis" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
browser = [
|
||||
{ name = "playwright" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" },
|
||||
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
||||
{ name = "redis", specifier = "==8.0.0" },
|
||||
]
|
||||
provides-extras = ["browser"]
|
||||
|
||||
[[package]]
|
||||
name = "surugaya-scraper-service"
|
||||
version = "0.1.0"
|
||||
@@ -535,6 +551,7 @@ dependencies = [
|
||||
{ name = "redis" },
|
||||
{ name = "selectolax" },
|
||||
{ name = "starlette" },
|
||||
{ name = "surugaya-common" },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
|
||||
@@ -554,9 +571,10 @@ requires-dist = [
|
||||
{ name = "pydantic-settings", specifier = ">=2.4.0,<3.0.0" },
|
||||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" },
|
||||
{ name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.24.0,<1.0.0" },
|
||||
{ name = "redis", specifier = ">=5.0.0,<6.0.0" },
|
||||
{ name = "redis", specifier = "==8.0.0" },
|
||||
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
|
||||
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" },
|
||||
{ name = "surugaya-common", editable = "packages/surugaya_common" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
|
||||
]
|
||||
provides-extras = ["dev"]
|
||||
|
||||
Reference in New Issue
Block a user