You've already forked pruchase_jhw
合并依赖
This commit is contained in:
+12
-1
@@ -9,7 +9,10 @@ authors = [
|
|||||||
{ name = "Antigravity Development Team" }
|
{ name = "Antigravity Development Team" }
|
||||||
]
|
]
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"playwright>=1.43.0",
|
# worker 自身已直接依赖 playwright,故 surugaya-common 无需 [browser] extra(extra 仅用于
|
||||||
|
# 给没有 playwright 的消费者拉取 playwright),保持不带 extra 更简洁。
|
||||||
|
"surugaya-common>=0.2.0",
|
||||||
|
"playwright>=1.54.0,<2.0.0",
|
||||||
"pydantic>=2.7.0",
|
"pydantic>=2.7.0",
|
||||||
"pydantic-settings>=2.2.0",
|
"pydantic-settings>=2.2.0",
|
||||||
"loguru>=0.7.2",
|
"loguru>=0.7.2",
|
||||||
@@ -30,3 +33,11 @@ build-backend = "hatchling.build"
|
|||||||
# 👇 在文件里加上这一行,告诉 uv 这只是个普通的脚本项目,不需要打包
|
# 👇 在文件里加上这一行,告诉 uv 这只是个普通的脚本项目,不需要打包
|
||||||
[tool.uv]
|
[tool.uv]
|
||||||
package = false
|
package = false
|
||||||
|
|
||||||
|
# Gitea PyPI registry: 安装(simple index)需 /simple 后缀;上传才用 .../pypi。
|
||||||
|
[[tool.uv.index]]
|
||||||
|
name = "jerryyan"
|
||||||
|
url = "https://git.jerryyan.net/api/packages/jp/pypi/simple"
|
||||||
|
|
||||||
|
[tool.uv.sources]
|
||||||
|
surugaya-common = { index = "jerryyan" }
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ from typing import Annotated, Any
|
|||||||
from pydantic import Field, field_validator
|
from pydantic import Field, field_validator
|
||||||
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
||||||
|
|
||||||
|
from surugaya_common import redis_keys
|
||||||
|
|
||||||
# 根据 APP_ENV 确定环境配置文件(默认为 dev)
|
# 根据 APP_ENV 确定环境配置文件(默认为 dev)
|
||||||
APP_ENV = os.getenv("APP_ENV", "dev").lower()
|
APP_ENV = os.getenv("APP_ENV", "dev").lower()
|
||||||
ENV_FILE = f".env.{APP_ENV}"
|
ENV_FILE = f".env.{APP_ENV}"
|
||||||
@@ -29,10 +31,10 @@ class Settings(BaseSettings):
|
|||||||
redis_port: int = Field(default=6379, alias="REDIS_PORT")
|
redis_port: int = Field(default=6379, alias="REDIS_PORT")
|
||||||
redis_password: str | None = Field(default=None, alias="REDIS_PASSWORD")
|
redis_password: str | None = Field(default=None, alias="REDIS_PASSWORD")
|
||||||
redis_db: int = Field(default=0, alias="REDIS_DB")
|
redis_db: int = Field(default=0, alias="REDIS_DB")
|
||||||
redis_stream_key: str = Field(default="jp_surugaya_purchase_task", alias="REDIS_STREAM_KEY")
|
redis_stream_key: str = Field(default=redis_keys.PURCHASE_TASK_STREAM, alias="REDIS_STREAM_KEY")
|
||||||
redis_consumer_group: str = Field(default="surugaya_purchase_task_group", alias="REDIS_CONSUMER_GROUP")
|
redis_consumer_group: str = Field(default="surugaya_purchase_task_group", alias="REDIS_CONSUMER_GROUP")
|
||||||
redis_consumer_name: str = Field(default="surugaya_purchase_task_worker", alias="REDIS_CONSUMER_NAME")
|
redis_consumer_name: str = Field(default="surugaya_purchase_task_worker", alias="REDIS_CONSUMER_NAME")
|
||||||
redis_delay_zset_key: str = Field(default="jp_surugaya_trade_delay", alias="REDIS_DELAY_ZSET_KEY")
|
redis_delay_zset_key: str = Field(default=redis_keys.TRADE_DELAY_ZSET, alias="REDIS_DELAY_ZSET_KEY")
|
||||||
redis_delay_poll_interval: int = Field(default=3600, alias="REDIS_DELAY_POLL_INTERVAL")
|
redis_delay_poll_interval: int = Field(default=3600, alias="REDIS_DELAY_POLL_INTERVAL")
|
||||||
redis_delay_batch_size: int = Field(default=10, alias="REDIS_DELAY_BATCH_SIZE")
|
redis_delay_batch_size: int = Field(default=10, alias="REDIS_DELAY_BATCH_SIZE")
|
||||||
|
|
||||||
|
|||||||
@@ -1 +0,0 @@
|
|||||||
"""页面解析器包。"""
|
|
||||||
@@ -1,18 +0,0 @@
|
|||||||
from .models import (
|
|
||||||
SurugayaOrderDetail,
|
|
||||||
SurugayaOrderItem,
|
|
||||||
SurugayaOrderStatus,
|
|
||||||
SurugayaOrderSummary,
|
|
||||||
SurugayaShippingInfo,
|
|
||||||
)
|
|
||||||
from .order_detail import SurugayaOrderDetailParser, parse_surugaya_order_detail
|
|
||||||
|
|
||||||
__all__ = [
|
|
||||||
"SurugayaOrderDetail",
|
|
||||||
"SurugayaOrderItem",
|
|
||||||
"SurugayaOrderStatus",
|
|
||||||
"SurugayaOrderSummary",
|
|
||||||
"SurugayaShippingInfo",
|
|
||||||
"SurugayaOrderDetailParser",
|
|
||||||
"parse_surugaya_order_detail",
|
|
||||||
]
|
|
||||||
@@ -1,69 +0,0 @@
|
|||||||
from typing import Any, Literal
|
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
|
||||||
|
|
||||||
|
|
||||||
class SurugayaOrderStatus(BaseModel):
|
|
||||||
"""交易状态表的结构化结果。"""
|
|
||||||
|
|
||||||
trade_status: str | None = None
|
|
||||||
status_message: str | None = None
|
|
||||||
# 保留日文原始键值,便于排障和后续补字段。
|
|
||||||
raw: dict[str, str] = Field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
class SurugayaOrderSummary(BaseModel):
|
|
||||||
"""订单摘要表的结构化结果。"""
|
|
||||||
|
|
||||||
trade_number: str | None = None
|
|
||||||
order_date: str | None = None
|
|
||||||
items_subtotal: int | None = None
|
|
||||||
item_count: int | None = None
|
|
||||||
payment_method: str | None = None
|
|
||||||
cash_on_delivery_fee: int | None = None
|
|
||||||
shipping_date: str | None = None
|
|
||||||
inquiry_number: str | None = None
|
|
||||||
shipping_and_handling_fee: int | None = None
|
|
||||||
digital_gift_certificate_amount: int | None = None
|
|
||||||
grand_total: int | None = None
|
|
||||||
# 保留订单摘要表的原始日文字段。
|
|
||||||
raw: dict[str, str] = Field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
class SurugayaShippingInfo(BaseModel):
|
|
||||||
"""收件信息表的结构化结果。"""
|
|
||||||
|
|
||||||
recipient_name: str | None = None
|
|
||||||
recipient_address: str | None = None
|
|
||||||
recipient_phone_number: str | None = None
|
|
||||||
recipient_email: str | None = None
|
|
||||||
# 保留配送信息表的原始日文字段。
|
|
||||||
raw: dict[str, str] = Field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
class SurugayaOrderItem(BaseModel):
|
|
||||||
"""商品明细表中的单行商品记录。"""
|
|
||||||
|
|
||||||
product_code: str | None = None
|
|
||||||
condition: str | None = None
|
|
||||||
branch_number: str | None = None
|
|
||||||
product_title: str | None = None
|
|
||||||
unit_price: int | None = None
|
|
||||||
quantity: int | None = None
|
|
||||||
discount_amount: int | None = None
|
|
||||||
line_total: int | None = None
|
|
||||||
remarks: str | None = None
|
|
||||||
# 每一行商品同时保留原始列名和值。
|
|
||||||
raw: dict[str, str] = Field(default_factory=dict)
|
|
||||||
|
|
||||||
|
|
||||||
class SurugayaOrderDetail(BaseModel):
|
|
||||||
"""骏河屋交易详情页的完整结构化结果。"""
|
|
||||||
|
|
||||||
site: Literal["surugaya"] = "surugaya"
|
|
||||||
status: SurugayaOrderStatus
|
|
||||||
summary: SurugayaOrderSummary
|
|
||||||
shipping: SurugayaShippingInfo
|
|
||||||
items: list[SurugayaOrderItem] = Field(default_factory=list)
|
|
||||||
# 以业务分段聚合保留原始表格结果。
|
|
||||||
raw_tables: dict[str, Any] = Field(default_factory=dict)
|
|
||||||
@@ -1,315 +0,0 @@
|
|||||||
import re
|
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
from playwright.async_api import Locator, Page
|
|
||||||
|
|
||||||
from .models import (
|
|
||||||
SurugayaOrderDetail,
|
|
||||||
SurugayaOrderItem,
|
|
||||||
SurugayaShippingInfo,
|
|
||||||
SurugayaOrderStatus,
|
|
||||||
SurugayaOrderSummary,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# 订单摘要表中的日文字段到英文字段映射。
|
|
||||||
SUMMARY_FIELD_MAP = {
|
|
||||||
"取引番号": "trade_number",
|
|
||||||
"注文日": "order_date",
|
|
||||||
"商品合計": "items_subtotal",
|
|
||||||
"商品点数": "item_count",
|
|
||||||
"支払方法": "payment_method",
|
|
||||||
"代引き手数料": "cash_on_delivery_fee",
|
|
||||||
"発送日": "shipping_date",
|
|
||||||
"お問い合わせ番号": "inquiry_number",
|
|
||||||
"送料・通信販売手数料": "shipping_and_handling_fee",
|
|
||||||
"電子商品券": "digital_gift_certificate_amount",
|
|
||||||
"総合計": "grand_total",
|
|
||||||
}
|
|
||||||
|
|
||||||
# 收件信息表的日文字段到英文字段映射。
|
|
||||||
SHIPPING_FIELD_MAP = {
|
|
||||||
"氏名": "recipient_name",
|
|
||||||
"住所": "recipient_address",
|
|
||||||
"電話番号": "recipient_phone_number",
|
|
||||||
"メールアドレス": "recipient_email",
|
|
||||||
}
|
|
||||||
|
|
||||||
# 商品明细表的列名到英文字段映射。
|
|
||||||
ITEM_FIELD_MAP = {
|
|
||||||
"品番": "product_id",
|
|
||||||
"状態": "status",
|
|
||||||
"枝番": "branch_number",
|
|
||||||
"商品タイトル": "product_title",
|
|
||||||
"単価": "unit_price",
|
|
||||||
"数量": "number",
|
|
||||||
"値引額": "discount_amount",
|
|
||||||
"金額": "product_total_amount",
|
|
||||||
"備考": "remarks",
|
|
||||||
}
|
|
||||||
|
|
||||||
# 需要按金额规则清洗的字段集合。
|
|
||||||
MONEY_FIELDS = {
|
|
||||||
"items_subtotal",
|
|
||||||
"cash_on_delivery_fee",
|
|
||||||
"shipping_and_handling_fee",
|
|
||||||
"digital_gift_certificate_amount",
|
|
||||||
"grand_total",
|
|
||||||
"unit_price",
|
|
||||||
"discount_amount",
|
|
||||||
"line_total",
|
|
||||||
}
|
|
||||||
|
|
||||||
# 需要按整数规则清洗的字段集合。
|
|
||||||
INT_FIELDS = {
|
|
||||||
"item_count",
|
|
||||||
"quantity",
|
|
||||||
}
|
|
||||||
|
|
||||||
class SurugayaOrderDetailParser:
|
|
||||||
"""骏河屋交易详情页解析器。"""
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
async def parse_page(cls, page: Page) -> SurugayaOrderDetail:
|
|
||||||
"""对外主入口:传入已加载详情页的 `Page`,返回结构化订单对象。"""
|
|
||||||
# 适配 `SurugayaOrderDetailParser.parse_page(page)` 这种类调用方式。
|
|
||||||
parser = cls()
|
|
||||||
if await page.locator("#pageCont").count() < 1:
|
|
||||||
raise ValueError("Surugaya order detail page not found")
|
|
||||||
return await parser._parse_page_content(page)
|
|
||||||
|
|
||||||
async def _parse_page_content(self, page: Page) -> SurugayaOrderDetail:
|
|
||||||
"""解析正常交易详情页中的 `#pageCont` 核心区域。"""
|
|
||||||
|
|
||||||
page_cont = page.locator("#pageCont").first
|
|
||||||
tables = page_cont.locator("table")
|
|
||||||
table_count = await tables.count()
|
|
||||||
if table_count < 4:
|
|
||||||
raise ValueError("Expected 4 trade tables under #pageCont")
|
|
||||||
|
|
||||||
status_table = tables.nth(0)
|
|
||||||
summary_table = tables.nth(1)
|
|
||||||
shipping_table = tables.nth(2)
|
|
||||||
items_table = tables.nth(3)
|
|
||||||
|
|
||||||
status = await self._parse_status_table(status_table)
|
|
||||||
summary = await self._parse_summary_table(summary_table)
|
|
||||||
shipping = await self._parse_shipping_table(shipping_table)
|
|
||||||
items = await self._parse_items_table(items_table)
|
|
||||||
|
|
||||||
return SurugayaOrderDetail(
|
|
||||||
status=status,
|
|
||||||
summary=summary,
|
|
||||||
shipping=shipping,
|
|
||||||
items=items,
|
|
||||||
raw_tables={
|
|
||||||
"status": status.raw,
|
|
||||||
"summary": summary.raw,
|
|
||||||
"shipping": shipping.raw,
|
|
||||||
"items": [item.raw for item in items],
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _parse_status_table(self, table: Locator) -> SurugayaOrderStatus:
|
|
||||||
"""解析第 1 张交易状态表。"""
|
|
||||||
rows = table.locator("tr")
|
|
||||||
row_count = await rows.count()
|
|
||||||
if row_count < 1:
|
|
||||||
return SurugayaOrderStatus()
|
|
||||||
|
|
||||||
first_row = await self._extract_row_cells(rows.nth(0))
|
|
||||||
raw: dict[str, str] = {}
|
|
||||||
trade_status = None
|
|
||||||
status_message = None
|
|
||||||
|
|
||||||
if len(first_row) >= 2 and first_row[0]["tag"] == "th" and first_row[1]["tag"] == "td":
|
|
||||||
label = first_row[0]["text"]
|
|
||||||
value = first_row[1]["text"]
|
|
||||||
raw[label] = value
|
|
||||||
lines = self._split_lines(value)
|
|
||||||
trade_status = lines[0] if lines else None
|
|
||||||
status_message = " ".join(lines[1:]) or None
|
|
||||||
|
|
||||||
return SurugayaOrderStatus(
|
|
||||||
trade_status=trade_status,
|
|
||||||
status_message=status_message,
|
|
||||||
raw=raw,
|
|
||||||
)
|
|
||||||
|
|
||||||
async def _parse_summary_table(self, table: Locator) -> SurugayaOrderSummary:
|
|
||||||
"""解析第 2 张订单摘要表。"""
|
|
||||||
raw_pairs = await self._parse_key_value_rows(table)
|
|
||||||
normalized = self._normalize_mapped_values(raw_pairs, SUMMARY_FIELD_MAP)
|
|
||||||
return SurugayaOrderSummary(**normalized, raw=raw_pairs)
|
|
||||||
|
|
||||||
async def _parse_shipping_table(self, table: Locator) -> SurugayaShippingInfo:
|
|
||||||
"""解析第 3 张收件信息表。"""
|
|
||||||
raw_pairs = await self._parse_key_value_rows(table)
|
|
||||||
normalized = self._normalize_mapped_values(raw_pairs, SHIPPING_FIELD_MAP)
|
|
||||||
return SurugayaShippingInfo(**normalized, raw=raw_pairs)
|
|
||||||
|
|
||||||
async def _parse_items_table(self, table: Locator) -> list[SurugayaOrderItem]:
|
|
||||||
"""解析第 4 张商品明细表。"""
|
|
||||||
rows = table.locator("tr")
|
|
||||||
row_count = await rows.count()
|
|
||||||
if row_count < 2:
|
|
||||||
return []
|
|
||||||
|
|
||||||
header_cells = await self._extract_row_cells(rows.nth(0))
|
|
||||||
headers = [cell["text"] for cell in header_cells if cell["tag"] == "th" and cell["text"]]
|
|
||||||
if not headers:
|
|
||||||
return []
|
|
||||||
|
|
||||||
items: list[SurugayaOrderItem] = []
|
|
||||||
expected_columns = len(headers)
|
|
||||||
|
|
||||||
for index in range(1, row_count):
|
|
||||||
cells = await self._extract_row_cells(rows.nth(index))
|
|
||||||
if not cells:
|
|
||||||
continue
|
|
||||||
|
|
||||||
td_cells = [cell for cell in cells if cell["tag"] == "td"]
|
|
||||||
if len(td_cells) != expected_columns:
|
|
||||||
continue
|
|
||||||
|
|
||||||
raw_item = {
|
|
||||||
header: td_cells[position]["text"]
|
|
||||||
for position, header in enumerate(headers)
|
|
||||||
}
|
|
||||||
|
|
||||||
if not any(raw_item.values()):
|
|
||||||
continue
|
|
||||||
|
|
||||||
normalized = self._normalize_mapped_values(raw_item, ITEM_FIELD_MAP)
|
|
||||||
items.append(SurugayaOrderItem(**normalized, raw=raw_item))
|
|
||||||
|
|
||||||
return items
|
|
||||||
|
|
||||||
async def _parse_key_value_rows(self, table: Locator) -> dict[str, str]:
|
|
||||||
"""通用解析:按 `th -> td` 配对方式抽取键值。"""
|
|
||||||
rows = table.locator("tr")
|
|
||||||
row_count = await rows.count()
|
|
||||||
result: dict[str, str] = {}
|
|
||||||
|
|
||||||
for index in range(row_count):
|
|
||||||
cells = await self._extract_row_cells(rows.nth(index))
|
|
||||||
if not cells:
|
|
||||||
continue
|
|
||||||
|
|
||||||
pointer = 0
|
|
||||||
while pointer < len(cells):
|
|
||||||
current = cells[pointer]
|
|
||||||
if current["tag"] != "th":
|
|
||||||
pointer += 1
|
|
||||||
continue
|
|
||||||
|
|
||||||
label = current["text"]
|
|
||||||
pointer += 1
|
|
||||||
if not label:
|
|
||||||
continue
|
|
||||||
|
|
||||||
while pointer < len(cells) and cells[pointer]["tag"] != "td":
|
|
||||||
pointer += 1
|
|
||||||
|
|
||||||
if pointer >= len(cells):
|
|
||||||
break
|
|
||||||
|
|
||||||
result[label] = cells[pointer]["text"]
|
|
||||||
pointer += 1
|
|
||||||
|
|
||||||
return result
|
|
||||||
|
|
||||||
async def _extract_row_cells(self, row: Locator) -> list[dict[str, str]]:
|
|
||||||
"""提取一行中的所有单元格,并保留标签名与纯文本。"""
|
|
||||||
return await row.locator(":scope > th, :scope > td").evaluate_all(
|
|
||||||
"""cells => cells.map((cell) => ({
|
|
||||||
tag: cell.tagName.toLowerCase(),
|
|
||||||
text: (cell.innerText || "").replace(/\\u00a0/g, " ").trim(),
|
|
||||||
}))"""
|
|
||||||
)
|
|
||||||
|
|
||||||
def _normalize_mapped_values(
|
|
||||||
self,
|
|
||||||
raw_data: dict[str, str],
|
|
||||||
field_map: dict[str, str],
|
|
||||||
) -> dict[str, Any]:
|
|
||||||
"""将日文键值映射并清洗为结构化字段。"""
|
|
||||||
normalized: dict[str, Any] = {}
|
|
||||||
|
|
||||||
for label, value in raw_data.items():
|
|
||||||
english_key = field_map.get(label)
|
|
||||||
if not english_key:
|
|
||||||
continue
|
|
||||||
|
|
||||||
cleaned_value = self._clean_text(value)
|
|
||||||
if english_key in MONEY_FIELDS:
|
|
||||||
normalized[english_key] = self._normalize_money(cleaned_value)
|
|
||||||
elif english_key in INT_FIELDS:
|
|
||||||
normalized[english_key] = self._normalize_int(cleaned_value)
|
|
||||||
elif english_key.endswith("_date"):
|
|
||||||
normalized[english_key] = self._normalize_date(cleaned_value)
|
|
||||||
else:
|
|
||||||
normalized[english_key] = cleaned_value
|
|
||||||
|
|
||||||
return normalized
|
|
||||||
|
|
||||||
def _split_lines(self, text: str | None) -> list[str]:
|
|
||||||
"""按行拆分文本,并清理每一行空白。"""
|
|
||||||
if not text:
|
|
||||||
return []
|
|
||||||
|
|
||||||
lines = [self._clean_text(part) for part in text.splitlines()]
|
|
||||||
return [line for line in lines if line]
|
|
||||||
|
|
||||||
def _clean_text(self, text: str | None) -> str | None:
|
|
||||||
"""清洗文本中的多余空白并统一空值。"""
|
|
||||||
if text is None:
|
|
||||||
return None
|
|
||||||
|
|
||||||
cleaned = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
|
|
||||||
return cleaned or None
|
|
||||||
|
|
||||||
def _normalize_money(self, text: str | None) -> int | None:
|
|
||||||
"""将 `¥1,234` / `1,234円` 这类金额文本转成整数。"""
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
|
|
||||||
digits = re.findall(r"-?\d[\d,]*", text)
|
|
||||||
if not digits:
|
|
||||||
return None
|
|
||||||
|
|
||||||
value = digits[0].replace(",", "")
|
|
||||||
try:
|
|
||||||
return int(value)
|
|
||||||
except ValueError:
|
|
||||||
return None
|
|
||||||
|
|
||||||
def _normalize_int(self, text: str | None) -> int | None:
|
|
||||||
"""提取文本中的首个整数值。"""
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
|
|
||||||
digits = re.findall(r"\d+", text)
|
|
||||||
if not digits:
|
|
||||||
return None
|
|
||||||
|
|
||||||
return int(digits[0])
|
|
||||||
|
|
||||||
def _normalize_date(self, text: str | None) -> str | None:
|
|
||||||
"""将 `YYYY.MM.DD` / `YYYY/MM/DD` 统一为 `YYYY-MM-DD`。"""
|
|
||||||
if not text:
|
|
||||||
return None
|
|
||||||
|
|
||||||
match = re.search(r"(\d{4})[./-](\d{2})[./-](\d{2})", text)
|
|
||||||
if not match:
|
|
||||||
return text
|
|
||||||
|
|
||||||
year, month, day = match.groups()
|
|
||||||
return f"{year}-{month}-{day}"
|
|
||||||
|
|
||||||
|
|
||||||
async def parse_surugaya_order_detail(page: Page) -> dict[str, Any]:
|
|
||||||
"""便捷函数:直接返回可 JSON 序列化的订单字典。"""
|
|
||||||
parser = SurugayaOrderDetailParser()
|
|
||||||
result = await parser.parse_page(page)
|
|
||||||
return result.model_dump(mode="json")
|
|
||||||
+4
-2
@@ -15,6 +15,8 @@ from src.browser.manager import BrowserManager
|
|||||||
from src.notifications.service import NotificationService
|
from src.notifications.service import NotificationService
|
||||||
from src.utils.video_recorder import TaskVideoRecorder
|
from src.utils.video_recorder import TaskVideoRecorder
|
||||||
|
|
||||||
|
from surugaya_common import redis_keys
|
||||||
|
|
||||||
|
|
||||||
class BasePurchaseTask(ABC):
|
class BasePurchaseTask(ABC):
|
||||||
"""
|
"""
|
||||||
@@ -173,7 +175,7 @@ class BasePurchaseTask(ABC):
|
|||||||
"""
|
"""
|
||||||
import json
|
import json
|
||||||
import httpx
|
import httpx
|
||||||
from src.utils.http_utils import generate_signed_headers
|
from surugaya_common.http_utils import generate_signed_headers
|
||||||
|
|
||||||
# 1. 准备基础数据
|
# 1. 准备基础数据
|
||||||
app_id = self.settings.appid or ""
|
app_id = self.settings.appid or ""
|
||||||
@@ -315,7 +317,7 @@ class BasePurchaseTask(ABC):
|
|||||||
"""
|
"""
|
||||||
生成Redis Hash表的key,包含节点ID和交易单号。
|
生成Redis Hash表的key,包含节点ID和交易单号。
|
||||||
"""
|
"""
|
||||||
return "jp_surugaya_trade"
|
return redis_keys.TRADE_HASH
|
||||||
|
|
||||||
def get_delay_interval_seconds(self) -> int:
|
def get_delay_interval_seconds(self) -> int:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -15,13 +15,14 @@ from playwright.async_api import Page
|
|||||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||||
|
|
||||||
from notifications import NotificationEvent
|
from notifications import NotificationEvent
|
||||||
from parsers.surugaya import SurugayaOrderDetail
|
|
||||||
from src.tasks.base import BasePurchaseTask
|
from src.tasks.base import BasePurchaseTask
|
||||||
from src.config.accounts import get_account_config
|
from src.config.accounts import get_account_config
|
||||||
from src.utils.page_utils import format_japanese_price, human_scroll
|
from src.utils.page_utils import format_japanese_price
|
||||||
from src.parsers.surugaya.order_detail import SurugayaOrderDetailParser
|
|
||||||
from task_queue.redis_manager import get_redis_client
|
from task_queue.redis_manager import get_redis_client
|
||||||
from utils.app_utils import is_valid_str
|
from surugaya_common.models import SurugayaOrderDetail
|
||||||
|
from surugaya_common.parsers.surugaya.order_detail import SurugayaOrderDetailParser
|
||||||
|
from surugaya_common.browser_utils import human_scroll
|
||||||
|
from surugaya_common.app_utils import is_valid_str
|
||||||
|
|
||||||
|
|
||||||
class TemplateMatchResult(TypedDict):
|
class TemplateMatchResult(TypedDict):
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ from typing import Any
|
|||||||
|
|
||||||
from loguru import logger
|
from loguru import logger
|
||||||
|
|
||||||
from parsers.surugaya import SurugayaOrderDetailParser
|
from surugaya_common.parsers.surugaya import SurugayaOrderDetailParser
|
||||||
from src.tasks.base import BasePurchaseTask
|
from src.tasks.base import BasePurchaseTask
|
||||||
from task_queue.redis_manager import get_redis_client
|
from task_queue.redis_manager import get_redis_client
|
||||||
|
|
||||||
|
|||||||
@@ -1,4 +0,0 @@
|
|||||||
|
|
||||||
def is_valid_str(s):
|
|
||||||
"""判断字符串是否有效,即非空且非空格。"""
|
|
||||||
return isinstance(s, str) and len(s.strip()) > 0
|
|
||||||
@@ -1,38 +0,0 @@
|
|||||||
import json
|
|
||||||
import time
|
|
||||||
import hmac
|
|
||||||
import secrets
|
|
||||||
import hashlib
|
|
||||||
from typing import Dict, Any
|
|
||||||
|
|
||||||
def generate_signed_headers(app_id: str, app_secret: str, data: Dict[str, Any], method: str = "POST") -> Dict[str, str]:
|
|
||||||
timestamp = str(int(time.time() * 1000))
|
|
||||||
nonce = secrets.token_hex(16)
|
|
||||||
|
|
||||||
payload_str = json.dumps(data, ensure_ascii=False)
|
|
||||||
payload_bytes = payload_str.encode("utf-8")
|
|
||||||
actual_body_hash = hashlib.sha256(payload_bytes).hexdigest()
|
|
||||||
|
|
||||||
string_to_sign = "\n".join([
|
|
||||||
method.upper(),
|
|
||||||
app_id,
|
|
||||||
timestamp,
|
|
||||||
nonce,
|
|
||||||
actual_body_hash
|
|
||||||
])
|
|
||||||
|
|
||||||
signature = hmac.new(
|
|
||||||
app_secret.encode("utf-8"),
|
|
||||||
string_to_sign.encode("utf-8"),
|
|
||||||
hashlib.sha256
|
|
||||||
).hexdigest()
|
|
||||||
|
|
||||||
headers = {
|
|
||||||
"X-Ca-Appid": app_id,
|
|
||||||
"X-Ca-Timestamp": timestamp,
|
|
||||||
"X-Ca-Nonce": nonce,
|
|
||||||
"X-Ca-Signature": signature,
|
|
||||||
"Content-Type": "application/json; charset=utf-8"
|
|
||||||
}
|
|
||||||
|
|
||||||
return headers
|
|
||||||
+3
-103
@@ -1,105 +1,5 @@
|
|||||||
from playwright.async_api import Page
|
|
||||||
import asyncio
|
|
||||||
import random
|
|
||||||
import re
|
import re
|
||||||
|
|
||||||
async def human_scroll(page: Page, mode: str, value: int = 0):
|
|
||||||
"""
|
|
||||||
模拟人类行为的网页滚动方法
|
|
||||||
:param page: Playwright 的 Page 对象
|
|
||||||
:param mode: 滚动模式 -> 'top'(滚动到顶部), 'bottom'(滚动到底部), 'random'(随机上下滚动), 'distance'(滚动指定距离)
|
|
||||||
:param value: 当 mode='distance' 时,代表要滚动的像素距离(正数向下,负数向上)
|
|
||||||
"""
|
|
||||||
|
|
||||||
async def smooth_scroll_step(pixel_distance: int):
|
|
||||||
"""核心拟真算法:将一个大距离拆分成带有加速度和随机抖动的微小步长"""
|
|
||||||
if pixel_distance == 0:
|
|
||||||
return
|
|
||||||
|
|
||||||
direction = 1 if pixel_distance > 0 else -1
|
|
||||||
remaining = abs(pixel_distance)
|
|
||||||
|
|
||||||
while remaining > 0:
|
|
||||||
# 模拟人类滚动时越接近目标速度越慢的特征(减速缓冲)
|
|
||||||
if remaining > 100:
|
|
||||||
step = random.randint(30, 70)
|
|
||||||
elif remaining > 30:
|
|
||||||
step = random.randint(10, 25)
|
|
||||||
else:
|
|
||||||
step = remaining
|
|
||||||
|
|
||||||
remaining -= step
|
|
||||||
scroll_y = step * direction
|
|
||||||
|
|
||||||
# 执行单步微调滚动
|
|
||||||
await page.evaluate(f"window.scrollBy(0, {scroll_y});")
|
|
||||||
|
|
||||||
# 随机微小停顿,模拟手指滑动的摩擦阻力(20ms - 50ms)
|
|
||||||
await asyncio.sleep(random.uniform(0.02, 0.05))
|
|
||||||
|
|
||||||
# 0.5% 的极低概率触发人类的“视线停留”
|
|
||||||
if random.random() < 0.005:
|
|
||||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
|
||||||
|
|
||||||
# ==================== 模式 0:滚动到顶部 ====================
|
|
||||||
if mode == 'top':
|
|
||||||
# print("[HumanScroll] 正在模拟向上滚动至页面顶部...")
|
|
||||||
last_scroll = -1
|
|
||||||
while True:
|
|
||||||
# 获取当前滚动条位置
|
|
||||||
current_scroll = await page.evaluate("window.scrollY")
|
|
||||||
|
|
||||||
# 防死循环:如果位置不再变化,说明已经到顶或页面无法滚动
|
|
||||||
if current_scroll <= 0 or current_scroll == last_scroll:
|
|
||||||
break
|
|
||||||
|
|
||||||
last_scroll = current_scroll
|
|
||||||
chunk = random.randint(300, 700)
|
|
||||||
await smooth_scroll_step(-chunk)
|
|
||||||
await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼
|
|
||||||
|
|
||||||
# ==================== 模式 1:滚动到底部 ====================
|
|
||||||
elif mode == 'bottom':
|
|
||||||
# print("[HumanScroll] 正在模拟向下滚动至页面底部...")
|
|
||||||
last_scroll = -1
|
|
||||||
while True:
|
|
||||||
# 合并 evaluate 请求,减少与浏览器的 IPC 通信开销
|
|
||||||
scroll_info = await page.evaluate("() => [window.scrollY + window.innerHeight, document.body.scrollHeight]")
|
|
||||||
current_scroll, total_height = scroll_info[0], scroll_info[1]
|
|
||||||
|
|
||||||
# 防死循环:如果位置不再变化,说明已经到底或页面无法滚动
|
|
||||||
if current_scroll >= total_height - 10 or current_scroll == last_scroll:
|
|
||||||
break
|
|
||||||
|
|
||||||
last_scroll = current_scroll
|
|
||||||
chunk = random.randint(300, 700)
|
|
||||||
await smooth_scroll_step(chunk)
|
|
||||||
await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼
|
|
||||||
|
|
||||||
# ==================== 模式 2:随机上下滚动几下 ====================
|
|
||||||
elif mode == 'random':
|
|
||||||
# print("[HumanScroll] 正在执行人类迷惑行为:随机上下滚动...")
|
|
||||||
# 随机决定滚动 3 到 6 下
|
|
||||||
scroll_times = random.randint(3, 6)
|
|
||||||
|
|
||||||
for _ in range(scroll_times):
|
|
||||||
# 70% 概率向下,30% 概率向上
|
|
||||||
if random.random() > 0.3:
|
|
||||||
distance = random.randint(200, 500) # 向下滚
|
|
||||||
else:
|
|
||||||
distance = random.randint(-400, -150) # 向上回滚
|
|
||||||
|
|
||||||
await smooth_scroll_step(distance)
|
|
||||||
await asyncio.sleep(random.uniform(0.6, 1.5)) # 停顿
|
|
||||||
|
|
||||||
# ==================== 模式 3:滚动指定距离 ====================
|
|
||||||
elif mode == 'distance':
|
|
||||||
# print(f"[HumanScroll] 正在精准模拟滚动指定距离: {value} 像素")
|
|
||||||
await smooth_scroll_step(value)
|
|
||||||
|
|
||||||
else:
|
|
||||||
raise ValueError("未知的滚动模式!请选择 'top', 'bottom', 'random' 或 'distance'")
|
|
||||||
|
|
||||||
|
|
||||||
def format_japanese_price(price_str: str | None) -> int:
|
def format_japanese_price(price_str: str | None) -> int:
|
||||||
"""
|
"""
|
||||||
@@ -108,9 +8,9 @@ def format_japanese_price(price_str: str | None) -> int:
|
|||||||
"""
|
"""
|
||||||
if not price_str:
|
if not price_str:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
digits = re.findall(r'\d', price_str)
|
digits = re.findall(r'\d', price_str)
|
||||||
if not digits:
|
if not digits:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
return int("".join(digits))
|
return int("".join(digits))
|
||||||
|
|||||||
@@ -172,6 +172,7 @@ dependencies = [
|
|||||||
{ name = "pyscreeze" },
|
{ name = "pyscreeze" },
|
||||||
{ name = "pyyaml" },
|
{ name = "pyyaml" },
|
||||||
{ name = "redis" },
|
{ name = "redis" },
|
||||||
|
{ name = "surugaya-common" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
@@ -181,13 +182,14 @@ requires-dist = [
|
|||||||
{ name = "numpy", specifier = ">=2.4.6" },
|
{ name = "numpy", specifier = ">=2.4.6" },
|
||||||
{ name = "opencv-python", specifier = ">=4.13.0.92" },
|
{ name = "opencv-python", specifier = ">=4.13.0.92" },
|
||||||
{ name = "pillow", specifier = ">=12.2.0" },
|
{ name = "pillow", specifier = ">=12.2.0" },
|
||||||
{ name = "playwright", specifier = ">=1.43.0" },
|
{ name = "playwright", specifier = ">=1.54.0,<2.0.0" },
|
||||||
{ name = "pyautogui", specifier = ">=0.9.54" },
|
{ name = "pyautogui", specifier = ">=0.9.54" },
|
||||||
{ name = "pydantic", specifier = ">=2.7.0" },
|
{ name = "pydantic", specifier = ">=2.7.0" },
|
||||||
{ name = "pydantic-settings", specifier = ">=2.2.0" },
|
{ name = "pydantic-settings", specifier = ">=2.2.0" },
|
||||||
{ name = "pyscreeze", specifier = ">=1.0.1" },
|
{ name = "pyscreeze", specifier = ">=1.0.1" },
|
||||||
{ name = "pyyaml", specifier = ">=6.0.1" },
|
{ name = "pyyaml", specifier = ">=6.0.1" },
|
||||||
{ name = "redis", specifier = "==8.0.0" },
|
{ name = "redis", specifier = "==8.0.0" },
|
||||||
|
{ name = "surugaya-common", specifier = ">=0.2.0", index = "https://git.jerryyan.top/api/packages/jp/pypi/simple" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -683,6 +685,19 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/50/56/ba4987a64ceaae27f629644c4c52d2291eeab0755486d976c4bd1c781d86/rubicon_objc-0.5.4-py3-none-any.whl", hash = "sha256:f7c755388d5709d7079152892137b13bcc6b4bdd796c9f0d9b3a1eb5bea715c4", size = 64553, upload-time = "2026-05-06T00:57:49.032Z" },
|
{ url = "https://files.pythonhosted.org/packages/50/56/ba4987a64ceaae27f629644c4c52d2291eeab0755486d976c4bd1c781d86/rubicon_objc-0.5.4-py3-none-any.whl", hash = "sha256:f7c755388d5709d7079152892137b13bcc6b4bdd796c9f0d9b3a1eb5bea715c4", size = 64553, upload-time = "2026-05-06T00:57:49.032Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "surugaya-common"
|
||||||
|
version = "0.2.0"
|
||||||
|
source = { registry = "https://git.jerryyan.top/api/packages/jp/pypi/simple" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "pydantic" },
|
||||||
|
{ name = "redis" },
|
||||||
|
]
|
||||||
|
sdist = { url = "https://git.jerryyan.net/api/packages/jp/pypi/files/surugaya-common/0.2.0/surugaya_common-0.2.0.tar.gz", hash = "sha256:168a5810672f5ead82957a9491611b64bf439f489b4eeaa287b302206712e2ab" }
|
||||||
|
wheels = [
|
||||||
|
{ url = "https://git.jerryyan.net/api/packages/jp/pypi/files/surugaya-common/0.2.0/surugaya_common-0.2.0-py3-none-any.whl", hash = "sha256:140587df6b60abf1f1fc3b592d0668a096ddd42f5216ee16d4b83a95783b58cd" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "typing-extensions"
|
name = "typing-extensions"
|
||||||
version = "4.15.0"
|
version = "4.15.0"
|
||||||
|
|||||||
Reference in New Issue
Block a user