You've already forked pruchase_jhw
合并依赖
This commit is contained in:
+12
-1
@@ -9,7 +9,10 @@ authors = [
|
||||
{ name = "Antigravity Development Team" }
|
||||
]
|
||||
dependencies = [
|
||||
"playwright>=1.43.0",
|
||||
# worker 自身已直接依赖 playwright，故 surugaya-common 无需 [browser] extra（extra 仅用于
|
||||
# 给没有 playwright 的消费者拉取 playwright），保持不带 extra 更简洁。
|
||||
"surugaya-common>=0.2.0",
|
||||
"playwright>=1.54.0,<2.0.0",
|
||||
"pydantic>=2.7.0",
|
||||
"pydantic-settings>=2.2.0",
|
||||
"loguru>=0.7.2",
|
||||
@@ -30,3 +33,11 @@ build-backend = "hatchling.build"
|
||||
# 👇 在文件里加上这一行，告诉 uv 这只是个普通的脚本项目，不需要打包
|
||||
[tool.uv]
|
||||
package = false
|
||||
|
||||
# Gitea PyPI registry: 安装(simple index)需 /simple 后缀；上传才用 .../pypi。
|
||||
[[tool.uv.index]]
|
||||
name = "jerryyan"
|
||||
url = "https://git.jerryyan.net/api/packages/jp/pypi/simple"
|
||||
|
||||
[tool.uv.sources]
|
||||
surugaya-common = { index = "jerryyan" }
|
||||
|
||||
@@ -5,6 +5,8 @@ from typing import Annotated, Any
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
||||
|
||||
from surugaya_common import redis_keys
|
||||
|
||||
# 根据 APP_ENV 确定环境配置文件（默认为 dev）
|
||||
APP_ENV = os.getenv("APP_ENV", "dev").lower()
|
||||
ENV_FILE = f".env.{APP_ENV}"
|
||||
@@ -29,10 +31,10 @@ class Settings(BaseSettings):
|
||||
redis_port: int = Field(default=6379, alias="REDIS_PORT")
|
||||
redis_password: str | None = Field(default=None, alias="REDIS_PASSWORD")
|
||||
redis_db: int = Field(default=0, alias="REDIS_DB")
|
||||
redis_stream_key: str = Field(default="jp_surugaya_purchase_task", alias="REDIS_STREAM_KEY")
|
||||
redis_stream_key: str = Field(default=redis_keys.PURCHASE_TASK_STREAM, alias="REDIS_STREAM_KEY")
|
||||
redis_consumer_group: str = Field(default="surugaya_purchase_task_group", alias="REDIS_CONSUMER_GROUP")
|
||||
redis_consumer_name: str = Field(default="surugaya_purchase_task_worker", alias="REDIS_CONSUMER_NAME")
|
||||
redis_delay_zset_key: str = Field(default="jp_surugaya_trade_delay", alias="REDIS_DELAY_ZSET_KEY")
|
||||
redis_delay_zset_key: str = Field(default=redis_keys.TRADE_DELAY_ZSET, alias="REDIS_DELAY_ZSET_KEY")
|
||||
redis_delay_poll_interval: int = Field(default=3600, alias="REDIS_DELAY_POLL_INTERVAL")
|
||||
redis_delay_batch_size: int = Field(default=10, alias="REDIS_DELAY_BATCH_SIZE")
|
||||
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
"""页面解析器包。"""
|
||||
@@ -1,18 +0,0 @@
|
||||
from .models import (
|
||||
SurugayaOrderDetail,
|
||||
SurugayaOrderItem,
|
||||
SurugayaOrderStatus,
|
||||
SurugayaOrderSummary,
|
||||
SurugayaShippingInfo,
|
||||
)
|
||||
from .order_detail import SurugayaOrderDetailParser, parse_surugaya_order_detail
|
||||
|
||||
__all__ = [
|
||||
"SurugayaOrderDetail",
|
||||
"SurugayaOrderItem",
|
||||
"SurugayaOrderStatus",
|
||||
"SurugayaOrderSummary",
|
||||
"SurugayaShippingInfo",
|
||||
"SurugayaOrderDetailParser",
|
||||
"parse_surugaya_order_detail",
|
||||
]
|
||||
@@ -1,69 +0,0 @@
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SurugayaOrderStatus(BaseModel):
|
||||
"""交易状态表的结构化结果。"""
|
||||
|
||||
trade_status: str | None = None
|
||||
status_message: str | None = None
|
||||
# 保留日文原始键值，便于排障和后续补字段。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaOrderSummary(BaseModel):
|
||||
"""订单摘要表的结构化结果。"""
|
||||
|
||||
trade_number: str | None = None
|
||||
order_date: str | None = None
|
||||
items_subtotal: int | None = None
|
||||
item_count: int | None = None
|
||||
payment_method: str | None = None
|
||||
cash_on_delivery_fee: int | None = None
|
||||
shipping_date: str | None = None
|
||||
inquiry_number: str | None = None
|
||||
shipping_and_handling_fee: int | None = None
|
||||
digital_gift_certificate_amount: int | None = None
|
||||
grand_total: int | None = None
|
||||
# 保留订单摘要表的原始日文字段。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaShippingInfo(BaseModel):
|
||||
"""收件信息表的结构化结果。"""
|
||||
|
||||
recipient_name: str | None = None
|
||||
recipient_address: str | None = None
|
||||
recipient_phone_number: str | None = None
|
||||
recipient_email: str | None = None
|
||||
# 保留配送信息表的原始日文字段。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaOrderItem(BaseModel):
|
||||
"""商品明细表中的单行商品记录。"""
|
||||
|
||||
product_code: str | None = None
|
||||
condition: str | None = None
|
||||
branch_number: str | None = None
|
||||
product_title: str | None = None
|
||||
unit_price: int | None = None
|
||||
quantity: int | None = None
|
||||
discount_amount: int | None = None
|
||||
line_total: int | None = None
|
||||
remarks: str | None = None
|
||||
# 每一行商品同时保留原始列名和值。
|
||||
raw: dict[str, str] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class SurugayaOrderDetail(BaseModel):
|
||||
"""骏河屋交易详情页的完整结构化结果。"""
|
||||
|
||||
site: Literal["surugaya"] = "surugaya"
|
||||
status: SurugayaOrderStatus
|
||||
summary: SurugayaOrderSummary
|
||||
shipping: SurugayaShippingInfo
|
||||
items: list[SurugayaOrderItem] = Field(default_factory=list)
|
||||
# 以业务分段聚合保留原始表格结果。
|
||||
raw_tables: dict[str, Any] = Field(default_factory=dict)
|
||||
@@ -1,315 +0,0 @@
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from playwright.async_api import Locator, Page
|
||||
|
||||
from .models import (
|
||||
SurugayaOrderDetail,
|
||||
SurugayaOrderItem,
|
||||
SurugayaShippingInfo,
|
||||
SurugayaOrderStatus,
|
||||
SurugayaOrderSummary,
|
||||
)
|
||||
|
||||
|
||||
# 订单摘要表中的日文字段到英文字段映射。
|
||||
SUMMARY_FIELD_MAP = {
|
||||
"取引番号": "trade_number",
|
||||
"注文日": "order_date",
|
||||
"商品合計": "items_subtotal",
|
||||
"商品点数": "item_count",
|
||||
"支払方法": "payment_method",
|
||||
"代引き手数料": "cash_on_delivery_fee",
|
||||
"発送日": "shipping_date",
|
||||
"お問い合わせ番号": "inquiry_number",
|
||||
"送料・通信販売手数料": "shipping_and_handling_fee",
|
||||
"電子商品券": "digital_gift_certificate_amount",
|
||||
"総合計": "grand_total",
|
||||
}
|
||||
|
||||
# 收件信息表的日文字段到英文字段映射。
|
||||
SHIPPING_FIELD_MAP = {
|
||||
"氏名": "recipient_name",
|
||||
"住所": "recipient_address",
|
||||
"電話番号": "recipient_phone_number",
|
||||
"メールアドレス": "recipient_email",
|
||||
}
|
||||
|
||||
# 商品明细表的列名到英文字段映射。
|
||||
ITEM_FIELD_MAP = {
|
||||
"品番": "product_id",
|
||||
"状態": "status",
|
||||
"枝番": "branch_number",
|
||||
"商品タイトル": "product_title",
|
||||
"単価": "unit_price",
|
||||
"数量": "number",
|
||||
"値引額": "discount_amount",
|
||||
"金額": "product_total_amount",
|
||||
"備考": "remarks",
|
||||
}
|
||||
|
||||
# 需要按金额规则清洗的字段集合。
|
||||
MONEY_FIELDS = {
|
||||
"items_subtotal",
|
||||
"cash_on_delivery_fee",
|
||||
"shipping_and_handling_fee",
|
||||
"digital_gift_certificate_amount",
|
||||
"grand_total",
|
||||
"unit_price",
|
||||
"discount_amount",
|
||||
"line_total",
|
||||
}
|
||||
|
||||
# 需要按整数规则清洗的字段集合。
|
||||
INT_FIELDS = {
|
||||
"item_count",
|
||||
"quantity",
|
||||
}
|
||||
|
||||
class SurugayaOrderDetailParser:
|
||||
"""骏河屋交易详情页解析器。"""
|
||||
|
||||
@classmethod
|
||||
async def parse_page(cls, page: Page) -> SurugayaOrderDetail:
|
||||
"""对外主入口：传入已加载详情页的 `Page`，返回结构化订单对象。"""
|
||||
# 适配 `SurugayaOrderDetailParser.parse_page(page)` 这种类调用方式。
|
||||
parser = cls()
|
||||
if await page.locator("#pageCont").count() < 1:
|
||||
raise ValueError("Surugaya order detail page not found")
|
||||
return await parser._parse_page_content(page)
|
||||
|
||||
async def _parse_page_content(self, page: Page) -> SurugayaOrderDetail:
|
||||
"""解析正常交易详情页中的 `#pageCont` 核心区域。"""
|
||||
|
||||
page_cont = page.locator("#pageCont").first
|
||||
tables = page_cont.locator("table")
|
||||
table_count = await tables.count()
|
||||
if table_count < 4:
|
||||
raise ValueError("Expected 4 trade tables under #pageCont")
|
||||
|
||||
status_table = tables.nth(0)
|
||||
summary_table = tables.nth(1)
|
||||
shipping_table = tables.nth(2)
|
||||
items_table = tables.nth(3)
|
||||
|
||||
status = await self._parse_status_table(status_table)
|
||||
summary = await self._parse_summary_table(summary_table)
|
||||
shipping = await self._parse_shipping_table(shipping_table)
|
||||
items = await self._parse_items_table(items_table)
|
||||
|
||||
return SurugayaOrderDetail(
|
||||
status=status,
|
||||
summary=summary,
|
||||
shipping=shipping,
|
||||
items=items,
|
||||
raw_tables={
|
||||
"status": status.raw,
|
||||
"summary": summary.raw,
|
||||
"shipping": shipping.raw,
|
||||
"items": [item.raw for item in items],
|
||||
},
|
||||
)
|
||||
|
||||
async def _parse_status_table(self, table: Locator) -> SurugayaOrderStatus:
|
||||
"""解析第 1 张交易状态表。"""
|
||||
rows = table.locator("tr")
|
||||
row_count = await rows.count()
|
||||
if row_count < 1:
|
||||
return SurugayaOrderStatus()
|
||||
|
||||
first_row = await self._extract_row_cells(rows.nth(0))
|
||||
raw: dict[str, str] = {}
|
||||
trade_status = None
|
||||
status_message = None
|
||||
|
||||
if len(first_row) >= 2 and first_row[0]["tag"] == "th" and first_row[1]["tag"] == "td":
|
||||
label = first_row[0]["text"]
|
||||
value = first_row[1]["text"]
|
||||
raw[label] = value
|
||||
lines = self._split_lines(value)
|
||||
trade_status = lines[0] if lines else None
|
||||
status_message = " ".join(lines[1:]) or None
|
||||
|
||||
return SurugayaOrderStatus(
|
||||
trade_status=trade_status,
|
||||
status_message=status_message,
|
||||
raw=raw,
|
||||
)
|
||||
|
||||
async def _parse_summary_table(self, table: Locator) -> SurugayaOrderSummary:
|
||||
"""解析第 2 张订单摘要表。"""
|
||||
raw_pairs = await self._parse_key_value_rows(table)
|
||||
normalized = self._normalize_mapped_values(raw_pairs, SUMMARY_FIELD_MAP)
|
||||
return SurugayaOrderSummary(**normalized, raw=raw_pairs)
|
||||
|
||||
async def _parse_shipping_table(self, table: Locator) -> SurugayaShippingInfo:
|
||||
"""解析第 3 张收件信息表。"""
|
||||
raw_pairs = await self._parse_key_value_rows(table)
|
||||
normalized = self._normalize_mapped_values(raw_pairs, SHIPPING_FIELD_MAP)
|
||||
return SurugayaShippingInfo(**normalized, raw=raw_pairs)
|
||||
|
||||
async def _parse_items_table(self, table: Locator) -> list[SurugayaOrderItem]:
|
||||
"""解析第 4 张商品明细表。"""
|
||||
rows = table.locator("tr")
|
||||
row_count = await rows.count()
|
||||
if row_count < 2:
|
||||
return []
|
||||
|
||||
header_cells = await self._extract_row_cells(rows.nth(0))
|
||||
headers = [cell["text"] for cell in header_cells if cell["tag"] == "th" and cell["text"]]
|
||||
if not headers:
|
||||
return []
|
||||
|
||||
items: list[SurugayaOrderItem] = []
|
||||
expected_columns = len(headers)
|
||||
|
||||
for index in range(1, row_count):
|
||||
cells = await self._extract_row_cells(rows.nth(index))
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
td_cells = [cell for cell in cells if cell["tag"] == "td"]
|
||||
if len(td_cells) != expected_columns:
|
||||
continue
|
||||
|
||||
raw_item = {
|
||||
header: td_cells[position]["text"]
|
||||
for position, header in enumerate(headers)
|
||||
}
|
||||
|
||||
if not any(raw_item.values()):
|
||||
continue
|
||||
|
||||
normalized = self._normalize_mapped_values(raw_item, ITEM_FIELD_MAP)
|
||||
items.append(SurugayaOrderItem(**normalized, raw=raw_item))
|
||||
|
||||
return items
|
||||
|
||||
async def _parse_key_value_rows(self, table: Locator) -> dict[str, str]:
|
||||
"""通用解析：按 `th -> td` 配对方式抽取键值。"""
|
||||
rows = table.locator("tr")
|
||||
row_count = await rows.count()
|
||||
result: dict[str, str] = {}
|
||||
|
||||
for index in range(row_count):
|
||||
cells = await self._extract_row_cells(rows.nth(index))
|
||||
if not cells:
|
||||
continue
|
||||
|
||||
pointer = 0
|
||||
while pointer < len(cells):
|
||||
current = cells[pointer]
|
||||
if current["tag"] != "th":
|
||||
pointer += 1
|
||||
continue
|
||||
|
||||
label = current["text"]
|
||||
pointer += 1
|
||||
if not label:
|
||||
continue
|
||||
|
||||
while pointer < len(cells) and cells[pointer]["tag"] != "td":
|
||||
pointer += 1
|
||||
|
||||
if pointer >= len(cells):
|
||||
break
|
||||
|
||||
result[label] = cells[pointer]["text"]
|
||||
pointer += 1
|
||||
|
||||
return result
|
||||
|
||||
async def _extract_row_cells(self, row: Locator) -> list[dict[str, str]]:
|
||||
"""提取一行中的所有单元格，并保留标签名与纯文本。"""
|
||||
return await row.locator(":scope > th, :scope > td").evaluate_all(
|
||||
"""cells => cells.map((cell) => ({
|
||||
tag: cell.tagName.toLowerCase(),
|
||||
text: (cell.innerText || "").replace(/\\u00a0/g, " ").trim(),
|
||||
}))"""
|
||||
)
|
||||
|
||||
def _normalize_mapped_values(
|
||||
self,
|
||||
raw_data: dict[str, str],
|
||||
field_map: dict[str, str],
|
||||
) -> dict[str, Any]:
|
||||
"""将日文键值映射并清洗为结构化字段。"""
|
||||
normalized: dict[str, Any] = {}
|
||||
|
||||
for label, value in raw_data.items():
|
||||
english_key = field_map.get(label)
|
||||
if not english_key:
|
||||
continue
|
||||
|
||||
cleaned_value = self._clean_text(value)
|
||||
if english_key in MONEY_FIELDS:
|
||||
normalized[english_key] = self._normalize_money(cleaned_value)
|
||||
elif english_key in INT_FIELDS:
|
||||
normalized[english_key] = self._normalize_int(cleaned_value)
|
||||
elif english_key.endswith("_date"):
|
||||
normalized[english_key] = self._normalize_date(cleaned_value)
|
||||
else:
|
||||
normalized[english_key] = cleaned_value
|
||||
|
||||
return normalized
|
||||
|
||||
def _split_lines(self, text: str | None) -> list[str]:
|
||||
"""按行拆分文本，并清理每一行空白。"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
lines = [self._clean_text(part) for part in text.splitlines()]
|
||||
return [line for line in lines if line]
|
||||
|
||||
def _clean_text(self, text: str | None) -> str | None:
|
||||
"""清洗文本中的多余空白并统一空值。"""
|
||||
if text is None:
|
||||
return None
|
||||
|
||||
cleaned = re.sub(r"\s+", " ", text.replace("\xa0", " ")).strip()
|
||||
return cleaned or None
|
||||
|
||||
def _normalize_money(self, text: str | None) -> int | None:
|
||||
"""将 `￥1,234` / `1,234円` 这类金额文本转成整数。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
digits = re.findall(r"-?\d[\d,]*", text)
|
||||
if not digits:
|
||||
return None
|
||||
|
||||
value = digits[0].replace(",", "")
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def _normalize_int(self, text: str | None) -> int | None:
|
||||
"""提取文本中的首个整数值。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
digits = re.findall(r"\d+", text)
|
||||
if not digits:
|
||||
return None
|
||||
|
||||
return int(digits[0])
|
||||
|
||||
def _normalize_date(self, text: str | None) -> str | None:
|
||||
"""将 `YYYY.MM.DD` / `YYYY/MM/DD` 统一为 `YYYY-MM-DD`。"""
|
||||
if not text:
|
||||
return None
|
||||
|
||||
match = re.search(r"(\d{4})[./-](\d{2})[./-](\d{2})", text)
|
||||
if not match:
|
||||
return text
|
||||
|
||||
year, month, day = match.groups()
|
||||
return f"{year}-{month}-{day}"
|
||||
|
||||
|
||||
async def parse_surugaya_order_detail(page: Page) -> dict[str, Any]:
|
||||
"""便捷函数：直接返回可 JSON 序列化的订单字典。"""
|
||||
parser = SurugayaOrderDetailParser()
|
||||
result = await parser.parse_page(page)
|
||||
return result.model_dump(mode="json")
|
||||
+4
-2
@@ -15,6 +15,8 @@ from src.browser.manager import BrowserManager
|
||||
from src.notifications.service import NotificationService
|
||||
from src.utils.video_recorder import TaskVideoRecorder
|
||||
|
||||
from surugaya_common import redis_keys
|
||||
|
||||
|
||||
class BasePurchaseTask(ABC):
|
||||
"""
|
||||
@@ -173,7 +175,7 @@ class BasePurchaseTask(ABC):
|
||||
"""
|
||||
import json
|
||||
import httpx
|
||||
from src.utils.http_utils import generate_signed_headers
|
||||
from surugaya_common.http_utils import generate_signed_headers
|
||||
|
||||
# 1. 准备基础数据
|
||||
app_id = self.settings.appid or ""
|
||||
@@ -315,7 +317,7 @@ class BasePurchaseTask(ABC):
|
||||
"""
|
||||
生成Redis Hash表的key，包含节点ID和交易单号。
|
||||
"""
|
||||
return "jp_surugaya_trade"
|
||||
return redis_keys.TRADE_HASH
|
||||
|
||||
def get_delay_interval_seconds(self) -> int:
|
||||
"""
|
||||
|
||||
@@ -15,13 +15,14 @@ from playwright.async_api import Page
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from notifications import NotificationEvent
|
||||
from parsers.surugaya import SurugayaOrderDetail
|
||||
from src.tasks.base import BasePurchaseTask
|
||||
from src.config.accounts import get_account_config
|
||||
from src.utils.page_utils import format_japanese_price, human_scroll
|
||||
from src.parsers.surugaya.order_detail import SurugayaOrderDetailParser
|
||||
from src.utils.page_utils import format_japanese_price
|
||||
from task_queue.redis_manager import get_redis_client
|
||||
from utils.app_utils import is_valid_str
|
||||
from surugaya_common.models import SurugayaOrderDetail
|
||||
from surugaya_common.parsers.surugaya.order_detail import SurugayaOrderDetailParser
|
||||
from surugaya_common.browser_utils import human_scroll
|
||||
from surugaya_common.app_utils import is_valid_str
|
||||
|
||||
|
||||
class TemplateMatchResult(TypedDict):
|
||||
|
||||
@@ -4,7 +4,7 @@ from typing import Any
|
||||
|
||||
from loguru import logger
|
||||
|
||||
from parsers.surugaya import SurugayaOrderDetailParser
|
||||
from surugaya_common.parsers.surugaya import SurugayaOrderDetailParser
|
||||
from src.tasks.base import BasePurchaseTask
|
||||
from task_queue.redis_manager import get_redis_client
|
||||
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
|
||||
def is_valid_str(s):
|
||||
"""判断字符串是否有效，即非空且非空格。"""
|
||||
return isinstance(s, str) and len(s.strip()) > 0
|
||||
@@ -1,38 +0,0 @@
|
||||
import json
|
||||
import time
|
||||
import hmac
|
||||
import secrets
|
||||
import hashlib
|
||||
from typing import Dict, Any
|
||||
|
||||
def generate_signed_headers(app_id: str, app_secret: str, data: Dict[str, Any], method: str = "POST") -> Dict[str, str]:
|
||||
timestamp = str(int(time.time() * 1000))
|
||||
nonce = secrets.token_hex(16)
|
||||
|
||||
payload_str = json.dumps(data, ensure_ascii=False)
|
||||
payload_bytes = payload_str.encode("utf-8")
|
||||
actual_body_hash = hashlib.sha256(payload_bytes).hexdigest()
|
||||
|
||||
string_to_sign = "\n".join([
|
||||
method.upper(),
|
||||
app_id,
|
||||
timestamp,
|
||||
nonce,
|
||||
actual_body_hash
|
||||
])
|
||||
|
||||
signature = hmac.new(
|
||||
app_secret.encode("utf-8"),
|
||||
string_to_sign.encode("utf-8"),
|
||||
hashlib.sha256
|
||||
).hexdigest()
|
||||
|
||||
headers = {
|
||||
"X-Ca-Appid": app_id,
|
||||
"X-Ca-Timestamp": timestamp,
|
||||
"X-Ca-Nonce": nonce,
|
||||
"X-Ca-Signature": signature,
|
||||
"Content-Type": "application/json; charset=utf-8"
|
||||
}
|
||||
|
||||
return headers
|
||||
@@ -1,105 +1,5 @@
|
||||
from playwright.async_api import Page
|
||||
import asyncio
|
||||
import random
|
||||
import re
|
||||
|
||||
async def human_scroll(page: Page, mode: str, value: int = 0):
|
||||
"""
|
||||
模拟人类行为的网页滚动方法
|
||||
:param page: Playwright 的 Page 对象
|
||||
:param mode: 滚动模式 -> 'top'(滚动到顶部), 'bottom'(滚动到底部), 'random'(随机上下滚动), 'distance'(滚动指定距离)
|
||||
:param value: 当 mode='distance' 时，代表要滚动的像素距离（正数向下，负数向上）
|
||||
"""
|
||||
|
||||
async def smooth_scroll_step(pixel_distance: int):
|
||||
"""核心拟真算法：将一个大距离拆分成带有加速度和随机抖动的微小步长"""
|
||||
if pixel_distance == 0:
|
||||
return
|
||||
|
||||
direction = 1 if pixel_distance > 0 else -1
|
||||
remaining = abs(pixel_distance)
|
||||
|
||||
while remaining > 0:
|
||||
# 模拟人类滚动时越接近目标速度越慢的特征（减速缓冲）
|
||||
if remaining > 100:
|
||||
step = random.randint(30, 70)
|
||||
elif remaining > 30:
|
||||
step = random.randint(10, 25)
|
||||
else:
|
||||
step = remaining
|
||||
|
||||
remaining -= step
|
||||
scroll_y = step * direction
|
||||
|
||||
# 执行单步微调滚动
|
||||
await page.evaluate(f"window.scrollBy(0, {scroll_y});")
|
||||
|
||||
# 随机微小停顿，模拟手指滑动的摩擦阻力（20ms - 50ms）
|
||||
await asyncio.sleep(random.uniform(0.02, 0.05))
|
||||
|
||||
# 0.5% 的极低概率触发人类的“视线停留”
|
||||
if random.random() < 0.005:
|
||||
await asyncio.sleep(random.uniform(0.3, 0.8))
|
||||
|
||||
# ==================== 模式 0：滚动到顶部 ====================
|
||||
if mode == 'top':
|
||||
# print("[HumanScroll] 正在模拟向上滚动至页面顶部...")
|
||||
last_scroll = -1
|
||||
while True:
|
||||
# 获取当前滚动条位置
|
||||
current_scroll = await page.evaluate("window.scrollY")
|
||||
|
||||
# 防死循环：如果位置不再变化，说明已经到顶或页面无法滚动
|
||||
if current_scroll <= 0 or current_scroll == last_scroll:
|
||||
break
|
||||
|
||||
last_scroll = current_scroll
|
||||
chunk = random.randint(300, 700)
|
||||
await smooth_scroll_step(-chunk)
|
||||
await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼
|
||||
|
||||
# ==================== 模式 1：滚动到底部 ====================
|
||||
elif mode == 'bottom':
|
||||
# print("[HumanScroll] 正在模拟向下滚动至页面底部...")
|
||||
last_scroll = -1
|
||||
while True:
|
||||
# 合并 evaluate 请求，减少与浏览器的 IPC 通信开销
|
||||
scroll_info = await page.evaluate("() => [window.scrollY + window.innerHeight, document.body.scrollHeight]")
|
||||
current_scroll, total_height = scroll_info[0], scroll_info[1]
|
||||
|
||||
# 防死循环：如果位置不再变化，说明已经到底或页面无法滚动
|
||||
if current_scroll >= total_height - 10 or current_scroll == last_scroll:
|
||||
break
|
||||
|
||||
last_scroll = current_scroll
|
||||
chunk = random.randint(300, 700)
|
||||
await smooth_scroll_step(chunk)
|
||||
await asyncio.sleep(random.uniform(0.5, 1.2)) # 每滑一下停顿看一眼
|
||||
|
||||
# ==================== 模式 2：随机上下滚动几下 ====================
|
||||
elif mode == 'random':
|
||||
# print("[HumanScroll] 正在执行人类迷惑行为：随机上下滚动...")
|
||||
# 随机决定滚动 3 到 6 下
|
||||
scroll_times = random.randint(3, 6)
|
||||
|
||||
for _ in range(scroll_times):
|
||||
# 70% 概率向下，30% 概率向上
|
||||
if random.random() > 0.3:
|
||||
distance = random.randint(200, 500) # 向下滚
|
||||
else:
|
||||
distance = random.randint(-400, -150) # 向上回滚
|
||||
|
||||
await smooth_scroll_step(distance)
|
||||
await asyncio.sleep(random.uniform(0.6, 1.5)) # 停顿
|
||||
|
||||
# ==================== 模式 3：滚动指定距离 ====================
|
||||
elif mode == 'distance':
|
||||
# print(f"[HumanScroll] 正在精准模拟滚动指定距离: {value} 像素")
|
||||
await smooth_scroll_step(value)
|
||||
|
||||
else:
|
||||
raise ValueError("未知的滚动模式！请选择 'top', 'bottom', 'random' 或 'distance'")
|
||||
|
||||
|
||||
def format_japanese_price(price_str: str | None) -> int:
|
||||
"""
|
||||
|
||||
@@ -172,6 +172,7 @@ dependencies = [
|
||||
{ name = "pyscreeze" },
|
||||
{ name = "pyyaml" },
|
||||
{ name = "redis" },
|
||||
{ name = "surugaya-common" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
@@ -181,13 +182,14 @@ requires-dist = [
|
||||
{ name = "numpy", specifier = ">=2.4.6" },
|
||||
{ name = "opencv-python", specifier = ">=4.13.0.92" },
|
||||
{ name = "pillow", specifier = ">=12.2.0" },
|
||||
{ name = "playwright", specifier = ">=1.43.0" },
|
||||
{ name = "playwright", specifier = ">=1.54.0,<2.0.0" },
|
||||
{ name = "pyautogui", specifier = ">=0.9.54" },
|
||||
{ name = "pydantic", specifier = ">=2.7.0" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.2.0" },
|
||||
{ name = "pyscreeze", specifier = ">=1.0.1" },
|
||||
{ name = "pyyaml", specifier = ">=6.0.1" },
|
||||
{ name = "redis", specifier = "==8.0.0" },
|
||||
{ name = "surugaya-common", specifier = ">=0.2.0", index = "https://git.jerryyan.top/api/packages/jp/pypi/simple" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -683,6 +685,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/50/56/ba4987a64ceaae27f629644c4c52d2291eeab0755486d976c4bd1c781d86/rubicon_objc-0.5.4-py3-none-any.whl", hash = "sha256:f7c755388d5709d7079152892137b13bcc6b4bdd796c9f0d9b3a1eb5bea715c4", size = 64553, upload-time = "2026-05-06T00:57:49.032Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "surugaya-common"
|
||||
version = "0.2.0"
|
||||
source = { registry = "https://git.jerryyan.top/api/packages/jp/pypi/simple" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
{ name = "redis" },
|
||||
]
|
||||
sdist = { url = "https://git.jerryyan.net/api/packages/jp/pypi/files/surugaya-common/0.2.0/surugaya_common-0.2.0.tar.gz", hash = "sha256:168a5810672f5ead82957a9491611b64bf439f489b4eeaa287b302206712e2ab" }
|
||||
wheels = [
|
||||
{ url = "https://git.jerryyan.net/api/packages/jp/pypi/files/surugaya-common/0.2.0/surugaya_common-0.2.0-py3-none-any.whl", hash = "sha256:140587df6b60abf1f1fc3b592d0668a096ddd42f5216ee16d4b83a95783b58cd" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.15.0"
|
||||
|
||||
Reference in New Issue
Block a user