common扩展
This commit is contained in:
+7
-8
@@ -9,6 +9,8 @@ from typing import Literal
|
|||||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from surugaya_common import browser_profile, urls
|
||||||
|
|
||||||
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||||
|
|
||||||
|
|
||||||
@@ -60,13 +62,10 @@ class Settings(BaseSettings):
|
|||||||
cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时
|
cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时
|
||||||
cloudflare_reuse_wait_timeout_seconds: float = 8.0 # 复用会话时消化挑战的短超时,超时则放弃复用直接重建
|
cloudflare_reuse_wait_timeout_seconds: float = 8.0 # 复用会话时消化挑战的短超时,超时则放弃复用直接重建
|
||||||
session_ttl_seconds: int = 900 # 会话过期时间(秒)
|
session_ttl_seconds: int = 900 # 会话过期时间(秒)
|
||||||
browser_locale: str = "ja-JP"
|
# 指纹参数默认值取自共享包,与 worker 保持一致
|
||||||
browser_timezone_id: str = "Asia/Tokyo"
|
browser_locale: str = browser_profile.LOCALE
|
||||||
browser_user_agent: str = (
|
browser_timezone_id: str = browser_profile.TIMEZONE_ID
|
||||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
browser_user_agent: str = browser_profile.DEFAULT_USER_AGENT
|
||||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
|
||||||
"Chrome/124.0.0.0 Safari/537.36"
|
|
||||||
)
|
|
||||||
browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式
|
browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式
|
||||||
|
|
||||||
# ---- Redis 配置(可选,未配置时使用进程内存)----
|
# ---- Redis 配置(可选,未配置时使用进程内存)----
|
||||||
@@ -82,7 +81,7 @@ class Settings(BaseSettings):
|
|||||||
proxy_password: str | None = None
|
proxy_password: str | None = None
|
||||||
|
|
||||||
# ---- 目标站点 ----
|
# ---- 目标站点 ----
|
||||||
base_url: str = "https://www.suruga-ya.jp"
|
base_url: str = urls.BASE_URL
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def browser_headless_effective(self) -> bool:
|
def browser_headless_effective(self) -> bool:
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ from typing import Any
|
|||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.core.errors import BrowserUnavailableError, ResourceBusyError
|
from app.core.errors import BrowserUnavailableError, ResourceBusyError
|
||||||
|
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -80,28 +81,17 @@ class BrowserPool:
|
|||||||
# 用 playwright-stealth 钩子包装 playwright:后续 new_context/new_page
|
# 用 playwright-stealth 钩子包装 playwright:后续 new_context/new_page
|
||||||
# 会自动应用 stealth(含 HTTP sec-ch-ua 客户端提示),并令 navigator.languages
|
# 会自动应用 stealth(含 HTTP sec-ch-ua 客户端提示),并令 navigator.languages
|
||||||
# 与日语环境保持一致,避免指纹自相矛盾。
|
# 与日语环境保持一致,避免指纹自相矛盾。
|
||||||
from playwright_stealth import Stealth
|
from surugaya_common.stealth import create_stealth
|
||||||
|
|
||||||
stealth = Stealth(
|
self._playwright_cm = create_stealth().use_async(async_playwright())
|
||||||
navigator_languages_override=("ja-JP", "ja", "en-US", "en"),
|
|
||||||
)
|
|
||||||
self._playwright_cm = stealth.use_async(async_playwright())
|
|
||||||
self._playwright = await self._playwright_cm.__aenter__()
|
self._playwright = await self._playwright_cm.__aenter__()
|
||||||
else:
|
else:
|
||||||
self._playwright_cm = None
|
self._playwright_cm = None
|
||||||
self._playwright = await async_playwright().start()
|
self._playwright = await async_playwright().start()
|
||||||
chromium = self._playwright.chromium
|
chromium = self._playwright.chromium
|
||||||
launch_args = [
|
launch_args = stealth_launch_args() + [
|
||||||
"--disable-blink-features=AutomationControlled",
|
|
||||||
"--disable-dev-shm-usage",
|
|
||||||
"--disable-web-security",
|
"--disable-web-security",
|
||||||
"--no-first-run",
|
"--no-first-run",
|
||||||
"--disable-infobars",
|
|
||||||
"--no-sandbox",
|
|
||||||
"--disable-setuid-sandbox",
|
|
||||||
"--disable-gpu",
|
|
||||||
"--use-mock-keychain",
|
|
||||||
"--password-store=basic",
|
|
||||||
]
|
]
|
||||||
# 本地可视化调试时强制窗口化,避免浏览器启动后不可见
|
# 本地可视化调试时强制窗口化,避免浏览器启动后不可见
|
||||||
if not headless:
|
if not headless:
|
||||||
@@ -305,13 +295,7 @@ class BrowserPool:
|
|||||||
async def _configure_page(context: Any) -> Any:
|
async def _configure_page(context: Any) -> Any:
|
||||||
"""在上下文中创建页面并设置反检测请求头。"""
|
"""在上下文中创建页面并设置反检测请求头。"""
|
||||||
page = await context.new_page()
|
page = await context.new_page()
|
||||||
await page.set_extra_http_headers(
|
await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
|
||||||
{
|
|
||||||
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
||||||
"Cache-Control": "no-cache",
|
|
||||||
"Pragma": "no-cache",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
return page
|
return page
|
||||||
|
|
||||||
async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any:
|
async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any:
|
||||||
|
|||||||
@@ -10,6 +10,9 @@ from selectolax.parser import HTMLParser
|
|||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.models.scrape import CargoRequest
|
from app.models.scrape import CargoRequest
|
||||||
from app.utils.parse_util import extract_product_id, format_price
|
from app.utils.parse_util import extract_product_id, format_price
|
||||||
|
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
|
||||||
|
from surugaya_common.browser_utils import clear_cart
|
||||||
|
from surugaya_common.urls import CARGO_DETAIL_URL, product_detail_url
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -29,16 +32,7 @@ class CargoService:
|
|||||||
"""
|
"""
|
||||||
async with async_playwright() as p:
|
async with async_playwright() as p:
|
||||||
headless = self._settings.browser_headless_effective
|
headless = self._settings.browser_headless_effective
|
||||||
launch_args = [
|
launch_args = stealth_launch_args()
|
||||||
"--disable-blink-features=AutomationControlled",
|
|
||||||
"--disable-infobars",
|
|
||||||
"--no-sandbox",
|
|
||||||
"--disable-setuid-sandbox",
|
|
||||||
"--disable-dev-shm-usage",
|
|
||||||
"--disable-gpu",
|
|
||||||
"--use-mock-keychain",
|
|
||||||
"--password-store=basic",
|
|
||||||
]
|
|
||||||
if not headless:
|
if not headless:
|
||||||
launch_args.extend(["--start-maximized", "--window-position=0,0"])
|
launch_args.extend(["--start-maximized", "--window-position=0,0"])
|
||||||
|
|
||||||
@@ -61,21 +55,17 @@ class CargoService:
|
|||||||
|
|
||||||
if self._settings.browser_stealth_enabled:
|
if self._settings.browser_stealth_enabled:
|
||||||
try:
|
try:
|
||||||
from playwright_stealth import Stealth
|
# launch_persistent_context 之外的裸 context 用 apply_stealth_async;
|
||||||
await Stealth().apply_stealth_async(context)
|
# 共享工厂带日语 languages override,与 locale/Accept-Language 对齐。
|
||||||
|
from surugaya_common.stealth import create_stealth
|
||||||
|
await create_stealth().apply_stealth_async(context)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("应用 playwright-stealth 失败: %s", e)
|
logger.warning("应用 playwright-stealth 失败: %s", e)
|
||||||
|
|
||||||
page = await context.new_page()
|
page = await context.new_page()
|
||||||
await page.set_extra_http_headers(
|
await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
|
||||||
{
|
|
||||||
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
|
|
||||||
"Cache-Control": "no-cache",
|
|
||||||
"Pragma": "no-cache",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
target_url = "https://www.suruga-ya.jp/cargo/detail"
|
target_url = CARGO_DETAIL_URL
|
||||||
# 先进入购物车页面
|
# 先进入购物车页面
|
||||||
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
|
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
|
||||||
|
|
||||||
@@ -85,7 +75,7 @@ class CargoService:
|
|||||||
# 清空购物车
|
# 清空购物车
|
||||||
await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟,模拟人工操作间隔
|
await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟,模拟人工操作间隔
|
||||||
if cart_goods_num > 0:
|
if cart_goods_num > 0:
|
||||||
await self.clear_cart(page)
|
await clear_cart(page)
|
||||||
|
|
||||||
# 将待采购的商品加入购物车
|
# 将待采购的商品加入购物车
|
||||||
item_list = request.payload.item_list
|
item_list = request.payload.item_list
|
||||||
@@ -101,7 +91,7 @@ class CargoService:
|
|||||||
continue
|
continue
|
||||||
|
|
||||||
logger.debug("进入商品 %s 详情页,准备购买数量: %s...", product_id, number)
|
logger.debug("进入商品 %s 详情页,准备购买数量: %s...", product_id, number)
|
||||||
detail_url = f"https://www.suruga-ya.jp/product/detail/{product_id}"
|
detail_url = product_detail_url(product_id)
|
||||||
await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
|
await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
|
||||||
|
|
||||||
# 判断是否缺货
|
# 判断是否缺货
|
||||||
@@ -327,57 +317,3 @@ class CargoService:
|
|||||||
"total_commission_fee": total_commission_fee
|
"total_commission_fee": total_commission_fee
|
||||||
}
|
}
|
||||||
|
|
||||||
async def clear_cart(self, page):
|
|
||||||
"""
|
|
||||||
清空购物车
|
|
||||||
"""
|
|
||||||
cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
|
|
||||||
if cart_goods_num < 1:
|
|
||||||
return
|
|
||||||
|
|
||||||
is_delete = 0
|
|
||||||
# 购物车商品数量超过3个后会出现"全削除"按钮
|
|
||||||
if cart_goods_num > 3:
|
|
||||||
try:
|
|
||||||
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
|
|
||||||
# 3. 显式等待按钮在页面上可见(容错处理:比如给它最多 3000 毫秒的加载时间)
|
|
||||||
# 如果按钮是一开始就在静态 HTML 里的,这一步也可以省略,直接用下面的 is_visible()
|
|
||||||
await delete_button.wait_for(state="visible", timeout=3000)
|
|
||||||
if await delete_button.is_visible():
|
|
||||||
logger.debug("检测到‘全削除’按钮存在,正在执行点击...")
|
|
||||||
await delete_button.click()
|
|
||||||
is_delete = 1
|
|
||||||
except Exception:
|
|
||||||
logger.debug("未检测到‘全削除’按钮。")
|
|
||||||
|
|
||||||
if is_delete == 1:
|
|
||||||
return
|
|
||||||
|
|
||||||
# 遍历删除
|
|
||||||
item_selector = "table.item_box tbody tr.item"
|
|
||||||
remove_selector = "table.item_box a.remove_product"
|
|
||||||
|
|
||||||
while True:
|
|
||||||
cart_goods_num = await page.locator(item_selector).count()
|
|
||||||
if cart_goods_num == 0:
|
|
||||||
break
|
|
||||||
|
|
||||||
remove_link = page.locator(remove_selector).first
|
|
||||||
|
|
||||||
try:
|
|
||||||
await remove_link.click()
|
|
||||||
# 删除按钮不会触发弹窗,直接等待购物车条目数减少即可。
|
|
||||||
await page.wait_for_function(
|
|
||||||
"""({selector, expectedCount}) => {
|
|
||||||
return document.querySelectorAll(selector).length < expectedCount;
|
|
||||||
}""",
|
|
||||||
arg={"selector": item_selector, "expectedCount": cart_goods_num},
|
|
||||||
timeout=15_000,
|
|
||||||
)
|
|
||||||
await page.wait_for_timeout(random.randint(800, 1500))
|
|
||||||
except PlaywrightTimeoutError:
|
|
||||||
latest_count = await page.locator(item_selector).count()
|
|
||||||
if latest_count < cart_goods_num:
|
|
||||||
continue
|
|
||||||
|
|
||||||
await page.wait_for_timeout(1000)
|
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ from app.services.cloudflare_session import CloudflareSessionManager
|
|||||||
from app.services.session_store import SessionStore
|
from app.services.session_store import SessionStore
|
||||||
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
|
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
|
||||||
from surugaya_common.app_utils import is_empty_str
|
from surugaya_common.app_utils import is_empty_str
|
||||||
|
from surugaya_common.urls import BASE_URL
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
@@ -303,7 +304,7 @@ class SurugayaClient:
|
|||||||
logger.info("开始获取并解析商品分类...")
|
logger.info("开始获取并解析商品分类...")
|
||||||
top_categories: list[CategoryData] = []
|
top_categories: list[CategoryData] = []
|
||||||
|
|
||||||
target_url = "https://www.suruga-ya.jp/"
|
target_url = f"{BASE_URL}/"
|
||||||
session = await self._session_manager.get_verified_session(target_url, True)
|
session = await self._session_manager.get_verified_session(target_url, True)
|
||||||
|
|
||||||
html_content = ""
|
html_content = ""
|
||||||
@@ -366,7 +367,7 @@ class SurugayaClient:
|
|||||||
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
|
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
|
||||||
for top_category in top_categories:
|
for top_category in top_categories:
|
||||||
try:
|
try:
|
||||||
category_url = f"https://www.suruga-ya.jp{top_category.href}"
|
category_url = f"{BASE_URL}{top_category.href}"
|
||||||
response = await page.goto(
|
response = await page.goto(
|
||||||
category_url,
|
category_url,
|
||||||
wait_until="domcontentloaded",
|
wait_until="domcontentloaded",
|
||||||
@@ -568,7 +569,7 @@ class SurugayaClient:
|
|||||||
"""将相对 URL 补全为完整的 https URL"""
|
"""将相对 URL 补全为完整的 https URL"""
|
||||||
if url.startswith("http://") or url.startswith("https://"):
|
if url.startswith("http://") or url.startswith("https://"):
|
||||||
return url
|
return url
|
||||||
return f"https://www.suruga-ya.jp{url}"
|
return f"{BASE_URL}{url}"
|
||||||
|
|
||||||
def get_breadcrumb_list(self, tree):
|
def get_breadcrumb_list(self, tree):
|
||||||
dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")
|
dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")
|
||||||
|
|||||||
@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
|
|||||||
|
|
||||||
[project]
|
[project]
|
||||||
name = "surugaya-common"
|
name = "surugaya-common"
|
||||||
version = "0.2.0"
|
version = "0.3.0"
|
||||||
description = "Shared contracts (Redis keys, future models) for jp_surugaya services"
|
description = "Shared contracts and browser utilities (Redis keys, models, parsers, fingerprint profile) for jp_surugaya services"
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"pydantic>=2.0.0,<3.0.0",
|
"pydantic>=2.0.0,<3.0.0",
|
||||||
@@ -15,7 +15,10 @@ dependencies = [
|
|||||||
# playwright 仅作为可选 extra:只有解析器/浏览器工具需要它,核心安装保持 light。
|
# playwright 仅作为可选 extra:只有解析器/浏览器工具需要它,核心安装保持 light。
|
||||||
# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集,确保 workspace 解析到单一 playwright。
|
# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集,确保 workspace 解析到单一 playwright。
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
browser = ["playwright>=1.54.0,<2.0.0"]
|
browser = [
|
||||||
|
"playwright>=1.54.0,<2.0.0",
|
||||||
|
"playwright-stealth>=1.0.0,<3.0.0",
|
||||||
|
]
|
||||||
|
|
||||||
[tool.hatch.build.targets.wheel]
|
[tool.hatch.build.targets.wheel]
|
||||||
packages = ["src/surugaya_common"]
|
packages = ["src/surugaya_common"]
|
||||||
|
|||||||
@@ -1,5 +1,10 @@
|
|||||||
"""Shared contracts for jp_surugaya services (Redis keys, future models)."""
|
"""Shared contracts and utilities for jp_surugaya services.
|
||||||
|
|
||||||
from surugaya_common import redis_keys
|
核心安装(无 extra)可用:redis_keys、urls、browser_profile、parse_utils、
|
||||||
|
app_utils、http_utils、models。
|
||||||
|
需安装 surugaya_common[browser]:browser_utils、stealth、parsers。
|
||||||
|
"""
|
||||||
|
|
||||||
__all__ = ["redis_keys"]
|
from surugaya_common import browser_profile, parse_utils, redis_keys, urls
|
||||||
|
|
||||||
|
__all__ = ["browser_profile", "parse_utils", "redis_keys", "urls"]
|
||||||
|
|||||||
@@ -0,0 +1,43 @@
|
|||||||
|
"""浏览器指纹与启动参数共享配置(纯常量,core 安装即可用,不依赖 playwright)。
|
||||||
|
|
||||||
|
API(jp_surugaya)与 worker(jp_pruchase_zdh_5)访问同一目标站点,
|
||||||
|
指纹参数必须保持一致:UA、语言、时区任何一处自相矛盾都可能触发风控
|
||||||
|
(参见 cf_clearance 复用因指纹不一致被判 403 的教训)。
|
||||||
|
"""
|
||||||
|
|
||||||
|
# 与日语环境一致的指纹参数
|
||||||
|
LOCALE = "ja-JP"
|
||||||
|
TIMEZONE_ID = "Asia/Tokyo"
|
||||||
|
NAVIGATOR_LANGUAGES = ("ja-JP", "ja", "en-US", "en")
|
||||||
|
ACCEPT_LANGUAGE = "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7"
|
||||||
|
|
||||||
|
# 两项目统一使用的 Windows Chrome User-Agent
|
||||||
|
DEFAULT_USER_AGENT = (
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/124.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
|
||||||
|
# 页面级反检测请求头(page.set_extra_http_headers 用)
|
||||||
|
DEFAULT_PAGE_HEADERS = {
|
||||||
|
"Accept-Language": ACCEPT_LANGUAGE,
|
||||||
|
"Cache-Control": "no-cache",
|
||||||
|
"Pragma": "no-cache",
|
||||||
|
}
|
||||||
|
|
||||||
|
# 两项目共用的 Chromium 反检测启动参数
|
||||||
|
STEALTH_LAUNCH_ARGS = (
|
||||||
|
"--disable-blink-features=AutomationControlled", # 浏览器层面移除 navigator.webdriver
|
||||||
|
"--disable-infobars", # 隐藏「Chrome 正受到自动测试软件控制」提示条
|
||||||
|
"--no-sandbox", # Docker/服务器环境兼容
|
||||||
|
"--disable-setuid-sandbox",
|
||||||
|
"--disable-dev-shm-usage", # 内存受限环境避免资源崩溃
|
||||||
|
"--disable-gpu", # 服务器无头模式稳定渲染
|
||||||
|
"--use-mock-keychain", # 避免本地密钥环弹窗(macOS/Linux)
|
||||||
|
"--password-store=basic",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def stealth_launch_args() -> list[str]:
|
||||||
|
"""返回共享启动参数的可变副本,调用方可在其上追加进程自有参数。"""
|
||||||
|
return list(STEALTH_LAUNCH_ARGS)
|
||||||
@@ -1,14 +1,81 @@
|
|||||||
"""浏览器拟真交互工具(需 playwright,安装 surugaya_common[browser])。
|
"""浏览器拟真交互工具(需 playwright,安装 surugaya_common[browser])。
|
||||||
|
|
||||||
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 human_scroll,逻辑保持一致。
|
- human_scroll:从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移,逻辑保持一致。
|
||||||
注:page_utils.py 中的纯函数 format_japanese_price 未迁移——它与 API 端
|
- clear_cart:worker 与 API cargo_service 各自的 clear_cart 近乎逐字重复,
|
||||||
app/utils/parse_util.py 的 format_price 语义不同,合并属行为变更,单独处理。
|
统一为本实现(以 worker 版为准,保留拟真随机等待)。
|
||||||
|
- format_japanese_price 已迁至 surugaya_common.parse_utils(纯 stdlib,不依赖 playwright)。
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from playwright.async_api import Page
|
from playwright.async_api import Page
|
||||||
|
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import logging
|
||||||
import random
|
import random
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 购物车页(/cargo/detail)的关键选择器
|
||||||
|
CART_ITEM_SELECTOR = "table.item_box tbody tr.item"
|
||||||
|
CART_REMOVE_LINK_SELECTOR = "table.item_box a.remove_product"
|
||||||
|
|
||||||
|
|
||||||
|
async def count_cart_items(page: Page) -> int:
|
||||||
|
"""统计购物车页当前的商品条目数(需已位于 /cargo/detail 页面)。"""
|
||||||
|
return await page.locator(CART_ITEM_SELECTOR).count()
|
||||||
|
|
||||||
|
|
||||||
|
async def clear_cart(page: Page) -> None:
|
||||||
|
"""清空购物车(需已位于 /cargo/detail 页面)。
|
||||||
|
|
||||||
|
- 商品数量超过 3 个时优先点击「全削除」按钮一次性清空
|
||||||
|
- 否则逐条点击删除链接,等待条目数减少后继续,含拟真随机等待
|
||||||
|
"""
|
||||||
|
logger.debug("正在清空购物车...")
|
||||||
|
cart_goods_num = await count_cart_items(page)
|
||||||
|
if cart_goods_num < 1:
|
||||||
|
return
|
||||||
|
|
||||||
|
# 购物车商品数量超过3个后会出现"全削除"按钮
|
||||||
|
if cart_goods_num > 3:
|
||||||
|
try:
|
||||||
|
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
|
||||||
|
# 显式等待按钮在页面上可见(容错处理:给它最多 3000 毫秒的加载时间)
|
||||||
|
await delete_button.wait_for(state="visible", timeout=3000)
|
||||||
|
if await delete_button.is_visible():
|
||||||
|
logger.debug("检测到'全削除'按钮存在,正在执行点击...")
|
||||||
|
await delete_button.click()
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
logger.debug("未检测到'全削除'按钮。")
|
||||||
|
|
||||||
|
# 遍历删除
|
||||||
|
await page.wait_for_timeout(random.randint(1000, 3000))
|
||||||
|
while True:
|
||||||
|
cart_goods_num = await page.locator(CART_ITEM_SELECTOR).count()
|
||||||
|
if cart_goods_num == 0:
|
||||||
|
break
|
||||||
|
|
||||||
|
remove_link = page.locator(CART_REMOVE_LINK_SELECTOR).first
|
||||||
|
try:
|
||||||
|
await remove_link.click()
|
||||||
|
# 删除按钮不会触发弹窗,直接等待购物车条目数减少即可。
|
||||||
|
await page.wait_for_function(
|
||||||
|
"""({selector, expectedCount}) => {
|
||||||
|
return document.querySelectorAll(selector).length < expectedCount;
|
||||||
|
}""",
|
||||||
|
arg={"selector": CART_ITEM_SELECTOR, "expectedCount": cart_goods_num},
|
||||||
|
timeout=15_000,
|
||||||
|
)
|
||||||
|
await page.wait_for_timeout(random.randint(800, 1500))
|
||||||
|
except PlaywrightTimeoutError:
|
||||||
|
latest_count = await page.locator(CART_ITEM_SELECTOR).count()
|
||||||
|
if latest_count < cart_goods_num:
|
||||||
|
logger.debug("购物车商品已删除,当前购物车数量:%s", latest_count)
|
||||||
|
continue
|
||||||
|
|
||||||
|
logger.warning("删除商品后数量未变化,等待后重试")
|
||||||
|
await page.wait_for_timeout(1000)
|
||||||
|
|
||||||
|
|
||||||
async def human_scroll(page: Page, mode: str, value: int = 0):
|
async def human_scroll(page: Page, mode: str, value: int = 0):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""纯文本解析工具(仅 stdlib)。
|
||||||
|
|
||||||
|
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 format_japanese_price。
|
||||||
|
"""
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
|
||||||
|
def format_japanese_price(price_str: str | None) -> int:
|
||||||
|
"""
|
||||||
|
将日本电商网站的价格字符串格式化为标准整数
|
||||||
|
支持的格式示例: "600円 (税込)", "1,250円", "¥ 3,000", "300"
|
||||||
|
|
||||||
|
注意:实现为提取字符串中所有数字字符后拼接,与 jp_surugaya API 端
|
||||||
|
parse_util.format_price(货币符号定位 + 价格范围取低价)语义不同,勿混用。
|
||||||
|
"""
|
||||||
|
if not price_str:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
digits = re.findall(r'\d', price_str)
|
||||||
|
if not digits:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
return int("".join(digits))
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
"""playwright-stealth 共享工厂(需安装 surugaya_common[browser])。
|
||||||
|
|
||||||
|
统一两项目的 stealth 指纹配置:navigator.languages 覆盖必须与日语
|
||||||
|
locale / Accept-Language 对齐,否则带 cf_clearance 的复用请求会因
|
||||||
|
指纹自相矛盾被 Cloudflare 判 403。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from playwright_stealth import Stealth
|
||||||
|
|
||||||
|
from surugaya_common.browser_profile import NAVIGATOR_LANGUAGES
|
||||||
|
|
||||||
|
|
||||||
|
def create_stealth() -> Stealth:
|
||||||
|
"""创建与日语环境对齐的 Stealth 实例。
|
||||||
|
|
||||||
|
用法约定:
|
||||||
|
- browser.new_context 流程:用 stealth.use_async(async_playwright()) 钩子
|
||||||
|
包装 playwright,new_context/new_page 自动应用 stealth(含 HTTP
|
||||||
|
sec-ch-ua 客户端提示)。jp_surugaya BrowserPool 即此用法。
|
||||||
|
- launch_persistent_context 流程:use_async 钩子不覆盖它(playwright-stealth
|
||||||
|
已知缺口),必须显式 await stealth.apply_stealth_async(context),
|
||||||
|
此路径只注入 init script,无 sec-ch-ua 等 HTTP 头伪装。
|
||||||
|
"""
|
||||||
|
return Stealth(navigator_languages_override=NAVIGATOR_LANGUAGES)
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
"""suruga-ya.jp 站点 URL 契约:两项目共享的页面地址与构造器。"""
|
||||||
|
|
||||||
|
BASE_URL = "https://www.suruga-ya.jp"
|
||||||
|
|
||||||
|
# 购物车详情页
|
||||||
|
CARGO_DETAIL_URL = f"{BASE_URL}/cargo/detail"
|
||||||
|
|
||||||
|
# 会员中心(登录入口)
|
||||||
|
MYPAGE_URL = f"{BASE_URL}/pcmypage"
|
||||||
|
|
||||||
|
|
||||||
|
def product_detail_url(product_id: str, tenpo_cd: str | None = None) -> str:
|
||||||
|
"""商品详情页地址;tenpo_cd 用于加盟店商品。"""
|
||||||
|
url = f"{BASE_URL}/product/detail/{product_id}"
|
||||||
|
if tenpo_cd:
|
||||||
|
url += f"?tenpo_cd={tenpo_cd}"
|
||||||
|
return url
|
||||||
|
|
||||||
|
|
||||||
|
def trade_detail_url(trade_code: str) -> str:
|
||||||
|
"""交易(订单)详情页地址。"""
|
||||||
|
return f"{BASE_URL}/pcmypage/action_sell_search/detail?trade_code={trade_code}"
|
||||||
+2
-3
@@ -6,15 +6,14 @@ requires-python = ">=3.12"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"fastapi>=0.115.0,<1.0.0",
|
"fastapi>=0.115.0,<1.0.0",
|
||||||
"loguru>=0.7.0,<1.0.0",
|
"loguru>=0.7.0,<1.0.0",
|
||||||
"playwright>=1.54.0,<2.0.0",
|
|
||||||
"playwright-stealth>=1.0.0,<3.0.0",
|
|
||||||
"pydantic>=2.0.0,<3.0.0",
|
"pydantic>=2.0.0,<3.0.0",
|
||||||
"pydantic-settings>=2.4.0,<3.0.0",
|
"pydantic-settings>=2.4.0,<3.0.0",
|
||||||
"redis==8.0.0",
|
"redis==8.0.0",
|
||||||
"selectolax>=0.3.21,<1.0.0",
|
"selectolax>=0.3.21,<1.0.0",
|
||||||
"starlette>=0.37.0,<1.0.0",
|
"starlette>=0.37.0,<1.0.0",
|
||||||
"uvicorn[standard]>=0.30.0,<1.0.0",
|
"uvicorn[standard]>=0.30.0,<1.0.0",
|
||||||
"surugaya-common",
|
# playwright 与 playwright-stealth 由 [browser] extra 提供,与 worker 保持同一来源
|
||||||
|
"surugaya-common[browser]",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
|
|||||||
@@ -0,0 +1,67 @@
|
|||||||
|
"""surugaya_common 共享契约回归测试:URL 构造与价格解析必须与迁移前字面量/行为一致。"""
|
||||||
|
|
||||||
|
from surugaya_common import browser_profile
|
||||||
|
from surugaya_common.parse_utils import format_japanese_price
|
||||||
|
from surugaya_common.urls import (
|
||||||
|
BASE_URL,
|
||||||
|
CARGO_DETAIL_URL,
|
||||||
|
MYPAGE_URL,
|
||||||
|
product_detail_url,
|
||||||
|
trade_detail_url,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestUrls:
|
||||||
|
def test_literals_match_pre_migration(self):
|
||||||
|
assert BASE_URL == "https://www.suruga-ya.jp"
|
||||||
|
assert CARGO_DETAIL_URL == "https://www.suruga-ya.jp/cargo/detail"
|
||||||
|
assert MYPAGE_URL == "https://www.suruga-ya.jp/pcmypage"
|
||||||
|
|
||||||
|
def test_product_detail_url(self):
|
||||||
|
assert product_detail_url("128001234") == "https://www.suruga-ya.jp/product/detail/128001234"
|
||||||
|
assert (
|
||||||
|
product_detail_url("128001234", tenpo_cd="400496")
|
||||||
|
== "https://www.suruga-ya.jp/product/detail/128001234?tenpo_cd=400496"
|
||||||
|
)
|
||||||
|
# 空字符串 tenpo_cd 与未提供等价(与迁移前 if tenpo_cd 判断一致)
|
||||||
|
assert product_detail_url("128001234", tenpo_cd="") == "https://www.suruga-ya.jp/product/detail/128001234"
|
||||||
|
|
||||||
|
def test_trade_detail_url(self):
|
||||||
|
assert (
|
||||||
|
trade_detail_url("T123456")
|
||||||
|
== "https://www.suruga-ya.jp/pcmypage/action_sell_search/detail?trade_code=T123456"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class TestFormatJapanesePrice:
|
||||||
|
def test_supported_formats(self):
|
||||||
|
assert format_japanese_price("600円 (税込)") == 600
|
||||||
|
assert format_japanese_price("1,250円") == 1250
|
||||||
|
assert format_japanese_price("¥ 3,000") == 3000
|
||||||
|
assert format_japanese_price("300") == 300
|
||||||
|
|
||||||
|
def test_empty_and_no_digit(self):
|
||||||
|
assert format_japanese_price(None) == 0
|
||||||
|
assert format_japanese_price("") == 0
|
||||||
|
assert format_japanese_price("税込") == 0
|
||||||
|
|
||||||
|
|
||||||
|
class TestBrowserProfile:
|
||||||
|
def test_fingerprint_consistency(self):
|
||||||
|
# UA / 语言 / 时区必须保持日语环境自洽,且与迁移前两仓字面量一致
|
||||||
|
assert browser_profile.DEFAULT_USER_AGENT == (
|
||||||
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||||
|
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||||
|
"Chrome/124.0.0.0 Safari/537.36"
|
||||||
|
)
|
||||||
|
assert browser_profile.LOCALE == "ja-JP"
|
||||||
|
assert browser_profile.TIMEZONE_ID == "Asia/Tokyo"
|
||||||
|
assert browser_profile.NAVIGATOR_LANGUAGES == ("ja-JP", "ja", "en-US", "en")
|
||||||
|
assert browser_profile.ACCEPT_LANGUAGE.startswith("ja-JP,ja;")
|
||||||
|
assert browser_profile.DEFAULT_PAGE_HEADERS["Accept-Language"] == browser_profile.ACCEPT_LANGUAGE
|
||||||
|
|
||||||
|
def test_stealth_launch_args_returns_copy(self):
|
||||||
|
args = browser_profile.stealth_launch_args()
|
||||||
|
assert "--disable-blink-features=AutomationControlled" in args
|
||||||
|
args.append("--extra")
|
||||||
|
assert "--extra" not in browser_profile.stealth_launch_args()
|
||||||
@@ -517,7 +517,7 @@ wheels = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "surugaya-common"
|
name = "surugaya-common"
|
||||||
version = "0.2.0"
|
version = "0.3.0"
|
||||||
source = { editable = "packages/surugaya_common" }
|
source = { editable = "packages/surugaya_common" }
|
||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "pydantic" },
|
{ name = "pydantic" },
|
||||||
@@ -527,11 +527,13 @@ dependencies = [
|
|||||||
[package.optional-dependencies]
|
[package.optional-dependencies]
|
||||||
browser = [
|
browser = [
|
||||||
{ name = "playwright" },
|
{ name = "playwright" },
|
||||||
|
{ name = "playwright-stealth" },
|
||||||
]
|
]
|
||||||
|
|
||||||
[package.metadata]
|
[package.metadata]
|
||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" },
|
{ name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" },
|
||||||
|
{ name = "playwright-stealth", marker = "extra == 'browser'", specifier = ">=1.0.0,<3.0.0" },
|
||||||
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
||||||
{ name = "redis", specifier = "==8.0.0" },
|
{ name = "redis", specifier = "==8.0.0" },
|
||||||
]
|
]
|
||||||
@@ -544,14 +546,12 @@ source = { virtual = "." }
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
{ name = "fastapi" },
|
{ name = "fastapi" },
|
||||||
{ name = "loguru" },
|
{ name = "loguru" },
|
||||||
{ name = "playwright" },
|
|
||||||
{ name = "playwright-stealth" },
|
|
||||||
{ name = "pydantic" },
|
{ name = "pydantic" },
|
||||||
{ name = "pydantic-settings" },
|
{ name = "pydantic-settings" },
|
||||||
{ name = "redis" },
|
{ name = "redis" },
|
||||||
{ name = "selectolax" },
|
{ name = "selectolax" },
|
||||||
{ name = "starlette" },
|
{ name = "starlette" },
|
||||||
{ name = "surugaya-common" },
|
{ name = "surugaya-common", extra = ["browser"] },
|
||||||
{ name = "uvicorn", extra = ["standard"] },
|
{ name = "uvicorn", extra = ["standard"] },
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -565,8 +565,6 @@ dev = [
|
|||||||
requires-dist = [
|
requires-dist = [
|
||||||
{ name = "fastapi", specifier = ">=0.115.0,<1.0.0" },
|
{ name = "fastapi", specifier = ">=0.115.0,<1.0.0" },
|
||||||
{ name = "loguru", specifier = ">=0.7.0,<1.0.0" },
|
{ name = "loguru", specifier = ">=0.7.0,<1.0.0" },
|
||||||
{ name = "playwright", specifier = ">=1.54.0,<2.0.0" },
|
|
||||||
{ name = "playwright-stealth", specifier = ">=1.0.0,<3.0.0" },
|
|
||||||
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
||||||
{ name = "pydantic-settings", specifier = ">=2.4.0,<3.0.0" },
|
{ name = "pydantic-settings", specifier = ">=2.4.0,<3.0.0" },
|
||||||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" },
|
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" },
|
||||||
@@ -574,7 +572,7 @@ requires-dist = [
|
|||||||
{ name = "redis", specifier = "==8.0.0" },
|
{ name = "redis", specifier = "==8.0.0" },
|
||||||
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
|
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
|
||||||
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" },
|
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" },
|
||||||
{ name = "surugaya-common", editable = "packages/surugaya_common" },
|
{ name = "surugaya-common", extras = ["browser"], editable = "packages/surugaya_common" },
|
||||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
|
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
|
||||||
]
|
]
|
||||||
provides-extras = ["dev"]
|
provides-extras = ["dev"]
|
||||||
|
|||||||
Reference in New Issue
Block a user