common扩展
This commit is contained in:
+7
-8
@@ -9,6 +9,8 @@ from typing import Literal
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
from pathlib import Path
|
||||
|
||||
from surugaya_common import browser_profile, urls
|
||||
|
||||
BASE_DIR = Path(__file__).resolve().parent.parent.parent
|
||||
|
||||
|
||||
@@ -60,13 +62,10 @@ class Settings(BaseSettings):
|
||||
cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时
|
||||
cloudflare_reuse_wait_timeout_seconds: float = 8.0 # 复用会话时消化挑战的短超时,超时则放弃复用直接重建
|
||||
session_ttl_seconds: int = 900 # 会话过期时间(秒)
|
||||
browser_locale: str = "ja-JP"
|
||||
browser_timezone_id: str = "Asia/Tokyo"
|
||||
browser_user_agent: str = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
# 指纹参数默认值取自共享包,与 worker 保持一致
|
||||
browser_locale: str = browser_profile.LOCALE
|
||||
browser_timezone_id: str = browser_profile.TIMEZONE_ID
|
||||
browser_user_agent: str = browser_profile.DEFAULT_USER_AGENT
|
||||
browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式
|
||||
|
||||
# ---- Redis 配置(可选,未配置时使用进程内存)----
|
||||
@@ -82,7 +81,7 @@ class Settings(BaseSettings):
|
||||
proxy_password: str | None = None
|
||||
|
||||
# ---- 目标站点 ----
|
||||
base_url: str = "https://www.suruga-ya.jp"
|
||||
base_url: str = urls.BASE_URL
|
||||
|
||||
@property
|
||||
def browser_headless_effective(self) -> bool:
|
||||
|
||||
@@ -15,6 +15,7 @@ from typing import Any
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import BrowserUnavailableError, ResourceBusyError
|
||||
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -80,28 +81,17 @@ class BrowserPool:
|
||||
# 用 playwright-stealth 钩子包装 playwright:后续 new_context/new_page
|
||||
# 会自动应用 stealth(含 HTTP sec-ch-ua 客户端提示),并令 navigator.languages
|
||||
# 与日语环境保持一致,避免指纹自相矛盾。
|
||||
from playwright_stealth import Stealth
|
||||
from surugaya_common.stealth import create_stealth
|
||||
|
||||
stealth = Stealth(
|
||||
navigator_languages_override=("ja-JP", "ja", "en-US", "en"),
|
||||
)
|
||||
self._playwright_cm = stealth.use_async(async_playwright())
|
||||
self._playwright_cm = create_stealth().use_async(async_playwright())
|
||||
self._playwright = await self._playwright_cm.__aenter__()
|
||||
else:
|
||||
self._playwright_cm = None
|
||||
self._playwright = await async_playwright().start()
|
||||
chromium = self._playwright.chromium
|
||||
launch_args = [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-dev-shm-usage",
|
||||
launch_args = stealth_launch_args() + [
|
||||
"--disable-web-security",
|
||||
"--no-first-run",
|
||||
"--disable-infobars",
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-gpu",
|
||||
"--use-mock-keychain",
|
||||
"--password-store=basic",
|
||||
]
|
||||
# 本地可视化调试时强制窗口化,避免浏览器启动后不可见
|
||||
if not headless:
|
||||
@@ -305,13 +295,7 @@ class BrowserPool:
|
||||
async def _configure_page(context: Any) -> Any:
|
||||
"""在上下文中创建页面并设置反检测请求头。"""
|
||||
page = await context.new_page()
|
||||
await page.set_extra_http_headers(
|
||||
{
|
||||
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Cache-Control": "no-cache",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
)
|
||||
await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
|
||||
return page
|
||||
|
||||
async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any:
|
||||
|
||||
@@ -10,6 +10,9 @@ from selectolax.parser import HTMLParser
|
||||
from app.core.config import Settings
|
||||
from app.models.scrape import CargoRequest
|
||||
from app.utils.parse_util import extract_product_id, format_price
|
||||
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
|
||||
from surugaya_common.browser_utils import clear_cart
|
||||
from surugaya_common.urls import CARGO_DETAIL_URL, product_detail_url
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,16 +32,7 @@ class CargoService:
|
||||
"""
|
||||
async with async_playwright() as p:
|
||||
headless = self._settings.browser_headless_effective
|
||||
launch_args = [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-infobars",
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--use-mock-keychain",
|
||||
"--password-store=basic",
|
||||
]
|
||||
launch_args = stealth_launch_args()
|
||||
if not headless:
|
||||
launch_args.extend(["--start-maximized", "--window-position=0,0"])
|
||||
|
||||
@@ -61,21 +55,17 @@ class CargoService:
|
||||
|
||||
if self._settings.browser_stealth_enabled:
|
||||
try:
|
||||
from playwright_stealth import Stealth
|
||||
await Stealth().apply_stealth_async(context)
|
||||
# launch_persistent_context 之外的裸 context 用 apply_stealth_async;
|
||||
# 共享工厂带日语 languages override,与 locale/Accept-Language 对齐。
|
||||
from surugaya_common.stealth import create_stealth
|
||||
await create_stealth().apply_stealth_async(context)
|
||||
except Exception as e:
|
||||
logger.warning("应用 playwright-stealth 失败: %s", e)
|
||||
|
||||
page = await context.new_page()
|
||||
await page.set_extra_http_headers(
|
||||
{
|
||||
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Cache-Control": "no-cache",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
)
|
||||
await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
|
||||
|
||||
target_url = "https://www.suruga-ya.jp/cargo/detail"
|
||||
target_url = CARGO_DETAIL_URL
|
||||
# 先进入购物车页面
|
||||
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
|
||||
|
||||
@@ -85,7 +75,7 @@ class CargoService:
|
||||
# 清空购物车
|
||||
await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟,模拟人工操作间隔
|
||||
if cart_goods_num > 0:
|
||||
await self.clear_cart(page)
|
||||
await clear_cart(page)
|
||||
|
||||
# 将待采购的商品加入购物车
|
||||
item_list = request.payload.item_list
|
||||
@@ -101,7 +91,7 @@ class CargoService:
|
||||
continue
|
||||
|
||||
logger.debug("进入商品 %s 详情页,准备购买数量: %s...", product_id, number)
|
||||
detail_url = f"https://www.suruga-ya.jp/product/detail/{product_id}"
|
||||
detail_url = product_detail_url(product_id)
|
||||
await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
|
||||
|
||||
# 判断是否缺货
|
||||
@@ -327,57 +317,3 @@ class CargoService:
|
||||
"total_commission_fee": total_commission_fee
|
||||
}
|
||||
|
||||
async def clear_cart(self, page):
|
||||
"""
|
||||
清空购物车
|
||||
"""
|
||||
cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
|
||||
if cart_goods_num < 1:
|
||||
return
|
||||
|
||||
is_delete = 0
|
||||
# 购物车商品数量超过3个后会出现"全削除"按钮
|
||||
if cart_goods_num > 3:
|
||||
try:
|
||||
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
|
||||
# 3. 显式等待按钮在页面上可见(容错处理:比如给它最多 3000 毫秒的加载时间)
|
||||
# 如果按钮是一开始就在静态 HTML 里的,这一步也可以省略,直接用下面的 is_visible()
|
||||
await delete_button.wait_for(state="visible", timeout=3000)
|
||||
if await delete_button.is_visible():
|
||||
logger.debug("检测到‘全削除’按钮存在,正在执行点击...")
|
||||
await delete_button.click()
|
||||
is_delete = 1
|
||||
except Exception:
|
||||
logger.debug("未检测到‘全削除’按钮。")
|
||||
|
||||
if is_delete == 1:
|
||||
return
|
||||
|
||||
# 遍历删除
|
||||
item_selector = "table.item_box tbody tr.item"
|
||||
remove_selector = "table.item_box a.remove_product"
|
||||
|
||||
while True:
|
||||
cart_goods_num = await page.locator(item_selector).count()
|
||||
if cart_goods_num == 0:
|
||||
break
|
||||
|
||||
remove_link = page.locator(remove_selector).first
|
||||
|
||||
try:
|
||||
await remove_link.click()
|
||||
# 删除按钮不会触发弹窗,直接等待购物车条目数减少即可。
|
||||
await page.wait_for_function(
|
||||
"""({selector, expectedCount}) => {
|
||||
return document.querySelectorAll(selector).length < expectedCount;
|
||||
}""",
|
||||
arg={"selector": item_selector, "expectedCount": cart_goods_num},
|
||||
timeout=15_000,
|
||||
)
|
||||
await page.wait_for_timeout(random.randint(800, 1500))
|
||||
except PlaywrightTimeoutError:
|
||||
latest_count = await page.locator(item_selector).count()
|
||||
if latest_count < cart_goods_num:
|
||||
continue
|
||||
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
@@ -26,6 +26,7 @@ from app.services.cloudflare_session import CloudflareSessionManager
|
||||
from app.services.session_store import SessionStore
|
||||
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
|
||||
from surugaya_common.app_utils import is_empty_str
|
||||
from surugaya_common.urls import BASE_URL
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -303,7 +304,7 @@ class SurugayaClient:
|
||||
logger.info("开始获取并解析商品分类...")
|
||||
top_categories: list[CategoryData] = []
|
||||
|
||||
target_url = "https://www.suruga-ya.jp/"
|
||||
target_url = f"{BASE_URL}/"
|
||||
session = await self._session_manager.get_verified_session(target_url, True)
|
||||
|
||||
html_content = ""
|
||||
@@ -366,7 +367,7 @@ class SurugayaClient:
|
||||
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
|
||||
for top_category in top_categories:
|
||||
try:
|
||||
category_url = f"https://www.suruga-ya.jp{top_category.href}"
|
||||
category_url = f"{BASE_URL}{top_category.href}"
|
||||
response = await page.goto(
|
||||
category_url,
|
||||
wait_until="domcontentloaded",
|
||||
@@ -568,7 +569,7 @@ class SurugayaClient:
|
||||
"""将相对 URL 补全为完整的 https URL"""
|
||||
if url.startswith("http://") or url.startswith("https://"):
|
||||
return url
|
||||
return f"https://www.suruga-ya.jp{url}"
|
||||
return f"{BASE_URL}{url}"
|
||||
|
||||
def get_breadcrumb_list(self, tree):
|
||||
dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")
|
||||
|
||||
@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "surugaya-common"
|
||||
version = "0.2.0"
|
||||
description = "Shared contracts (Redis keys, future models) for jp_surugaya services"
|
||||
version = "0.3.0"
|
||||
description = "Shared contracts and browser utilities (Redis keys, models, parsers, fingerprint profile) for jp_surugaya services"
|
||||
requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"pydantic>=2.0.0,<3.0.0",
|
||||
@@ -15,7 +15,10 @@ dependencies = [
|
||||
# playwright 仅作为可选 extra:只有解析器/浏览器工具需要它,核心安装保持 light。
|
||||
# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集,确保 workspace 解析到单一 playwright。
|
||||
[project.optional-dependencies]
|
||||
browser = ["playwright>=1.54.0,<2.0.0"]
|
||||
browser = [
|
||||
"playwright>=1.54.0,<2.0.0",
|
||||
"playwright-stealth>=1.0.0,<3.0.0",
|
||||
]
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/surugaya_common"]
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
"""Shared contracts for jp_surugaya services (Redis keys, future models)."""
|
||||
"""Shared contracts and utilities for jp_surugaya services.
|
||||
|
||||
from surugaya_common import redis_keys
|
||||
核心安装(无 extra)可用:redis_keys、urls、browser_profile、parse_utils、
|
||||
app_utils、http_utils、models。
|
||||
需安装 surugaya_common[browser]:browser_utils、stealth、parsers。
|
||||
"""
|
||||
|
||||
__all__ = ["redis_keys"]
|
||||
from surugaya_common import browser_profile, parse_utils, redis_keys, urls
|
||||
|
||||
__all__ = ["browser_profile", "parse_utils", "redis_keys", "urls"]
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""浏览器指纹与启动参数共享配置(纯常量,core 安装即可用,不依赖 playwright)。
|
||||
|
||||
API(jp_surugaya)与 worker(jp_pruchase_zdh_5)访问同一目标站点,
|
||||
指纹参数必须保持一致:UA、语言、时区任何一处自相矛盾都可能触发风控
|
||||
(参见 cf_clearance 复用因指纹不一致被判 403 的教训)。
|
||||
"""
|
||||
|
||||
# 与日语环境一致的指纹参数
|
||||
LOCALE = "ja-JP"
|
||||
TIMEZONE_ID = "Asia/Tokyo"
|
||||
NAVIGATOR_LANGUAGES = ("ja-JP", "ja", "en-US", "en")
|
||||
ACCEPT_LANGUAGE = "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7"
|
||||
|
||||
# 两项目统一使用的 Windows Chrome User-Agent
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
|
||||
# 页面级反检测请求头(page.set_extra_http_headers 用)
|
||||
DEFAULT_PAGE_HEADERS = {
|
||||
"Accept-Language": ACCEPT_LANGUAGE,
|
||||
"Cache-Control": "no-cache",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
|
||||
# 两项目共用的 Chromium 反检测启动参数
|
||||
STEALTH_LAUNCH_ARGS = (
|
||||
"--disable-blink-features=AutomationControlled", # 浏览器层面移除 navigator.webdriver
|
||||
"--disable-infobars", # 隐藏「Chrome 正受到自动测试软件控制」提示条
|
||||
"--no-sandbox", # Docker/服务器环境兼容
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage", # 内存受限环境避免资源崩溃
|
||||
"--disable-gpu", # 服务器无头模式稳定渲染
|
||||
"--use-mock-keychain", # 避免本地密钥环弹窗(macOS/Linux)
|
||||
"--password-store=basic",
|
||||
)
|
||||
|
||||
|
||||
def stealth_launch_args() -> list[str]:
|
||||
"""返回共享启动参数的可变副本,调用方可在其上追加进程自有参数。"""
|
||||
return list(STEALTH_LAUNCH_ARGS)
|
||||
@@ -1,14 +1,81 @@
|
||||
"""浏览器拟真交互工具(需 playwright,安装 surugaya_common[browser])。
|
||||
|
||||
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 human_scroll,逻辑保持一致。
|
||||
注:page_utils.py 中的纯函数 format_japanese_price 未迁移——它与 API 端
|
||||
app/utils/parse_util.py 的 format_price 语义不同,合并属行为变更,单独处理。
|
||||
- human_scroll:从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移,逻辑保持一致。
|
||||
- clear_cart:worker 与 API cargo_service 各自的 clear_cart 近乎逐字重复,
|
||||
统一为本实现(以 worker 版为准,保留拟真随机等待)。
|
||||
- format_japanese_price 已迁至 surugaya_common.parse_utils(纯 stdlib,不依赖 playwright)。
|
||||
"""
|
||||
|
||||
from playwright.async_api import Page
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 购物车页(/cargo/detail)的关键选择器
|
||||
CART_ITEM_SELECTOR = "table.item_box tbody tr.item"
|
||||
CART_REMOVE_LINK_SELECTOR = "table.item_box a.remove_product"
|
||||
|
||||
|
||||
async def count_cart_items(page: Page) -> int:
|
||||
"""统计购物车页当前的商品条目数(需已位于 /cargo/detail 页面)。"""
|
||||
return await page.locator(CART_ITEM_SELECTOR).count()
|
||||
|
||||
|
||||
async def clear_cart(page: Page) -> None:
|
||||
"""清空购物车(需已位于 /cargo/detail 页面)。
|
||||
|
||||
- 商品数量超过 3 个时优先点击「全削除」按钮一次性清空
|
||||
- 否则逐条点击删除链接,等待条目数减少后继续,含拟真随机等待
|
||||
"""
|
||||
logger.debug("正在清空购物车...")
|
||||
cart_goods_num = await count_cart_items(page)
|
||||
if cart_goods_num < 1:
|
||||
return
|
||||
|
||||
# 购物车商品数量超过3个后会出现"全削除"按钮
|
||||
if cart_goods_num > 3:
|
||||
try:
|
||||
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
|
||||
# 显式等待按钮在页面上可见(容错处理:给它最多 3000 毫秒的加载时间)
|
||||
await delete_button.wait_for(state="visible", timeout=3000)
|
||||
if await delete_button.is_visible():
|
||||
logger.debug("检测到'全削除'按钮存在,正在执行点击...")
|
||||
await delete_button.click()
|
||||
return
|
||||
except Exception:
|
||||
logger.debug("未检测到'全削除'按钮。")
|
||||
|
||||
# 遍历删除
|
||||
await page.wait_for_timeout(random.randint(1000, 3000))
|
||||
while True:
|
||||
cart_goods_num = await page.locator(CART_ITEM_SELECTOR).count()
|
||||
if cart_goods_num == 0:
|
||||
break
|
||||
|
||||
remove_link = page.locator(CART_REMOVE_LINK_SELECTOR).first
|
||||
try:
|
||||
await remove_link.click()
|
||||
# 删除按钮不会触发弹窗,直接等待购物车条目数减少即可。
|
||||
await page.wait_for_function(
|
||||
"""({selector, expectedCount}) => {
|
||||
return document.querySelectorAll(selector).length < expectedCount;
|
||||
}""",
|
||||
arg={"selector": CART_ITEM_SELECTOR, "expectedCount": cart_goods_num},
|
||||
timeout=15_000,
|
||||
)
|
||||
await page.wait_for_timeout(random.randint(800, 1500))
|
||||
except PlaywrightTimeoutError:
|
||||
latest_count = await page.locator(CART_ITEM_SELECTOR).count()
|
||||
if latest_count < cart_goods_num:
|
||||
logger.debug("购物车商品已删除,当前购物车数量:%s", latest_count)
|
||||
continue
|
||||
|
||||
logger.warning("删除商品后数量未变化,等待后重试")
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
|
||||
async def human_scroll(page: Page, mode: str, value: int = 0):
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
"""纯文本解析工具(仅 stdlib)。
|
||||
|
||||
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 format_japanese_price。
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
|
||||
def format_japanese_price(price_str: str | None) -> int:
|
||||
"""
|
||||
将日本电商网站的价格字符串格式化为标准整数
|
||||
支持的格式示例: "600円 (税込)", "1,250円", "¥ 3,000", "300"
|
||||
|
||||
注意:实现为提取字符串中所有数字字符后拼接,与 jp_surugaya API 端
|
||||
parse_util.format_price(货币符号定位 + 价格范围取低价)语义不同,勿混用。
|
||||
"""
|
||||
if not price_str:
|
||||
return 0
|
||||
|
||||
digits = re.findall(r'\d', price_str)
|
||||
if not digits:
|
||||
return 0
|
||||
|
||||
return int("".join(digits))
|
||||
@@ -0,0 +1,24 @@
|
||||
"""playwright-stealth 共享工厂(需安装 surugaya_common[browser])。
|
||||
|
||||
统一两项目的 stealth 指纹配置:navigator.languages 覆盖必须与日语
|
||||
locale / Accept-Language 对齐,否则带 cf_clearance 的复用请求会因
|
||||
指纹自相矛盾被 Cloudflare 判 403。
|
||||
"""
|
||||
|
||||
from playwright_stealth import Stealth
|
||||
|
||||
from surugaya_common.browser_profile import NAVIGATOR_LANGUAGES
|
||||
|
||||
|
||||
def create_stealth() -> Stealth:
|
||||
"""创建与日语环境对齐的 Stealth 实例。
|
||||
|
||||
用法约定:
|
||||
- browser.new_context 流程:用 stealth.use_async(async_playwright()) 钩子
|
||||
包装 playwright,new_context/new_page 自动应用 stealth(含 HTTP
|
||||
sec-ch-ua 客户端提示)。jp_surugaya BrowserPool 即此用法。
|
||||
- launch_persistent_context 流程:use_async 钩子不覆盖它(playwright-stealth
|
||||
已知缺口),必须显式 await stealth.apply_stealth_async(context),
|
||||
此路径只注入 init script,无 sec-ch-ua 等 HTTP 头伪装。
|
||||
"""
|
||||
return Stealth(navigator_languages_override=NAVIGATOR_LANGUAGES)
|
||||
@@ -0,0 +1,22 @@
|
||||
"""suruga-ya.jp 站点 URL 契约:两项目共享的页面地址与构造器。"""
|
||||
|
||||
BASE_URL = "https://www.suruga-ya.jp"
|
||||
|
||||
# 购物车详情页
|
||||
CARGO_DETAIL_URL = f"{BASE_URL}/cargo/detail"
|
||||
|
||||
# 会员中心(登录入口)
|
||||
MYPAGE_URL = f"{BASE_URL}/pcmypage"
|
||||
|
||||
|
||||
def product_detail_url(product_id: str, tenpo_cd: str | None = None) -> str:
|
||||
"""商品详情页地址;tenpo_cd 用于加盟店商品。"""
|
||||
url = f"{BASE_URL}/product/detail/{product_id}"
|
||||
if tenpo_cd:
|
||||
url += f"?tenpo_cd={tenpo_cd}"
|
||||
return url
|
||||
|
||||
|
||||
def trade_detail_url(trade_code: str) -> str:
|
||||
"""交易(订单)详情页地址。"""
|
||||
return f"{BASE_URL}/pcmypage/action_sell_search/detail?trade_code={trade_code}"
|
||||
+2
-3
@@ -6,15 +6,14 @@ requires-python = ">=3.12"
|
||||
dependencies = [
|
||||
"fastapi>=0.115.0,<1.0.0",
|
||||
"loguru>=0.7.0,<1.0.0",
|
||||
"playwright>=1.54.0,<2.0.0",
|
||||
"playwright-stealth>=1.0.0,<3.0.0",
|
||||
"pydantic>=2.0.0,<3.0.0",
|
||||
"pydantic-settings>=2.4.0,<3.0.0",
|
||||
"redis==8.0.0",
|
||||
"selectolax>=0.3.21,<1.0.0",
|
||||
"starlette>=0.37.0,<1.0.0",
|
||||
"uvicorn[standard]>=0.30.0,<1.0.0",
|
||||
"surugaya-common",
|
||||
# playwright 与 playwright-stealth 由 [browser] extra 提供,与 worker 保持同一来源
|
||||
"surugaya-common[browser]",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""surugaya_common 共享契约回归测试:URL 构造与价格解析必须与迁移前字面量/行为一致。"""
|
||||
|
||||
from surugaya_common import browser_profile
|
||||
from surugaya_common.parse_utils import format_japanese_price
|
||||
from surugaya_common.urls import (
|
||||
BASE_URL,
|
||||
CARGO_DETAIL_URL,
|
||||
MYPAGE_URL,
|
||||
product_detail_url,
|
||||
trade_detail_url,
|
||||
)
|
||||
|
||||
|
||||
class TestUrls:
|
||||
def test_literals_match_pre_migration(self):
|
||||
assert BASE_URL == "https://www.suruga-ya.jp"
|
||||
assert CARGO_DETAIL_URL == "https://www.suruga-ya.jp/cargo/detail"
|
||||
assert MYPAGE_URL == "https://www.suruga-ya.jp/pcmypage"
|
||||
|
||||
def test_product_detail_url(self):
|
||||
assert product_detail_url("128001234") == "https://www.suruga-ya.jp/product/detail/128001234"
|
||||
assert (
|
||||
product_detail_url("128001234", tenpo_cd="400496")
|
||||
== "https://www.suruga-ya.jp/product/detail/128001234?tenpo_cd=400496"
|
||||
)
|
||||
# 空字符串 tenpo_cd 与未提供等价(与迁移前 if tenpo_cd 判断一致)
|
||||
assert product_detail_url("128001234", tenpo_cd="") == "https://www.suruga-ya.jp/product/detail/128001234"
|
||||
|
||||
def test_trade_detail_url(self):
|
||||
assert (
|
||||
trade_detail_url("T123456")
|
||||
== "https://www.suruga-ya.jp/pcmypage/action_sell_search/detail?trade_code=T123456"
|
||||
)
|
||||
|
||||
|
||||
class TestFormatJapanesePrice:
|
||||
def test_supported_formats(self):
|
||||
assert format_japanese_price("600円 (税込)") == 600
|
||||
assert format_japanese_price("1,250円") == 1250
|
||||
assert format_japanese_price("¥ 3,000") == 3000
|
||||
assert format_japanese_price("300") == 300
|
||||
|
||||
def test_empty_and_no_digit(self):
|
||||
assert format_japanese_price(None) == 0
|
||||
assert format_japanese_price("") == 0
|
||||
assert format_japanese_price("税込") == 0
|
||||
|
||||
|
||||
class TestBrowserProfile:
|
||||
def test_fingerprint_consistency(self):
|
||||
# UA / 语言 / 时区必须保持日语环境自洽,且与迁移前两仓字面量一致
|
||||
assert browser_profile.DEFAULT_USER_AGENT == (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
assert browser_profile.LOCALE == "ja-JP"
|
||||
assert browser_profile.TIMEZONE_ID == "Asia/Tokyo"
|
||||
assert browser_profile.NAVIGATOR_LANGUAGES == ("ja-JP", "ja", "en-US", "en")
|
||||
assert browser_profile.ACCEPT_LANGUAGE.startswith("ja-JP,ja;")
|
||||
assert browser_profile.DEFAULT_PAGE_HEADERS["Accept-Language"] == browser_profile.ACCEPT_LANGUAGE
|
||||
|
||||
def test_stealth_launch_args_returns_copy(self):
|
||||
args = browser_profile.stealth_launch_args()
|
||||
assert "--disable-blink-features=AutomationControlled" in args
|
||||
args.append("--extra")
|
||||
assert "--extra" not in browser_profile.stealth_launch_args()
|
||||
@@ -517,7 +517,7 @@ wheels = [
|
||||
|
||||
[[package]]
|
||||
name = "surugaya-common"
|
||||
version = "0.2.0"
|
||||
version = "0.3.0"
|
||||
source = { editable = "packages/surugaya_common" }
|
||||
dependencies = [
|
||||
{ name = "pydantic" },
|
||||
@@ -527,11 +527,13 @@ dependencies = [
|
||||
[package.optional-dependencies]
|
||||
browser = [
|
||||
{ name = "playwright" },
|
||||
{ name = "playwright-stealth" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" },
|
||||
{ name = "playwright-stealth", marker = "extra == 'browser'", specifier = ">=1.0.0,<3.0.0" },
|
||||
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
||||
{ name = "redis", specifier = "==8.0.0" },
|
||||
]
|
||||
@@ -544,14 +546,12 @@ source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "fastapi" },
|
||||
{ name = "loguru" },
|
||||
{ name = "playwright" },
|
||||
{ name = "playwright-stealth" },
|
||||
{ name = "pydantic" },
|
||||
{ name = "pydantic-settings" },
|
||||
{ name = "redis" },
|
||||
{ name = "selectolax" },
|
||||
{ name = "starlette" },
|
||||
{ name = "surugaya-common" },
|
||||
{ name = "surugaya-common", extra = ["browser"] },
|
||||
{ name = "uvicorn", extra = ["standard"] },
|
||||
]
|
||||
|
||||
@@ -565,8 +565,6 @@ dev = [
|
||||
requires-dist = [
|
||||
{ name = "fastapi", specifier = ">=0.115.0,<1.0.0" },
|
||||
{ name = "loguru", specifier = ">=0.7.0,<1.0.0" },
|
||||
{ name = "playwright", specifier = ">=1.54.0,<2.0.0" },
|
||||
{ name = "playwright-stealth", specifier = ">=1.0.0,<3.0.0" },
|
||||
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
||||
{ name = "pydantic-settings", specifier = ">=2.4.0,<3.0.0" },
|
||||
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" },
|
||||
@@ -574,7 +572,7 @@ requires-dist = [
|
||||
{ name = "redis", specifier = "==8.0.0" },
|
||||
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
|
||||
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" },
|
||||
{ name = "surugaya-common", editable = "packages/surugaya_common" },
|
||||
{ name = "surugaya-common", extras = ["browser"], editable = "packages/surugaya_common" },
|
||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
|
||||
]
|
||||
provides-extras = ["dev"]
|
||||
|
||||
Reference in New Issue
Block a user