common扩展

This commit is contained in:
2026-06-11 14:38:23 +08:00
parent 919840a893
commit 02a4484211
14 changed files with 299 additions and 127 deletions
+7 -8
View File
@@ -9,6 +9,8 @@ from typing import Literal
from pydantic_settings import BaseSettings, SettingsConfigDict from pydantic_settings import BaseSettings, SettingsConfigDict
from pathlib import Path from pathlib import Path
from surugaya_common import browser_profile, urls
BASE_DIR = Path(__file__).resolve().parent.parent.parent BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -60,13 +62,10 @@ class Settings(BaseSettings):
cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时 cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时
cloudflare_reuse_wait_timeout_seconds: float = 8.0 # 复用会话时消化挑战的短超时,超时则放弃复用直接重建 cloudflare_reuse_wait_timeout_seconds: float = 8.0 # 复用会话时消化挑战的短超时,超时则放弃复用直接重建
session_ttl_seconds: int = 900 # 会话过期时间(秒) session_ttl_seconds: int = 900 # 会话过期时间(秒)
browser_locale: str = "ja-JP" # 指纹参数默认值取自共享包,与 worker 保持一致
browser_timezone_id: str = "Asia/Tokyo" browser_locale: str = browser_profile.LOCALE
browser_user_agent: str = ( browser_timezone_id: str = browser_profile.TIMEZONE_ID
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " browser_user_agent: str = browser_profile.DEFAULT_USER_AGENT
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式 browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式
# ---- Redis 配置(可选,未配置时使用进程内存)---- # ---- Redis 配置(可选,未配置时使用进程内存)----
@@ -82,7 +81,7 @@ class Settings(BaseSettings):
proxy_password: str | None = None proxy_password: str | None = None
# ---- 目标站点 ---- # ---- 目标站点 ----
base_url: str = "https://www.suruga-ya.jp" base_url: str = urls.BASE_URL
@property @property
def browser_headless_effective(self) -> bool: def browser_headless_effective(self) -> bool:
+5 -21
View File
@@ -15,6 +15,7 @@ from typing import Any
from app.core.config import Settings from app.core.config import Settings
from app.core.errors import BrowserUnavailableError, ResourceBusyError from app.core.errors import BrowserUnavailableError, ResourceBusyError
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -80,28 +81,17 @@ class BrowserPool:
# 用 playwright-stealth 钩子包装 playwright:后续 new_context/new_page # 用 playwright-stealth 钩子包装 playwright:后续 new_context/new_page
# 会自动应用 stealth(含 HTTP sec-ch-ua 客户端提示),并令 navigator.languages # 会自动应用 stealth(含 HTTP sec-ch-ua 客户端提示),并令 navigator.languages
# 与日语环境保持一致,避免指纹自相矛盾。 # 与日语环境保持一致,避免指纹自相矛盾。
from playwright_stealth import Stealth from surugaya_common.stealth import create_stealth
stealth = Stealth( self._playwright_cm = create_stealth().use_async(async_playwright())
navigator_languages_override=("ja-JP", "ja", "en-US", "en"),
)
self._playwright_cm = stealth.use_async(async_playwright())
self._playwright = await self._playwright_cm.__aenter__() self._playwright = await self._playwright_cm.__aenter__()
else: else:
self._playwright_cm = None self._playwright_cm = None
self._playwright = await async_playwright().start() self._playwright = await async_playwright().start()
chromium = self._playwright.chromium chromium = self._playwright.chromium
launch_args = [ launch_args = stealth_launch_args() + [
"--disable-blink-features=AutomationControlled",
"--disable-dev-shm-usage",
"--disable-web-security", "--disable-web-security",
"--no-first-run", "--no-first-run",
"--disable-infobars",
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-gpu",
"--use-mock-keychain",
"--password-store=basic",
] ]
# 本地可视化调试时强制窗口化,避免浏览器启动后不可见 # 本地可视化调试时强制窗口化,避免浏览器启动后不可见
if not headless: if not headless:
@@ -305,13 +295,7 @@ class BrowserPool:
async def _configure_page(context: Any) -> Any: async def _configure_page(context: Any) -> Any:
"""在上下文中创建页面并设置反检测请求头。""" """在上下文中创建页面并设置反检测请求头。"""
page = await context.new_page() page = await context.new_page()
await page.set_extra_http_headers( await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
{
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
}
)
return page return page
async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any: async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any:
+12 -76
View File
@@ -10,6 +10,9 @@ from selectolax.parser import HTMLParser
from app.core.config import Settings from app.core.config import Settings
from app.models.scrape import CargoRequest from app.models.scrape import CargoRequest
from app.utils.parse_util import extract_product_id, format_price from app.utils.parse_util import extract_product_id, format_price
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
from surugaya_common.browser_utils import clear_cart
from surugaya_common.urls import CARGO_DETAIL_URL, product_detail_url
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -29,16 +32,7 @@ class CargoService:
""" """
async with async_playwright() as p: async with async_playwright() as p:
headless = self._settings.browser_headless_effective headless = self._settings.browser_headless_effective
launch_args = [ launch_args = stealth_launch_args()
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
"--use-mock-keychain",
"--password-store=basic",
]
if not headless: if not headless:
launch_args.extend(["--start-maximized", "--window-position=0,0"]) launch_args.extend(["--start-maximized", "--window-position=0,0"])
@@ -61,21 +55,17 @@ class CargoService:
if self._settings.browser_stealth_enabled: if self._settings.browser_stealth_enabled:
try: try:
from playwright_stealth import Stealth # launch_persistent_context 之外的裸 context 用 apply_stealth_async;
await Stealth().apply_stealth_async(context) # 共享工厂带日语 languages override,与 locale/Accept-Language 对齐。
from surugaya_common.stealth import create_stealth
await create_stealth().apply_stealth_async(context)
except Exception as e: except Exception as e:
logger.warning("应用 playwright-stealth 失败: %s", e) logger.warning("应用 playwright-stealth 失败: %s", e)
page = await context.new_page() page = await context.new_page()
await page.set_extra_http_headers( await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
{
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
}
)
target_url = "https://www.suruga-ya.jp/cargo/detail" target_url = CARGO_DETAIL_URL
# 先进入购物车页面 # 先进入购物车页面
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000) await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
@@ -85,7 +75,7 @@ class CargoService:
# 清空购物车 # 清空购物车
await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟,模拟人工操作间隔 await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟,模拟人工操作间隔
if cart_goods_num > 0: if cart_goods_num > 0:
await self.clear_cart(page) await clear_cart(page)
# 将待采购的商品加入购物车 # 将待采购的商品加入购物车
item_list = request.payload.item_list item_list = request.payload.item_list
@@ -101,7 +91,7 @@ class CargoService:
continue continue
logger.debug("进入商品 %s 详情页,准备购买数量: %s...", product_id, number) logger.debug("进入商品 %s 详情页,准备购买数量: %s...", product_id, number)
detail_url = f"https://www.suruga-ya.jp/product/detail/{product_id}" detail_url = product_detail_url(product_id)
await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000) await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
# 判断是否缺货 # 判断是否缺货
@@ -327,57 +317,3 @@ class CargoService:
"total_commission_fee": total_commission_fee "total_commission_fee": total_commission_fee
} }
async def clear_cart(self, page):
"""
清空购物车
"""
cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
if cart_goods_num < 1:
return
is_delete = 0
# 购物车商品数量超过3个后会出现"全削除"按钮
if cart_goods_num > 3:
try:
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
# 3. 显式等待按钮在页面上可见(容错处理:比如给它最多 3000 毫秒的加载时间)
# 如果按钮是一开始就在静态 HTML 里的,这一步也可以省略,直接用下面的 is_visible()
await delete_button.wait_for(state="visible", timeout=3000)
if await delete_button.is_visible():
logger.debug("检测到‘全削除’按钮存在,正在执行点击...")
await delete_button.click()
is_delete = 1
except Exception:
logger.debug("未检测到‘全削除’按钮。")
if is_delete == 1:
return
# 遍历删除
item_selector = "table.item_box tbody tr.item"
remove_selector = "table.item_box a.remove_product"
while True:
cart_goods_num = await page.locator(item_selector).count()
if cart_goods_num == 0:
break
remove_link = page.locator(remove_selector).first
try:
await remove_link.click()
# 删除按钮不会触发弹窗,直接等待购物车条目数减少即可。
await page.wait_for_function(
"""({selector, expectedCount}) => {
return document.querySelectorAll(selector).length < expectedCount;
}""",
arg={"selector": item_selector, "expectedCount": cart_goods_num},
timeout=15_000,
)
await page.wait_for_timeout(random.randint(800, 1500))
except PlaywrightTimeoutError:
latest_count = await page.locator(item_selector).count()
if latest_count < cart_goods_num:
continue
await page.wait_for_timeout(1000)
+4 -3
View File
@@ -26,6 +26,7 @@ from app.services.cloudflare_session import CloudflareSessionManager
from app.services.session_store import SessionStore from app.services.session_store import SessionStore
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
from surugaya_common.app_utils import is_empty_str from surugaya_common.app_utils import is_empty_str
from surugaya_common.urls import BASE_URL
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@@ -303,7 +304,7 @@ class SurugayaClient:
logger.info("开始获取并解析商品分类...") logger.info("开始获取并解析商品分类...")
top_categories: list[CategoryData] = [] top_categories: list[CategoryData] = []
target_url = "https://www.suruga-ya.jp/" target_url = f"{BASE_URL}/"
session = await self._session_manager.get_verified_session(target_url, True) session = await self._session_manager.get_verified_session(target_url, True)
html_content = "" html_content = ""
@@ -366,7 +367,7 @@ class SurugayaClient:
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page): async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
for top_category in top_categories: for top_category in top_categories:
try: try:
category_url = f"https://www.suruga-ya.jp{top_category.href}" category_url = f"{BASE_URL}{top_category.href}"
response = await page.goto( response = await page.goto(
category_url, category_url,
wait_until="domcontentloaded", wait_until="domcontentloaded",
@@ -568,7 +569,7 @@ class SurugayaClient:
"""将相对 URL 补全为完整的 https URL""" """将相对 URL 补全为完整的 https URL"""
if url.startswith("http://") or url.startswith("https://"): if url.startswith("http://") or url.startswith("https://"):
return url return url
return f"https://www.suruga-ya.jp{url}" return f"{BASE_URL}{url}"
def get_breadcrumb_list(self, tree): def get_breadcrumb_list(self, tree):
dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item") dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")
+6 -3
View File
@@ -4,8 +4,8 @@ build-backend = "hatchling.build"
[project] [project]
name = "surugaya-common" name = "surugaya-common"
version = "0.2.0" version = "0.3.0"
description = "Shared contracts (Redis keys, future models) for jp_surugaya services" description = "Shared contracts and browser utilities (Redis keys, models, parsers, fingerprint profile) for jp_surugaya services"
requires-python = ">=3.12" requires-python = ">=3.12"
dependencies = [ dependencies = [
"pydantic>=2.0.0,<3.0.0", "pydantic>=2.0.0,<3.0.0",
@@ -15,7 +15,10 @@ dependencies = [
# playwright 仅作为可选 extra:只有解析器/浏览器工具需要它,核心安装保持 light。 # playwright 仅作为可选 extra:只有解析器/浏览器工具需要它,核心安装保持 light。
# 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集,确保 workspace 解析到单一 playwright。 # 范围取 jp_surugaya(>=1.54,<2) 与 worker(>=1.43) 的交集,确保 workspace 解析到单一 playwright。
[project.optional-dependencies] [project.optional-dependencies]
browser = ["playwright>=1.54.0,<2.0.0"] browser = [
"playwright>=1.54.0,<2.0.0",
"playwright-stealth>=1.0.0,<3.0.0",
]
[tool.hatch.build.targets.wheel] [tool.hatch.build.targets.wheel]
packages = ["src/surugaya_common"] packages = ["src/surugaya_common"]
@@ -1,5 +1,10 @@
"""Shared contracts for jp_surugaya services (Redis keys, future models).""" """Shared contracts and utilities for jp_surugaya services.
from surugaya_common import redis_keys 核心安装(无 extra)可用:redis_keys、urls、browser_profile、parse_utils、
app_utils、http_utils、models。
需安装 surugaya_common[browser]:browser_utils、stealth、parsers。
"""
__all__ = ["redis_keys"] from surugaya_common import browser_profile, parse_utils, redis_keys, urls
__all__ = ["browser_profile", "parse_utils", "redis_keys", "urls"]
@@ -0,0 +1,43 @@
"""浏览器指纹与启动参数共享配置(纯常量,core 安装即可用,不依赖 playwright)。
API(jp_surugaya)与 worker(jp_pruchase_zdh_5)访问同一目标站点,
指纹参数必须保持一致:UA、语言、时区任何一处自相矛盾都可能触发风控
(参见 cf_clearance 复用因指纹不一致被判 403 的教训)。
"""
# 与日语环境一致的指纹参数
LOCALE = "ja-JP"
TIMEZONE_ID = "Asia/Tokyo"
NAVIGATOR_LANGUAGES = ("ja-JP", "ja", "en-US", "en")
ACCEPT_LANGUAGE = "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7"
# 两项目统一使用的 Windows Chrome User-Agent
DEFAULT_USER_AGENT = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
# 页面级反检测请求头(page.set_extra_http_headers 用)
DEFAULT_PAGE_HEADERS = {
"Accept-Language": ACCEPT_LANGUAGE,
"Cache-Control": "no-cache",
"Pragma": "no-cache",
}
# 两项目共用的 Chromium 反检测启动参数
STEALTH_LAUNCH_ARGS = (
"--disable-blink-features=AutomationControlled", # 浏览器层面移除 navigator.webdriver
"--disable-infobars", # 隐藏「Chrome 正受到自动测试软件控制」提示条
"--no-sandbox", # Docker/服务器环境兼容
"--disable-setuid-sandbox",
"--disable-dev-shm-usage", # 内存受限环境避免资源崩溃
"--disable-gpu", # 服务器无头模式稳定渲染
"--use-mock-keychain", # 避免本地密钥环弹窗(macOS/Linux)
"--password-store=basic",
)
def stealth_launch_args() -> list[str]:
"""返回共享启动参数的可变副本,调用方可在其上追加进程自有参数。"""
return list(STEALTH_LAUNCH_ARGS)
@@ -1,14 +1,81 @@
"""浏览器拟真交互工具(需 playwright,安装 surugaya_common[browser])。 """浏览器拟真交互工具(需 playwright,安装 surugaya_common[browser])。
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 human_scroll,逻辑保持一致。 - human_scroll:从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移,逻辑保持一致。
注:page_utils.py 中的纯函数 format_japanese_price 未迁移——它与 API 端 - clear_cart:worker 与 API cargo_service 各自的 clear_cart 近乎逐字重复,
app/utils/parse_util.py 的 format_price 语义不同,合并属行为变更,单独处理 统一为本实现(以 worker 版为准,保留拟真随机等待)
- format_japanese_price 已迁至 surugaya_common.parse_utils(纯 stdlib,不依赖 playwright)。
""" """
from playwright.async_api import Page from playwright.async_api import Page
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
import asyncio import asyncio
import logging
import random import random
logger = logging.getLogger(__name__)
# 购物车页(/cargo/detail)的关键选择器
CART_ITEM_SELECTOR = "table.item_box tbody tr.item"
CART_REMOVE_LINK_SELECTOR = "table.item_box a.remove_product"
async def count_cart_items(page: Page) -> int:
"""统计购物车页当前的商品条目数(需已位于 /cargo/detail 页面)。"""
return await page.locator(CART_ITEM_SELECTOR).count()
async def clear_cart(page: Page) -> None:
"""清空购物车(需已位于 /cargo/detail 页面)。
- 商品数量超过 3 个时优先点击「全削除」按钮一次性清空
- 否则逐条点击删除链接,等待条目数减少后继续,含拟真随机等待
"""
logger.debug("正在清空购物车...")
cart_goods_num = await count_cart_items(page)
if cart_goods_num < 1:
return
# 购物车商品数量超过3个后会出现"全削除"按钮
if cart_goods_num > 3:
try:
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
# 显式等待按钮在页面上可见(容错处理:给它最多 3000 毫秒的加载时间)
await delete_button.wait_for(state="visible", timeout=3000)
if await delete_button.is_visible():
logger.debug("检测到'全削除'按钮存在,正在执行点击...")
await delete_button.click()
return
except Exception:
logger.debug("未检测到'全削除'按钮。")
# 遍历删除
await page.wait_for_timeout(random.randint(1000, 3000))
while True:
cart_goods_num = await page.locator(CART_ITEM_SELECTOR).count()
if cart_goods_num == 0:
break
remove_link = page.locator(CART_REMOVE_LINK_SELECTOR).first
try:
await remove_link.click()
# 删除按钮不会触发弹窗,直接等待购物车条目数减少即可。
await page.wait_for_function(
"""({selector, expectedCount}) => {
return document.querySelectorAll(selector).length < expectedCount;
}""",
arg={"selector": CART_ITEM_SELECTOR, "expectedCount": cart_goods_num},
timeout=15_000,
)
await page.wait_for_timeout(random.randint(800, 1500))
except PlaywrightTimeoutError:
latest_count = await page.locator(CART_ITEM_SELECTOR).count()
if latest_count < cart_goods_num:
logger.debug("购物车商品已删除,当前购物车数量:%s", latest_count)
continue
logger.warning("删除商品后数量未变化,等待后重试")
await page.wait_for_timeout(1000)
async def human_scroll(page: Page, mode: str, value: int = 0): async def human_scroll(page: Page, mode: str, value: int = 0):
""" """
@@ -0,0 +1,24 @@
"""纯文本解析工具(仅 stdlib)。
从 jp_pruchase_zdh_5/src/utils/page_utils.py 忠实迁移 format_japanese_price。
"""
import re
def format_japanese_price(price_str: str | None) -> int:
"""
将日本电商网站的价格字符串格式化为标准整数
支持的格式示例: "600円 (税込)", "1,250円", "¥ 3,000", "300"
注意:实现为提取字符串中所有数字字符后拼接,与 jp_surugaya API 端
parse_util.format_price(货币符号定位 + 价格范围取低价)语义不同,勿混用。
"""
if not price_str:
return 0
digits = re.findall(r'\d', price_str)
if not digits:
return 0
return int("".join(digits))
@@ -0,0 +1,24 @@
"""playwright-stealth 共享工厂(需安装 surugaya_common[browser])。
统一两项目的 stealth 指纹配置:navigator.languages 覆盖必须与日语
locale / Accept-Language 对齐,否则带 cf_clearance 的复用请求会因
指纹自相矛盾被 Cloudflare 判 403。
"""
from playwright_stealth import Stealth
from surugaya_common.browser_profile import NAVIGATOR_LANGUAGES
def create_stealth() -> Stealth:
"""创建与日语环境对齐的 Stealth 实例。
用法约定:
- browser.new_context 流程:用 stealth.use_async(async_playwright()) 钩子
包装 playwright,new_context/new_page 自动应用 stealth(含 HTTP
sec-ch-ua 客户端提示)。jp_surugaya BrowserPool 即此用法。
- launch_persistent_context 流程:use_async 钩子不覆盖它(playwright-stealth
已知缺口),必须显式 await stealth.apply_stealth_async(context),
此路径只注入 init script,无 sec-ch-ua 等 HTTP 头伪装。
"""
return Stealth(navigator_languages_override=NAVIGATOR_LANGUAGES)
@@ -0,0 +1,22 @@
"""suruga-ya.jp 站点 URL 契约:两项目共享的页面地址与构造器。"""
BASE_URL = "https://www.suruga-ya.jp"
# 购物车详情页
CARGO_DETAIL_URL = f"{BASE_URL}/cargo/detail"
# 会员中心(登录入口)
MYPAGE_URL = f"{BASE_URL}/pcmypage"
def product_detail_url(product_id: str, tenpo_cd: str | None = None) -> str:
"""商品详情页地址;tenpo_cd 用于加盟店商品。"""
url = f"{BASE_URL}/product/detail/{product_id}"
if tenpo_cd:
url += f"?tenpo_cd={tenpo_cd}"
return url
def trade_detail_url(trade_code: str) -> str:
"""交易(订单)详情页地址。"""
return f"{BASE_URL}/pcmypage/action_sell_search/detail?trade_code={trade_code}"
+2 -3
View File
@@ -6,15 +6,14 @@ requires-python = ">=3.12"
dependencies = [ dependencies = [
"fastapi>=0.115.0,<1.0.0", "fastapi>=0.115.0,<1.0.0",
"loguru>=0.7.0,<1.0.0", "loguru>=0.7.0,<1.0.0",
"playwright>=1.54.0,<2.0.0",
"playwright-stealth>=1.0.0,<3.0.0",
"pydantic>=2.0.0,<3.0.0", "pydantic>=2.0.0,<3.0.0",
"pydantic-settings>=2.4.0,<3.0.0", "pydantic-settings>=2.4.0,<3.0.0",
"redis==8.0.0", "redis==8.0.0",
"selectolax>=0.3.21,<1.0.0", "selectolax>=0.3.21,<1.0.0",
"starlette>=0.37.0,<1.0.0", "starlette>=0.37.0,<1.0.0",
"uvicorn[standard]>=0.30.0,<1.0.0", "uvicorn[standard]>=0.30.0,<1.0.0",
"surugaya-common", # playwright 与 playwright-stealth 由 [browser] extra 提供,与 worker 保持同一来源
"surugaya-common[browser]",
] ]
[project.optional-dependencies] [project.optional-dependencies]
+67
View File
@@ -0,0 +1,67 @@
"""surugaya_common 共享契约回归测试:URL 构造与价格解析必须与迁移前字面量/行为一致。"""
from surugaya_common import browser_profile
from surugaya_common.parse_utils import format_japanese_price
from surugaya_common.urls import (
BASE_URL,
CARGO_DETAIL_URL,
MYPAGE_URL,
product_detail_url,
trade_detail_url,
)
class TestUrls:
def test_literals_match_pre_migration(self):
assert BASE_URL == "https://www.suruga-ya.jp"
assert CARGO_DETAIL_URL == "https://www.suruga-ya.jp/cargo/detail"
assert MYPAGE_URL == "https://www.suruga-ya.jp/pcmypage"
def test_product_detail_url(self):
assert product_detail_url("128001234") == "https://www.suruga-ya.jp/product/detail/128001234"
assert (
product_detail_url("128001234", tenpo_cd="400496")
== "https://www.suruga-ya.jp/product/detail/128001234?tenpo_cd=400496"
)
# 空字符串 tenpo_cd 与未提供等价(与迁移前 if tenpo_cd 判断一致)
assert product_detail_url("128001234", tenpo_cd="") == "https://www.suruga-ya.jp/product/detail/128001234"
def test_trade_detail_url(self):
assert (
trade_detail_url("T123456")
== "https://www.suruga-ya.jp/pcmypage/action_sell_search/detail?trade_code=T123456"
)
class TestFormatJapanesePrice:
def test_supported_formats(self):
assert format_japanese_price("600円 (税込)") == 600
assert format_japanese_price("1,250円") == 1250
assert format_japanese_price("¥ 3,000") == 3000
assert format_japanese_price("300") == 300
def test_empty_and_no_digit(self):
assert format_japanese_price(None) == 0
assert format_japanese_price("") == 0
assert format_japanese_price("税込") == 0
class TestBrowserProfile:
def test_fingerprint_consistency(self):
# UA / 语言 / 时区必须保持日语环境自洽,且与迁移前两仓字面量一致
assert browser_profile.DEFAULT_USER_AGENT == (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
assert browser_profile.LOCALE == "ja-JP"
assert browser_profile.TIMEZONE_ID == "Asia/Tokyo"
assert browser_profile.NAVIGATOR_LANGUAGES == ("ja-JP", "ja", "en-US", "en")
assert browser_profile.ACCEPT_LANGUAGE.startswith("ja-JP,ja;")
assert browser_profile.DEFAULT_PAGE_HEADERS["Accept-Language"] == browser_profile.ACCEPT_LANGUAGE
def test_stealth_launch_args_returns_copy(self):
args = browser_profile.stealth_launch_args()
assert "--disable-blink-features=AutomationControlled" in args
args.append("--extra")
assert "--extra" not in browser_profile.stealth_launch_args()
Generated
+5 -7
View File
@@ -517,7 +517,7 @@ wheels = [
[[package]] [[package]]
name = "surugaya-common" name = "surugaya-common"
version = "0.2.0" version = "0.3.0"
source = { editable = "packages/surugaya_common" } source = { editable = "packages/surugaya_common" }
dependencies = [ dependencies = [
{ name = "pydantic" }, { name = "pydantic" },
@@ -527,11 +527,13 @@ dependencies = [
[package.optional-dependencies] [package.optional-dependencies]
browser = [ browser = [
{ name = "playwright" }, { name = "playwright" },
{ name = "playwright-stealth" },
] ]
[package.metadata] [package.metadata]
requires-dist = [ requires-dist = [
{ name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" }, { name = "playwright", marker = "extra == 'browser'", specifier = ">=1.54.0,<2.0.0" },
{ name = "playwright-stealth", marker = "extra == 'browser'", specifier = ">=1.0.0,<3.0.0" },
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
{ name = "redis", specifier = "==8.0.0" }, { name = "redis", specifier = "==8.0.0" },
] ]
@@ -544,14 +546,12 @@ source = { virtual = "." }
dependencies = [ dependencies = [
{ name = "fastapi" }, { name = "fastapi" },
{ name = "loguru" }, { name = "loguru" },
{ name = "playwright" },
{ name = "playwright-stealth" },
{ name = "pydantic" }, { name = "pydantic" },
{ name = "pydantic-settings" }, { name = "pydantic-settings" },
{ name = "redis" }, { name = "redis" },
{ name = "selectolax" }, { name = "selectolax" },
{ name = "starlette" }, { name = "starlette" },
{ name = "surugaya-common" }, { name = "surugaya-common", extra = ["browser"] },
{ name = "uvicorn", extra = ["standard"] }, { name = "uvicorn", extra = ["standard"] },
] ]
@@ -565,8 +565,6 @@ dev = [
requires-dist = [ requires-dist = [
{ name = "fastapi", specifier = ">=0.115.0,<1.0.0" }, { name = "fastapi", specifier = ">=0.115.0,<1.0.0" },
{ name = "loguru", specifier = ">=0.7.0,<1.0.0" }, { name = "loguru", specifier = ">=0.7.0,<1.0.0" },
{ name = "playwright", specifier = ">=1.54.0,<2.0.0" },
{ name = "playwright-stealth", specifier = ">=1.0.0,<3.0.0" },
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" }, { name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
{ name = "pydantic-settings", specifier = ">=2.4.0,<3.0.0" }, { name = "pydantic-settings", specifier = ">=2.4.0,<3.0.0" },
{ name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0,<9.0.0" },
@@ -574,7 +572,7 @@ requires-dist = [
{ name = "redis", specifier = "==8.0.0" }, { name = "redis", specifier = "==8.0.0" },
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" }, { name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" }, { name = "starlette", specifier = ">=0.37.0,<1.0.0" },
{ name = "surugaya-common", editable = "packages/surugaya_common" }, { name = "surugaya-common", extras = ["browser"], editable = "packages/surugaya_common" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
] ]
provides-extras = ["dev"] provides-extras = ["dev"]