jp/surugaya
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

common扩展

Browse Source
This commit is contained in:
Jerry Yan
2026-06-11 14:38:23 +08:00
parent 919840a893
commit 02a4484211
14 changed files with 299 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/core/config.py
+7 -8
View File
@@ -9,6 +9,8 @@ from typing import Literal
from pydantic_settings import BaseSettings, SettingsConfigDict
from pathlib import Path
from surugaya_common import browser_profile, urls
BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -60,13 +62,10 @@ class Settings(BaseSettings):
cloudflare_wait_timeout_seconds: float = 60.0 # 等待 Cloudflare 挑战超时
cloudflare_reuse_wait_timeout_seconds: float = 8.0 # 复用会话时消化挑战的短超时，超时则放弃复用直接重建
session_ttl_seconds: int = 900 # 会话过期时间（秒）
browser_locale: str = "ja-JP"
browser_timezone_id: str = "Asia/Tokyo"
browser_user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/124.0.0.0 Safari/537.36"
)
# 指纹参数默认值取自共享包，与 worker 保持一致
browser_locale: str = browser_profile.LOCALE
browser_timezone_id: str = browser_profile.TIMEZONE_ID
browser_user_agent: str = browser_profile.DEFAULT_USER_AGENT
browser_stealth_enabled: bool = True # 是否启用浏览器 Stealth 模式
# ---- Redis 配置（可选，未配置时使用进程内存）----
@@ -82,7 +81,7 @@ class Settings(BaseSettings):
proxy_password: str | None = None
# ---- 目标站点 ----
base_url: str = "https://www.suruga-ya.jp"
base_url: str = urls.BASE_URL
@property
def browser_headless_effective(self) -> bool:
app/services/browser_pool.py
+5 -21
View File
@@ -15,6 +15,7 @@ from typing import Any
from app.core.config import Settings
from app.core.errors import BrowserUnavailableError, ResourceBusyError
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
logger = logging.getLogger(__name__)
@@ -80,28 +81,17 @@ class BrowserPool:
# 用 playwright-stealth 钩子包装 playwright：后续 new_context/new_page
# 会自动应用 stealth（含 HTTP sec-ch-ua 客户端提示），并令 navigator.languages
# 与日语环境保持一致，避免指纹自相矛盾。
from playwright_stealth import Stealth
from surugaya_common.stealth import create_stealth
stealth = Stealth(
navigator_languages_override=("ja-JP", "ja", "en-US", "en"),
)
self._playwright_cm = stealth.use_async(async_playwright())
self._playwright_cm = create_stealth().use_async(async_playwright())
self._playwright = await self._playwright_cm.__aenter__()
else:
self._playwright_cm = None
self._playwright = await async_playwright().start()
chromium = self._playwright.chromium
launch_args = [
"--disable-blink-features=AutomationControlled",
"--disable-dev-shm-usage",
launch_args = stealth_launch_args() + [
"--disable-web-security",
"--no-first-run",
"--disable-infobars",
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-gpu",
"--use-mock-keychain",
"--password-store=basic",
]
# 本地可视化调试时强制窗口化，避免浏览器启动后不可见
if not headless:
@@ -305,13 +295,7 @@ class BrowserPool:
async def _configure_page(context: Any) -> Any:
"""在上下文中创建页面并设置反检测请求头。"""
page = await context.new_page()
await page.set_extra_http_headers(
{
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
}
)
await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
return page
async def create_persistent_context(self, storage_state: dict[str, Any] | None = None) -> Any:
app/services/cargo_service.py
+12 -76
View File
@@ -10,6 +10,9 @@ from selectolax.parser import HTMLParser
from app.core.config import Settings
from app.models.scrape import CargoRequest
from app.utils.parse_util import extract_product_id, format_price
from surugaya_common.browser_profile import DEFAULT_PAGE_HEADERS, stealth_launch_args
from surugaya_common.browser_utils import clear_cart
from surugaya_common.urls import CARGO_DETAIL_URL, product_detail_url
logger = logging.getLogger(__name__)
@@ -29,16 +32,7 @@ class CargoService:
"""
async with async_playwright() as p:
headless = self._settings.browser_headless_effective
launch_args = [
"--disable-blink-features=AutomationControlled",
"--disable-infobars",
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-dev-shm-usage",
"--disable-gpu",
"--use-mock-keychain",
"--password-store=basic",
]
launch_args = stealth_launch_args()
if not headless:
launch_args.extend(["--start-maximized", "--window-position=0,0"])
@@ -61,21 +55,17 @@ class CargoService:
if self._settings.browser_stealth_enabled:
try:
from playwright_stealth import Stealth
await Stealth().apply_stealth_async(context)
# launch_persistent_context 之外的裸 context 用 apply_stealth_async；
# 共享工厂带日语 languages override，与 locale/Accept-Language 对齐。
from surugaya_common.stealth import create_stealth
await create_stealth().apply_stealth_async(context)
except Exception as e:
logger.warning("应用 playwright-stealth 失败: %s", e)
page = await context.new_page()
await page.set_extra_http_headers(
{
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
}
)
await page.set_extra_http_headers(DEFAULT_PAGE_HEADERS)
target_url = "https://www.suruga-ya.jp/cargo/detail"
target_url = CARGO_DETAIL_URL
# 先进入购物车页面
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
@@ -85,7 +75,7 @@ class CargoService:
# 清空购物车
await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟，模拟人工操作间隔
if cart_goods_num > 0:
await self.clear_cart(page)
await clear_cart(page)
# 将待采购的商品加入购物车
item_list = request.payload.item_list
@@ -101,7 +91,7 @@ class CargoService:
continue
logger.debug("进入商品 %s 详情页，准备购买数量: %s...", product_id, number)
detail_url = f"https://www.suruga-ya.jp/product/detail/{product_id}"
detail_url = product_detail_url(product_id)
await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
# 判断是否缺货
@@ -327,57 +317,3 @@ class CargoService:
"total_commission_fee": total_commission_fee
}
async def clear_cart(self, page):
"""
清空购物车
"""
cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
if cart_goods_num < 1:
return
is_delete = 0
# 购物车商品数量超过3个后会出现"全削除"按钮
if cart_goods_num > 3:
try:
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
# 3. 显式等待按钮在页面上可见（容错处理：比如给它最多 3000 毫秒的加载时间）
# 如果按钮是一开始就在静态 HTML 里的，这一步也可以省略，直接用下面的 is_visible()
await delete_button.wait_for(state="visible", timeout=3000)
if await delete_button.is_visible():
logger.debug("检测到‘全削除’按钮存在，正在执行点击...")
await delete_button.click()
is_delete = 1
except Exception:
logger.debug("未检测到‘全削除’按钮。")
if is_delete == 1:
return
# 遍历删除
item_selector = "table.item_box tbody tr.item"
remove_selector = "table.item_box a.remove_product"
while True:
cart_goods_num = await page.locator(item_selector).count()
if cart_goods_num == 0:
break
remove_link = page.locator(remove_selector).first
try:
await remove_link.click()
# 删除按钮不会触发弹窗，直接等待购物车条目数减少即可。
await page.wait_for_function(
"""({selector, expectedCount}) => {
return document.querySelectorAll(selector).length < expectedCount;
}""",
arg={"selector": item_selector, "expectedCount": cart_goods_num},
timeout=15_000,
)
await page.wait_for_timeout(random.randint(800, 1500))
except PlaywrightTimeoutError:
latest_count = await page.locator(item_selector).count()
if latest_count < cart_goods_num:
continue
await page.wait_for_timeout(1000)
app/services/surugaya_client.py
+4 -3
View File
@@ -26,6 +26,7 @@ from app.services.cloudflare_session import CloudflareSessionManager
from app.services.session_store import SessionStore
from app.utils.parse_util import format_price, get_shipping_fee, surugaya_photo_url_to_cdn
from surugaya_common.app_utils import is_empty_str
from surugaya_common.urls import BASE_URL
logger = logging.getLogger(__name__)
@@ -303,7 +304,7 @@ class SurugayaClient:
logger.info("开始获取并解析商品分类...")
top_categories: list[CategoryData] = []
target_url = "https://www.suruga-ya.jp/"
target_url = f"{BASE_URL}/"
session = await self._session_manager.get_verified_session(target_url, True)
html_content = ""
@@ -366,7 +367,7 @@ class SurugayaClient:
async with self._browser_pool.page_session(storage_state=session.storage_state) as (_, page):
for top_category in top_categories:
try:
category_url = f"https://www.suruga-ya.jp{top_category.href}"
category_url = f"{BASE_URL}{top_category.href}"
response = await page.goto(
category_url,
wait_until="domcontentloaded",
@@ -568,7 +569,7 @@ class SurugayaClient:
"""将相对 URL 补全为完整的 https URL"""
if url.startswith("http://") or url.startswith("https://"):
return url
return f"https://www.suruga-ya.jp{url}"
return f"{BASE_URL}{url}"
def get_breadcrumb_list(self, tree):
dom_list = tree.css(".block-blocksurugayabreadcrum nav .breadcrumb .breadcrumb-item")