Init
This commit is contained in:
@@ -0,0 +1,383 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
|
||||
from playwright.async_api import async_playwright, Page
|
||||
from playwright.async_api import TimeoutError as PlaywrightTimeoutError
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models.scrape import CargoRequest
|
||||
from app.utils.parse_util import extract_product_id, format_price
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CargoService:
|
||||
"""购物车服务"""
|
||||
|
||||
def __init__(self, settings: Settings):
|
||||
self._settings = settings
|
||||
|
||||
async def get_cargo_info(self, request: CargoRequest) -> dict:
|
||||
"""获取购物车信息
|
||||
|
||||
每次访问都会启动一个全新的 Playwright 浏览器实例，
|
||||
创建全新的上下文，避免任何状态残留。
|
||||
先将指定的商品加入购物车，然后再进入购物车页面解析信息。
|
||||
"""
|
||||
async with async_playwright() as p:
|
||||
headless = self._settings.browser_headless_effective
|
||||
launch_args = [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-infobars",
|
||||
"--no-sandbox",
|
||||
"--disable-setuid-sandbox",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-gpu",
|
||||
"--use-mock-keychain",
|
||||
"--password-store=basic",
|
||||
]
|
||||
if not headless:
|
||||
launch_args.extend(["--start-maximized", "--window-position=0,0"])
|
||||
|
||||
# 启动全新的浏览器进程
|
||||
browser = await p.chromium.launch(
|
||||
headless=headless,
|
||||
channel=self._settings.browser_channel,
|
||||
proxy=self._settings.proxy,
|
||||
args=launch_args,
|
||||
)
|
||||
|
||||
try:
|
||||
# 创建全新上下文
|
||||
context = await browser.new_context(
|
||||
locale=self._settings.browser_locale,
|
||||
timezone_id=self._settings.browser_timezone_id,
|
||||
user_agent=self._settings.browser_user_agent,
|
||||
viewport=None if not headless else {"width": 1440, "height": 900},
|
||||
)
|
||||
|
||||
if self._settings.browser_stealth_enabled:
|
||||
try:
|
||||
from playwright_stealth import Stealth
|
||||
await Stealth().apply_stealth_async(context)
|
||||
except Exception as e:
|
||||
logger.warning("应用 playwright-stealth 失败: %s", e)
|
||||
|
||||
page = await context.new_page()
|
||||
await page.set_extra_http_headers(
|
||||
{
|
||||
"Accept-Language": "ja-JP,ja;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Cache-Control": "no-cache",
|
||||
"Pragma": "no-cache",
|
||||
}
|
||||
)
|
||||
|
||||
target_url = "https://www.suruga-ya.jp/cargo/detail"
|
||||
# 先进入购物车页面
|
||||
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
|
||||
|
||||
# 判断购物车是否有商品
|
||||
cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
|
||||
|
||||
# 清空购物车
|
||||
await asyncio.sleep(random.uniform(1, 3)) # 加入随机延迟，模拟人工操作间隔
|
||||
if cart_goods_num > 0:
|
||||
await self.clear_cart(page)
|
||||
|
||||
# 将待采购的商品加入购物车
|
||||
item_list = request.payload.item_list
|
||||
for index, item in enumerate(item_list):
|
||||
product_id = item.id
|
||||
if not product_id:
|
||||
logger.warning("索引 %s 的商品未提供 id，跳过。", index)
|
||||
continue
|
||||
|
||||
number = item.number or 1
|
||||
if number <= 0:
|
||||
logger.warning("索引 %s 的商品数量为 0，跳过。", index)
|
||||
continue
|
||||
|
||||
logger.debug("进入商品 %s 详情页，准备购买数量: %s...", product_id, number)
|
||||
detail_url = f"https://www.suruga-ya.jp/product/detail/{product_id}"
|
||||
await page.goto(detail_url, wait_until="domcontentloaded", timeout=20000)
|
||||
|
||||
# 判断是否缺货
|
||||
out_of_stock_dom = page.locator("div.out-of-stock-text")
|
||||
is_out_of_stock = await out_of_stock_dom.is_visible()
|
||||
if is_out_of_stock:
|
||||
logger.warning("商品 %s 缺货，跳过", product_id)
|
||||
continue
|
||||
|
||||
# 提取商品最大购买数量
|
||||
try:
|
||||
quantity_select = page.locator("#quantity_selection")
|
||||
await quantity_select.wait_for(state="visible", timeout=5000)
|
||||
last_option_locator = quantity_select.locator("option").last
|
||||
buy_number_limit_str = await last_option_locator.get_attribute("value")
|
||||
buy_number_limit = int(buy_number_limit_str) if buy_number_limit_str else 0
|
||||
logger.debug("商品 %s 最大购买数量: %s", product_id, buy_number_limit)
|
||||
if 0 < buy_number_limit < number:
|
||||
number = buy_number_limit
|
||||
except PlaywrightTimeoutError:
|
||||
logger.debug("商品 %s 没有数量选择框，默认购买数量: %s", product_id, number)
|
||||
|
||||
try:
|
||||
btn_buy = page.locator("button.btn_buy").first
|
||||
await btn_buy.wait_for(state="visible", timeout=5000)
|
||||
|
||||
if await btn_buy.is_disabled():
|
||||
logger.warning("商品 %s 已售罄或无法购买（按钮被禁用），跳过", product_id)
|
||||
continue
|
||||
|
||||
# 根据 number 循环点击加入购物车
|
||||
for i in range(number):
|
||||
logger.debug("点击加入购物车按钮，第 %s 次", i + 1)
|
||||
await btn_buy.click()
|
||||
if i < number - 1:
|
||||
# 多次点击之间加入随机延迟，防止被识别为机器人或触发频率限制
|
||||
await asyncio.sleep(random.uniform(0.8, 1.5))
|
||||
except PlaywrightTimeoutError:
|
||||
logger.warning("商品 %s 找不到购买按钮(可能是已下架/缺货)，跳过", product_id)
|
||||
continue
|
||||
|
||||
logger.info("再次访问购物车页面: %s", target_url)
|
||||
await page.goto(target_url, wait_until="domcontentloaded", timeout=15000)
|
||||
|
||||
# 解析购物车
|
||||
try:
|
||||
primary = page.locator("#container #main3 #primary")
|
||||
if not await primary.is_visible(timeout=10000):
|
||||
logger.warning("购物车商品元素未找到，跳过解析")
|
||||
raise Exception("购物车商品获取失败")
|
||||
|
||||
result = await self.parse_cargo_info(page)
|
||||
except Exception as e:
|
||||
logger.warning("购物车价格元素未找到，跳过解析: %s", e)
|
||||
raise Exception("购物车价格获取失败")
|
||||
|
||||
return result
|
||||
finally:
|
||||
await browser.close()
|
||||
|
||||
async def parse_cargo_info(self, page: Page):
|
||||
"""
|
||||
解析商品总额，运费和手续费金额等信息
|
||||
"""
|
||||
logger.debug("解析购物车信息")
|
||||
total_goods_amount = 0
|
||||
total_shipping_fee = 0
|
||||
total_commission_fee = 0
|
||||
items: list[dict] = []
|
||||
|
||||
total_price_dom = page.locator("#total_box div.total_price")
|
||||
try:
|
||||
await total_price_dom.first.wait_for(state="attached", timeout=20000)
|
||||
|
||||
total_price_str = await total_price_dom.locator("p.price_name .total_price").first.text_content()
|
||||
if total_price_str:
|
||||
total_goods_amount = format_price(total_price_str.strip()) or 0
|
||||
logger.debug("商品总额: %s", total_goods_amount)
|
||||
|
||||
delivery_charge_html = await total_price_dom.locator("p.delivery_charge").first.inner_html()
|
||||
if delivery_charge_html:
|
||||
shipping_match = re.search(r"([\d,]+)(?=円の送料)", delivery_charge_html)
|
||||
if shipping_match:
|
||||
total_shipping_fee = int(shipping_match.group(1).replace(",", ""))
|
||||
logger.debug("运费: %s", total_shipping_fee)
|
||||
|
||||
handling_match = re.search(r"通信販売手数料.*?:([\d,]+)(?=円)", delivery_charge_html)
|
||||
if handling_match:
|
||||
total_commission_fee = int(handling_match.group(1).replace(",", ""))
|
||||
logger.debug("手续费: %s", total_commission_fee)
|
||||
except PlaywrightTimeoutError:
|
||||
pass
|
||||
|
||||
table = page.locator("#container #main3 #primary table.item_box")
|
||||
await table.wait_for(state="visible", timeout=10000)
|
||||
|
||||
rows = table.locator("tr.item_row")
|
||||
for i in range(await rows.count()):
|
||||
row = rows.nth(i)
|
||||
logger.debug("解析商品 %s", i)
|
||||
|
||||
link_node = row.locator("td.photo_box a").first
|
||||
href = await link_node.get_attribute("href") or ""
|
||||
|
||||
img_node = row.locator("td.photo_box img").first
|
||||
goods_image = await img_node.get_attribute("src") or ""
|
||||
|
||||
name_node = row.locator("td.item_detail .item_name a").first
|
||||
goods_name = (await name_node.text_content() or "").strip()
|
||||
|
||||
price_node = row.locator("td.price").first
|
||||
price_text = (await price_node.text_content() or "").strip()
|
||||
goods_price = format_price(price_text) or 0
|
||||
|
||||
quantity = 1
|
||||
qty_input = row.locator("input[name*='qty'], input[name*='quantity']").first
|
||||
if await qty_input.count():
|
||||
qty_val = (await qty_input.get_attribute("value") or "").strip()
|
||||
if qty_val.isdigit():
|
||||
quantity = int(qty_val)
|
||||
else:
|
||||
qty_select = row.locator("select[name*='qty'], select[name*='quantity'], select.quantity").first
|
||||
if await qty_select.count():
|
||||
selected = qty_select.locator("option[selected]").first
|
||||
if await selected.count():
|
||||
qty_val = (await selected.get_attribute("value") or "").strip()
|
||||
if qty_val.isdigit():
|
||||
quantity = int(qty_val)
|
||||
|
||||
items.append(
|
||||
{
|
||||
"goods_id": extract_product_id(href),
|
||||
"goods_name": goods_name,
|
||||
"goods_price": goods_price,
|
||||
"goods_image": goods_image,
|
||||
"quantity": quantity,
|
||||
}
|
||||
)
|
||||
|
||||
if total_goods_amount <= 0:
|
||||
total_goods_amount = sum(item["goods_price"] * item["quantity"] for item in items)
|
||||
|
||||
return {
|
||||
"total_goods_amount": total_goods_amount,
|
||||
"total_shipping_fee": total_shipping_fee,
|
||||
"total_commission_fee": total_commission_fee,
|
||||
"items": items,
|
||||
}
|
||||
|
||||
def _parse_cargo_html(self, html: str) -> dict:
|
||||
tree = HTMLParser(html)
|
||||
|
||||
items = []
|
||||
# 根据常见的骏河屋购物车结构进行解析
|
||||
item_nodes = tree.css("table.cart_table tr.item_row, .cart-item")
|
||||
if not item_nodes:
|
||||
item_nodes = tree.css("div.item_box") # Fallback
|
||||
|
||||
for node in item_nodes:
|
||||
# 解析商品ID
|
||||
id_node = node.css_first("input[name='goods_id'], .goods_id")
|
||||
goods_id = id_node.attributes.get("value", "") if id_node else ""
|
||||
if not goods_id:
|
||||
# 尝试从链接提取
|
||||
link = node.css_first("a[href*='/product/detail/']")
|
||||
if link:
|
||||
href = link.attributes.get("href", "")
|
||||
import re
|
||||
match = re.search(r'/product/detail/([A-Za-z0-9]+)', href)
|
||||
if match:
|
||||
goods_id = match.group(1)
|
||||
|
||||
if not goods_id:
|
||||
continue
|
||||
|
||||
# 解析商品名称
|
||||
name_node = node.css_first(".item_name, .title, p.name")
|
||||
goods_name = name_node.text().strip() if name_node else f"product-{goods_id}"
|
||||
|
||||
# 解析商品价格
|
||||
price_node = node.css_first(".price, .item_price, td.price")
|
||||
price_text = price_node.text().strip() if price_node else "0"
|
||||
goods_price = format_price(price_text) or 0
|
||||
|
||||
# 解析商品图片
|
||||
img_node = node.css_first("img")
|
||||
goods_image = img_node.attributes.get("src", "") if img_node else ""
|
||||
|
||||
items.append({
|
||||
"goods_id": goods_id,
|
||||
"goods_name": goods_name,
|
||||
"goods_price": goods_price,
|
||||
"goods_image": goods_image,
|
||||
"number": 1
|
||||
})
|
||||
|
||||
# 提取金额信息
|
||||
total_goods_amount = 0
|
||||
total_shipping_fee = 0
|
||||
total_commission_fee = 0
|
||||
|
||||
# 寻找包含金额的总计区域
|
||||
summary_nodes = tree.css(".summary_box, .total_box, table.total_table tr")
|
||||
for node in summary_nodes:
|
||||
text = node.text().replace(" ", "").replace("\n", "")
|
||||
if "商品合計" in text or "小計" in text:
|
||||
price_node = node.css_first(".price, td:last-child")
|
||||
if price_node:
|
||||
total_goods_amount = format_price(price_node.text().strip()) or 0
|
||||
elif "送料" in text:
|
||||
price_node = node.css_first(".price, td:last-child")
|
||||
if price_node:
|
||||
total_shipping_fee = format_price(price_node.text().strip()) or 0
|
||||
elif "通信販売手数料" in text or "代引手数料" in text or "手数料" in text:
|
||||
price_node = node.css_first(".price, td:last-child")
|
||||
if price_node:
|
||||
total_commission_fee = format_price(price_node.text().strip()) or 0
|
||||
|
||||
return {
|
||||
"items": items,
|
||||
"total_goods_amount": total_goods_amount,
|
||||
"total_shipping_fee": total_shipping_fee,
|
||||
"total_commission_fee": total_commission_fee
|
||||
}
|
||||
|
||||
async def clear_cart(self, page):
|
||||
"""
|
||||
清空购物车
|
||||
"""
|
||||
cart_goods_num = await page.locator("table.item_box tbody tr.item").count()
|
||||
if cart_goods_num < 1:
|
||||
return
|
||||
|
||||
is_delete = 0
|
||||
# 购物车商品数量超过3个后会出现"全削除"按钮
|
||||
if cart_goods_num > 3:
|
||||
try:
|
||||
delete_button = page.locator("#total_box .delbtn-all").get_by_role("button", name="全削除")
|
||||
# 3. 显式等待按钮在页面上可见（容错处理：比如给它最多 3000 毫秒的加载时间）
|
||||
# 如果按钮是一开始就在静态 HTML 里的，这一步也可以省略，直接用下面的 is_visible()
|
||||
await delete_button.wait_for(state="visible", timeout=3000)
|
||||
if await delete_button.is_visible():
|
||||
logger.debug("检测到‘全削除’按钮存在，正在执行点击...")
|
||||
await delete_button.click()
|
||||
is_delete = 1
|
||||
except Exception:
|
||||
logger.debug("未检测到‘全削除’按钮。")
|
||||
|
||||
if is_delete == 1:
|
||||
return
|
||||
|
||||
# 遍历删除
|
||||
item_selector = "table.item_box tbody tr.item"
|
||||
remove_selector = "table.item_box a.remove_product"
|
||||
|
||||
while True:
|
||||
cart_goods_num = await page.locator(item_selector).count()
|
||||
if cart_goods_num == 0:
|
||||
break
|
||||
|
||||
remove_link = page.locator(remove_selector).first
|
||||
|
||||
try:
|
||||
await remove_link.click()
|
||||
# 删除按钮不会触发弹窗，直接等待购物车条目数减少即可。
|
||||
await page.wait_for_function(
|
||||
"""({selector, expectedCount}) => {
|
||||
return document.querySelectorAll(selector).length < expectedCount;
|
||||
}""",
|
||||
arg={"selector": item_selector, "expectedCount": cart_goods_num},
|
||||
timeout=15_000,
|
||||
)
|
||||
await page.wait_for_timeout(random.randint(800, 1500))
|
||||
except PlaywrightTimeoutError:
|
||||
latest_count = await page.locator(item_selector).count()
|
||||
if latest_count < cart_goods_num:
|
||||
continue
|
||||
|
||||
await page.wait_for_timeout(1000)
|
||||
Reference in New Issue
Block a user