workspace
This commit is contained in:
@@ -28,6 +28,7 @@ from app.utils.parse_util import (
|
|||||||
get_handle_fee,
|
get_handle_fee,
|
||||||
get_shipping_fee,
|
get_shipping_fee,
|
||||||
)
|
)
|
||||||
|
from surugaya_common import redis_keys
|
||||||
|
|
||||||
router = APIRouter(prefix="/api", tags=["scrape"])
|
router = APIRouter(prefix="/api", tags=["scrape"])
|
||||||
|
|
||||||
@@ -202,7 +203,7 @@ async def order_add_purchase_task(
|
|||||||
code=500,
|
code=500,
|
||||||
)
|
)
|
||||||
|
|
||||||
stream_name = "jp_surugaya_purchase_task_dev" if payload.is_dev == "1" else "jp_surugaya_purchase_task"
|
stream_name = redis_keys.purchase_task_stream(payload.is_dev)
|
||||||
params_str = payload.model_dump_json(by_alias=True)
|
params_str = payload.model_dump_json(by_alias=True)
|
||||||
|
|
||||||
await container.session_store._redis.xadd(
|
await container.session_store._redis.xadd(
|
||||||
@@ -259,8 +260,7 @@ async def order_add_trade_monitor(
|
|||||||
code=400,
|
code=400,
|
||||||
)
|
)
|
||||||
|
|
||||||
queue_name = "jp_surugaya_trade_delay_dev" if payload.is_dev == "1" else "jp_surugaya_trade_delay"
|
queue_name = redis_keys.trade_delay_zset(node_id, payload.is_dev)
|
||||||
queue_name += f":{node_id}"
|
|
||||||
|
|
||||||
score = int(time.time()) + 3
|
score = int(time.time()) + 3
|
||||||
member = json.dumps(payload.payload)
|
member = json.dumps(payload.payload)
|
||||||
@@ -306,18 +306,18 @@ async def trade(
|
|||||||
)
|
)
|
||||||
|
|
||||||
trade_data = await container.session_store._redis.hget(
|
trade_data = await container.session_store._redis.hget(
|
||||||
name="jp_surugaya_trade",
|
name=redis_keys.trade_hash(),
|
||||||
key=trade_code,
|
key=trade_code,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not trade_data:
|
if not trade_data:
|
||||||
# 去重:同一 trade_code 在 3600s 内不重复下发
|
# 去重:同一 trade_code 在 3600s 内不重复下发
|
||||||
dedup_key = f"jp_surugaya_trade_pending:{trade_code}"
|
dedup_key = redis_keys.trade_pending_key(trade_code)
|
||||||
dedup_exists = await container.session_store._redis.get(dedup_key)
|
dedup_exists = await container.session_store._redis.get(dedup_key)
|
||||||
if not dedup_exists:
|
if not dedup_exists:
|
||||||
await container.session_store._redis.setex(dedup_key, 3600, "1")
|
await container.session_store._redis.setex(dedup_key, 3600, "1")
|
||||||
await container.session_store._redis.xadd(
|
await container.session_store._redis.xadd(
|
||||||
name="jp_surugaya_trade_fetch",
|
name=redis_keys.trade_fetch_stream(),
|
||||||
fields={"payload": json.dumps({"trade_code": trade_code})},
|
fields={"payload": json.dumps({"trade_code": trade_code})},
|
||||||
maxlen=100,
|
maxlen=100,
|
||||||
approximate=True,
|
approximate=True,
|
||||||
|
|||||||
@@ -0,0 +1,21 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "surugaya-common"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Shared contracts (Redis keys, future models) for jp_surugaya services"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
dependencies = [
|
||||||
|
"pydantic>=2.0.0,<3.0.0",
|
||||||
|
"redis>=5.0.0,<6.0.0",
|
||||||
|
]
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/surugaya_common"]
|
||||||
|
|
||||||
|
# NOTE (deferred): redis pin here intentionally matches the API service (<6).
|
||||||
|
# The worker's redis==8 requirement is NOT introduced in this increment, so there
|
||||||
|
# is no conflict to resolve yet. Do not add fastapi/playwright/opencv/numpy/
|
||||||
|
# pyautogui/pillow here — surugaya_common must stay light.
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
"""Shared contracts for jp_surugaya services (Redis keys, future models)."""
|
||||||
|
|
||||||
|
from surugaya_common import redis_keys
|
||||||
|
|
||||||
|
__all__ = ["redis_keys"]
|
||||||
@@ -0,0 +1,80 @@
|
|||||||
|
"""Centralized Redis key contract shared across jp_surugaya services.
|
||||||
|
|
||||||
|
Single source of truth for every Redis key schema. Helper functions reproduce
|
||||||
|
the EXACT literals currently produced inline by the API service so behaviour is
|
||||||
|
byte-for-byte identical after refactoring.
|
||||||
|
|
||||||
|
Current literals (as of this extraction):
|
||||||
|
purchase task stream : jp_surugaya_purchase_task[_dev]
|
||||||
|
trade delay zset : jp_surugaya_trade_delay[_dev]:{node_id}
|
||||||
|
trade hash : jp_surugaya_trade
|
||||||
|
trade pending dedup : jp_surugaya_trade_pending:{trade_code}
|
||||||
|
trade fetch stream : jp_surugaya_trade_fetch
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
# --- Base key constants (no suffixes applied) -------------------------------
|
||||||
|
|
||||||
|
PURCHASE_TASK_STREAM: str = "jp_surugaya_purchase_task"
|
||||||
|
TRADE_DELAY_ZSET: str = "jp_surugaya_trade_delay"
|
||||||
|
TRADE_HASH: str = "jp_surugaya_trade"
|
||||||
|
TRADE_PENDING_PREFIX: str = "jp_surugaya_trade_pending"
|
||||||
|
TRADE_FETCH_STREAM: str = "jp_surugaya_trade_fetch"
|
||||||
|
|
||||||
|
# Suffix appended to dev-mode keys.
|
||||||
|
_DEV_SUFFIX: str = "_dev"
|
||||||
|
|
||||||
|
|
||||||
|
def _is_dev(is_dev: object) -> bool:
|
||||||
|
"""Normalize the dev flag.
|
||||||
|
|
||||||
|
The API passes payload.is_dev which is the string "1" for dev mode.
|
||||||
|
Accepts the raw string flag ("1"), a bool, or anything truthy-comparable
|
||||||
|
while preserving the exact original semantics (is_dev == "1").
|
||||||
|
"""
|
||||||
|
if isinstance(is_dev, str):
|
||||||
|
return is_dev == "1"
|
||||||
|
return bool(is_dev)
|
||||||
|
|
||||||
|
|
||||||
|
def purchase_task_stream(is_dev: object = None) -> str:
|
||||||
|
"""Redis Stream key for the purchase task queue.
|
||||||
|
|
||||||
|
Reproduces:
|
||||||
|
"jp_surugaya_purchase_task_dev" if is_dev == "1" else "jp_surugaya_purchase_task"
|
||||||
|
"""
|
||||||
|
if _is_dev(is_dev):
|
||||||
|
return f"{PURCHASE_TASK_STREAM}{_DEV_SUFFIX}"
|
||||||
|
return PURCHASE_TASK_STREAM
|
||||||
|
|
||||||
|
|
||||||
|
def trade_delay_zset(node_id: object, is_dev: object = None) -> str:
|
||||||
|
"""Redis Sorted Set key for delayed trade monitoring, scoped by node_id.
|
||||||
|
|
||||||
|
Reproduces:
|
||||||
|
base = "jp_surugaya_trade_delay_dev" if is_dev == "1" else "jp_surugaya_trade_delay"
|
||||||
|
base + f":{node_id}"
|
||||||
|
"""
|
||||||
|
base = TRADE_DELAY_ZSET
|
||||||
|
if _is_dev(is_dev):
|
||||||
|
base = f"{base}{_DEV_SUFFIX}"
|
||||||
|
return f"{base}:{node_id}"
|
||||||
|
|
||||||
|
|
||||||
|
def trade_hash() -> str:
|
||||||
|
"""Redis Hash key storing trade order details keyed by trade_code."""
|
||||||
|
return TRADE_HASH
|
||||||
|
|
||||||
|
|
||||||
|
def trade_pending_key(trade_code: object) -> str:
|
||||||
|
"""TTL dedup key for a pending trade fetch.
|
||||||
|
|
||||||
|
Reproduces: f"jp_surugaya_trade_pending:{trade_code}"
|
||||||
|
"""
|
||||||
|
return f"{TRADE_PENDING_PREFIX}:{trade_code}"
|
||||||
|
|
||||||
|
|
||||||
|
def trade_fetch_stream() -> str:
|
||||||
|
"""Redis Stream key for trade fetch tasks."""
|
||||||
|
return TRADE_FETCH_STREAM
|
||||||
@@ -14,6 +14,7 @@ dependencies = [
|
|||||||
"selectolax>=0.3.21,<1.0.0",
|
"selectolax>=0.3.21,<1.0.0",
|
||||||
"starlette>=0.37.0,<1.0.0",
|
"starlette>=0.37.0,<1.0.0",
|
||||||
"uvicorn[standard]>=0.30.0,<1.0.0",
|
"uvicorn[standard]>=0.30.0,<1.0.0",
|
||||||
|
"surugaya-common",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
@@ -29,3 +30,9 @@ testpaths = ["tests"]
|
|||||||
|
|
||||||
[tool.uv]
|
[tool.uv]
|
||||||
package = false
|
package = false
|
||||||
|
|
||||||
|
[tool.uv.workspace]
|
||||||
|
members = ["packages/*"]
|
||||||
|
|
||||||
|
[tool.uv.sources]
|
||||||
|
surugaya-common = { workspace = true }
|
||||||
|
|||||||
@@ -2,6 +2,12 @@ version = 1
|
|||||||
revision = 3
|
revision = 3
|
||||||
requires-python = ">=3.12"
|
requires-python = ">=3.12"
|
||||||
|
|
||||||
|
[manifest]
|
||||||
|
members = [
|
||||||
|
"surugaya-common",
|
||||||
|
"surugaya-scraper-service",
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "annotated-doc"
|
name = "annotated-doc"
|
||||||
version = "0.0.4"
|
version = "0.0.4"
|
||||||
@@ -521,6 +527,21 @@ wheels = [
|
|||||||
{ url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" },
|
{ url = "https://files.pythonhosted.org/packages/81/0d/13d1d239a25cbfb19e740db83143e95c772a1fe10202dda4b76792b114dd/starlette-0.52.1-py3-none-any.whl", hash = "sha256:0029d43eb3d273bc4f83a08720b4912ea4b071087a3b48db01b7c839f7954d74", size = 74272, upload-time = "2026-01-18T13:34:09.188Z" },
|
||||||
]
|
]
|
||||||
|
|
||||||
|
[[package]]
|
||||||
|
name = "surugaya-common"
|
||||||
|
version = "0.1.0"
|
||||||
|
source = { editable = "packages/surugaya_common" }
|
||||||
|
dependencies = [
|
||||||
|
{ name = "pydantic" },
|
||||||
|
{ name = "redis" },
|
||||||
|
]
|
||||||
|
|
||||||
|
[package.metadata]
|
||||||
|
requires-dist = [
|
||||||
|
{ name = "pydantic", specifier = ">=2.0.0,<3.0.0" },
|
||||||
|
{ name = "redis", specifier = ">=5.0.0,<6.0.0" },
|
||||||
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "surugaya-scraper-service"
|
name = "surugaya-scraper-service"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
@@ -535,6 +556,7 @@ dependencies = [
|
|||||||
{ name = "redis" },
|
{ name = "redis" },
|
||||||
{ name = "selectolax" },
|
{ name = "selectolax" },
|
||||||
{ name = "starlette" },
|
{ name = "starlette" },
|
||||||
|
{ name = "surugaya-common" },
|
||||||
{ name = "uvicorn", extra = ["standard"] },
|
{ name = "uvicorn", extra = ["standard"] },
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -557,6 +579,7 @@ requires-dist = [
|
|||||||
{ name = "redis", specifier = ">=5.0.0,<6.0.0" },
|
{ name = "redis", specifier = ">=5.0.0,<6.0.0" },
|
||||||
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
|
{ name = "selectolax", specifier = ">=0.3.21,<1.0.0" },
|
||||||
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" },
|
{ name = "starlette", specifier = ">=0.37.0,<1.0.0" },
|
||||||
|
{ name = "surugaya-common", editable = "packages/surugaya_common" },
|
||||||
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
|
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0,<1.0.0" },
|
||||||
]
|
]
|
||||||
provides-extras = ["dev"]
|
provides-extras = ["dev"]
|
||||||
|
|||||||
Reference in New Issue
Block a user