You've already forked FrameTour-RenderWorker
feat(cache): 添加素材缓存功能以避免重复下载
- 新增素材缓存配置选项包括启用状态、缓存目录和最大缓存大小 - 实现 MaterialCache 类提供缓存存储和检索功能 - 修改 download_file 方法支持缓存下载模式 - 添加缓存清理机制使用 LRU 策略管理磁盘空间 - 配置默认值优化本地开发体验 - 实现缓存统计和监控功能
This commit is contained in:
@@ -19,6 +19,7 @@ from domain.task import Task
|
||||
from domain.result import TaskResult, ErrorCode
|
||||
from domain.config import WorkerConfig
|
||||
from services import storage
|
||||
from services.cache import MaterialCache
|
||||
from constant import (
|
||||
HW_ACCEL_NONE, HW_ACCEL_QSV, HW_ACCEL_CUDA,
|
||||
VIDEO_ENCODE_PARAMS, VIDEO_ENCODE_PARAMS_QSV, VIDEO_ENCODE_PARAMS_CUDA
|
||||
@@ -260,6 +261,11 @@ class BaseHandler(TaskHandler, ABC):
|
||||
"""
|
||||
self.config = config
|
||||
self.api_client = api_client
|
||||
self.material_cache = MaterialCache(
|
||||
cache_dir=config.cache_dir,
|
||||
enabled=config.cache_enabled,
|
||||
max_size_gb=config.cache_max_size_gb
|
||||
)
|
||||
|
||||
def get_video_encode_args(self) -> List[str]:
|
||||
"""
|
||||
@@ -333,14 +339,15 @@ class BaseHandler(TaskHandler, ABC):
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to cleanup work directory {work_dir}: {e}")
|
||||
|
||||
def download_file(self, url: str, dest: str, timeout: int = None) -> bool:
|
||||
def download_file(self, url: str, dest: str, timeout: int = None, use_cache: bool = True) -> bool:
|
||||
"""
|
||||
下载文件
|
||||
下载文件(支持缓存)
|
||||
|
||||
Args:
|
||||
url: 文件 URL
|
||||
dest: 目标路径
|
||||
timeout: 超时时间(秒)
|
||||
use_cache: 是否使用缓存(默认 True)
|
||||
|
||||
Returns:
|
||||
是否成功
|
||||
@@ -349,7 +356,13 @@ class BaseHandler(TaskHandler, ABC):
|
||||
timeout = self.config.download_timeout
|
||||
|
||||
try:
|
||||
result = storage.download_file(url, dest, timeout=timeout)
|
||||
if use_cache:
|
||||
# 使用缓存下载
|
||||
result = self.material_cache.get_or_download(url, dest, timeout=timeout)
|
||||
else:
|
||||
# 直接下载(不走缓存)
|
||||
result = storage.download_file(url, dest, timeout=timeout)
|
||||
|
||||
if result:
|
||||
file_size = os.path.getsize(dest) if os.path.exists(dest) else 0
|
||||
logger.debug(f"Downloaded: {url} -> {dest} ({file_size} bytes)")
|
||||
|
||||
Reference in New Issue
Block a user