feat(cache): 添加素材缓存功能以避免重复下载

- 新增素材缓存配置选项包括启用状态、缓存目录和最大缓存大小
- 实现 MaterialCache 类提供缓存存储和检索功能
- 修改 download_file 方法支持缓存下载模式
- 添加缓存清理机制使用 LRU 策略管理磁盘空间
- 配置默认值优化本地开发体验
- 实现缓存统计和监控功能
This commit is contained in:
2026-01-17 15:07:12 +08:00
parent d5cd0dca03
commit fe757408b6
3 changed files with 321 additions and 4 deletions

View File

@@ -19,6 +19,7 @@ from domain.task import Task
from domain.result import TaskResult, ErrorCode
from domain.config import WorkerConfig
from services import storage
from services.cache import MaterialCache
from constant import (
HW_ACCEL_NONE, HW_ACCEL_QSV, HW_ACCEL_CUDA,
VIDEO_ENCODE_PARAMS, VIDEO_ENCODE_PARAMS_QSV, VIDEO_ENCODE_PARAMS_CUDA
@@ -260,6 +261,11 @@ class BaseHandler(TaskHandler, ABC):
"""
self.config = config
self.api_client = api_client
self.material_cache = MaterialCache(
cache_dir=config.cache_dir,
enabled=config.cache_enabled,
max_size_gb=config.cache_max_size_gb
)
def get_video_encode_args(self) -> List[str]:
"""
@@ -333,14 +339,15 @@ class BaseHandler(TaskHandler, ABC):
except Exception as e:
logger.warning(f"Failed to cleanup work directory {work_dir}: {e}")
def download_file(self, url: str, dest: str, timeout: int = None) -> bool:
def download_file(self, url: str, dest: str, timeout: int = None, use_cache: bool = True) -> bool:
"""
下载文件
下载文件(支持缓存)
Args:
url: 文件 URL
dest: 目标路径
timeout: 超时时间(秒)
use_cache: 是否使用缓存(默认 True)
Returns:
是否成功
@@ -349,7 +356,13 @@ class BaseHandler(TaskHandler, ABC):
timeout = self.config.download_timeout
try:
result = storage.download_file(url, dest, timeout=timeout)
if use_cache:
# 使用缓存下载
result = self.material_cache.get_or_download(url, dest, timeout=timeout)
else:
# 直接下载(不走缓存)
result = storage.download_file(url, dest, timeout=timeout)
if result:
file_size = os.path.getsize(dest) if os.path.exists(dest) else 0
logger.debug(f"Downloaded: {url} -> {dest} ({file_size} bytes)")