Files
VptPassiveAdapter/fs/file_list_cache.go
Jerry Yan 10e39a506c feat(task): 优化文件列表获取逻辑并添加缓存机制
- 实现按时间前缀获取文件列表,支持小时级目录检索
- 添加降级机制,当时间前缀方式无法找到文件时回退到按天目录
- 在适配器层添加单例模式和客户端连接池管理
- 为S3和AliOSS适配器添加文件列表缓存功能
- 修复跨天任务处理逻辑,约束业务不支持跨天操作
- 优化文件去重逻辑,避免重复处理相同文件
- 添加详细的链路追踪和错误处理机制
2025-12-29 18:39:24 +08:00

161 lines
3.3 KiB
Go

package fs
import (
"ZhenTuLocalPassiveAdapter/config"
"ZhenTuLocalPassiveAdapter/dto"
"container/list"
"fmt"
"sync"
"time"
"github.com/patrickmn/go-cache"
"golang.org/x/sync/singleflight"
)
const (
defaultFileListCacheTTLSeconds = 30
defaultFileListCacheMaxEntries = 256
fileListCacheCleanupInterval = 1 * time.Minute
)
func getFileListCacheTTL() time.Duration {
ttlSeconds := config.Config.Record.Cache.FileListTTLSeconds
if ttlSeconds <= 0 {
ttlSeconds = defaultFileListCacheTTLSeconds
}
return time.Duration(ttlSeconds) * time.Second
}
func getFileListCacheMaxEntries() int {
maxEntries := config.Config.Record.Cache.FileListMaxEntries
if maxEntries <= 0 {
maxEntries = defaultFileListCacheMaxEntries
}
return maxEntries
}
type fileListCache struct {
cache *cache.Cache
maxEntries int
mu sync.Mutex
lru *list.List
elements map[string]*list.Element
group singleflight.Group
}
func newFileListCache(ttl time.Duration, maxEntries int) *fileListCache {
if ttl <= 0 {
ttl = defaultFileListCacheTTLSeconds * time.Second
}
if maxEntries <= 0 {
maxEntries = defaultFileListCacheMaxEntries
}
c := cache.New(ttl, fileListCacheCleanupInterval)
result := &fileListCache{
cache: c,
maxEntries: maxEntries,
lru: list.New(),
elements: make(map[string]*list.Element, maxEntries),
}
c.OnEvicted(func(key string, value any) {
result.removeFromLru(key)
})
return result
}
func (c *fileListCache) GetOrLoad(key string, loader func() ([]dto.File, error)) ([]dto.File, bool, bool, error) {
if files, ok := c.Get(key); ok {
return files, true, false, nil
}
value, err, shared := c.group.Do(key, func() (any, error) {
if files, ok := c.Get(key); ok {
return files, nil
}
files, err := loader()
if err != nil {
return nil, err
}
c.Set(key, files)
return files, nil
})
if err != nil {
return nil, false, shared, err
}
files, ok := value.([]dto.File)
if !ok {
return nil, false, shared, fmt.Errorf("缓存返回类型错误: key=%s", key)
}
return files, false, shared, nil
}
func (c *fileListCache) Get(key string) ([]dto.File, bool) {
value, ok := c.cache.Get(key)
if !ok {
c.removeFromLru(key)
return nil, false
}
files, ok := value.([]dto.File)
if !ok {
c.cache.Delete(key)
c.removeFromLru(key)
return nil, false
}
c.touch(key)
return files, true
}
func (c *fileListCache) Set(key string, files []dto.File) {
c.cache.Set(key, files, cache.DefaultExpiration)
c.touch(key)
c.evictIfNeeded()
}
func (c *fileListCache) touch(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.elements[key]; ok {
c.lru.MoveToFront(el)
return
}
c.elements[key] = c.lru.PushFront(key)
}
func (c *fileListCache) removeFromLru(key string) {
c.mu.Lock()
defer c.mu.Unlock()
el, ok := c.elements[key]
if !ok {
return
}
c.lru.Remove(el)
delete(c.elements, key)
}
func (c *fileListCache) evictIfNeeded() {
for {
c.mu.Lock()
if len(c.elements) <= c.maxEntries {
c.mu.Unlock()
return
}
el := c.lru.Back()
if el == nil {
c.mu.Unlock()
return
}
key := el.Value.(string)
c.lru.Remove(el)
delete(c.elements, key)
c.mu.Unlock()
// 注意:不要在持有 c.mu 时调用 c.cache.Delete,否则会和 OnEvicted 回调产生锁顺序死锁风险
c.cache.Delete(key)
}
}