You've already forked VptPassiveAdapter
- 实现按时间前缀获取文件列表,支持小时级目录检索 - 添加降级机制,当时间前缀方式无法找到文件时回退到按天目录 - 在适配器层添加单例模式和客户端连接池管理 - 为S3和AliOSS适配器添加文件列表缓存功能 - 修复跨天任务处理逻辑,约束业务不支持跨天操作 - 优化文件去重逻辑,避免重复处理相同文件 - 添加详细的链路追踪和错误处理机制
161 lines
3.3 KiB
Go
161 lines
3.3 KiB
Go
package fs
|
|
|
|
import (
|
|
"ZhenTuLocalPassiveAdapter/config"
|
|
"ZhenTuLocalPassiveAdapter/dto"
|
|
"container/list"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/patrickmn/go-cache"
|
|
"golang.org/x/sync/singleflight"
|
|
)
|
|
|
|
const (
|
|
defaultFileListCacheTTLSeconds = 30
|
|
defaultFileListCacheMaxEntries = 256
|
|
fileListCacheCleanupInterval = 1 * time.Minute
|
|
)
|
|
|
|
func getFileListCacheTTL() time.Duration {
|
|
ttlSeconds := config.Config.Record.Cache.FileListTTLSeconds
|
|
if ttlSeconds <= 0 {
|
|
ttlSeconds = defaultFileListCacheTTLSeconds
|
|
}
|
|
return time.Duration(ttlSeconds) * time.Second
|
|
}
|
|
|
|
func getFileListCacheMaxEntries() int {
|
|
maxEntries := config.Config.Record.Cache.FileListMaxEntries
|
|
if maxEntries <= 0 {
|
|
maxEntries = defaultFileListCacheMaxEntries
|
|
}
|
|
return maxEntries
|
|
}
|
|
|
|
type fileListCache struct {
|
|
cache *cache.Cache
|
|
maxEntries int
|
|
|
|
mu sync.Mutex
|
|
lru *list.List
|
|
elements map[string]*list.Element
|
|
|
|
group singleflight.Group
|
|
}
|
|
|
|
func newFileListCache(ttl time.Duration, maxEntries int) *fileListCache {
|
|
if ttl <= 0 {
|
|
ttl = defaultFileListCacheTTLSeconds * time.Second
|
|
}
|
|
if maxEntries <= 0 {
|
|
maxEntries = defaultFileListCacheMaxEntries
|
|
}
|
|
|
|
c := cache.New(ttl, fileListCacheCleanupInterval)
|
|
result := &fileListCache{
|
|
cache: c,
|
|
maxEntries: maxEntries,
|
|
lru: list.New(),
|
|
elements: make(map[string]*list.Element, maxEntries),
|
|
}
|
|
c.OnEvicted(func(key string, value any) {
|
|
result.removeFromLru(key)
|
|
})
|
|
return result
|
|
}
|
|
|
|
func (c *fileListCache) GetOrLoad(key string, loader func() ([]dto.File, error)) ([]dto.File, bool, bool, error) {
|
|
if files, ok := c.Get(key); ok {
|
|
return files, true, false, nil
|
|
}
|
|
|
|
value, err, shared := c.group.Do(key, func() (any, error) {
|
|
if files, ok := c.Get(key); ok {
|
|
return files, nil
|
|
}
|
|
files, err := loader()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
c.Set(key, files)
|
|
return files, nil
|
|
})
|
|
if err != nil {
|
|
return nil, false, shared, err
|
|
}
|
|
files, ok := value.([]dto.File)
|
|
if !ok {
|
|
return nil, false, shared, fmt.Errorf("缓存返回类型错误: key=%s", key)
|
|
}
|
|
return files, false, shared, nil
|
|
}
|
|
|
|
func (c *fileListCache) Get(key string) ([]dto.File, bool) {
|
|
value, ok := c.cache.Get(key)
|
|
if !ok {
|
|
c.removeFromLru(key)
|
|
return nil, false
|
|
}
|
|
files, ok := value.([]dto.File)
|
|
if !ok {
|
|
c.cache.Delete(key)
|
|
c.removeFromLru(key)
|
|
return nil, false
|
|
}
|
|
c.touch(key)
|
|
return files, true
|
|
}
|
|
|
|
func (c *fileListCache) Set(key string, files []dto.File) {
|
|
c.cache.Set(key, files, cache.DefaultExpiration)
|
|
c.touch(key)
|
|
c.evictIfNeeded()
|
|
}
|
|
|
|
func (c *fileListCache) touch(key string) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
|
|
if el, ok := c.elements[key]; ok {
|
|
c.lru.MoveToFront(el)
|
|
return
|
|
}
|
|
c.elements[key] = c.lru.PushFront(key)
|
|
}
|
|
|
|
func (c *fileListCache) removeFromLru(key string) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
|
|
el, ok := c.elements[key]
|
|
if !ok {
|
|
return
|
|
}
|
|
c.lru.Remove(el)
|
|
delete(c.elements, key)
|
|
}
|
|
|
|
func (c *fileListCache) evictIfNeeded() {
|
|
for {
|
|
c.mu.Lock()
|
|
if len(c.elements) <= c.maxEntries {
|
|
c.mu.Unlock()
|
|
return
|
|
}
|
|
el := c.lru.Back()
|
|
if el == nil {
|
|
c.mu.Unlock()
|
|
return
|
|
}
|
|
key := el.Value.(string)
|
|
c.lru.Remove(el)
|
|
delete(c.elements, key)
|
|
c.mu.Unlock()
|
|
|
|
// 注意:不要在持有 c.mu 时调用 c.cache.Delete,否则会和 OnEvicted 回调产生锁顺序死锁风险
|
|
c.cache.Delete(key)
|
|
}
|
|
}
|