feat(task): 优化文件列表获取逻辑并添加缓存机制

- 实现按时间前缀获取文件列表，支持小时级目录检索 - 添加降级机制，当时间前缀方式无法找到文件时回退到按天目录 - 在适配器层添加单例模式和客户端连接池管理 - 为S3和AliOSS适配器添加文件列表缓存功能 - 修复跨天任务处理逻辑，约束业务不支持跨天操作 - 优化文件去重逻辑，避免重复处理相同文件 - 添加详细的链路追踪和错误处理机制
2025-12-29 18:39:24 +08:00
parent 686401162f
commit 10e39a506c
5 changed files with 178 additions and 47 deletions
--- a/core/task.go
+++ b/core/task.go
@@ -13,6 +13,7 @@ import (
 	"go.opentelemetry.io/otel/codes"
 	"go.uber.org/zap"
 	"path"
+	"time"
 )

 var tracer = otel.Tracer("task")
@@ -25,11 +26,8 @@ func HandleTask(ctx context.Context, device config.DeviceMapping, task dto.Task)
 	span.SetAttributes(attribute.String("task", util.ToJson(task)))
 	span.SetAttributes(attribute.String("device.no", device.DeviceNo))
 	span.SetAttributes(attribute.String("device.name", device.Name))
-	fileList, err := adapter.GetFileList(
-		subCtx,
-		path.Join(device.Name, task.StartTime.Format("2006"+config.Config.FileName.DateSeparator+"01"+config.Config.FileName.DateSeparator+"02")),
-		task.StartTime,
-	)
+	dateDirPath := path.Join(device.Name, task.StartTime.Format("2006"+config.Config.FileName.DateSeparator+"01"+config.Config.FileName.DateSeparator+"02"))
+	fileList, usedTimePrefix, err := getFileListForTask(subCtx, adapter, dateDirPath, task)
 	if err != nil {
 		span.SetAttributes(attribute.String("error", err.Error()))
 		span.SetStatus(codes.Error, "获取文件列表失败")
@@ -38,14 +36,60 @@ func HandleTask(ctx context.Context, device config.DeviceMapping, task dto.Task)
 			zap.Error(err))
 		return nil, err
 	}
+
 	files := util.FilterAndSortFiles(subCtx, fileList, task.StartTime, task.EndTime)
 	if len(files) == 0 {
-		span.SetStatus(codes.Error, "没有找到文件")
-		return nil, fmt.Errorf("没有找到文件")
+		if usedTimePrefix {
+			span.SetAttributes(attribute.Bool("fileList.fallback", true))
+			fallbackFileList, err := adapter.GetFileList(subCtx, dateDirPath, task.StartTime)
+			if err != nil {
+				span.SetAttributes(attribute.String("error", err.Error()))
+				span.SetStatus(codes.Error, "获取文件列表失败")
+				return nil, err
+			}
+			fileList = fallbackFileList
+			files = util.FilterAndSortFiles(subCtx, fileList, task.StartTime, task.EndTime)
+		}
+		if len(files) == 0 {
+			span.SetStatus(codes.Error, "没有找到文件")
+			return nil, fmt.Errorf("没有找到文件")
+		}
 	}
 	span.SetAttributes(attribute.Int("fileCount", len(files)))
+
+	// 如果过滤后的文件列表无法覆盖任务开始时间，说明“按小时前缀”可能漏掉了前一小时的尾巴片段，降级为按天目录列举
+	if usedTimePrefix && files[0].StartTime.After(task.StartTime) {
+		span.SetAttributes(attribute.Bool("fileList.fallback", true))
+		fallbackFileList, err := adapter.GetFileList(subCtx, dateDirPath, task.StartTime)
+		if err != nil {
+			span.SetAttributes(attribute.String("error", err.Error()))
+			span.SetStatus(codes.Error, "获取文件列表失败")
+			return nil, err
+		}
+		fileList = fallbackFileList
+		files = util.FilterAndSortFiles(subCtx, fileList, task.StartTime, task.EndTime)
+		if len(files) == 0 {
+			span.SetStatus(codes.Error, "没有找到文件")
+			return nil, fmt.Errorf("没有找到文件")
+		}
+	}
+
 	constructTask, err := util.CheckFileCoverageAndConstructTask(subCtx, files, task.StartTime, task.EndTime, task)
 	if err != nil {
+		if usedTimePrefix {
+			span.SetAttributes(attribute.Bool("fileList.fallback", true))
+			fallbackFileList, fallbackErr := adapter.GetFileList(subCtx, dateDirPath, task.StartTime)
+			if fallbackErr != nil {
+				span.SetAttributes(attribute.String("error", fallbackErr.Error()))
+				span.SetStatus(codes.Error, "获取文件列表失败")
+				return nil, fallbackErr
+			}
+			files = util.FilterAndSortFiles(subCtx, fallbackFileList, task.StartTime, task.EndTime)
+			constructTask, err = util.CheckFileCoverageAndConstructTask(subCtx, files, task.StartTime, task.EndTime, task)
+		}
+		if err == nil {
+			goto runFfmpeg
+		}
 		span.SetAttributes(attribute.String("error", err.Error()))
 		span.SetStatus(codes.Error, "文件片段检查失败")
 		logger.Error("文件片段检查失败",
@@ -53,6 +97,8 @@ func HandleTask(ctx context.Context, device config.DeviceMapping, task dto.Task)
 			zap.Error(err))
 		return nil, err
 	}
+
+runFfmpeg:
 	ok := util.RunFfmpegTask(subCtx, constructTask)
 	if !ok {
 		span.SetAttributes(attribute.String("error", "ffmpeg任务执行失败"))
@@ -66,3 +112,57 @@ func HandleTask(ctx context.Context, device config.DeviceMapping, task dto.Task)
 		URL:          constructTask.OutputFile,
 	}, nil
 }
+
+func getFileListForTask(ctx context.Context, adapter fs.Adapter, dateDirPath string, task dto.Task) ([]dto.File, bool, error) {
+	storageType := config.Config.Record.Storage.Type
+	if storageType != "s3" && storageType != "alioss" {
+		fileList, err := adapter.GetFileList(ctx, dateDirPath, task.StartTime)
+		return fileList, false, err
+	}
+
+	lookBack := time.Duration(config.Config.Record.Duration) * time.Second
+	if lookBack < time.Minute {
+		lookBack = time.Minute
+	}
+	earliestStart := task.StartTime.Add(-lookBack)
+	if !isSameDate(earliestStart, task.StartTime) {
+		earliestStart = task.StartTime
+	}
+	if !isSameDate(task.EndTime, task.StartTime) {
+		// 约束：按业务约定不跨天；如果出现跨天，直接回退到按天目录列举（但仍只列举开始日期目录）
+		fileList, err := adapter.GetFileList(ctx, dateDirPath, task.StartTime)
+		return fileList, false, err
+	}
+
+	startHour := earliestStart.Hour()
+	endHour := task.EndTime.Hour()
+	if endHour < startHour {
+		// 理论上只有跨天才会出现；按业务约定不跨天，直接回退
+		fileList, err := adapter.GetFileList(ctx, dateDirPath, task.StartTime)
+		return fileList, false, err
+	}
+
+	seen := make(map[string]struct{}, (endHour-startHour+1)*512)
+	var result []dto.File
+	for hour := startHour; hour <= endHour; hour++ {
+		hourPrefix := fmt.Sprintf("%02d", hour)
+		dirPathWithHourPrefix := path.Join(dateDirPath, hourPrefix)
+		fileList, err := adapter.GetFileList(ctx, dirPathWithHourPrefix, task.StartTime)
+		if err != nil {
+			return nil, true, err
+		}
+		for _, file := range fileList {
+			key := file.Path + "/" + file.Name
+			if _, ok := seen[key]; ok {
+				continue
+			}
+			seen[key] = struct{}{}
+			result = append(result, file)
+		}
+	}
+	return result, true, nil
+}
+
+func isSameDate(a, b time.Time) bool {
+	return a.Year() == b.Year() && a.Month() == b.Month() && a.Day() == b.Day()
+}