package fs import ( "ZhenTuLocalPassiveAdapter/config" "ZhenTuLocalPassiveAdapter/dto" "ZhenTuLocalPassiveAdapter/logger" "ZhenTuLocalPassiveAdapter/util" "context" "fmt" "github.com/aws/aws-sdk-go-v2/credentials" "go.uber.org/zap" "path" "sort" "sync" "time" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" ) type S3Adapter struct { StorageConfig config.StorageConfig fileListCacheOnce sync.Once fileListCache *fileListCache clientOnce sync.Once clientErr error s3Client *s3.Client } func (s *S3Adapter) getClient() (*s3.Client, error) { s.clientOnce.Do(func() { const defaultRegion = "us-east-1" resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { return aws.Endpoint{ PartitionID: "aws", URL: s.StorageConfig.S3.Endpoint, // or where ever you ran minio SigningRegion: defaultRegion, HostnameImmutable: true, }, nil }) creds := credentials.NewStaticCredentialsProvider(s.StorageConfig.S3.AkId, s.StorageConfig.S3.AkSec, "") cfg := aws.Config{ Credentials: creds, Region: defaultRegion, EndpointResolver: resolver, } s.s3Client = s3.NewFromConfig(cfg) if s.s3Client == nil { s.clientErr = fmt.Errorf("创建S3客户端失败") } }) if s.clientErr != nil { return nil, s.clientErr } if s.s3Client == nil { return nil, fmt.Errorf("S3客户端未初始化") } return s.s3Client, nil } func (s *S3Adapter) getFileListCache() *fileListCache { s.fileListCacheOnce.Do(func() { s.fileListCache = newFileListCache(getFileListCacheTTL(), getFileListCacheMaxEntries()) }) return s.fileListCache } func (s *S3Adapter) GetFileList(ctx context.Context, dirPath string, relDt time.Time) ([]dto.File, error) { _, span := tracer.Start(ctx, "GetFileList_s3") defer span.End() span.SetAttributes(attribute.String("path", dirPath)) span.SetAttributes(attribute.String("relativeDate", relDt.Format("2006-01-02"))) if s.StorageConfig.S3.Bucket == "" { span.SetAttributes(attribute.String("error", "未配置S3存储桶")) span.SetStatus(codes.Error, "未配置S3存储桶") return nil, fmt.Errorf("未配置S3存储桶") } cacheKey := fmt.Sprintf("%s_%s", dirPath, relDt.Format("2006-01-02")) fileListCache := s.getFileListCache() if cachedFiles, ok := fileListCache.Get(cacheKey); ok { logger.Debug("获取已缓存列表", zap.String("cacheKey", cacheKey)) span.SetAttributes(attribute.Bool("cache.hit", true)) span.SetAttributes(attribute.Int("file.count", len(cachedFiles))) span.SetStatus(codes.Ok, "文件读取成功") return cachedFiles, nil } fileList, hit, shared, err := fileListCache.GetOrLoad(cacheKey, func() ([]dto.File, error) { listObjectsInput := &s3.ListObjectsV2Input{ Bucket: aws.String(s.StorageConfig.S3.Bucket), Prefix: aws.String(path.Join(s.StorageConfig.S3.Prefix, dirPath)), MaxKeys: aws.Int32(1000), } client, err := s.getClient() if err != nil { return nil, err } var resultFiles []dto.File var continuationToken *string for { if continuationToken != nil { listObjectsInput.ContinuationToken = continuationToken } result, err := client.ListObjectsV2(context.TODO(), listObjectsInput) if err != nil { return nil, err } for _, object := range result.Contents { key := *object.Key if !util.IsVideoFile(path.Base(key)) { continue } startTime, stopTime, err := util.ParseStartStopTime(path.Base(key), relDt) if err != nil { continue } if stopTime.IsZero() { stopTime = startTime } if startTime.Equal(stopTime) { stopTime = stopTime.Add(time.Second * time.Duration(config.Config.Record.Duration)) } presignClient := s3.NewPresignClient(client) request, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ Bucket: aws.String(s.StorageConfig.S3.Bucket), Key: aws.String(key), }, func(presignOptions *s3.PresignOptions) { presignOptions.Expires = 10 * time.Minute }) if err != nil { logger.Error("生成预签名URL失败", zap.Error(err)) continue } resultFiles = append(resultFiles, dto.File{ BasePath: s.StorageConfig.S3.Bucket, Name: path.Base(key), Path: path.Dir(key), Url: request.URL, StartTime: startTime, EndTime: stopTime, }) } if !*result.IsTruncated { break } continuationToken = result.NextContinuationToken } sort.Slice(resultFiles, func(i, j int) bool { return resultFiles[i].StartTime.Before(resultFiles[j].StartTime) }) return resultFiles, nil }) if err != nil { span.SetAttributes(attribute.String("error", err.Error())) span.SetStatus(codes.Error, "文件列表读取失败") return nil, err } span.SetAttributes(attribute.Bool("cache.shared", shared)) span.SetAttributes(attribute.Int("file.count", len(fileList))) span.SetStatus(codes.Ok, "文件读取成功") if !hit && !shared { logger.Debug("缓存文件列表", zap.String("cacheKey", cacheKey)) } return fileList, nil }