VptPassiveAdapter/fs/s3_adapter.go

package fs

import (
	"ZhenTuLocalPassiveAdapter/config"
	"ZhenTuLocalPassiveAdapter/dto"
	"ZhenTuLocalPassiveAdapter/logger"
	"ZhenTuLocalPassiveAdapter/util"
	"context"
	"fmt"
	"github.com/aws/aws-sdk-go-v2/credentials"
	"go.uber.org/zap"
	"path"
	"sort"
	"sync"
	"time"

	"github.com/aws/aws-sdk-go-v2/aws"
	"github.com/aws/aws-sdk-go-v2/service/s3"
	"go.opentelemetry.io/otel/attribute"
	"go.opentelemetry.io/otel/codes"
)

type S3Adapter struct {
	StorageConfig     config.StorageConfig
	fileListCacheOnce sync.Once
	fileListCache     *fileListCache

	clientOnce sync.Once
	clientErr  error
	s3Client   *s3.Client
}

func (s *S3Adapter) getClient() (*s3.Client, error) {
	s.clientOnce.Do(func() {
		const defaultRegion = "us-east-1"
		resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
			return aws.Endpoint{
				PartitionID:       "aws",
				URL:               s.StorageConfig.S3.Endpoint, // or where ever you ran minio
				SigningRegion:     defaultRegion,
				HostnameImmutable: true,
			}, nil
		})
		creds := credentials.NewStaticCredentialsProvider(s.StorageConfig.S3.AkId, s.StorageConfig.S3.AkSec, "")
		cfg := aws.Config{
			Credentials:      creds,
			Region:           defaultRegion,
			EndpointResolver: resolver,
		}
		s.s3Client = s3.NewFromConfig(cfg)
		if s.s3Client == nil {
			s.clientErr = fmt.Errorf("创建S3客户端失败")
		}
	})
	if s.clientErr != nil {
		return nil, s.clientErr
	}
	if s.s3Client == nil {
		return nil, fmt.Errorf("S3客户端未初始化")
	}
	return s.s3Client, nil
}

func (s *S3Adapter) getFileListCache() *fileListCache {
	s.fileListCacheOnce.Do(func() {
		s.fileListCache = newFileListCache(getFileListCacheTTL(), getFileListCacheMaxEntries())
	})
	return s.fileListCache
}

func (s *S3Adapter) GetFileList(ctx context.Context, dirPath string, relDt time.Time) ([]dto.File, error) {
	_, span := tracer.Start(ctx, "GetFileList_s3")
	defer span.End()

	span.SetAttributes(attribute.String("path", dirPath))
	span.SetAttributes(attribute.String("relativeDate", relDt.Format("2006-01-02")))
	if s.StorageConfig.S3.Bucket == "" {
		span.SetAttributes(attribute.String("error", "未配置S3存储桶"))
		span.SetStatus(codes.Error, "未配置S3存储桶")
		return nil, fmt.Errorf("未配置S3存储桶")
	}

	cacheKey := fmt.Sprintf("%s_%s", dirPath, relDt.Format("2006-01-02"))
	fileListCache := s.getFileListCache()
	if cachedFiles, ok := fileListCache.Get(cacheKey); ok {
		logger.Debug("获取已缓存列表", zap.String("cacheKey", cacheKey))
		span.SetAttributes(attribute.Bool("cache.hit", true))
		span.SetAttributes(attribute.Int("file.count", len(cachedFiles)))
		span.SetStatus(codes.Ok, "文件读取成功")
		return cachedFiles, nil
	}

	fileList, hit, shared, err := fileListCache.GetOrLoad(cacheKey, func() ([]dto.File, error) {
		listObjectsInput := &s3.ListObjectsV2Input{
			Bucket:  aws.String(s.StorageConfig.S3.Bucket),
			Prefix:  aws.String(path.Join(s.StorageConfig.S3.Prefix, dirPath)),
			MaxKeys: aws.Int32(1000),
		}

		client, err := s.getClient()
		if err != nil {
			return nil, err
		}

		var resultFiles []dto.File
		var continuationToken *string

		for {
			if continuationToken != nil {
				listObjectsInput.ContinuationToken = continuationToken
			}

			result, err := client.ListObjectsV2(context.TODO(), listObjectsInput)
			if err != nil {
				return nil, err
			}

			for _, object := range result.Contents {
				key := *object.Key
				if !util.IsVideoFile(path.Base(key)) {
					continue
				}
				startTime, stopTime, err := util.ParseStartStopTime(path.Base(key), relDt)
				if err != nil {
					continue
				}
				if stopTime.IsZero() {
					stopTime = startTime
				}
				if startTime.Equal(stopTime) {
					stopTime = stopTime.Add(time.Second * time.Duration(config.Config.Record.Duration))
				}
				presignClient := s3.NewPresignClient(client)
				request, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{
					Bucket: aws.String(s.StorageConfig.S3.Bucket),
					Key:    aws.String(key),
				}, func(presignOptions *s3.PresignOptions) {
					presignOptions.Expires = 10 * time.Minute
				})
				if err != nil {
					logger.Error("生成预签名URL失败", zap.Error(err))
					continue
				}
				resultFiles = append(resultFiles, dto.File{
					BasePath:  s.StorageConfig.S3.Bucket,
					Name:      path.Base(key),
					Path:      path.Dir(key),
					Url:       request.URL,
					StartTime: startTime,
					EndTime:   stopTime,
				})
			}

			if !*result.IsTruncated {
				break
			}
			continuationToken = result.NextContinuationToken
		}

		sort.Slice(resultFiles, func(i, j int) bool {
			return resultFiles[i].StartTime.Before(resultFiles[j].StartTime)
		})
		return resultFiles, nil
	})
	if err != nil {
		span.SetAttributes(attribute.String("error", err.Error()))
		span.SetStatus(codes.Error, "文件列表读取失败")
		return nil, err
	}

	span.SetAttributes(attribute.Bool("cache.shared", shared))
	span.SetAttributes(attribute.Int("file.count", len(fileList)))
	span.SetStatus(codes.Ok, "文件读取成功")

	if !hit && !shared {
		logger.Debug("缓存文件列表", zap.String("cacheKey", cacheKey))
	}
	return fileList, nil
}