diff --git a/fs/s3_adapter.go b/fs/s3_adapter.go index 286fe10..009cfa5 100644 --- a/fs/s3_adapter.go +++ b/fs/s3_adapter.go @@ -57,44 +57,59 @@ func (s *S3Adapter) GetFileList(dirPath string, relDt time.Time) ([]dto.File, er if err != nil { return nil, err } - result, err := client.ListObjectsV2(context.TODO(), listObjectsInput) - if err != nil { - return nil, err - } var fileList []dto.File - for _, object := range result.Contents { - key := *object.Key - if !util.IsVideoFile(path.Base(key)) { - continue + var continuationToken *string + + for { + if continuationToken != nil { + listObjectsInput.ContinuationToken = continuationToken } - startTime, stopTime, err := util.ParseStartStopTime(path.Base(key), relDt) + + result, err := client.ListObjectsV2(context.TODO(), listObjectsInput) if err != nil { - continue + return nil, err } - if startTime.Equal(stopTime) || stopTime.IsZero() { - stopTime = stopTime.Add(time.Second * time.Duration(config.Config.Record.Duration)) + + for _, object := range result.Contents { + key := *object.Key + if !util.IsVideoFile(path.Base(key)) { + continue + } + startTime, stopTime, err := util.ParseStartStopTime(path.Base(key), relDt) + if err != nil { + continue + } + if startTime.Equal(stopTime) || stopTime.IsZero() { + stopTime = stopTime.Add(time.Second * time.Duration(config.Config.Record.Duration)) + } + presignClient := s3.NewPresignClient(client) + request, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String(s.StorageConfig.S3.Bucket), + Key: aws.String(key), + }, func(presignOptions *s3.PresignOptions) { + presignOptions.Expires = 10 * time.Minute + }) + if err != nil { + log.Println("Error presigning GetObject request:", err) + continue + } + fileList = append(fileList, dto.File{ + BasePath: s.StorageConfig.S3.Bucket, + Name: path.Base(key), + Path: path.Dir(key), + Url: request.URL, + StartTime: startTime, + EndTime: stopTime, + }) } - presignClient := s3.NewPresignClient(client) - request, err := presignClient.PresignGetObject(context.TODO(), &s3.GetObjectInput{ - Bucket: aws.String(s.StorageConfig.S3.Bucket), - Key: aws.String(key), - }, func(presignOptions *s3.PresignOptions) { - presignOptions.Expires = 10 * time.Minute - }) - if err != nil { - log.Println("Error presigning GetObject request:", err) - continue + + if !*result.IsTruncated { + break } - fileList = append(fileList, dto.File{ - BasePath: s.StorageConfig.S3.Bucket, - Name: path.Base(key), - Path: path.Dir(key), - Url: request.URL, - StartTime: startTime, - EndTime: stopTime, - }) + continuationToken = result.NextContinuationToken } + sort.Slice(fileList, func(i, j int) bool { return fileList[i].StartTime.Before(fileList[j].StartTime) })