You've already forked FrameTour-BE
Jerry Yan
95c82cfcf2
- 移除默认存储配置常量 DEFAULT_STORAGE - 简化 UploadStage 中的存储适配器获取逻辑，直接使用 StorageFactory.use() - 移除降级到默认存储的处理机制 - 在 PuzzleGenerateServiceImpl 中复用存储适配器实例 - 移除 SourceRepository 中的 StorageUnsupportedException 导入 - 移除 GoodsServiceImpl 中的 StorageType 枚举导入 - 移除 SourceServiceImpl 中的 ScenicService 依赖注入 - 移除 PrinterServiceImpl 中的复杂存储适配器配置逻辑 - 在 TaskTaskServiceImpl 中统一使用景点存储适配器 - 在 FaceCleaner 中添加新的存储清理逻辑，使用独立的图片存储适配器 - 添加 sourceImageUrlMap 和 sourceScenicIdMap 来优化文件清理逻辑
356 lines
17 KiB
Java
356 lines
17 KiB
Java
package com.ycwl.basic.task;
|
|
|
|
import cn.hutool.core.date.DateUnit;
|
|
import cn.hutool.core.date.DateUtil;
|
|
import com.ycwl.basic.constant.StorageConstant;
|
|
import com.ycwl.basic.facebody.adapter.IFaceBodyAdapter;
|
|
import com.ycwl.basic.integration.scenic.dto.scenic.ScenicV2DTO;
|
|
import com.ycwl.basic.mapper.FaceMapper;
|
|
import com.ycwl.basic.mapper.FaceSampleMapper;
|
|
import com.ycwl.basic.mapper.SourceMapper;
|
|
import com.ycwl.basic.mapper.VideoMapper;
|
|
import com.ycwl.basic.model.pc.face.entity.FaceEntity;
|
|
import com.ycwl.basic.model.pc.face.req.FaceReqQuery;
|
|
import com.ycwl.basic.model.pc.faceSample.entity.FaceSampleEntity;
|
|
import com.ycwl.basic.model.pc.faceSample.req.FaceSampleReqQuery;
|
|
import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO;
|
|
import com.ycwl.basic.integration.common.manager.ScenicConfigManager;
|
|
import com.ycwl.basic.model.pc.scenic.req.ScenicReqQuery;
|
|
import com.ycwl.basic.model.pc.source.req.SourceReqQuery;
|
|
import com.ycwl.basic.model.pc.source.resp.SourceRespVO;
|
|
import com.ycwl.basic.model.pc.video.req.VideoReqQuery;
|
|
import com.ycwl.basic.model.pc.video.resp.VideoRespVO;
|
|
import com.ycwl.basic.repository.ScenicRepository;
|
|
import com.ycwl.basic.service.pc.ScenicService;
|
|
import com.ycwl.basic.storage.StorageFactory;
|
|
import com.ycwl.basic.storage.adapters.IStorageAdapter;
|
|
import com.ycwl.basic.storage.entity.StorageFileObject;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.context.annotation.Profile;
|
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.HashMap;
|
|
import java.util.List;
|
|
import java.util.Map;
|
|
import java.util.Objects;
|
|
|
|
import static com.ycwl.basic.constant.FaceConstant.USER_FACE_DB_NAME;
|
|
|
|
@Component
|
|
@EnableScheduling
|
|
@Slf4j
|
|
@Profile("prod")
|
|
public class FaceCleaner {
|
|
@Autowired
|
|
private FaceSampleMapper faceSampleMapper;
|
|
@Autowired
|
|
private SourceMapper sourceMapper;
|
|
@Autowired
|
|
private VideoMapper videoMapper;
|
|
@Autowired
|
|
private ScenicRepository scenicRepository;
|
|
@Autowired
|
|
private FaceMapper faceMapper;
|
|
@Autowired
|
|
private ScenicService scenicService;
|
|
public static final List<String> disableDeleteScenicIds = List.of("3955650120997015552");
|
|
|
|
@Scheduled(cron = "0 0 1 * * ?")
|
|
public void deleteExpireSample(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.parseLong(scenic.getId());
|
|
log.info("当前景区{}，开始删除人脸样本", scenicId);
|
|
IFaceBodyAdapter adapter = scenicService.getScenicFaceBodyAdapter(scenicId);
|
|
ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId);
|
|
Integer sampleStoreDay = scenicConfig.getInteger("sample_store_day");
|
|
if (sampleStoreDay == null) {
|
|
log.info("当前景区{}，人脸样本保存天数未设置，默认7天", scenic.getId());
|
|
sampleStoreDay = 7;
|
|
}
|
|
Date sampleEndDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -sampleStoreDay);
|
|
List<FaceSampleEntity> faceSampleList = faceSampleMapper.listEntityBeforeDate(scenicId, sampleEndDate);
|
|
if (faceSampleList.isEmpty()) {
|
|
log.info("当前景区{}，人脸样本为空", scenic.getId());
|
|
return;
|
|
}
|
|
faceSampleList.forEach(faceSample -> {
|
|
boolean success = adapter.deleteFace(scenic.getId(), faceSample.getId().toString());
|
|
if (success) {
|
|
log.info("当前景区{}，人脸样本ID{}，删除成功", scenic.getId(), faceSample.getId());
|
|
faceSampleMapper.deleteById(faceSample.getId());
|
|
} else {
|
|
log.info("当前景区{}，人脸样本ID{}，删除失败", scenic.getId(), faceSample.getId());
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 45 2 * * ?")
|
|
public void deleteExpireFace() {
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.parseLong(scenic.getId());
|
|
log.info("当前景区{}，开始删除用户人脸", scenic.getId());
|
|
ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId);
|
|
IFaceBodyAdapter adapter = scenicService.getScenicFaceBodyAdapter(scenicId);
|
|
Integer faceStoreDay = scenicConfig.getInteger("face_store_day");
|
|
if (faceStoreDay == null) {
|
|
log.info("当前景区{}，人脸样本保存天数未设置，默认3天", scenic.getName());
|
|
faceStoreDay = 3;
|
|
}
|
|
FaceReqQuery req = new FaceReqQuery();
|
|
req.setScenicId(scenicId);
|
|
Date faceEndDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -faceStoreDay);
|
|
List<FaceEntity> list = faceMapper.listUnpaidEntityBeforeDate(scenicId, faceEndDate);
|
|
list.forEach(face -> {
|
|
boolean result = adapter.deleteFace(USER_FACE_DB_NAME+face.getScenicId(), face.getId().toString());
|
|
if (result) {
|
|
log.info("当前景区{}，人脸样本ID{}，删除成功", scenic.getId(), face.getId());
|
|
} else {
|
|
result = adapter.deleteFace(USER_FACE_DB_NAME+face.getScenicId(), face.getId().toString());
|
|
if (result) {
|
|
log.info("当前景区{}，人脸样本ID{}，删除成功", scenic.getId(), face.getId());
|
|
} else {
|
|
log.info("当前景区{}，人脸样本ID{}，删除失败", scenic.getId(), face.getId());
|
|
}
|
|
}
|
|
faceMapper.forceDeleteById(face.getId());
|
|
});
|
|
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 0 1 * * ?")
|
|
public void deleteNotBuySource(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId);
|
|
if (scenicConfig == null) {
|
|
log.info("当前景区{}，无配置信息", scenic.getName());
|
|
return;
|
|
}
|
|
if (scenicConfig.getInteger("user_source_expire_day") == null) {
|
|
log.info("当前景区{}，人脸样本过期天数未设置", scenic.getName());
|
|
return;
|
|
}
|
|
int expireDay = scenicConfig.getInteger("user_source_expire_day");
|
|
Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -expireDay);
|
|
int deleteCount = sourceMapper.deleteNotBuyRelations(scenicId, endDate);
|
|
log.info("当前景区{}，删除关联素材{}个", scenic.getName(), deleteCount);
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 15 1 * * ?")
|
|
public void deleteNotBuyVideos(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId);
|
|
if (scenicConfig == null) {
|
|
log.info("当前景区{}，无配置信息", scenic.getName());
|
|
return;
|
|
}
|
|
if (scenicConfig.getInteger("video_store_day") == null) {
|
|
log.info("当前景区{}，VLOG过期天数未设置", scenic.getName());
|
|
return;
|
|
}
|
|
int expireDay = scenicConfig.getInteger("video_store_day");
|
|
Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -expireDay);
|
|
int deleteCount = videoMapper.deleteNotBuyRelations(scenicId, endDate);
|
|
int deleteVideoCount = videoMapper.deleteUselessVideo();
|
|
log.info("当前景区{}，删除VLOG关系{}个，删除VLOG记录{}个", scenic.getName(), deleteCount, deleteVideoCount);
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 30 1 * * ?")
|
|
public void deleteExpiredSource(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId);
|
|
if (scenicConfig == null) {
|
|
log.info("当前景区{}，无配置信息", scenic.getName());
|
|
return;
|
|
}
|
|
int imageSourceExpireDay = 7;
|
|
int videoSourceExpireDay = 7;
|
|
if (scenicConfig.getInteger("image_source_store_day") != null) {
|
|
imageSourceExpireDay = scenicConfig.getInteger("image_source_store_day");
|
|
} else {
|
|
log.info("当前景区{}，原始素材保存天数未设置，默认7天", scenic.getName());
|
|
}
|
|
if (scenicConfig.getInteger("video_source_store_day") != null) {
|
|
videoSourceExpireDay = scenicConfig.getInteger("video_source_store_day");
|
|
} else {
|
|
log.info("当前景区{}，原始素材保存天数未设置，默认7天", scenic.getName());
|
|
}
|
|
if (Boolean.TRUE.equals(scenicConfig.getBoolean("disable_source_video"))) {
|
|
return;
|
|
}
|
|
if (Boolean.TRUE.equals(scenicConfig.getBoolean("disable_source_image"))) {
|
|
return;
|
|
}
|
|
log.info("当前景区{}，开始删除原始素材", scenic.getName());
|
|
Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -videoSourceExpireDay);
|
|
int deleteVideoSourceCount = sourceMapper.deleteNotRelateSource(1, endDate);
|
|
log.info("当前景区{}，删除原始视频素材{}个", scenic.getName(), deleteVideoSourceCount);
|
|
log.info("当前景区{}，开始删除原始图片素材", scenic.getName());
|
|
Date endDate2 = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -imageSourceExpireDay);
|
|
int deleteImageSourceCount = sourceMapper.deleteNotRelateSource(2, endDate2);
|
|
log.info("当前景区{}，删除原始图片素材{}个", scenic.getName(), deleteImageSourceCount);
|
|
int deleteSourceCount = sourceMapper.deleteUselessSource();
|
|
log.info("当前景区{}，删除无用素材{}个", scenic.getName(), deleteSourceCount);
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 0 2 * * ?")
|
|
public void clearOss(){
|
|
cleanSourceOss();
|
|
cleanVideoOss();
|
|
}
|
|
public void cleanSourceOss() {
|
|
log.info("开始清理源视频素材文件");
|
|
List<SourceRespVO> list = sourceMapper.list(new SourceReqQuery());
|
|
Map<Long, String> sourceImageUrlMap = new HashMap<>();
|
|
Map<Long, Long> sourceScenicIdMap = new HashMap<>();
|
|
list.forEach(item -> {
|
|
if (item.getId() != null && item.getScenicId() != null) {
|
|
sourceScenicIdMap.put(item.getId(), item.getScenicId());
|
|
}
|
|
if (item.getId() != null) {
|
|
sourceImageUrlMap.put(item.getId(), item.getUrl());
|
|
}
|
|
});
|
|
ArrayList<String> adapterIdentity = new ArrayList<>();
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
if (disableDeleteScenicIds.contains(scenic.getId())) {
|
|
log.info("景区【{}】禁止删除文件，跳过！", scenic.getName());
|
|
return;
|
|
}
|
|
IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId);
|
|
String identity = adapter.identity();
|
|
if (!adapterIdentity.contains(identity)) {
|
|
log.info("因为Identity相同，跳过");
|
|
adapterIdentity.add(identity);
|
|
} else {
|
|
return;
|
|
}
|
|
log.info("开始清理视频文件");
|
|
List<StorageFileObject> fileObjectList = adapter.listDir(StorageConstant.VIDEO_PIECE_PATH);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) {
|
|
return;
|
|
}
|
|
}
|
|
if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getName()))){
|
|
log.info("删除文件：{}", fileObject);
|
|
adapter.deleteFile(fileObject.getFullPath());
|
|
} else {
|
|
log.info("文件存在关系：{}，未删除", fileObject);
|
|
}
|
|
});
|
|
});
|
|
|
|
log.info("开始清理图片文件");
|
|
IStorageAdapter imageAdapter = StorageFactory.use();
|
|
List<StorageFileObject> fileObjectList = imageAdapter.listDir(StorageConstant.PHOTO_PATH);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) {
|
|
return;
|
|
}
|
|
}
|
|
|
|
String name = fileObject.getName();
|
|
if (name == null) {
|
|
return;
|
|
}
|
|
int underscoreIndex = name.indexOf('_');
|
|
if (underscoreIndex <= 0) {
|
|
return;
|
|
}
|
|
Long sourceId;
|
|
try {
|
|
sourceId = Long.parseLong(name.substring(0, underscoreIndex));
|
|
} catch (NumberFormatException e) {
|
|
return;
|
|
}
|
|
Long scenicId = sourceScenicIdMap.get(sourceId);
|
|
if (scenicId == null || disableDeleteScenicIds.contains(scenicId.toString())) {
|
|
return;
|
|
}
|
|
|
|
String imageUrl = sourceImageUrlMap.get(sourceId);
|
|
if (imageUrl != null && imageUrl.contains(name)) {
|
|
log.info("文件存在关系：{}，未删除", fileObject);
|
|
return;
|
|
}
|
|
log.info("删除文件：{}", fileObject);
|
|
imageAdapter.deleteFile(fileObject.getFullPath());
|
|
});
|
|
}
|
|
public void cleanVideoOss() {
|
|
log.info("开始清理视频文件");
|
|
List<VideoRespVO> list = videoMapper.list(new VideoReqQuery());
|
|
ArrayList<String> adapterIdentity = new ArrayList<>();
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
if (disableDeleteScenicIds.contains(scenic.getId())) {
|
|
log.info("景区【{}】禁止删除文件，跳过！", scenic.getName());
|
|
return;
|
|
}
|
|
IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId);
|
|
String identity = adapter.identity();
|
|
if (!adapterIdentity.contains(identity)) {
|
|
adapterIdentity.add(identity);
|
|
} else {
|
|
log.info("因为Identity相同，跳过");
|
|
return;
|
|
}
|
|
log.info("开始清理视频文件");
|
|
List<StorageFileObject> fileObjectList = adapter.listDir(StorageConstant.VLOG_PATH);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) {
|
|
return;
|
|
}
|
|
}
|
|
if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getName()))){
|
|
log.info("删除文件：{}", fileObject);
|
|
adapter.deleteFile(fileObject.getFullPath());
|
|
} else {
|
|
log.info("文件存在关系：{}，未删除", fileObject);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
}
|