You've already forked FrameTour-BE
Jerry Yan
f2ac6aaea0
- 移除 ScenicMapper 接口，将相关方法移至 ScenicRepository - 修改景区列表查询逻辑，使用 ScenicRepository 的 list 方法 - 优化景区详情获取方式，使用 ScenicRepository 的 getScenicBasic 方法 - 重构缓存机制，增加对景区基本信息的缓存 - 优化 AppScenicService 和 ScenicService接口，使用 ScenicV2DTO 替代 ScenicRespV
341 lines
17 KiB
Java
341 lines
17 KiB
Java
package com.ycwl.basic.task;
|
|
|
|
import cn.hutool.core.date.DateUnit;
|
|
import cn.hutool.core.date.DateUtil;
|
|
import com.ycwl.basic.constant.StorageConstant;
|
|
import com.ycwl.basic.facebody.adapter.IFaceBodyAdapter;
|
|
import com.ycwl.basic.integration.scenic.dto.scenic.ScenicV2DTO;
|
|
import com.ycwl.basic.mapper.FaceMapper;
|
|
import com.ycwl.basic.mapper.FaceSampleMapper;
|
|
import com.ycwl.basic.mapper.SourceMapper;
|
|
import com.ycwl.basic.mapper.VideoMapper;
|
|
import com.ycwl.basic.model.pc.face.entity.FaceEntity;
|
|
import com.ycwl.basic.model.pc.face.req.FaceReqQuery;
|
|
import com.ycwl.basic.model.pc.faceSample.entity.FaceSampleEntity;
|
|
import com.ycwl.basic.model.pc.faceSample.req.FaceSampleReqQuery;
|
|
import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO;
|
|
import com.ycwl.basic.model.pc.scenic.entity.ScenicConfigEntity;
|
|
import com.ycwl.basic.model.pc.scenic.req.ScenicReqQuery;
|
|
import com.ycwl.basic.model.pc.source.req.SourceReqQuery;
|
|
import com.ycwl.basic.model.pc.source.resp.SourceRespVO;
|
|
import com.ycwl.basic.model.pc.video.req.VideoReqQuery;
|
|
import com.ycwl.basic.model.pc.video.resp.VideoRespVO;
|
|
import com.ycwl.basic.repository.ScenicRepository;
|
|
import com.ycwl.basic.service.pc.ScenicService;
|
|
import com.ycwl.basic.storage.StorageFactory;
|
|
import com.ycwl.basic.storage.adapters.IStorageAdapter;
|
|
import com.ycwl.basic.storage.entity.StorageFileObject;
|
|
import lombok.extern.slf4j.Slf4j;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.context.annotation.Profile;
|
|
import org.springframework.scheduling.annotation.EnableScheduling;
|
|
import org.springframework.scheduling.annotation.Scheduled;
|
|
import org.springframework.stereotype.Component;
|
|
|
|
import java.util.ArrayList;
|
|
import java.util.Date;
|
|
import java.util.List;
|
|
import java.util.Objects;
|
|
|
|
import static com.ycwl.basic.constant.FaceConstant.USER_FACE_DB_NAME;
|
|
import static com.ycwl.basic.constant.StorageConstant.VIID_FACE;
|
|
|
|
@Component
|
|
@EnableScheduling
|
|
@Slf4j
|
|
@Profile("prod")
|
|
public class FaceCleaner {
|
|
@Autowired
|
|
private FaceSampleMapper faceSampleMapper;
|
|
@Autowired
|
|
private SourceMapper sourceMapper;
|
|
@Autowired
|
|
private VideoMapper videoMapper;
|
|
@Autowired
|
|
private ScenicRepository scenicRepository;
|
|
@Autowired
|
|
private FaceMapper faceMapper;
|
|
@Autowired
|
|
private ScenicService scenicService;
|
|
public static final List<String> disableDeleteScenicIds = List.of("3955650120997015552");
|
|
|
|
@Scheduled(cron = "0 0 1 * * ?")
|
|
public void deleteExpireSample(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.parseLong(scenic.getId());
|
|
log.info("当前景区{}，开始删除人脸样本", scenicId);
|
|
IFaceBodyAdapter adapter = scenicService.getScenicFaceBodyAdapter(scenicId);
|
|
ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId);
|
|
Integer sampleStoreDay = scenicConfig.getSampleStoreDay();
|
|
if (sampleStoreDay == null) {
|
|
log.info("当前景区{}，人脸样本保存天数未设置，默认7天", scenic.getId());
|
|
sampleStoreDay = 7;
|
|
}
|
|
Date sampleEndDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -sampleStoreDay);
|
|
List<FaceSampleEntity> faceSampleList = faceSampleMapper.listEntityBeforeDate(scenicId, sampleEndDate);
|
|
if (faceSampleList.isEmpty()) {
|
|
log.info("当前景区{}，人脸样本为空", scenic.getId());
|
|
return;
|
|
}
|
|
faceSampleList.forEach(faceSample -> {
|
|
boolean success = adapter.deleteFace(scenic.getId(), faceSample.getId().toString());
|
|
if (success) {
|
|
log.info("当前景区{}，人脸样本ID{}，删除成功", scenic.getId(), faceSample.getId());
|
|
faceSampleMapper.deleteById(faceSample.getId());
|
|
} else {
|
|
log.info("当前景区{}，人脸样本ID{}，删除失败", scenic.getId(), faceSample.getId());
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 45 2 * * ?")
|
|
public void deleteExpireFace() {
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.parseLong(scenic.getId());
|
|
log.info("当前景区{}，开始删除用户人脸", scenic.getId());
|
|
ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId);
|
|
IFaceBodyAdapter adapter = scenicService.getScenicFaceBodyAdapter(scenicId);
|
|
Integer faceStoreDay = scenicConfig.getFaceStoreDay();
|
|
if (faceStoreDay == null) {
|
|
log.info("当前景区{}，人脸样本保存天数未设置，默认3天", scenic.getName());
|
|
faceStoreDay = 3;
|
|
}
|
|
FaceReqQuery req = new FaceReqQuery();
|
|
req.setScenicId(scenicId);
|
|
Date faceEndDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -faceStoreDay);
|
|
List<FaceEntity> list = faceMapper.listUnpaidEntityBeforeDate(scenicId, faceEndDate);
|
|
list.forEach(face -> {
|
|
boolean result = adapter.deleteFace(USER_FACE_DB_NAME+face.getScenicId(), face.getId().toString());
|
|
if (result) {
|
|
log.info("当前景区{}，人脸样本ID{}，删除成功", scenic.getId(), face.getId());
|
|
} else {
|
|
result = adapter.deleteFace(USER_FACE_DB_NAME+face.getScenicId(), face.getId().toString());
|
|
if (result) {
|
|
log.info("当前景区{}，人脸样本ID{}，删除成功", scenic.getId(), face.getId());
|
|
} else {
|
|
log.info("当前景区{}，人脸样本ID{}，删除失败", scenic.getId(), face.getId());
|
|
}
|
|
}
|
|
faceMapper.forceDeleteById(face.getId());
|
|
});
|
|
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 0 1 * * ?")
|
|
public void deleteNotBuySource(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId);
|
|
if (scenicConfig == null) {
|
|
log.info("当前景区{}，无配置信息", scenic.getName());
|
|
return;
|
|
}
|
|
if (scenicConfig.getUserSourceExpireDay() == null) {
|
|
log.info("当前景区{}，人脸样本过期天数未设置", scenic.getName());
|
|
return;
|
|
}
|
|
int expireDay = scenicConfig.getUserSourceExpireDay();
|
|
Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -expireDay);
|
|
int deleteCount = sourceMapper.deleteNotBuyRelations(scenicId, endDate);
|
|
log.info("当前景区{}，删除关联素材{}个", scenic.getName(), deleteCount);
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 15 1 * * ?")
|
|
public void deleteNotBuyVideos(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId);
|
|
if (scenicConfig == null) {
|
|
log.info("当前景区{}，无配置信息", scenic.getName());
|
|
return;
|
|
}
|
|
if (scenicConfig.getVideoStoreDay() == null) {
|
|
log.info("当前景区{}，VLOG过期天数未设置", scenic.getName());
|
|
return;
|
|
}
|
|
int expireDay = scenicConfig.getVideoStoreDay();
|
|
Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -expireDay);
|
|
int deleteCount = videoMapper.deleteNotBuyRelations(scenicId, endDate);
|
|
int deleteVideoCount = videoMapper.deleteUselessVideo();
|
|
log.info("当前景区{}，删除VLOG关系{}个，删除VLOG记录{}个", scenic.getName(), deleteCount, deleteVideoCount);
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 30 1 * * ?")
|
|
public void deleteExpiredSource(){
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.parallelStream().forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId);
|
|
if (scenicConfig == null) {
|
|
log.info("当前景区{}，无配置信息", scenic.getName());
|
|
return;
|
|
}
|
|
int imageSourceExpireDay = 7;
|
|
int videoSourceExpireDay = 7;
|
|
if (scenicConfig.getImageSourceStoreDay() != null) {
|
|
imageSourceExpireDay = scenicConfig.getImageSourceStoreDay();
|
|
} else {
|
|
log.info("当前景区{}，原始素材保存天数未设置，默认7天", scenic.getName());
|
|
}
|
|
if (scenicConfig.getVideoSourceStoreDay() != null) {
|
|
videoSourceExpireDay = scenicConfig.getVideoSourceStoreDay();
|
|
} else {
|
|
log.info("当前景区{}，原始素材保存天数未设置，默认7天", scenic.getName());
|
|
}
|
|
if (Boolean.TRUE.equals(scenicConfig.getDisableSourceVideo())) {
|
|
return;
|
|
}
|
|
if (Boolean.TRUE.equals(scenicConfig.getDisableSourceImage())) {
|
|
return;
|
|
}
|
|
log.info("当前景区{}，开始删除原始素材", scenic.getName());
|
|
Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -videoSourceExpireDay);
|
|
int deleteVideoSourceCount = sourceMapper.deleteNotRelateSource(1, endDate);
|
|
log.info("当前景区{}，删除原始视频素材{}个", scenic.getName(), deleteVideoSourceCount);
|
|
log.info("当前景区{}，开始删除原始图片素材", scenic.getName());
|
|
Date endDate2 = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -imageSourceExpireDay);
|
|
int deleteImageSourceCount = sourceMapper.deleteNotRelateSource(2, endDate2);
|
|
log.info("当前景区{}，删除原始图片素材{}个", scenic.getName(), deleteImageSourceCount);
|
|
int deleteSourceCount = sourceMapper.deleteUselessSource();
|
|
log.info("当前景区{}，删除无用素材{}个", scenic.getName(), deleteSourceCount);
|
|
});
|
|
}
|
|
|
|
@Scheduled(cron = "0 0 2 * * ?")
|
|
public void clearOss(){
|
|
cleanFaceSampleOss();
|
|
cleanSourceOss();
|
|
cleanVideoOss();
|
|
}
|
|
private void cleanFaceSampleOss() {
|
|
log.info("开始清理人脸文件");
|
|
List<FaceSampleRespVO> list = faceSampleMapper.list(new FaceSampleReqQuery());
|
|
IStorageAdapter adapter = StorageFactory.use("faces");
|
|
List<StorageFileObject> fileObjectList = adapter.listDir(VIID_FACE);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) < 1) {
|
|
return;
|
|
}
|
|
}
|
|
if(list.parallelStream().noneMatch(faceSampleRespVO -> faceSampleRespVO.getFaceUrl().contains(fileObject.getName()))){
|
|
log.info("删除人脸文件：{}", fileObject);
|
|
adapter.deleteFile(fileObject.getFullPath());
|
|
}
|
|
});
|
|
}
|
|
public void cleanSourceOss() {
|
|
log.info("开始清理源视频素材文件");
|
|
List<SourceRespVO> list = sourceMapper.list(new SourceReqQuery());
|
|
ArrayList<String> adapterIdentity = new ArrayList<>();
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
if (disableDeleteScenicIds.contains(scenic.getId())) {
|
|
log.info("景区【{}】禁止删除文件，跳过！", scenic.getName());
|
|
return;
|
|
}
|
|
IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId);
|
|
String identity = adapter.identity();
|
|
if (!adapterIdentity.contains(identity)) {
|
|
log.info("因为Identity相同，跳过");
|
|
adapterIdentity.add(identity);
|
|
} else {
|
|
return;
|
|
}
|
|
log.info("开始清理视频文件");
|
|
List<StorageFileObject> fileObjectList = adapter.listDir(StorageConstant.VIDEO_PIECE_PATH);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) {
|
|
return;
|
|
}
|
|
}
|
|
if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getName()))){
|
|
log.info("删除文件：{}", fileObject);
|
|
adapter.deleteFile(fileObject.getFullPath());
|
|
} else {
|
|
log.info("文件存在关系：{}，未删除", fileObject);
|
|
}
|
|
});
|
|
log.info("开始清理图片文件");
|
|
fileObjectList = adapter.listDir(StorageConstant.PHOTO_PATH);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) {
|
|
return;
|
|
}
|
|
}
|
|
if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getUrl())).noneMatch(videoRespVO -> videoRespVO.getUrl().contains(fileObject.getName()))){
|
|
log.info("删除文件：{}", fileObject);
|
|
adapter.deleteFile(fileObject.getFullPath());
|
|
} else {
|
|
log.info("文件存在关系：{}，未删除", fileObject);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
public void cleanVideoOss() {
|
|
log.info("开始清理视频文件");
|
|
List<VideoRespVO> list = videoMapper.list(new VideoReqQuery());
|
|
ArrayList<String> adapterIdentity = new ArrayList<>();
|
|
ScenicReqQuery query = new ScenicReqQuery();
|
|
query.setPageSize(1000);
|
|
List<ScenicV2DTO> scenicList = scenicRepository.list(query);
|
|
scenicList.forEach(scenic -> {
|
|
Long scenicId = Long.valueOf(scenic.getId());
|
|
if (disableDeleteScenicIds.contains(scenic.getId())) {
|
|
log.info("景区【{}】禁止删除文件，跳过！", scenic.getName());
|
|
return;
|
|
}
|
|
IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId);
|
|
String identity = adapter.identity();
|
|
if (!adapterIdentity.contains(identity)) {
|
|
adapterIdentity.add(identity);
|
|
} else {
|
|
log.info("因为Identity相同，跳过");
|
|
return;
|
|
}
|
|
log.info("开始清理视频文件");
|
|
List<StorageFileObject> fileObjectList = adapter.listDir(StorageConstant.VLOG_PATH);
|
|
fileObjectList.parallelStream().forEach(fileObject -> {
|
|
if (fileObject.getModifyTime() != null) {
|
|
// 如果是一天以内修改的，则跳过
|
|
if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) {
|
|
return;
|
|
}
|
|
}
|
|
if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getName()))){
|
|
log.info("删除文件：{}", fileObject);
|
|
adapter.deleteFile(fileObject.getFullPath());
|
|
} else {
|
|
log.info("文件存在关系：{}，未删除", fileObject);
|
|
}
|
|
});
|
|
});
|
|
}
|
|
}
|