package com.ycwl.basic.task; import cn.hutool.core.date.DateUnit; import cn.hutool.core.date.DateUtil; import com.ycwl.basic.constant.StorageConstant; import com.ycwl.basic.facebody.adapter.IFaceBodyAdapter; import com.ycwl.basic.integration.scenic.dto.scenic.ScenicV2DTO; import com.ycwl.basic.mapper.FaceMapper; import com.ycwl.basic.mapper.FaceSampleMapper; import com.ycwl.basic.mapper.SourceMapper; import com.ycwl.basic.mapper.VideoMapper; import com.ycwl.basic.model.pc.face.entity.FaceEntity; import com.ycwl.basic.model.pc.face.req.FaceReqQuery; import com.ycwl.basic.model.pc.faceSample.entity.FaceSampleEntity; import com.ycwl.basic.model.pc.faceSample.req.FaceSampleReqQuery; import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO; import com.ycwl.basic.integration.common.manager.ScenicConfigManager; import com.ycwl.basic.model.pc.scenic.req.ScenicReqQuery; import com.ycwl.basic.model.pc.source.req.SourceReqQuery; import com.ycwl.basic.model.pc.source.resp.SourceRespVO; import com.ycwl.basic.model.pc.video.req.VideoReqQuery; import com.ycwl.basic.model.pc.video.resp.VideoRespVO; import com.ycwl.basic.repository.ScenicRepository; import com.ycwl.basic.service.pc.ScenicService; import com.ycwl.basic.storage.StorageFactory; import com.ycwl.basic.storage.adapters.IStorageAdapter; import com.ycwl.basic.storage.entity.StorageFileObject; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Profile; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; import static com.ycwl.basic.constant.FaceConstant.USER_FACE_DB_NAME; @Component @EnableScheduling @Slf4j @Profile("prod") public class FaceCleaner { @Autowired private FaceSampleMapper faceSampleMapper; @Autowired private SourceMapper sourceMapper; @Autowired private VideoMapper videoMapper; @Autowired private ScenicRepository scenicRepository; @Autowired private FaceMapper faceMapper; @Autowired private ScenicService scenicService; public static final List disableDeleteScenicIds = List.of("3955650120997015552"); @Scheduled(cron = "0 0 1 * * ?") public void deleteExpireSample(){ ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.parallelStream().forEach(scenic -> { Long scenicId = Long.parseLong(scenic.getId()); log.info("当前景区{},开始删除人脸样本", scenicId); IFaceBodyAdapter adapter = scenicService.getScenicFaceBodyAdapter(scenicId); ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId); Integer sampleStoreDay = scenicConfig.getInteger("sample_store_day"); if (sampleStoreDay == null) { log.info("当前景区{},人脸样本保存天数未设置,默认7天", scenic.getId()); sampleStoreDay = 7; } Date sampleEndDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -sampleStoreDay); List faceSampleList = faceSampleMapper.listEntityBeforeDate(scenicId, sampleEndDate); if (faceSampleList.isEmpty()) { log.info("当前景区{},人脸样本为空", scenic.getId()); return; } faceSampleList.forEach(faceSample -> { boolean success = adapter.deleteFace(scenic.getId(), faceSample.getId().toString()); if (success) { log.info("当前景区{},人脸样本ID{},删除成功", scenic.getId(), faceSample.getId()); faceSampleMapper.deleteById(faceSample.getId()); } else { log.info("当前景区{},人脸样本ID{},删除失败", scenic.getId(), faceSample.getId()); } }); }); } @Scheduled(cron = "0 45 2 * * ?") public void deleteExpireFace() { ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.parallelStream().forEach(scenic -> { Long scenicId = Long.parseLong(scenic.getId()); log.info("当前景区{},开始删除用户人脸", scenic.getId()); ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId); IFaceBodyAdapter adapter = scenicService.getScenicFaceBodyAdapter(scenicId); Integer faceStoreDay = scenicConfig.getInteger("face_store_day"); if (faceStoreDay == null) { log.info("当前景区{},人脸样本保存天数未设置,默认3天", scenic.getName()); faceStoreDay = 3; } FaceReqQuery req = new FaceReqQuery(); req.setScenicId(scenicId); Date faceEndDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -faceStoreDay); List list = faceMapper.listUnpaidEntityBeforeDate(scenicId, faceEndDate); list.forEach(face -> { boolean result = adapter.deleteFace(USER_FACE_DB_NAME+face.getScenicId(), face.getId().toString()); if (result) { log.info("当前景区{},人脸样本ID{},删除成功", scenic.getId(), face.getId()); } else { result = adapter.deleteFace(USER_FACE_DB_NAME+face.getScenicId(), face.getId().toString()); if (result) { log.info("当前景区{},人脸样本ID{},删除成功", scenic.getId(), face.getId()); } else { log.info("当前景区{},人脸样本ID{},删除失败", scenic.getId(), face.getId()); } } faceMapper.forceDeleteById(face.getId()); }); }); } @Scheduled(cron = "0 0 1 * * ?") public void deleteNotBuySource(){ ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.parallelStream().forEach(scenic -> { Long scenicId = Long.valueOf(scenic.getId()); ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId); if (scenicConfig == null) { log.info("当前景区{},无配置信息", scenic.getName()); return; } if (scenicConfig.getInteger("user_source_expire_day") == null) { log.info("当前景区{},人脸样本过期天数未设置", scenic.getName()); return; } int expireDay = scenicConfig.getInteger("user_source_expire_day"); Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -expireDay); int deleteCount = sourceMapper.deleteNotBuyRelations(scenicId, endDate); log.info("当前景区{},删除关联素材{}个", scenic.getName(), deleteCount); }); } @Scheduled(cron = "0 15 1 * * ?") public void deleteNotBuyVideos(){ ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.parallelStream().forEach(scenic -> { Long scenicId = Long.valueOf(scenic.getId()); ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId); if (scenicConfig == null) { log.info("当前景区{},无配置信息", scenic.getName()); return; } if (scenicConfig.getInteger("video_store_day") == null) { log.info("当前景区{},VLOG过期天数未设置", scenic.getName()); return; } int expireDay = scenicConfig.getInteger("video_store_day"); Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -expireDay); int deleteCount = videoMapper.deleteNotBuyRelations(scenicId, endDate); int deleteVideoCount = videoMapper.deleteUselessVideo(); log.info("当前景区{},删除VLOG关系{}个,删除VLOG记录{}个", scenic.getName(), deleteCount, deleteVideoCount); }); } @Scheduled(cron = "0 30 1 * * ?") public void deleteExpiredSource(){ ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.parallelStream().forEach(scenic -> { Long scenicId = Long.valueOf(scenic.getId()); ScenicConfigManager scenicConfig = scenicRepository.getScenicConfigManager(scenicId); if (scenicConfig == null) { log.info("当前景区{},无配置信息", scenic.getName()); return; } int imageSourceExpireDay = 7; int videoSourceExpireDay = 7; if (scenicConfig.getInteger("image_source_store_day") != null) { imageSourceExpireDay = scenicConfig.getInteger("image_source_store_day"); } else { log.info("当前景区{},原始素材保存天数未设置,默认7天", scenic.getName()); } if (scenicConfig.getInteger("video_source_store_day") != null) { videoSourceExpireDay = scenicConfig.getInteger("video_source_store_day"); } else { log.info("当前景区{},原始素材保存天数未设置,默认7天", scenic.getName()); } if (Boolean.TRUE.equals(scenicConfig.getBoolean("disable_source_video"))) { return; } if (Boolean.TRUE.equals(scenicConfig.getBoolean("disable_source_image"))) { return; } log.info("当前景区{},开始删除原始素材", scenic.getName()); Date endDate = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -videoSourceExpireDay); int deleteVideoSourceCount = sourceMapper.deleteNotRelateSource(1, endDate); log.info("当前景区{},删除原始视频素材{}个", scenic.getName(), deleteVideoSourceCount); log.info("当前景区{},开始删除原始图片素材", scenic.getName()); Date endDate2 = DateUtil.offsetDay(DateUtil.beginOfDay(new Date()), -imageSourceExpireDay); int deleteImageSourceCount = sourceMapper.deleteNotRelateSource(2, endDate2); log.info("当前景区{},删除原始图片素材{}个", scenic.getName(), deleteImageSourceCount); int deleteSourceCount = sourceMapper.deleteUselessSource(); log.info("当前景区{},删除无用素材{}个", scenic.getName(), deleteSourceCount); }); } @Scheduled(cron = "0 0 2 * * ?") public void clearOss(){ cleanSourceOss(); cleanVideoOss(); } public void cleanSourceOss() { log.info("开始清理源视频素材文件"); List list = sourceMapper.list(new SourceReqQuery()); Map sourceImageUrlMap = new HashMap<>(); Map sourceScenicIdMap = new HashMap<>(); list.forEach(item -> { if (item.getId() != null && item.getScenicId() != null) { sourceScenicIdMap.put(item.getId(), item.getScenicId()); } if (item.getId() != null) { sourceImageUrlMap.put(item.getId(), item.getUrl()); } }); ArrayList adapterIdentity = new ArrayList<>(); ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.forEach(scenic -> { Long scenicId = Long.valueOf(scenic.getId()); if (disableDeleteScenicIds.contains(scenic.getId())) { log.info("景区【{}】禁止删除文件,跳过!", scenic.getName()); return; } IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId); String identity = adapter.identity(); if (!adapterIdentity.contains(identity)) { log.info("因为Identity相同,跳过"); adapterIdentity.add(identity); } else { return; } log.info("开始清理视频文件"); List fileObjectList = adapter.listDir(StorageConstant.VIDEO_PIECE_PATH); fileObjectList.parallelStream().forEach(fileObject -> { if (fileObject.getModifyTime() != null) { // 如果是一天以内修改的,则跳过 if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) { return; } } if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getName()))){ log.info("删除文件:{}", fileObject); adapter.deleteFile(fileObject.getFullPath()); } else { log.info("文件存在关系:{},未删除", fileObject); } }); }); log.info("开始清理图片文件"); IStorageAdapter imageAdapter = StorageFactory.use(); List fileObjectList = imageAdapter.listDir(StorageConstant.PHOTO_PATH); fileObjectList.parallelStream().forEach(fileObject -> { if (fileObject.getModifyTime() != null) { // 如果是一天以内修改的,则跳过 if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) { return; } } String name = fileObject.getName(); if (name == null) { return; } int underscoreIndex = name.indexOf('_'); if (underscoreIndex <= 0) { return; } Long sourceId; try { sourceId = Long.parseLong(name.substring(0, underscoreIndex)); } catch (NumberFormatException e) { return; } Long scenicId = sourceScenicIdMap.get(sourceId); if (scenicId == null || disableDeleteScenicIds.contains(scenicId.toString())) { return; } String imageUrl = sourceImageUrlMap.get(sourceId); if (imageUrl != null && imageUrl.contains(name)) { log.info("文件存在关系:{},未删除", fileObject); return; } log.info("删除文件:{}", fileObject); imageAdapter.deleteFile(fileObject.getFullPath()); }); } public void cleanVideoOss() { log.info("开始清理视频文件"); List list = videoMapper.list(new VideoReqQuery()); ArrayList adapterIdentity = new ArrayList<>(); ScenicReqQuery query = new ScenicReqQuery(); query.setPageSize(1000); List scenicList = scenicRepository.list(query); scenicList.forEach(scenic -> { Long scenicId = Long.valueOf(scenic.getId()); if (disableDeleteScenicIds.contains(scenic.getId())) { log.info("景区【{}】禁止删除文件,跳过!", scenic.getName()); return; } IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId); String identity = adapter.identity(); if (!adapterIdentity.contains(identity)) { adapterIdentity.add(identity); } else { log.info("因为Identity相同,跳过"); return; } log.info("开始清理视频文件"); List fileObjectList = adapter.listDir(StorageConstant.VLOG_PATH); fileObjectList.parallelStream().forEach(fileObject -> { if (fileObject.getModifyTime() != null) { // 如果是一天以内修改的,则跳过 if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) <= 1) { return; } } if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getName()))){ log.info("删除文件:{}", fileObject); adapter.deleteFile(fileObject.getFullPath()); } else { log.info("文件存在关系:{},未删除", fileObject); } }); }); } }