From 7892c0f5cc59a98929451a3e30632034e753d879 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Wed, 5 Feb 2025 11:28:50 +0800 Subject: [PATCH] =?UTF-8?q?bug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../basic/controller/viid/ViidController.java | 8 +- .../basic/controller/wvp/WvpController.java | 1 + .../exception/CustomExceptionHandle.java | 6 + .../ycwl/basic/mapper/FaceSampleMapper.java | 2 + .../pc/scenic/entity/ScenicConfigEntity.java | 2 + .../basic/ratelimiter/FixedRateLimiter.java | 32 +++++ .../ratelimiter/SlidingWindowRateLimiter.java | 5 +- .../basic/repository/TemplateRepository.java | 3 + .../impl/mobile/AppStatisticsServiceImpl.java | 13 +- .../ycwl/basic/service/task/TaskService.java | 2 + .../task/impl/TaskFaceServiceImpl.java | 114 ++++++++++----- .../task/impl/TaskTaskServiceImpl.java | 135 ++++++++++++++++-- .../ycwl/basic/task/DynamicTaskGenerator.java | 10 +- .../java/com/ycwl/basic/task/FaceCleaner.java | 31 +++- .../ycwl/basic/task/VideoTaskGenerator.java | 33 ++--- .../com/ycwl/basic/utils/JwtTokenUtil.java | 2 +- .../resources/mapper/FaceSampleMapper.xml | 5 + src/main/resources/mapper/OrderMapper.xml | 5 +- src/main/resources/mapper/ScenicMapper.xml | 4 +- src/main/resources/mapper/SourceMapper.xml | 2 +- .../resources/mapper/StatisticsMapper.xml | 4 +- 21 files changed, 324 insertions(+), 95 deletions(-) create mode 100644 src/main/java/com/ycwl/basic/ratelimiter/FixedRateLimiter.java diff --git a/src/main/java/com/ycwl/basic/controller/viid/ViidController.java b/src/main/java/com/ycwl/basic/controller/viid/ViidController.java index 96b005f..06424d8 100644 --- a/src/main/java/com/ycwl/basic/controller/viid/ViidController.java +++ b/src/main/java/com/ycwl/basic/controller/viid/ViidController.java @@ -233,6 +233,8 @@ public class ViidController { } if (shotTime == null) { shotTime = new Date(); + } else if (Math.abs(shotTime.getTime() - System.currentTimeMillis()) > + 24 * 60 * 60 * 1000) { + shotTime = new Date(); } Long scenicId = device.getScenicId(); if (scenicId == null) { @@ -269,8 +271,10 @@ public class ViidController { String url = adapter.uploadFile(file, "user-face", UUID.randomUUID() + "." + ext); faceSample.setFaceUrl(url); faceSampleMapper.add(faceSample); - DynamicTaskGenerator.addTask(faceSample.getId()); - taskFaceService.addFaceSample(faceSample.getId()); + new Thread(() -> { + taskFaceService.addFaceSample(faceSample.getId()); + DynamicTaskGenerator.addTask(faceSample.getId()); + }).start(); for (SubImageInfoObject _subImage : type14ImageList) { facePosition.setImgHeight(_subImage.getHeight()); facePosition.setImgWidth(_subImage.getWidth()); diff --git a/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java b/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java index 354e41c..363a2a1 100644 --- a/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java +++ b/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java @@ -18,6 +18,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.Date; import java.util.List; @Slf4j diff --git a/src/main/java/com/ycwl/basic/exception/CustomExceptionHandle.java b/src/main/java/com/ycwl/basic/exception/CustomExceptionHandle.java index 73f00a0..5936407 100644 --- a/src/main/java/com/ycwl/basic/exception/CustomExceptionHandle.java +++ b/src/main/java/com/ycwl/basic/exception/CustomExceptionHandle.java @@ -62,6 +62,12 @@ public class CustomExceptionHandle { return ApiResponse.buildResponse(bizException.getCode(), bizException.getMsg()); } + @ExceptionHandler(value =IOException.class) + public ApiResponse handle(IOException e) { + LOGGER.error("系统异常 -> {}", e.getMessage(), e); + return ApiResponse.buildResult(BizCodeEnum.SERVER_UNKONWN_ERROR); + } + /** * 异常统一返回处理 */ diff --git a/src/main/java/com/ycwl/basic/mapper/FaceSampleMapper.java b/src/main/java/com/ycwl/basic/mapper/FaceSampleMapper.java index 63a6411..35e8d81 100644 --- a/src/main/java/com/ycwl/basic/mapper/FaceSampleMapper.java +++ b/src/main/java/com/ycwl/basic/mapper/FaceSampleMapper.java @@ -6,6 +6,7 @@ import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO; import org.apache.ibatis.annotations.Mapper; import org.apache.ibatis.annotations.Param; +import java.util.Date; import java.util.List; /** @@ -25,4 +26,5 @@ public interface FaceSampleMapper { List listByIds(List list); FaceSampleEntity getEntity(Long faceSampleId); + List listEntity(Long scenicId, Date endDate); } diff --git a/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java b/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java index 1fa8b17..316e810 100644 --- a/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java +++ b/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java @@ -56,4 +56,6 @@ public class ScenicConfigEntity { private Integer disableSourceImage; private Integer templateNewVideoType; private Integer antiScreenRecordType; + private Integer videoSourceStoreDay; + private Integer imageSourceStoreDay; } diff --git a/src/main/java/com/ycwl/basic/ratelimiter/FixedRateLimiter.java b/src/main/java/com/ycwl/basic/ratelimiter/FixedRateLimiter.java new file mode 100644 index 0000000..70546c4 --- /dev/null +++ b/src/main/java/com/ycwl/basic/ratelimiter/FixedRateLimiter.java @@ -0,0 +1,32 @@ +package com.ycwl.basic.ratelimiter; + +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.Semaphore; +import java.util.concurrent.TimeUnit; + +public class FixedRateLimiter { + private final Semaphore semaphore = new Semaphore(1); + private final ScheduledExecutorService scheduler = new ScheduledThreadPoolExecutor(1); + + public FixedRateLimiter(int rate, TimeUnit timeUnit) { + // 启动一个线程每0.5秒释放一个许可 + scheduler.scheduleAtFixedRate(() -> { + synchronized (semaphore) { + if (semaphore.availablePermits() < 1) { + semaphore.release(1); + } + } + }, rate, rate, timeUnit); + } + + public void acquire() throws InterruptedException { + synchronized (semaphore) { + semaphore.acquire(); + } + } + + public void shutdown() { + scheduler.shutdown(); + } +} diff --git a/src/main/java/com/ycwl/basic/ratelimiter/SlidingWindowRateLimiter.java b/src/main/java/com/ycwl/basic/ratelimiter/SlidingWindowRateLimiter.java index 54c2069..e815ee3 100644 --- a/src/main/java/com/ycwl/basic/ratelimiter/SlidingWindowRateLimiter.java +++ b/src/main/java/com/ycwl/basic/ratelimiter/SlidingWindowRateLimiter.java @@ -12,14 +12,15 @@ public class SlidingWindowRateLimiter { public SlidingWindowRateLimiter(int maxRequestsPerSecond) { this.semaphore = new Semaphore(maxRequestsPerSecond); + int rate = 1000000 / maxRequestsPerSecond; scheduler.scheduleAtFixedRate(() -> { if (semaphore.availablePermits() < maxRequestsPerSecond) { semaphore.release(1); } - }, 0, (1000 / maxRequestsPerSecond), TimeUnit.MILLISECONDS); + }, rate, rate, TimeUnit.MICROSECONDS); } - public void aquire() throws InterruptedException { + public void acquire() throws InterruptedException { semaphore.acquire(); } diff --git a/src/main/java/com/ycwl/basic/repository/TemplateRepository.java b/src/main/java/com/ycwl/basic/repository/TemplateRepository.java index 6e1d9cc..b92ef0c 100644 --- a/src/main/java/com/ycwl/basic/repository/TemplateRepository.java +++ b/src/main/java/com/ycwl/basic/repository/TemplateRepository.java @@ -73,6 +73,9 @@ public class TemplateRepository { return JSONObject.parseObject(redisTemplate.opsForValue().get(String.format(TEMPLATE_CACHE_KEY, templateId)), TemplateRespVO.class); } TemplateRespVO template = templateMapper.getById(templateId); + if (template == null) { + return null; + } if (null == template.getPid() || template.getPid() == 0) { template.setChildren(templateMapper.getByPid(templateId)); redisTemplate.opsForValue().set(String.format(TEMPLATE_CACHE_KEY, templateId), JSONObject.toJSONString(template)); diff --git a/src/main/java/com/ycwl/basic/service/impl/mobile/AppStatisticsServiceImpl.java b/src/main/java/com/ycwl/basic/service/impl/mobile/AppStatisticsServiceImpl.java index a6ae382..0dbba64 100644 --- a/src/main/java/com/ycwl/basic/service/impl/mobile/AppStatisticsServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/impl/mobile/AppStatisticsServiceImpl.java @@ -358,14 +358,17 @@ public class AppStatisticsServiceImpl implements AppStatisticsService { //上一个周期的支付订单金额 vo.setPreviousOrderAmount(orderAmountDf.format(orderAmount)); // 计算预览_支付转化率 - if(pay==0){ + if(preview==0){ vo.setPreviousPreviewPay("0.00"); + }else { + BigDecimal previewPay = new BigDecimal(pay).divide(new BigDecimal(preview), 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); + vo.setPreviousPreviewPay(df.format(previewPay)); + } + if(scanCode==0){ vo.setPreviousScanCodePay("0.00"); }else { - BigDecimal previewPay = new BigDecimal(preview).divide(new BigDecimal(pay), 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); - vo.setNowPreviewPay(df.format(previewPay)); - BigDecimal scanCodePay = new BigDecimal(scanCode).divide(new BigDecimal(pay), 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); - vo.setNowScanCodePay(df.format(scanCodePay)); + BigDecimal scanCodePay = new BigDecimal(pay).divide(new BigDecimal(scanCode), 4, RoundingMode.HALF_UP).multiply(new BigDecimal(100)); + vo.setPreviousScanCodePay(df.format(scanCodePay)); } } } diff --git a/src/main/java/com/ycwl/basic/service/task/TaskService.java b/src/main/java/com/ycwl/basic/service/task/TaskService.java index 8ac2791..39333ce 100644 --- a/src/main/java/com/ycwl/basic/service/task/TaskService.java +++ b/src/main/java/com/ycwl/basic/service/task/TaskService.java @@ -25,6 +25,8 @@ public interface TaskService { void taskStart(Long taskId, WorkerAuthReqVo req); + void forceCreateTaskByFaceIdAndTempalteId(Long faceId, Long templateId); + void autoCreateTaskByFaceId(Long id); Date getTaskShotDate(Long taskId); diff --git a/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java b/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java index f47b912..0c180cf 100644 --- a/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java @@ -3,6 +3,7 @@ package com.ycwl.basic.service.task.impl; import cn.hutool.core.date.DateUtil; import com.alibaba.fastjson.JSON; import com.aliyuncs.exceptions.ClientException; +import com.aliyuncs.exceptions.ServerException; import com.aliyuncs.facebody.model.v20191230.AddFaceEntityRequest; import com.aliyuncs.facebody.model.v20191230.AddFaceRequest; import com.aliyuncs.facebody.model.v20191230.AddFaceResponse; @@ -15,6 +16,7 @@ import com.aliyuncs.facebody.model.v20191230.ListFaceEntitiesRequest; import com.aliyuncs.facebody.model.v20191230.ListFaceEntitiesResponse; import com.aliyuncs.facebody.model.v20191230.SearchFaceRequest; import com.aliyuncs.facebody.model.v20191230.SearchFaceResponse; +import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.ycwl.basic.biz.OrderBiz; import com.ycwl.basic.config.FaceDetectConfig; import com.ycwl.basic.constant.FaceConstant; @@ -33,12 +35,16 @@ import com.ycwl.basic.model.pc.faceSample.entity.FaceSampleEntity; import com.ycwl.basic.model.pc.faceSample.req.FaceSampleReqQuery; import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO; import com.ycwl.basic.model.pc.scenic.entity.ScenicConfigEntity; +import com.ycwl.basic.model.pc.scenic.req.ScenicReqQuery; +import com.ycwl.basic.model.pc.scenic.resp.ScenicRespVO; import com.ycwl.basic.model.pc.source.entity.MemberSourceEntity; import com.ycwl.basic.model.pc.source.entity.SourceEntity; import com.ycwl.basic.model.task.resp.AddFaceSampleRespVo; import com.ycwl.basic.model.task.resp.SearchFaceRespVo; +import com.ycwl.basic.ratelimiter.FixedRateLimiter; import com.ycwl.basic.ratelimiter.SlidingWindowRateLimiter; import com.ycwl.basic.repository.FaceRepository; +import com.ycwl.basic.repository.ScenicRepository; import com.ycwl.basic.service.task.TaskFaceService; import com.ycwl.basic.storage.StorageFactory; import com.ycwl.basic.storage.adapters.IStorageAdapter; @@ -49,6 +55,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; import com.aliyuncs.DefaultAcsClient; import com.aliyuncs.IAcsClient; @@ -63,6 +70,7 @@ import java.util.Date; import java.util.List; import java.util.Random; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.stream.Collectors; @Slf4j @@ -87,11 +95,14 @@ public class TaskFaceServiceImpl implements TaskFaceService { @Autowired private OrderBiz orderBiz; // 阿里云人脸检索限制qps=2 - private final SlidingWindowRateLimiter addEntityLimiter = new SlidingWindowRateLimiter(1); + private final FixedRateLimiter addEntityLimiter = new FixedRateLimiter(600, TimeUnit.MILLISECONDS); // 阿里云人脸检索限制qps=5 - private final SlidingWindowRateLimiter searchFaceLimiter = new SlidingWindowRateLimiter(4); - private final SlidingWindowRateLimiter deleteDbLimiter = new SlidingWindowRateLimiter(1); - private final SlidingWindowRateLimiter deleteEntityLimiter = new SlidingWindowRateLimiter(1); + private final FixedRateLimiter searchFaceLimiter = new FixedRateLimiter(200, TimeUnit.MILLISECONDS); + private final FixedRateLimiter deleteDbLimiter = new FixedRateLimiter(600, TimeUnit.MILLISECONDS); + private final FixedRateLimiter deleteEntityLimiter = new FixedRateLimiter(600, TimeUnit.MILLISECONDS); + + @Autowired + private ScenicRepository scenicRepository; private IAcsClient getClient() { DefaultProfile profile = DefaultProfile.getProfile( @@ -168,7 +179,7 @@ public class TaskFaceServiceImpl implements TaskFaceService { // request.setQualityScoreThreshold(60f); FaceDetectLog log = FaceDetectLog.quickCreate("预留字段", request); try { - searchFaceLimiter.aquire(); + searchFaceLimiter.acquire(); } catch (InterruptedException ignored) { } try { @@ -243,7 +254,7 @@ public class TaskFaceServiceImpl implements TaskFaceService { request.setEntityId(entityId); IAcsClient client = getClient(); try { - addEntityLimiter.aquire(); + addEntityLimiter.acquire(); } catch (InterruptedException ignored) { } try { @@ -270,38 +281,67 @@ public class TaskFaceServiceImpl implements TaskFaceService { @Override public void batchDeleteExpiredFace(Long scenicId) { - FaceSampleReqQuery query = new FaceSampleReqQuery(); - query.setDeviceId(scenicId); - faceSampleMapper.list(query); - ScenicConfigEntity scenicConfig = scenicMapper.getConfig(scenicId); - if (scenicConfig == null) { - return; - } + log.info("当前景区{}，开始删除人脸样本", scenicId); + ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId); Integer sampleStoreDay = scenicConfig.getSampleStoreDay(); if (sampleStoreDay == null) { - sampleStoreDay = 3; + log.info("当前景区{}，人脸样本保存天数未设置，默认7天", scenicId); + sampleStoreDay = 7; } - Date thatDay = DateUtil.offsetDay(new Date(), -sampleStoreDay); - Date dayEnd = DateUtil.endOfDay(thatDay); - query.setEndTime(dayEnd); - IAcsClient client = getClient(); - faceSampleMapper.list(query).forEach(faceSampleEntity -> { - String entityId = generateEntityId(faceSampleEntity); - DeleteFaceEntityRequest request = new DeleteFaceEntityRequest(); - request.setDbName(scenicId.toString()); - request.setEntityId(entityId); - try { - deleteEntityLimiter.aquire(); - } catch (InterruptedException ignored) { - } - try { - client.getAcsResponse(request); - } catch (ClientException e) { - return; - } finally { - faceSampleMapper.deleteById(faceSampleEntity.getId()); + Date endDate = DateUtils.addDateDays(new Date(), -(sampleStoreDay + 1)); + List faceSampleList = faceSampleMapper.listEntity(scenicId, endDate); + if (faceSampleList.isEmpty()) { + log.info("当前景区{}，人脸样本为空", scenicId); + return; + } + faceSampleList.forEach(faceSample -> { + boolean success = deleteFaceSample(String.valueOf(scenicId), generateEntityId(faceSample)); + if (success) { + log.info("当前景区{}，人脸样本ID{}，删除成功", scenicId, faceSample.getId()); + faceSampleMapper.deleteById(faceSample.getId()); + } else { + log.info("当前景区{}，人脸样本ID{}，删除失败", scenicId, faceSample.getId()); } }); + ListFaceEntitiesRequest listFaceEntitiesRequest = new ListFaceEntitiesRequest(); + listFaceEntitiesRequest.setDbName(String.valueOf(scenicId)); + listFaceEntitiesRequest.setOrder("asc"); + try { + IAcsClient client = getClient(); + while (true) { + AtomicBoolean flag = new AtomicBoolean(false); + ListFaceEntitiesResponse listFaceEntitiesResponse = client.getAcsResponse(listFaceEntitiesRequest); + if (listFaceEntitiesResponse == null || listFaceEntitiesResponse.getData() == null || listFaceEntitiesResponse.getData().getEntities() == null || listFaceEntitiesResponse.getData().getEntities().isEmpty()) { + break; + } + listFaceEntitiesResponse.getData().getEntities().forEach(entity -> { + String entityId = entity.getEntityId(); + String[] split = entityId.split("_"); + if (split.length != 2) { + return; + } + String deviceId = split[0]; + if (StringUtils.isBlank(deviceId)) { + return; + } + String dateString = split[1]; + if (StringUtils.isBlank(dateString)) { + return; + } + if (DateUtils.parse(dateString, DATE_FORMAT).before(endDate)) { + flag.set(true); + log.info("当前景区{}，开始删除人脸样本：{}", scenicId, entity.getEntityId()); + deleteFaceSample(String.valueOf(scenicId), entity.getEntityId()); + } else { + log.info("当前景区{}，人脸样本：{}未过期", scenicId, entity.getEntityId()); + } + }); + if (!flag.get()) { + break; + } + } + } catch (Exception ignored) { + } } @Override @@ -321,7 +361,7 @@ public class TaskFaceServiceImpl implements TaskFaceService { deleteFaceEntityRequest.setDbName(entity.getDbName()); deleteFaceEntityRequest.setEntityId(entity.getEntityId()); try { - deleteEntityLimiter.aquire(); + deleteEntityLimiter.acquire(); } catch (InterruptedException ignored) { } try { @@ -334,7 +374,7 @@ public class TaskFaceServiceImpl implements TaskFaceService { DeleteFaceDbRequest deleteFaceDbRequest = new DeleteFaceDbRequest(); deleteFaceDbRequest.setName(dbName); try { - deleteDbLimiter.aquire(); + deleteDbLimiter.acquire(); } catch (InterruptedException ignored) { } client.getAcsResponse(deleteFaceDbRequest); @@ -426,10 +466,12 @@ public class TaskFaceServiceImpl implements TaskFaceService { deleteFaceEntityRequest.setDbName(dbName); deleteFaceEntityRequest.setEntityId(entityId); try { + deleteEntityLimiter.acquire(); + log.info("删除{}人脸实体：{}", dbName, entityId); IAcsClient client = getClient(); client.getAcsResponse(deleteFaceEntityRequest); return true; - } catch (ClientException e) { + } catch (Exception e) { log.error("删除人脸样本失败！", e); return false; } diff --git a/src/main/java/com/ycwl/basic/service/task/impl/TaskTaskServiceImpl.java b/src/main/java/com/ycwl/basic/service/task/impl/TaskTaskServiceImpl.java index 60ebddd..4f4a4a1 100644 --- a/src/main/java/com/ycwl/basic/service/task/impl/TaskTaskServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/task/impl/TaskTaskServiceImpl.java @@ -16,7 +16,6 @@ import com.ycwl.basic.mapper.TaskMapper; import com.ycwl.basic.mapper.TemplateMapper; import com.ycwl.basic.mapper.VideoMapper; import com.ycwl.basic.model.mobile.order.IsBuyRespVO; -import com.ycwl.basic.model.mobile.order.PriceObj; import com.ycwl.basic.model.pc.face.entity.FaceEntity; import com.ycwl.basic.model.pc.face.resp.FaceRespVO; import com.ycwl.basic.model.pc.faceSample.entity.FaceSampleEntity; @@ -26,11 +25,9 @@ import com.ycwl.basic.model.pc.renderWorker.entity.RenderWorkerEntity; import com.ycwl.basic.model.pc.scenic.entity.ScenicConfigEntity; import com.ycwl.basic.model.pc.scenic.entity.ScenicEntity; import com.ycwl.basic.model.pc.source.entity.SourceEntity; -import com.ycwl.basic.model.pc.source.resp.SourceRespVO; import com.ycwl.basic.model.pc.task.entity.TaskEntity; import com.ycwl.basic.model.pc.task.req.TaskReqQuery; import com.ycwl.basic.model.pc.task.resp.TaskRespVO; -import com.ycwl.basic.model.pc.template.entity.TemplateConfigEntity; import com.ycwl.basic.model.pc.template.resp.TemplateRespVO; import com.ycwl.basic.model.pc.video.entity.MemberVideoEntity; import com.ycwl.basic.model.pc.video.entity.VideoEntity; @@ -69,7 +66,6 @@ import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; @@ -161,6 +157,9 @@ public class TaskTaskServiceImpl implements TaskService { req.getTemplateList().forEach(template -> { if (StringUtils.isNumeric(template.getId())) { TemplateRespVO dbTemplate = templateRepository.getTemplate(Long.parseLong(template.getId())); + if (dbTemplate == null) { + return; + } if (!dbTemplate.getUpdateTime().equals(template.getUpdateTime())) { updTemplateList.add(dbTemplate); } @@ -171,13 +170,17 @@ public class TaskTaskServiceImpl implements TaskService { } try { if (lock.tryLock(2, TimeUnit.SECONDS)) { - List taskList = taskMapper.selectNotRunning(); - resp.setTasks(taskList); - resp.setTemplates(updTemplateList); - taskList.forEach(task -> { - taskMapper.assignToWorker(task.getId(), worker.getId()); - videoTaskRepository.clearTaskCache(task.getId()); - }); + try { + List taskList = taskMapper.selectNotRunning(); + resp.setTasks(taskList); + resp.setTemplates(updTemplateList); + taskList.forEach(task -> { + taskMapper.assignToWorker(task.getId(), worker.getId()); + videoTaskRepository.clearTaskCache(task.getId()); + }); + } finally { + lock.unlock(); + } } } catch (InterruptedException ignored) { } @@ -214,6 +217,113 @@ public class TaskTaskServiceImpl implements TaskService { videoTaskRepository.clearTaskCache(taskUpdate.getId()); } + @Override + public void forceCreateTaskByFaceIdAndTempalteId(Long faceId, Long templateId) { + FaceEntity face = faceRepository.getFace(faceId); + if (face == null) { + log.info("faceId:{} is not exist", faceId); + return; + } + List faceSampleList = faceRepository.getFaceSampleList(faceId); + if (faceSampleList.isEmpty()) { + log.info("faceId:{} sample list not exist", faceId); + } + List faceSampleIds = faceSampleList.stream().map(FaceSampleEntity::getId).collect(Collectors.toList()); + List sourceList = sourceMapper.listVideoByScenicFaceRelation(face.getScenicId(), faceId); + VideoPieceGetter.Task task = new VideoPieceGetter.Task(); + task.faceId = faceId; + task.faceSampleIds = faceSampleIds; + task.templateId = templateId; + task.memberId = face.getMemberId(); + task.callback = () -> { + log.info("task callback"); + List videoSourceList = sourceMapper.listVideoByScenicFaceRelation(face.getScenicId(), faceId); + Map> sourcesMap = videoSourceList.stream() + .peek(item -> item.setUrl(item.getVideoUrl())) + .collect(Collectors.groupingBy(item -> item.getDeviceId().toString())); + if (sourcesMap.isEmpty()) { + // 主动禁止没有视频源视频生成 + log.info("task callback: 没有视频源"); + return; + } + sourcesMap.forEach((key, value) -> { + // 每个value只保留第一个 + value.removeIf(item -> !value.get(0).equals(item)); + }); + TaskReqQuery taskReqQuery = new TaskReqQuery(); + taskReqQuery.setFaceId(faceId); + taskReqQuery.setTemplateId(templateId); + taskReqQuery.setTaskParams(JSON.toJSONString(sourcesMap)); + List list = taskMapper.list(taskReqQuery); + + MemberVideoEntity memberVideoEntity = new MemberVideoEntity(); + memberVideoEntity.setMemberId(face.getMemberId()); + memberVideoEntity.setScenicId(face.getScenicId()); + memberVideoEntity.setFaceId(faceId); + memberVideoEntity.setTemplateId(templateId); + memberVideoEntity.setIsBuy(0); + if (list.isEmpty()) { + log.info("创建任务! faceId:{},templateId:{},taskParams:{}", faceId, templateId, sourcesMap); + ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(face.getScenicId()); + TaskEntity taskEntity = null; + if (Integer.valueOf(0).equals(scenicConfig.getTemplateNewVideoType())) { + log.info("景区{}启用：templateNewVideoType：全新视频原位替换", face.getScenicId()); + taskReqQuery.setTemplateId(templateId); + List templateTaskList = taskMapper.listEntity(taskReqQuery); + if (!templateTaskList.isEmpty()) { + taskEntity = templateTaskList.get(0); + log.info("已有旧生成的视频：{}", taskEntity); + MemberVideoEntity taskVideoRelation = videoMapper.queryRelationByMemberTask(face.getMemberId(), taskEntity.getId()); + if (taskVideoRelation != null) { + log.info("已有旧关联记录的视频：{}", taskVideoRelation); + memberVideoEntity.setIsBuy(taskVideoRelation.getIsBuy()); + memberVideoEntity.setOrderId(taskVideoRelation.getOrderId()); + } + taskMapper.deleteById(taskEntity.getId()); + } + } + if (taskEntity == null) { + taskEntity = new TaskEntity(); + taskEntity.setId(SnowFlakeUtil.getLongId()); + taskEntity.setScenicId(face.getScenicId()); + taskEntity.setFaceId(faceId); + taskEntity.setTemplateId(templateId); + taskEntity.setAutomatic(0); + } + taskEntity.setWorkerId(null); + taskEntity.setStatus(0); + taskEntity.setTaskParams(JSON.toJSONString(sourcesMap)); + taskMapper.add(taskEntity); + memberVideoEntity.setTaskId(taskEntity.getId()); + } else { + log.info("重复task! faceId:{},templateId:{},taskParams:{}", faceId, templateId, sourcesMap); + memberVideoEntity.setTaskId(list.get(0).getId()); + VideoEntity video = videoMapper.findByTaskId(list.get(0).getId()); + if (video != null) { + IsBuyRespVO isBuy = orderBiz.isBuy(face.getMemberId(), list.get(0).getScenicId(), 0, video.getId()); + if (isBuy.isBuy()) { + memberVideoEntity.setIsBuy(1); + memberVideoEntity.setOrderId(isBuy.getOrderId()); + } + if (isBuy.isFree()) { + memberVideoEntity.setIsBuy(1); + } + memberVideoEntity.setVideoId(video.getId()); + } + } + videoMapper.addRelation(memberVideoEntity); + }; + if (faceSampleIds.isEmpty()) { + // 没有人脸样本 + if (!sourceList.isEmpty()) { + // 但是有原片 + task.callback.onInvoke(); + } + } else { + VideoPieceGetter.addTask(task); + } + } + @Override public void autoCreateTaskByFaceId(Long faceId) { FaceRespVO faceRespVO = faceMapper.getById(faceId); @@ -410,6 +520,7 @@ public class TaskTaskServiceImpl implements TaskService { taskUpdate.setWorkerId(worker.getId()); taskMapper.setSuccess(taskUpdate); videoTaskRepository.clearTaskCache(taskUpdate.getId()); + VideoEntity video = videoMapper.findByTaskId(taskId); if (video != null) { video.setVideoUrl(task.getVideoUrl()); @@ -503,7 +614,7 @@ public class TaskTaskServiceImpl implements TaskService { taskMapper.update(updateTask); videoTaskRepository.clearTaskCache(updateTask.getId()); } - return adapter.getUrlForUpload(filename); + return adapter.getUrlForUpload(new Date(System.currentTimeMillis() + 1000 * 60 * 60), "video/mp4", filename); } public void sendVideoGeneratedServiceNotification(Long taskId) { diff --git a/src/main/java/com/ycwl/basic/task/DynamicTaskGenerator.java b/src/main/java/com/ycwl/basic/task/DynamicTaskGenerator.java index be558bf..8c97dda 100644 --- a/src/main/java/com/ycwl/basic/task/DynamicTaskGenerator.java +++ b/src/main/java/com/ycwl/basic/task/DynamicTaskGenerator.java @@ -96,26 +96,26 @@ public class DynamicTaskGenerator { if (task == null) { return; } - log.info("开始执行任务：{}", task); // 根据人脸照片获取人脸样本ID FaceSampleRespVO faceSample = faceSampleMapper.getById(task.getFaceSampleId()); if (faceSample == null) { - log.info("人脸样本ID{}不存在", task.getFaceSampleId()); + log.debug("人脸样本ID{}不存在", task.getFaceSampleId()); return; } if (faceSample.getScore() == null) { - log.info("人脸样本ID{}人脸质量为空", task.getFaceSampleId()); + log.debug("人脸样本ID{}人脸质量为空", task.getFaceSampleId()); return; } ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(faceSample.getScenicId()); if (scenicConfig == null) { - log.info("当前景区{}，无配置", faceSample.getScenicId()); + log.debug("当前景区{}，无配置", faceSample.getScenicId()); return; } if (!Integer.valueOf(1).equals(scenicConfig.getBookRoutine()) && !Integer.valueOf(3).equals(scenicConfig.getBookRoutine())) { - log.info("当前景区{}未启用预约流程，跳过", faceSample.getScenicId()); + log.debug("当前景区{}未启用预约流程，跳过", faceSample.getScenicId()); return; } + log.info("开始执行任务：{}", task); SearchFaceRespVo userDbSearchResult = faceService.searchFace(USER_FACE_DB_NAME+faceSample.getScenicId(), faceSample.getFaceUrl()); // 如果人脸样本ID在人脸样本库中，则创建任务 if (!userDbSearchResult.getSampleListIds().isEmpty()) { diff --git a/src/main/java/com/ycwl/basic/task/FaceCleaner.java b/src/main/java/com/ycwl/basic/task/FaceCleaner.java index 44a54fc..30102f7 100644 --- a/src/main/java/com/ycwl/basic/task/FaceCleaner.java +++ b/src/main/java/com/ycwl/basic/task/FaceCleaner.java @@ -1,9 +1,16 @@ package com.ycwl.basic.task; +import com.ycwl.basic.mapper.FaceSampleMapper; import com.ycwl.basic.mapper.ScenicMapper; +import com.ycwl.basic.mapper.SourceMapper; +import com.ycwl.basic.model.pc.faceSample.req.FaceSampleReqQuery; +import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO; import com.ycwl.basic.model.pc.scenic.req.ScenicReqQuery; import com.ycwl.basic.model.pc.scenic.resp.ScenicRespVO; import com.ycwl.basic.service.task.TaskFaceService; +import com.ycwl.basic.storage.StorageFactory; +import com.ycwl.basic.storage.adapters.IStorageAdapter; +import com.ycwl.basic.storage.entity.StorageFileObject; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.EnableScheduling; @@ -20,7 +27,10 @@ public class FaceCleaner { private ScenicMapper scenicMapper; @Autowired private TaskFaceService faceService; - + @Autowired + private FaceSampleMapper faceSampleMapper; + @Autowired + private SourceMapper sourceMapper; @Scheduled(cron = "0 0 4 * * ?") public void clean(){ @@ -31,4 +41,23 @@ public class FaceCleaner { faceService.batchDeleteExpiredFace(scenic.getId()); }); } + + @Scheduled(cron = "0 0 3 * * ?") + public void deleteExpiredSource(){ + + } + + @Scheduled(cron = "0 0 5 * * ?") + public void clear(){ + log.info("开始清理人脸文件"); + List faceSampleRespVOS = faceSampleMapper.list(new FaceSampleReqQuery()); + IStorageAdapter adapter = StorageFactory.use("faces"); + List fileObjectList = adapter.listDir("user-face"); + fileObjectList.parallelStream().forEach(fileObject -> { + if(faceSampleRespVOS.parallelStream().noneMatch(faceSampleRespVO -> faceSampleRespVO.getFaceUrl().contains(fileObject.getFullPath()))){ + log.info("删除人脸文件：{}", fileObject); + adapter.deleteFile(fileObject.getFullPath()); + } + }); + } } diff --git a/src/main/java/com/ycwl/basic/task/VideoTaskGenerator.java b/src/main/java/com/ycwl/basic/task/VideoTaskGenerator.java index c8f204c..c0911e6 100644 --- a/src/main/java/com/ycwl/basic/task/VideoTaskGenerator.java +++ b/src/main/java/com/ycwl/basic/task/VideoTaskGenerator.java @@ -13,7 +13,9 @@ import com.ycwl.basic.repository.ScenicRepository; import com.ycwl.basic.repository.TemplateRepository; import com.ycwl.basic.service.task.TaskFaceService; import com.ycwl.basic.service.task.impl.TaskTaskServiceImpl; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Profile; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -21,8 +23,10 @@ import org.springframework.stereotype.Component; import java.util.Date; import java.util.List; +@Slf4j @EnableScheduling @Component +@Profile("prod") public class VideoTaskGenerator { @Autowired private FaceMapper faceMapper; @@ -55,31 +59,10 @@ public class VideoTaskGenerator { taskFaceService.searchFace(face.getId()); boolean canAutoGenerate = templateBiz.determineTemplateCanAutoGenerate(templateId, face.getId(), false); if (canAutoGenerate) { - taskTaskService.autoCreateTaskByFaceId(face.getId()); - } - }); - } - - - @Scheduled(cron = "0 30 4 * * *") - public void generateVideoTaskZTJQ() { - // 指定，获取指定日期的未完成人脸样本，并生成任务 - Long scenicId = 3930324797233434624L; - List contentList = templateMapper.listFor(scenicId); - if (contentList.isEmpty()) { - return; - } - Long templateId = contentList.get(0).getTemplateId(); - FaceReqQuery query = new FaceReqQuery(); - query.setScenicId(scenicId); - query.setStartTime(DateUtil.beginOfDay(new Date())); - query.setEndTime(DateUtil.endOfDay(new Date())); - List list = faceMapper.list(query); - list.stream().parallel().forEach(face -> { - taskFaceService.searchFace(face.getId()); - boolean canAutoGenerate = templateBiz.determineTemplateCanAutoGenerate(templateId, face.getId(), false); - if (canAutoGenerate) { - taskTaskService.autoCreateTaskByFaceId(face.getId()); + log.info("task callback: 自动生成"); + taskTaskService.forceCreateTaskByFaceIdAndTempalteId(face.getId(), templateId); + } else { + log.info("task callback: 不自动生成"); } }); } diff --git a/src/main/java/com/ycwl/basic/utils/JwtTokenUtil.java b/src/main/java/com/ycwl/basic/utils/JwtTokenUtil.java index b644770..46422b3 100644 --- a/src/main/java/com/ycwl/basic/utils/JwtTokenUtil.java +++ b/src/main/java/com/ycwl/basic/utils/JwtTokenUtil.java @@ -96,7 +96,7 @@ public class JwtTokenUtil { public static void main(String[] args) throws Exception { JwtInfo jwtInfo = new JwtInfo(); - jwtInfo.setUserId(3936121342868459520L); + jwtInfo.setUserId(3950649683084447744L); jwtInfo.setName("微信用户"); System.out.println(generateToken(jwtInfo, 86400)); } diff --git a/src/main/resources/mapper/FaceSampleMapper.xml b/src/main/resources/mapper/FaceSampleMapper.xml index 1453042..71722db 100644 --- a/src/main/resources/mapper/FaceSampleMapper.xml +++ b/src/main/resources/mapper/FaceSampleMapper.xml @@ -101,4 +101,9 @@ from face_sample where id = #{id} + diff --git a/src/main/resources/mapper/OrderMapper.xml b/src/main/resources/mapper/OrderMapper.xml index 9dee49e..9118a30 100644 --- a/src/main/resources/mapper/OrderMapper.xml +++ b/src/main/resources/mapper/OrderMapper.xml @@ -70,7 +70,7 @@ LEFT JOIN video v ON mv.video_id = v.id ), member_source_data AS ( - SELECT ms.member_id, ms.source_id, ms.face_id, f.face_url, s.video_url, s.url + SELECT ms.member_id, ms.type, ms.source_id, ms.face_id, f.face_url, s.video_url, s.url FROM member_source ms LEFT JOIN face f ON ms.face_id = f.id LEFT JOIN source s ON ms.source_id = s.id @@ -79,6 +79,7 @@ oi.id AS oiId, oi.order_id AS orderId, oi.goods_id, + msd.source_id, sc.id AS scenic_id, sc.name AS scenic_name, CASE oi.goods_type @@ -113,7 +114,7 @@ LEFT JOIN `order` o ON oi.order_id = o.id LEFT JOIN scenic sc ON o.scenic_id = sc.id LEFT JOIN member_video_data mvd ON o.member_id = mvd.member_id AND oi.goods_id = mvd.video_id - LEFT JOIN member_source_data msd ON o.member_id = msd.member_id AND oi.goods_id = msd.face_id + LEFT JOIN member_source_data msd ON o.member_id = msd.member_id AND oi.goods_id = msd.face_id AND msd.type = oi.goods_type WHERE oi.order_id = #{id}; diff --git a/src/main/resources/mapper/ScenicMapper.xml b/src/main/resources/mapper/ScenicMapper.xml index 2e27627..7305cb1 100644 --- a/src/main/resources/mapper/ScenicMapper.xml +++ b/src/main/resources/mapper/ScenicMapper.xml @@ -92,7 +92,9 @@ template_new_video_type=#{templateNewVideoType}, anti_screen_record_type=#{antiScreenRecordType}, disable_source_video=#{disableSourceVideo}, - disable_source_image=#{disableSourceImage} + disable_source_image=#{disableSourceImage}, + video_source_store_day=#{videoSourceStoreDay}, + image_source_store_day=#{imageSourceStoreDay} where id = #{id} diff --git a/src/main/resources/mapper/SourceMapper.xml b/src/main/resources/mapper/SourceMapper.xml index da79e3d..660d890 100644 --- a/src/main/resources/mapper/SourceMapper.xml +++ b/src/main/resources/mapper/SourceMapper.xml @@ -121,7 +121,7 @@ and ms.is_buy = #{isBuy} and ms.type = #{type} and ms.face_id = #{faceId} - order by so.create_time desc + order by so.create_time asc