diff --git a/src/main/java/com/ycwl/basic/controller/pc/BrokerController.java b/src/main/java/com/ycwl/basic/controller/pc/BrokerController.java index 7c92641..9a61793 100644 --- a/src/main/java/com/ycwl/basic/controller/pc/BrokerController.java +++ b/src/main/java/com/ycwl/basic/controller/pc/BrokerController.java @@ -3,7 +3,6 @@ package com.ycwl.basic.controller.pc; import com.ycwl.basic.model.pc.broker.entity.BrokerEntity; import com.ycwl.basic.model.pc.broker.req.BrokerRecordReqQuery; import com.ycwl.basic.model.pc.broker.req.BrokerReqQuery; -import com.ycwl.basic.model.pc.broker.resp.BrokerRecordRespVO; import com.ycwl.basic.model.pc.broker.resp.BrokerRespVO; import com.ycwl.basic.model.pc.broker.resp.DailySummaryRespVO; import com.ycwl.basic.service.pc.BrokerRecordService; @@ -131,7 +130,7 @@ public class BrokerController { try { WxMpUtil.generateWXAQRCode(appId, appSecret, appState, path, filePath); File file = new File(filePath); - String s = adapter.uploadFile(file, filePath); + String s = adapter.uploadFile(null, file, filePath); file.delete(); adapter.setAcl(StorageAcl.PUBLIC_READ, filePath); return ApiResponse.success(s); diff --git a/src/main/java/com/ycwl/basic/controller/pc/ScenicController.java b/src/main/java/com/ycwl/basic/controller/pc/ScenicController.java index b0146de..6d6fe17 100644 --- a/src/main/java/com/ycwl/basic/controller/pc/ScenicController.java +++ b/src/main/java/com/ycwl/basic/controller/pc/ScenicController.java @@ -121,7 +121,7 @@ public class ScenicController { try { WxMpUtil.generateWXAQRCode(appId, appSecret, appState, path, filePath); File file = new File(filePath); - String s = adapter.uploadFile(file, filePath); + String s = adapter.uploadFile(null, file, filePath); file.delete(); adapter.setAcl(StorageAcl.PUBLIC_READ, filePath); return ApiResponse.success(s); diff --git a/src/main/java/com/ycwl/basic/controller/proxy/ProxyController.java b/src/main/java/com/ycwl/basic/controller/proxy/ProxyController.java new file mode 100644 index 0000000..64e18bd --- /dev/null +++ b/src/main/java/com/ycwl/basic/controller/proxy/ProxyController.java @@ -0,0 +1,75 @@ +package com.ycwl.basic.controller.proxy; + +import com.ycwl.basic.annotation.IgnoreToken; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Enumeration; +import java.util.List; +import java.util.Map; + +@RestController +public class ProxyController { + @IgnoreToken + @RequestMapping(value = "/proxy", method = RequestMethod.GET) + public void proxy(@RequestParam(value = "url") String url, + HttpServletRequest request, + HttpServletResponse response) throws IOException { + if (!url.startsWith("http://") && !url.startsWith("https://")) { + url = "http://" + url; // 或根据业务逻辑选择默认协议 + } + + // 新增User-Agent检测逻辑 + String userAgent = request.getHeader("User-Agent"); + if (userAgent != null && userAgent.contains("Lavf/")) { + response.sendRedirect(url); + return; + } + // 创建HTTP连接 + URL urlObj = new URL(url); + HttpURLConnection connection = (HttpURLConnection) urlObj.openConnection(); + + // 设置请求方法和请求头 + connection.setRequestMethod("GET"); + for (Enumeration headers = request.getHeaderNames(); headers.hasMoreElements();) { + String headerName = headers.nextElement(); + connection.addRequestProperty(headerName, request.getHeader(headerName)); + } + + // 处理响应 + int responseCode = connection.getResponseCode(); + response.setStatus(responseCode); + + // 转发响应头 + Map> headerFields = connection.getHeaderFields(); + for (Map.Entry> entry : headerFields.entrySet()) { + if (entry.getKey() != null) { + for (String value : entry.getValue()) { + response.addHeader(entry.getKey(), value); + } + } + } + + // 流式传输响应体 + try (InputStream inputStream = connection.getInputStream(); + OutputStream outputStream = response.getOutputStream()) { + byte[] buffer = new byte[4096]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, bytesRead); + } + } catch (IOException e) { + return; + } + } +} diff --git a/src/main/java/com/ycwl/basic/controller/vpt/VptController.java b/src/main/java/com/ycwl/basic/controller/vpt/VptController.java index eed355f..b41412e 100644 --- a/src/main/java/com/ycwl/basic/controller/vpt/VptController.java +++ b/src/main/java/com/ycwl/basic/controller/vpt/VptController.java @@ -40,7 +40,7 @@ public class VptController { } @PostMapping("/scenic/{scenicId}/{taskId}/uploadUrl") public String uploadUrl(@PathVariable("scenicId") Long scenicId, @PathVariable("taskId") Long taskId) { - IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId); + IStorageAdapter adapter = scenicService.getScenicTmpStorageAdapter(scenicId); String filename = StorageUtil.joinPath(StorageConstant.VIDEO_PIECE_PATH, taskId.toString() + ".mp4"); String urlForUpload = adapter.getUrlForUpload(new Date(System.currentTimeMillis() + 1000 * 60 * 60), "video/mp4", filename); urlForUpload = urlForUpload.replace("-internal.aliyuncs.com", ".aliyuncs.com"); @@ -48,7 +48,7 @@ public class VptController { } @PostMapping("/scenic/{scenicId}/{taskId}/success") public ApiResponse success(@PathVariable("scenicId") Long scenicId, @PathVariable("taskId") Long taskId, @RequestBody FileObject fileObject) { - IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId); + IStorageAdapter adapter = scenicService.getScenicTmpStorageAdapter(scenicId); String filename = StorageUtil.joinPath(StorageConstant.VIDEO_PIECE_PATH, taskId.toString() + ".mp4"); fileObject.setUrl(adapter.getUrl(filename)); adapter.setAcl(StorageAcl.PUBLIC_READ, filename); diff --git a/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java b/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java index ce240fe..c9a0562 100644 --- a/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java +++ b/src/main/java/com/ycwl/basic/controller/wvp/WvpController.java @@ -45,7 +45,7 @@ public class WvpController { @PostMapping("/scenic/{scenicId}/{taskId}/uploadUrl") public String uploadUrl(@PathVariable("scenicId") Long scenicId, @PathVariable("taskId") Long taskId) { - IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId); + IStorageAdapter adapter = scenicService.getScenicTmpStorageAdapter(scenicId); String filename = StorageUtil.joinPath(StorageConstant.VIDEO_PIECE_PATH, taskId.toString() + ".mp4"); String urlForUpload = adapter.getUrlForUpload(new Date(System.currentTimeMillis() + 1000 * 60 * 60), "video/mp4", filename); urlForUpload = urlForUpload.replace("-internal.aliyuncs.com", ".aliyuncs.com"); @@ -53,7 +53,7 @@ public class WvpController { } @PostMapping("/scenic/{scenicId}/{taskId}/success") public ApiResponse success(@PathVariable("scenicId") Long scenicId, @PathVariable("taskId") Long taskId, @RequestBody FileObject fileObject) { - IStorageAdapter adapter = scenicService.getScenicStorageAdapter(scenicId); + IStorageAdapter adapter = scenicService.getScenicTmpStorageAdapter(scenicId); String filename = StorageUtil.joinPath(StorageConstant.VIDEO_PIECE_PATH, taskId.toString() + ".mp4"); fileObject.setUrl(adapter.getUrl(filename)); adapter.setAcl(StorageAcl.PUBLIC_READ, filename); diff --git a/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java b/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java index 8313866..253056d 100644 --- a/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java +++ b/src/main/java/com/ycwl/basic/model/pc/scenic/entity/ScenicConfigEntity.java @@ -71,6 +71,8 @@ public class ScenicConfigEntity { private Float faceScoreThreshold; private StorageType storeType; private String storeConfigJson; + private StorageType tmpStoreType; + private String tmpStoreConfigJson; private BigDecimal brokerDirectRate; private Integer faceDetectHelperThreshold; diff --git a/src/main/java/com/ycwl/basic/service/mobile/impl/GoodsServiceImpl.java b/src/main/java/com/ycwl/basic/service/mobile/impl/GoodsServiceImpl.java index 4622280..d607baf 100644 --- a/src/main/java/com/ycwl/basic/service/mobile/impl/GoodsServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/mobile/impl/GoodsServiceImpl.java @@ -46,8 +46,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.web.context.request.RequestContextHolder; +import org.springframework.web.context.request.ServletRequestAttributes; import java.io.File; +import java.net.MalformedURLException; +import java.net.URL; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -199,7 +203,20 @@ public class GoodsServiceImpl implements GoodsService { goodsDetailVO.setGoodsType(sourceType); goodsDetailVO.setSourceType(sourceType); goodsDetailVO.setGoodsId(sourceRespVO.getId()); - goodsDetailVO.setVideoUrl(sourceRespVO.getVideoUrl()); + if (sourceRespVO.getVideoUrl() != null) { + try { + URL url = new URL(sourceRespVO.getVideoUrl()); + if (StringUtils.startsWith(url.getHost(), "100.64.")) { + // 内网地址，需要代理 + goodsDetailVO.setVideoUrl("https://zhentuai.com/proxy?url=" + sourceRespVO.getVideoUrl()); + } else { + goodsDetailVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } catch (MalformedURLException e) { + log.warn("url地址解析异常：{}", sourceRespVO.getVideoUrl(), e); + goodsDetailVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } goodsDetailVO.setUrl(sourceRespVO.getUrl()); goodsDetailVO.setCreateTime(sourceRespVO.getCreateTime()); goodsDetailVO.setIsFree(sourceRespVO.getIsFree()); @@ -430,10 +447,7 @@ public class GoodsServiceImpl implements GoodsService { public ApiResponse sourceGoodsInfo(Long sourceId) { SourceRespVO sourceRespVO = sourceMapper.getById(sourceId); if (sourceRespVO == null) { - sourceRespVO = sourceMapper.getById(sourceId); - if (sourceRespVO == null) { - return ApiResponse.fail("该视频不存在"); - } + return ApiResponse.fail("该视频不存在"); } GoodsDetailVO goodsDetailVO = new GoodsDetailVO(); goodsDetailVO.setGoodsName("原片"); @@ -441,7 +455,20 @@ public class GoodsServiceImpl implements GoodsService { goodsDetailVO.setScenicName(sourceRespVO.getScenicName()); goodsDetailVO.setGoodsType(sourceRespVO.getType()); goodsDetailVO.setGoodsId(sourceRespVO.getId()); - goodsDetailVO.setVideoUrl(sourceRespVO.getVideoUrl()); + if (sourceRespVO.getVideoUrl() != null) { + try { + URL url = new URL(sourceRespVO.getVideoUrl()); + if (StringUtils.startsWith(url.getHost(), "100.64.")) { + // 内网地址，需要代理 + goodsDetailVO.setVideoUrl("https://zhentuai.com/proxy?url=" + sourceRespVO.getVideoUrl()); + } else { + goodsDetailVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } catch (MalformedURLException e) { + log.warn("url地址解析异常：{}", sourceRespVO.getVideoUrl(), e); + goodsDetailVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } goodsDetailVO.setTemplateCoverUrl(sourceRespVO.getUrl()); goodsDetailVO.setCreateTime(sourceRespVO.getCreateTime()); return ApiResponse.success(goodsDetailVO); @@ -522,7 +549,7 @@ public class GoodsServiceImpl implements GoodsService { log.error("process error", e); return; } - String url = adapter.uploadFile(watermarkedFile, StorageConstant.PHOTO_WATERMARKED_PATH, watermarkedFile.getName()); + String url = adapter.uploadFile(null, watermarkedFile, StorageConstant.PHOTO_WATERMARKED_PATH, watermarkedFile.getName()); adapter.setAcl(StorageAcl.PUBLIC_READ, StorageConstant.PHOTO_WATERMARKED_PATH, watermarkedFile.getName()); sourceMapper.addSourceWatermark(item.getGoodsId(), null, ImageWatermarkOperatorEnum.WATERMARK.getType(), url); tmpFile.add(watermarkedFile); @@ -634,7 +661,7 @@ public class GoodsServiceImpl implements GoodsService { log.error("process error", e); return; } - String url = adapter.uploadFile(watermarkedFile, StorageConstant.PHOTO_WATERMARKED_PATH, watermarkedFile.getName()); + String url = adapter.uploadFile(null, watermarkedFile, StorageConstant.PHOTO_WATERMARKED_PATH, watermarkedFile.getName()); adapter.setAcl(StorageAcl.PUBLIC_READ, StorageConstant.PHOTO_WATERMARKED_PATH, watermarkedFile.getName()); sourceMapper.addSourceWatermark(item.getGoodsId(), face.getId(), type.getType(), url); tmpFile.add(watermarkedFile); diff --git a/src/main/java/com/ycwl/basic/service/pc/ScenicService.java b/src/main/java/com/ycwl/basic/service/pc/ScenicService.java index 47d9ba1..781fda8 100644 --- a/src/main/java/com/ycwl/basic/service/pc/ScenicService.java +++ b/src/main/java/com/ycwl/basic/service/pc/ScenicService.java @@ -36,5 +36,7 @@ public interface ScenicService { IStorageAdapter getScenicStorageAdapter(Long scenicId); + IStorageAdapter getScenicTmpStorageAdapter(Long scenicId); + IFaceBodyAdapter getScenicFaceBodyAdapter(Long scenicId); } diff --git a/src/main/java/com/ycwl/basic/service/pc/impl/ScenicServiceImpl.java b/src/main/java/com/ycwl/basic/service/pc/impl/ScenicServiceImpl.java index 37c2ca5..a61aa0e 100644 --- a/src/main/java/com/ycwl/basic/service/pc/impl/ScenicServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/pc/impl/ScenicServiceImpl.java @@ -17,6 +17,7 @@ import com.ycwl.basic.service.pc.ScenicService; import com.ycwl.basic.service.task.TaskFaceService; import com.ycwl.basic.storage.StorageFactory; import com.ycwl.basic.storage.adapters.IStorageAdapter; +import com.ycwl.basic.storage.exceptions.StorageUnsupportedException; import com.ycwl.basic.utils.ApiResponse; import com.ycwl.basic.utils.SnowFlakeUtil; import lombok.extern.slf4j.Slf4j; @@ -197,6 +198,7 @@ public class ScenicServiceImpl implements ScenicService { scenicRepository.clearCache(config.getScenicId()); scenicFaceBodyAdapterMap.remove(config.getScenicId()); scenicStorageAdapterMap.remove(config.getScenicId()); + scenicTmpStorageAdapterMap.remove(config.getScenicId()); } @@ -207,14 +209,37 @@ public class ScenicServiceImpl implements ScenicService { IStorageAdapter adapter; ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId); if (scenicConfig != null && scenicConfig.getStoreType() != null) { - adapter = StorageFactory.get(scenicConfig.getStoreType()); - adapter.loadConfig(JSONObject.parseObject(scenicConfig.getStoreConfigJson(), Map.class)); + try { + adapter = StorageFactory.get(scenicConfig.getStoreType()); + adapter.loadConfig(JSONObject.parseObject(scenicConfig.getStoreConfigJson(), Map.class)); + } catch (StorageUnsupportedException ignored) { + return StorageFactory.use("video"); + } } else { adapter = StorageFactory.use("video"); } return adapter; }); } + private static final Map scenicTmpStorageAdapterMap = new ConcurrentHashMap<>(); + @Override + public IStorageAdapter getScenicTmpStorageAdapter(Long scenicId) { + return scenicTmpStorageAdapterMap.computeIfAbsent(scenicId, (key) -> { + IStorageAdapter adapter; + ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId); + if (scenicConfig != null && scenicConfig.getTmpStoreType() != null) { + try { + adapter = StorageFactory.get(scenicConfig.getTmpStoreType()); + adapter.loadConfig(JSONObject.parseObject(scenicConfig.getTmpStoreConfigJson(), Map.class)); + } catch (StorageUnsupportedException ignored) { + return getScenicStorageAdapter(scenicId); + } + } else { + return getScenicStorageAdapter(scenicId); + } + return adapter; + }); + } private static final Map scenicFaceBodyAdapterMap = new ConcurrentHashMap<>(); @Override public IFaceBodyAdapter getScenicFaceBodyAdapter(Long scenicId) { diff --git a/src/main/java/com/ycwl/basic/service/pc/impl/SourceServiceImpl.java b/src/main/java/com/ycwl/basic/service/pc/impl/SourceServiceImpl.java index 52c55a3..ddce799 100644 --- a/src/main/java/com/ycwl/basic/service/pc/impl/SourceServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/pc/impl/SourceServiceImpl.java @@ -11,9 +11,12 @@ import com.ycwl.basic.service.pc.SourceService; import com.ycwl.basic.task.VideoPieceGetter; import com.ycwl.basic.utils.ApiResponse; import com.ycwl.basic.utils.SnowFlakeUtil; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.net.MalformedURLException; +import java.net.URL; import java.util.Collections; import java.util.List; @@ -32,18 +35,63 @@ public class SourceServiceImpl implements SourceService { public ApiResponse> pageQuery(SourceReqQuery sourceReqQuery) { PageHelper.startPage(sourceReqQuery.getPageNum(), sourceReqQuery.getPageSize()); List list = sourceMapper.list(sourceReqQuery); + list.forEach(sourceRespVO -> { + if (sourceRespVO.getVideoUrl() != null) { + try { + URL url = new URL(sourceRespVO.getVideoUrl()); + if (StringUtils.startsWith(url.getHost(), "100.64.")) { + // 内网地址，需要代理 + sourceRespVO.setVideoUrl("https://zhentuai.com/proxy?url=" + sourceRespVO.getVideoUrl()); + } else { + sourceRespVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } catch (MalformedURLException e) { + sourceRespVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } + }); PageInfo pageInfo = new PageInfo<>(list); return ApiResponse.success(pageInfo); } @Override public ApiResponse> list(SourceReqQuery sourceReqQuery) { - return ApiResponse.success(sourceMapper.list(sourceReqQuery)); + List list = sourceMapper.list(sourceReqQuery); + list.forEach(sourceRespVO -> { + if (sourceRespVO.getVideoUrl() != null) { + try { + URL url = new URL(sourceRespVO.getVideoUrl()); + if (StringUtils.startsWith(url.getHost(), "100.64.")) { + // 内网地址，需要代理 + sourceRespVO.setVideoUrl("https://zhentuai.com/proxy?url=" + sourceRespVO.getVideoUrl()); + } else { + sourceRespVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } catch (MalformedURLException e) { + sourceRespVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } + }); + return ApiResponse.success(list); } @Override public ApiResponse getById(Long id, Long userId) { - return ApiResponse.success(sourceMapper.userGetById(id, userId)); + SourceRespVO sourceRespVO = sourceMapper.userGetById(id, userId); + if (sourceRespVO.getVideoUrl() != null) { + try { + URL url = new URL(sourceRespVO.getVideoUrl()); + if (StringUtils.startsWith(url.getHost(), "100.64.")) { + // 内网地址，需要代理 + sourceRespVO.setVideoUrl("https://zhentuai.com/proxy?url=" + sourceRespVO.getVideoUrl()); + } else { + sourceRespVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } catch (MalformedURLException e) { + sourceRespVO.setVideoUrl(sourceRespVO.getVideoUrl()); + } + } + return ApiResponse.success(sourceRespVO); } @Override diff --git a/src/main/java/com/ycwl/basic/service/task/TaskFaceService.java b/src/main/java/com/ycwl/basic/service/task/TaskFaceService.java index e944cb3..b450ac1 100644 --- a/src/main/java/com/ycwl/basic/service/task/TaskFaceService.java +++ b/src/main/java/com/ycwl/basic/service/task/TaskFaceService.java @@ -10,8 +10,6 @@ public interface TaskFaceService { SearchFaceRespVo searchFace(IFaceBodyAdapter adapter, String dbName, String faceUrl, String reason); - String uploadFile(MultipartFile file, Long userId); - boolean deleteFaceSample(Long scenicId, String dbName, String entityId); boolean assureFaceDb(IFaceBodyAdapter faceBodyAdapter, String dbName); diff --git a/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java b/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java index 83a398a..04b1236 100644 --- a/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java +++ b/src/main/java/com/ycwl/basic/service/task/impl/TaskFaceServiceImpl.java @@ -289,30 +289,6 @@ public class TaskFaceServiceImpl implements TaskFaceService { } } - @Override - public String uploadFile(MultipartFile file, Long userId) { - if (file.isEmpty()) { - throw new RuntimeException("文件不存在！"); - } - String originalFilename = file.getOriginalFilename(); - //获取文件名后缀 - String suffix = originalFilename.split("\\.")[1]; - if ("Jpeg".equals(suffix)) { - suffix = "jpg"; - } - //文件储存路径 - String filePath = StorageUtil.joinPath("user-faces", DateUtils.format(new Date(),"yyyy-MM-dd")); - // 生成文件名 - String fileName= userId + "." + suffix; - IStorageAdapter adapter = StorageFactory.use("faces"); - try { - return adapter.uploadFile(file.getInputStream(), filePath, fileName); - } catch (IOException e) { - log.error("文件上传失败！", e); - return null; - } - } - @Override public boolean deleteFaceSample(Long scenicId, String dbName, String entityId) { ScenicConfigEntity scenicConfig = scenicRepository.getScenicConfig(scenicId); diff --git a/src/main/java/com/ycwl/basic/storage/adapters/AStorageAdapter.java b/src/main/java/com/ycwl/basic/storage/adapters/AStorageAdapter.java index eb06c2e..e9bfbd8 100644 --- a/src/main/java/com/ycwl/basic/storage/adapters/AStorageAdapter.java +++ b/src/main/java/com/ycwl/basic/storage/adapters/AStorageAdapter.java @@ -1,6 +1,7 @@ package com.ycwl.basic.storage.adapters; import com.ycwl.basic.storage.exceptions.UploadFileFailedException; +import lombok.extern.slf4j.Slf4j; import org.springframework.web.multipart.MultipartFile; import java.io.File; @@ -9,16 +10,17 @@ import java.io.FileNotFoundException; import java.io.InputStream; import java.util.Date; +@Slf4j public abstract class AStorageAdapter implements IStorageAdapter { @Override - public String uploadFile(File file, String ...path) { + public String uploadFile(String contentType, File file, String ...path) { if (file == null) { return null; } try { InputStream inputStream = new FileInputStream(file); - return uploadFile(inputStream, path); + return uploadFile(contentType, inputStream, path); } catch (FileNotFoundException e) { throw new UploadFileFailedException("文件不存在"); } @@ -31,8 +33,9 @@ public abstract class AStorageAdapter implements IStorageAdapter { } try { InputStream inputStream = file.getInputStream(); - return uploadFile(inputStream, path); + return uploadFile(file.getContentType(), inputStream, path); } catch (Exception e) { + log.warn("文件上传失败", e); throw new UploadFileFailedException("文件上传失败"); } } diff --git a/src/main/java/com/ycwl/basic/storage/adapters/AliOssAdapter.java b/src/main/java/com/ycwl/basic/storage/adapters/AliOssAdapter.java index ffd1440..8c663fc 100644 --- a/src/main/java/com/ycwl/basic/storage/adapters/AliOssAdapter.java +++ b/src/main/java/com/ycwl/basic/storage/adapters/AliOssAdapter.java @@ -62,7 +62,7 @@ final public class AliOssAdapter extends AStorageAdapter { } @Override - public String uploadFile(InputStream inputStream, String ...path) { + public String uploadFile(String contentType, InputStream inputStream, String ...path) { if (inputStream == null) { return null; } @@ -71,6 +71,9 @@ final public class AliOssAdapter extends AStorageAdapter { OSS ossClient = wrapper.getOSSClient(); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(inputStream.available()); + if (StringUtils.isNotBlank(contentType)) { + metadata.setContentType(contentType); + } PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBucketName(), fullPath, inputStream); ossClient.putObject(putObjectRequest); return getUrl(path); diff --git a/src/main/java/com/ycwl/basic/storage/adapters/AwsOssAdapter.java b/src/main/java/com/ycwl/basic/storage/adapters/AwsOssAdapter.java index 45bdba9..9622919 100644 --- a/src/main/java/com/ycwl/basic/storage/adapters/AwsOssAdapter.java +++ b/src/main/java/com/ycwl/basic/storage/adapters/AwsOssAdapter.java @@ -55,7 +55,7 @@ public class AwsOssAdapter extends AStorageAdapter { } @Override - public String uploadFile(InputStream inputStream, String... path) { + public String uploadFile(String contentType, InputStream inputStream, String... path) { if (inputStream == null) { return null; } @@ -64,6 +64,9 @@ public class AwsOssAdapter extends AStorageAdapter { AmazonS3Client s3Client = wrapper.getS3Client(); ObjectMetadata metadata = new ObjectMetadata(); metadata.setContentLength(inputStream.available()); + if (StringUtils.isNotBlank(contentType)) { + metadata.setContentType(contentType); + } PutObjectRequest putObjectRequest = new PutObjectRequest(config.getBucketName(), fullPath, inputStream, metadata); putObjectRequest.withCannedAcl(CannedAccessControlList.PublicRead); // 设置访问权限，让所有用户都允许访问 s3Client.putObject(putObjectRequest); diff --git a/src/main/java/com/ycwl/basic/storage/adapters/IStorageAdapter.java b/src/main/java/com/ycwl/basic/storage/adapters/IStorageAdapter.java index af3b22e..d0f8e09 100644 --- a/src/main/java/com/ycwl/basic/storage/adapters/IStorageAdapter.java +++ b/src/main/java/com/ycwl/basic/storage/adapters/IStorageAdapter.java @@ -14,8 +14,8 @@ import java.util.Map; public interface IStorageAdapter { void loadConfig(Map config); void setConfig(StorageConfig config); - String uploadFile(InputStream inputStream, String ...path); - String uploadFile(File file, String ...path); + String uploadFile(String contentType, InputStream inputStream, String ...path); + String uploadFile(String contentType, File file, String ...path); String uploadFile(MultipartFile file, String ...path); boolean deleteFile(String ...path); String getUrl(String ...path); diff --git a/src/main/java/com/ycwl/basic/storage/adapters/LocalStorageAdapter.java b/src/main/java/com/ycwl/basic/storage/adapters/LocalStorageAdapter.java index 684c97d..a32644f 100644 --- a/src/main/java/com/ycwl/basic/storage/adapters/LocalStorageAdapter.java +++ b/src/main/java/com/ycwl/basic/storage/adapters/LocalStorageAdapter.java @@ -22,7 +22,7 @@ public class LocalStorageAdapter extends AStorageAdapter{ } @Override - public String uploadFile(InputStream inputStream, String... path) { + public String uploadFile(String contentType, InputStream inputStream, String... path) { return ""; } diff --git a/src/main/java/com/ycwl/basic/storage/entity/StorageFileObject.java b/src/main/java/com/ycwl/basic/storage/entity/StorageFileObject.java index 381c56e..d6372ad 100644 --- a/src/main/java/com/ycwl/basic/storage/entity/StorageFileObject.java +++ b/src/main/java/com/ycwl/basic/storage/entity/StorageFileObject.java @@ -2,12 +2,15 @@ package com.ycwl.basic.storage.entity; import lombok.Data; +import java.util.Date; + @Data public class StorageFileObject { private String path; private String name; private Long size; private Object rawObject; + private Date modifyTime; public String getFullPath() { return path + "/" + name; diff --git a/src/main/java/com/ycwl/basic/task/FaceCleaner.java b/src/main/java/com/ycwl/basic/task/FaceCleaner.java index 9d1bf97..bb6338f 100644 --- a/src/main/java/com/ycwl/basic/task/FaceCleaner.java +++ b/src/main/java/com/ycwl/basic/task/FaceCleaner.java @@ -1,5 +1,6 @@ package com.ycwl.basic.task; +import cn.hutool.core.date.DateUnit; import cn.hutool.core.date.DateUtil; import com.ycwl.basic.constant.StorageConstant; import com.ycwl.basic.facebody.adapter.IFaceBodyAdapter; @@ -211,6 +212,12 @@ public class FaceCleaner { IStorageAdapter adapter = StorageFactory.use("faces"); List fileObjectList = adapter.listDir("user-face"); fileObjectList.parallelStream().forEach(fileObject -> { + if (fileObject.getModifyTime() != null) { + // 如果是一天以内修改的，则跳过 + if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) < 1) { + return; + } + } if(faceSampleRespVOS.parallelStream().noneMatch(faceSampleRespVO -> faceSampleRespVO.getFaceUrl().contains(fileObject.getFullPath()))){ log.info("删除人脸文件：{}", fileObject); adapter.deleteFile(fileObject.getFullPath()); @@ -225,6 +232,12 @@ public class FaceCleaner { log.info("开始清理视频文件"); List fileObjectList = adapter.listDir(StorageConstant.VIDEO_PIECE_PATH); fileObjectList.parallelStream().forEach(fileObject -> { + if (fileObject.getModifyTime() != null) { + // 如果是一天以内修改的，则跳过 + if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) < 1) { + return; + } + } if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getFullPath()))){ log.info("删除文件：{}", fileObject); adapter.deleteFile(fileObject.getFullPath()); @@ -235,6 +248,12 @@ public class FaceCleaner { log.info("开始清理图片文件"); fileObjectList = adapter.listDir(StorageConstant.PHOTO_PATH); fileObjectList.parallelStream().forEach(fileObject -> { + if (fileObject.getModifyTime() != null) { + // 如果是一天以内修改的，则跳过 + if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) < 1) { + return; + } + } if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getUrl())).noneMatch(videoRespVO -> videoRespVO.getUrl().contains(fileObject.getFullPath()))){ log.info("删除文件：{}", fileObject); adapter.deleteFile(fileObject.getFullPath()); @@ -252,6 +271,12 @@ public class FaceCleaner { log.info("开始清理视频文件"); List fileObjectList = adapter.listDir(StorageConstant.VLOG_PATH); fileObjectList.parallelStream().forEach(fileObject -> { + if (fileObject.getModifyTime() != null) { + // 如果是一天以内修改的，则跳过 + if (DateUtil.between(fileObject.getModifyTime(), new Date(), DateUnit.DAY) < 1) { + return; + } + } if (list.parallelStream().filter(videoRespVO -> Objects.nonNull(videoRespVO.getVideoUrl())).noneMatch(videoRespVO -> videoRespVO.getVideoUrl().contains(fileObject.getFullPath()))){ log.info("删除文件：{}", fileObject); adapter.deleteFile(fileObject.getFullPath()); diff --git a/src/main/java/com/ycwl/basic/task/VideoPieceGetter.java b/src/main/java/com/ycwl/basic/task/VideoPieceGetter.java index 3ef1228..5ee903c 100644 --- a/src/main/java/com/ycwl/basic/task/VideoPieceGetter.java +++ b/src/main/java/com/ycwl/basic/task/VideoPieceGetter.java @@ -3,11 +3,11 @@ package com.ycwl.basic.task; import cn.hutool.core.thread.ThreadFactoryBuilder; import com.ycwl.basic.biz.OrderBiz; import com.ycwl.basic.biz.TaskStatusBiz; +import com.ycwl.basic.constant.StorageConstant; import com.ycwl.basic.device.DeviceFactory; import com.ycwl.basic.device.entity.common.FileObject; import com.ycwl.basic.device.operator.IDeviceStorageOperator; import com.ycwl.basic.model.pc.faceSample.entity.FaceSampleEntity; -import com.ycwl.basic.model.pc.faceSample.resp.FaceSampleRespVO; import com.ycwl.basic.repository.DeviceRepository; import com.ycwl.basic.mapper.FaceSampleMapper; import com.ycwl.basic.mapper.SourceMapper; @@ -20,9 +20,9 @@ import com.ycwl.basic.repository.TemplateRepository; import com.ycwl.basic.storage.StorageFactory; import com.ycwl.basic.storage.adapters.IStorageAdapter; import com.ycwl.basic.utils.SnowFlakeUtil; +import com.ycwl.basic.utils.VideoReUploader; import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.EnableScheduling; import org.springframework.scheduling.annotation.Scheduled; @@ -65,6 +65,8 @@ public class VideoPieceGetter { private TemplateRepository templateRepository; @Autowired private TaskStatusBiz taskStatusBiz; + @Autowired + private VideoReUploader videoReUploader; @Data public static class Task { @@ -285,7 +287,7 @@ public class VideoPieceGetter { } log.info("视频裁切成功"); IStorageAdapter adapter = StorageFactory.use("assets"); - url = adapter.uploadFile(outFile, "video-source", outFile.getName()); + url = adapter.uploadFile("video/mp4", outFile, StorageConstant.VIDEO_PIECE_PATH, outFile.getName()); // 上传成功后删除文件 outFile.delete(); } @@ -320,6 +322,7 @@ public class VideoPieceGetter { sourceMapper.addRelation(videoSource); } sourceMapper.add(sourceEntity); + videoReUploader.addTask(sourceEntity.getVideoUrl(), sourceEntity.getId()); } else { // 有原视频 if (task.memberId != null && task.faceId != null) { diff --git a/src/main/java/com/ycwl/basic/utils/ImageUtils.java b/src/main/java/com/ycwl/basic/utils/ImageUtils.java index 1bca534..64ee649 100644 --- a/src/main/java/com/ycwl/basic/utils/ImageUtils.java +++ b/src/main/java/com/ycwl/basic/utils/ImageUtils.java @@ -45,8 +45,7 @@ public class ImageUtils { @Override public String getContentType() { - // TODO - implementation depends on your requirements - return header.split(":")[0]; + return ""; } @Override diff --git a/src/main/java/com/ycwl/basic/utils/VideoReUploader.java b/src/main/java/com/ycwl/basic/utils/VideoReUploader.java new file mode 100644 index 0000000..f29ea0d --- /dev/null +++ b/src/main/java/com/ycwl/basic/utils/VideoReUploader.java @@ -0,0 +1,93 @@ +package com.ycwl.basic.utils; + +import cn.hutool.core.thread.ThreadFactoryBuilder; +import cn.hutool.http.HttpUtil; +import com.ycwl.basic.constant.StorageConstant; +import com.ycwl.basic.mapper.SourceMapper; +import com.ycwl.basic.model.pc.source.entity.SourceEntity; +import com.ycwl.basic.service.pc.ScenicService; +import com.ycwl.basic.storage.adapters.IStorageAdapter; +import com.ycwl.basic.storage.utils.StorageUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +@Component +@Slf4j +public class VideoReUploader { + private static final ThreadFactory threadFactory = new ThreadFactoryBuilder() + .setNamePrefix("Vid-ReUp-") + .build(); + private static final ThreadPoolExecutor executor = new ThreadPoolExecutor(8, 1024, 0L, TimeUnit.MILLISECONDS, + new ArrayBlockingQueue<>(1024), + threadFactory + ); + + @Autowired + private SourceMapper sourceMapper; + @Autowired + private ScenicService scenicService; + + public void addTask(String url, Long sourceId) { + try { + URL _url = new URL(url); + if (!StringUtils.startsWith(_url.getHost(), "100.64.")) { + return; + } + } catch (MalformedURLException ignored) { + return; + } + SourceEntity entity = sourceMapper.getEntity(sourceId); + if (entity == null) { + return; + } + if (entity.getScenicId() == null) { + return; + } + if (entity.getType() != 1) { + return; + } + String tmpFilePath = UUID.randomUUID().toString(); + executor.execute(() -> { + // 先下载，后上传 + File dstFile = new File(tmpFilePath); + log.info("下载视频：{}；sourceId：{}", url, sourceId); + long size = HttpUtil.downloadFile(url, dstFile); + log.info("下载视频完成：{}；大小：{}；sourceId：{}", url, size, sourceId); + String dstFilePath = StorageUtil.joinPath(StorageConstant.VIDEO_PIECE_PATH, entity.getId().toString() + ".mp4"); + IStorageAdapter dstAdapter = scenicService.getScenicStorageAdapter(entity.getScenicId()); + try { + log.info("开始上传：{}；sourceId：{}", dstFilePath, sourceId); + String newUrl = dstAdapter.uploadFile("video/mp4", dstFile, dstFilePath); + log.info("上传成功：{}；sourceId：{}", newUrl, sourceId); + SourceEntity updateEntity = new SourceEntity(); + updateEntity.setId(sourceId); + updateEntity.setVideoUrl(newUrl); + sourceMapper.update(updateEntity); + } catch (Exception e) { + log.info("上传失败：{}；sourceId：{}", dstFilePath, sourceId, e); + } finally { + try { + dstFile.delete(); + } catch (Exception ignored) { + } + } + }); + } +} diff --git a/src/main/resources/mapper/ScenicMapper.xml b/src/main/resources/mapper/ScenicMapper.xml index 184142f..2fa83a0 100644 --- a/src/main/resources/mapper/ScenicMapper.xml +++ b/src/main/resources/mapper/ScenicMapper.xml @@ -109,6 +109,8 @@ face_detect_helper_threshold=#{faceDetectHelperThreshold}, store_type=#{storeType}, store_config_json=#{storeConfigJson}, + tmp_store_type=#{tmpStoreType}, + tmp_store_config_json=#{tmpStoreConfigJson}, broker_direct_rate=#{brokerDirectRate}, watermark_type=#{watermarkType}, watermark_scenic_text=#{watermarkScenicText},