feat(database): 迁移统计数据查询到ClickHouse

- 添加ClickHouse数据源配置和相关依赖 - 实现ClickHouse统计查询服务和MySQL兜底方案 - 新增扫码统计、订单统计等数据查询接口 - 重构分销员数据统计逻辑，整合MySQL和ClickHouse数据源 - 更新应用配置文件以支持ClickHouse启用开关 - 修改分布式任务统计以支持跨库查询场景
2026-01-04 10:34:17 +08:00
parent 32297dc29c
commit aec5e57df7
19 changed files with 944 additions and 20 deletions
--- a/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java
+++ b/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java
@@ -0,0 +1,118 @@
+package com.ycwl.basic.clickhouse.service.impl;
+
+import com.ycwl.basic.clickhouse.mapper.ClickHouseStatsMapper;
+import com.ycwl.basic.clickhouse.service.StatsQueryService;
+import com.ycwl.basic.mapper.TaskMapper;
+import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * ClickHouse 统计数据查询服务实现
+ * 当 clickhouse.enabled=true 时启用
+ */
+@Slf4j
+@Service
+@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true")
+public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
+
+    @Autowired
+    private ClickHouseStatsMapper clickHouseStatsMapper;
+
+    @Autowired
+    private TaskMapper taskMapper;
+
+    @Override
+    public Integer countPreviewVideoOfMember(CommonQueryReq query) {
+        return clickHouseStatsMapper.countPreviewVideoOfMember(query);
+    }
+
+    @Override
+    public Integer countScanCodeOfMember(CommonQueryReq query) {
+        return clickHouseStatsMapper.countScanCodeOfMember(query);
+    }
+
+    @Override
+    public Integer countPushOfMember(CommonQueryReq query) {
+        return clickHouseStatsMapper.countPushOfMember(query);
+    }
+
+    @Override
+    public Integer countUploadFaceOfMember(CommonQueryReq query) {
+        return clickHouseStatsMapper.countUploadFaceOfMember(query);
+    }
+
+    @Override
+    public Integer countCompleteVideoOfMember(CommonQueryReq query) {
+        // 从 ClickHouse 获取 face_id 列表，然后在 MySQL 中查询完成的任务
+        List<String> faceIds = clickHouseStatsMapper.listFaceIdsWithUpload(query);
+        if (faceIds == null || faceIds.isEmpty()) {
+            return 0;
+        }
+        // 在 MySQL 中统计已完成任务的用户数
+        return taskMapper.countCompletedTaskMembersByFaceIds(faceIds);
+    }
+
+    @Override
+    public Integer countCompleteOfVideo(CommonQueryReq query) {
+        // 从 ClickHouse 获取 face_id 列表，然后在 MySQL 中查询完成的任务数
+        List<String> faceIds = clickHouseStatsMapper.listFaceIdsWithUpload(query);
+        if (faceIds == null || faceIds.isEmpty()) {
+            return 0;
+        }
+        // 在 MySQL 中统计已完成的任务数
+        return taskMapper.countCompletedTasksByFaceIds(faceIds);
+    }
+
+    @Override
+    public Integer countTotalVisitorOfMember(CommonQueryReq query) {
+        return clickHouseStatsMapper.countTotalVisitorOfMember(query);
+    }
+
+    @Override
+    public Integer countPreviewOfVideo(CommonQueryReq query) {
+        return clickHouseStatsMapper.countPreviewOfVideo(query);
+    }
+
+    @Override
+    public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
+        return clickHouseStatsMapper.getBrokerIdListForUser(memberId, startTime, endTime);
+    }
+
+    @Override
+    public Long getUserRecentEnterType(Long memberId, Date endTime) {
+        return clickHouseStatsMapper.getUserRecentEnterType(memberId, endTime);
+    }
+
+    @Override
+    public List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime) {
+        return clickHouseStatsMapper.getProjectIdListForUser(memberId, startTime, endTime);
+    }
+
+    @Override
+    public Integer countBrokerScanCount(Long brokerId) {
+        return clickHouseStatsMapper.countBrokerScanCount(brokerId);
+    }
+
+    @Override
+    public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
+        return clickHouseStatsMapper.getDailyScanStats(brokerId, startTime, endTime);
+    }
+
+    @Override
+    public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
+        return clickHouseStatsMapper.scanCodeMemberChartByHour(query);
+    }
+
+    @Override
+    public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
+        return clickHouseStatsMapper.scanCodeMemberChartByDate(query);
+    }
+}
--- a/src/main/java/com/ycwl/basic/clickhouse/service/impl/MySqlStatsQueryServiceImpl.java
+++ b/src/main/java/com/ycwl/basic/clickhouse/service/impl/MySqlStatsQueryServiceImpl.java
@@ -0,0 +1,101 @@
+package com.ycwl.basic.clickhouse.service.impl;
+
+import com.ycwl.basic.clickhouse.service.StatsQueryService;
+import com.ycwl.basic.mapper.StatisticsMapper;
+import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+
+/**
+ * MySQL 统计数据查询服务实现
+ * 当 clickhouse.enabled 未启用时使用此实现（兜底）
+ */
+@Slf4j
+@Service
+@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "false", matchIfMissing = true)
+public class MySqlStatsQueryServiceImpl implements StatsQueryService {
+
+    @Autowired
+    private StatisticsMapper statisticsMapper;
+
+    @Override
+    public Integer countPreviewVideoOfMember(CommonQueryReq query) {
+        return statisticsMapper.countPreviewVideoOfMember(query);
+    }
+
+    @Override
+    public Integer countScanCodeOfMember(CommonQueryReq query) {
+        return statisticsMapper.countScanCodeOfMember(query);
+    }
+
+    @Override
+    public Integer countPushOfMember(CommonQueryReq query) {
+        return statisticsMapper.countPushOfMember(query);
+    }
+
+    @Override
+    public Integer countUploadFaceOfMember(CommonQueryReq query) {
+        return statisticsMapper.countUploadFaceOfMember(query);
+    }
+
+    @Override
+    public Integer countCompleteVideoOfMember(CommonQueryReq query) {
+        return statisticsMapper.countCompleteVideoOfMember(query);
+    }
+
+    @Override
+    public Integer countCompleteOfVideo(CommonQueryReq query) {
+        return statisticsMapper.countCompleteOfVideo(query);
+    }
+
+    @Override
+    public Integer countTotalVisitorOfMember(CommonQueryReq query) {
+        return statisticsMapper.countTotalVisitorOfMember(query);
+    }
+
+    @Override
+    public Integer countPreviewOfVideo(CommonQueryReq query) {
+        return statisticsMapper.countPreviewOfVideo(query);
+    }
+
+    @Override
+    public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
+        return statisticsMapper.getBrokerIdListForUser(memberId, startTime, endTime);
+    }
+
+    @Override
+    public Long getUserRecentEnterType(Long memberId, Date endTime) {
+        return statisticsMapper.getUserRecentEnterType(memberId, endTime);
+    }
+
+    @Override
+    public List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime) {
+        return statisticsMapper.getProjectIdListForUser(memberId, startTime, endTime);
+    }
+
+    @Override
+    public Integer countBrokerScanCount(Long brokerId) {
+        return statisticsMapper.countBrokerScanCount(brokerId);
+    }
+
+    @Override
+    public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
+        return statisticsMapper.getDailyScanStats(brokerId, startTime, endTime);
+    }
+
+    @Override
+    public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
+        return statisticsMapper.scanCodeMemberChartByHour(query);
+    }
+
+    @Override
+    public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
+        return statisticsMapper.scanCodeMemberChartByDate(query);
+    }
+}