feat(database): 迁移统计数据查询到ClickHouse

- 添加ClickHouse数据源配置和相关依赖
- 实现ClickHouse统计查询服务和MySQL兜底方案
- 新增扫码统计、订单统计等数据查询接口
- 重构分销员数据统计逻辑,整合MySQL和ClickHouse数据源
- 更新应用配置文件以支持ClickHouse启用开关
- 修改分布式任务统计以支持跨库查询场景
This commit is contained in:
2026-01-04 10:34:17 +08:00
parent 32297dc29c
commit aec5e57df7
19 changed files with 944 additions and 20 deletions

View File

@@ -0,0 +1,90 @@
package com.ycwl.basic.clickhouse.service;
import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
/**
* 统计数据查询服务接口
* 用于抽象 t_stats 和 t_stats_record 表的查询
* 支持 MySQL 和 ClickHouse 两种实现
*/
public interface StatsQueryService {
/**
* 统计预览视频人数
*/
Integer countPreviewVideoOfMember(CommonQueryReq query);
/**
* 统计扫码访问人数
*/
Integer countScanCodeOfMember(CommonQueryReq query);
/**
* 统计推送订阅人数
*/
Integer countPushOfMember(CommonQueryReq query);
/**
* 统计上传头像人数
*/
Integer countUploadFaceOfMember(CommonQueryReq query);
/**
* 统计生成视频人数
*/
Integer countCompleteVideoOfMember(CommonQueryReq query);
/**
* 统计生成视频条数
*/
Integer countCompleteOfVideo(CommonQueryReq query);
/**
* 统计总访问人数
*/
Integer countTotalVisitorOfMember(CommonQueryReq query);
/**
* 统计预览视频条数
*/
Integer countPreviewOfVideo(CommonQueryReq query);
/**
* 获取用户分销员 ID 列表
*/
List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime);
/**
* 获取用户最近进入类型
*/
Long getUserRecentEnterType(Long memberId, Date endTime);
/**
* 获取用户项目 ID 列表
*/
List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime);
/**
* 统计分销员扫码次数
*/
Integer countBrokerScanCount(Long brokerId);
/**
* 按日期统计分销员扫码数据
*/
List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime);
/**
* 按小时统计扫码人数
*/
List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query);
/**
* 按日期统计扫码人数
*/
List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query);
}

View File

@@ -0,0 +1,118 @@
package com.ycwl.basic.clickhouse.service.impl;
import com.ycwl.basic.clickhouse.mapper.ClickHouseStatsMapper;
import com.ycwl.basic.clickhouse.service.StatsQueryService;
import com.ycwl.basic.mapper.TaskMapper;
import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
/**
* ClickHouse 统计数据查询服务实现
* 当 clickhouse.enabled=true 时启用
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true")
public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Autowired
private ClickHouseStatsMapper clickHouseStatsMapper;
@Autowired
private TaskMapper taskMapper;
@Override
public Integer countPreviewVideoOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countPreviewVideoOfMember(query);
}
@Override
public Integer countScanCodeOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countScanCodeOfMember(query);
}
@Override
public Integer countPushOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countPushOfMember(query);
}
@Override
public Integer countUploadFaceOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countUploadFaceOfMember(query);
}
@Override
public Integer countCompleteVideoOfMember(CommonQueryReq query) {
// 从 ClickHouse 获取 face_id 列表,然后在 MySQL 中查询完成的任务
List<String> faceIds = clickHouseStatsMapper.listFaceIdsWithUpload(query);
if (faceIds == null || faceIds.isEmpty()) {
return 0;
}
// 在 MySQL 中统计已完成任务的用户数
return taskMapper.countCompletedTaskMembersByFaceIds(faceIds);
}
@Override
public Integer countCompleteOfVideo(CommonQueryReq query) {
// 从 ClickHouse 获取 face_id 列表,然后在 MySQL 中查询完成的任务数
List<String> faceIds = clickHouseStatsMapper.listFaceIdsWithUpload(query);
if (faceIds == null || faceIds.isEmpty()) {
return 0;
}
// 在 MySQL 中统计已完成的任务数
return taskMapper.countCompletedTasksByFaceIds(faceIds);
}
@Override
public Integer countTotalVisitorOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countTotalVisitorOfMember(query);
}
@Override
public Integer countPreviewOfVideo(CommonQueryReq query) {
return clickHouseStatsMapper.countPreviewOfVideo(query);
}
@Override
public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
return clickHouseStatsMapper.getBrokerIdListForUser(memberId, startTime, endTime);
}
@Override
public Long getUserRecentEnterType(Long memberId, Date endTime) {
return clickHouseStatsMapper.getUserRecentEnterType(memberId, endTime);
}
@Override
public List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime) {
return clickHouseStatsMapper.getProjectIdListForUser(memberId, startTime, endTime);
}
@Override
public Integer countBrokerScanCount(Long brokerId) {
return clickHouseStatsMapper.countBrokerScanCount(brokerId);
}
@Override
public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
return clickHouseStatsMapper.getDailyScanStats(brokerId, startTime, endTime);
}
@Override
public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
return clickHouseStatsMapper.scanCodeMemberChartByHour(query);
}
@Override
public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
return clickHouseStatsMapper.scanCodeMemberChartByDate(query);
}
}

View File

@@ -0,0 +1,101 @@
package com.ycwl.basic.clickhouse.service.impl;
import com.ycwl.basic.clickhouse.service.StatsQueryService;
import com.ycwl.basic.mapper.StatisticsMapper;
import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Service;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
/**
* MySQL 统计数据查询服务实现
* 当 clickhouse.enabled 未启用时使用此实现(兜底)
*/
@Slf4j
@Service
@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "false", matchIfMissing = true)
public class MySqlStatsQueryServiceImpl implements StatsQueryService {
@Autowired
private StatisticsMapper statisticsMapper;
@Override
public Integer countPreviewVideoOfMember(CommonQueryReq query) {
return statisticsMapper.countPreviewVideoOfMember(query);
}
@Override
public Integer countScanCodeOfMember(CommonQueryReq query) {
return statisticsMapper.countScanCodeOfMember(query);
}
@Override
public Integer countPushOfMember(CommonQueryReq query) {
return statisticsMapper.countPushOfMember(query);
}
@Override
public Integer countUploadFaceOfMember(CommonQueryReq query) {
return statisticsMapper.countUploadFaceOfMember(query);
}
@Override
public Integer countCompleteVideoOfMember(CommonQueryReq query) {
return statisticsMapper.countCompleteVideoOfMember(query);
}
@Override
public Integer countCompleteOfVideo(CommonQueryReq query) {
return statisticsMapper.countCompleteOfVideo(query);
}
@Override
public Integer countTotalVisitorOfMember(CommonQueryReq query) {
return statisticsMapper.countTotalVisitorOfMember(query);
}
@Override
public Integer countPreviewOfVideo(CommonQueryReq query) {
return statisticsMapper.countPreviewOfVideo(query);
}
@Override
public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
return statisticsMapper.getBrokerIdListForUser(memberId, startTime, endTime);
}
@Override
public Long getUserRecentEnterType(Long memberId, Date endTime) {
return statisticsMapper.getUserRecentEnterType(memberId, endTime);
}
@Override
public List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime) {
return statisticsMapper.getProjectIdListForUser(memberId, startTime, endTime);
}
@Override
public Integer countBrokerScanCount(Long brokerId) {
return statisticsMapper.countBrokerScanCount(brokerId);
}
@Override
public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
return statisticsMapper.getDailyScanStats(brokerId, startTime, endTime);
}
@Override
public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
return statisticsMapper.scanCodeMemberChartByHour(query);
}
@Override
public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
return statisticsMapper.scanCodeMemberChartByDate(query);
}
}