refactor(clickhouse): 将统计数据查询从 MyBatis 迁移到 JDBC 模板

- 移除 ClickHouseStatsMapper 接口及 XML 映射文件
- 使用 NamedParameterJdbcTemplate 替代 MyBatis 实现数据查询
- 添加日期格式化工具类处理 ClickHouse 时间格式
- 重构所有统计查询方法使用原生 SQL 字符串构建
- 添加 MySQL 主数据源配置确保多数据源正确配置
- 升级 ClickHouse JDBC 驱动版本到 0.8.5
- 解决 0.6.x 版本参数绑定问题通过手动 SQL 构建
- 保持原有查询逻辑不变仅改变实现方式
This commit is contained in:
2026-01-04 13:17:01 +08:00
parent e1023b6ea8
commit d1962ed615
6 changed files with 341 additions and 416 deletions

View File

@@ -308,7 +308,7 @@
<dependency> <dependency>
<groupId>com.clickhouse</groupId> <groupId>com.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId> <artifactId>clickhouse-jdbc</artifactId>
<version>0.6.0</version> <version>0.8.5</version>
<classifier>all</classifier> <classifier>all</classifier>
</dependency> </dependency>
</dependencies> </dependencies>

View File

@@ -1,95 +0,0 @@
package com.ycwl.basic.clickhouse.mapper;
import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
/**
* ClickHouse 统计数据 Mapper
* 用于查询 t_stats 和 t_stats_record 表
*/
@Mapper
public interface ClickHouseStatsMapper {
/**
* 统计预览视频人数
*/
Integer countPreviewVideoOfMember(CommonQueryReq query);
/**
* 统计扫码访问人数
*/
Integer countScanCodeOfMember(CommonQueryReq query);
/**
* 统计推送订阅人数
*/
Integer countPushOfMember(CommonQueryReq query);
/**
* 统计上传头像人数
*/
Integer countUploadFaceOfMember(CommonQueryReq query);
/**
* 统计生成视频人数
* 注意:需要关联 MySQL 中的 task 表,此处只返回 face_id 列表
*/
List<String> listFaceIdsWithUpload(CommonQueryReq query);
/**
* 统计总访问人数
*/
Integer countTotalVisitorOfMember(CommonQueryReq query);
/**
* 统计预览视频条数
*/
Integer countPreviewOfVideo(CommonQueryReq query);
/**
* 获取用户分销员 ID 列表
*/
List<Long> getBrokerIdListForUser(@Param("memberId") Long memberId,
@Param("startTime") Date startTime,
@Param("endTime") Date endTime);
/**
* 获取用户最近进入类型
*/
Long getUserRecentEnterType(@Param("memberId") Long memberId,
@Param("endTime") Date endTime);
/**
* 获取用户项目 ID 列表
*/
List<Long> getProjectIdListForUser(@Param("memberId") Long memberId,
@Param("startTime") Date startTime,
@Param("endTime") Date endTime);
/**
* 统计分销员扫码次数
*/
Integer countBrokerScanCount(@Param("brokerId") Long brokerId);
/**
* 按日期统计分销员扫码数据(用于 BrokerRecordMapper.getDailySummaryByBrokerId)
*/
List<HashMap<String, Object>> getDailyScanStats(@Param("brokerId") Long brokerId,
@Param("startTime") Date startTime,
@Param("endTime") Date endTime);
/**
* 按小时统计扫码人数
*/
List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query);
/**
* 按日期统计扫码人数
*/
List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query);
}

View File

@@ -1,15 +1,17 @@
package com.ycwl.basic.clickhouse.service.impl; package com.ycwl.basic.clickhouse.service.impl;
import com.ycwl.basic.clickhouse.mapper.ClickHouseStatsMapper;
import com.ycwl.basic.clickhouse.service.StatsQueryService; import com.ycwl.basic.clickhouse.service.StatsQueryService;
import com.ycwl.basic.mapper.TaskMapper; import com.ycwl.basic.mapper.TaskMapper;
import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq; import com.ycwl.basic.model.mobile.statistic.req.CommonQueryReq;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.util.Collections; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
@@ -17,102 +19,367 @@ import java.util.List;
/** /**
* ClickHouse 统计数据查询服务实现 * ClickHouse 统计数据查询服务实现
* 当 clickhouse.enabled=true 时启用 * 当 clickhouse.enabled=true 时启用
*
* 注意:ClickHouse JDBC 驱动 0.6.x 对参数绑定支持有问题,
* 因此使用字符串格式化方式构建 SQL(参数均为内部生成的数值或日期,无 SQL 注入风险)
*/ */
@Slf4j @Slf4j
@Service @Service
@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true")
public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd");
private static final SimpleDateFormat DATETIME_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
@Autowired @Autowired
private ClickHouseStatsMapper clickHouseStatsMapper; @Qualifier("clickHouseJdbcTemplate")
private NamedParameterJdbcTemplate namedJdbcTemplate;
private JdbcTemplate jdbcTemplate;
@Autowired @Autowired
private TaskMapper taskMapper; private TaskMapper taskMapper;
private JdbcTemplate getJdbcTemplate() {
if (jdbcTemplate == null) {
jdbcTemplate = namedJdbcTemplate.getJdbcTemplate();
}
return jdbcTemplate;
}
/**
* 格式化日期时间为 ClickHouse 可识别的字符串
*/
private String formatDateTime(Date date) {
return date != null ? "'" + DATETIME_FORMAT.format(date) + "'" : null;
}
/**
* 格式化日期为 ClickHouse 可识别的字符串
*/
private String formatDate(Date date) {
return date != null ? "'" + DATE_FORMAT.format(date) + "'" : null;
}
@Override @Override
public Integer countPreviewVideoOfMember(CommonQueryReq query) { public Integer countPreviewVideoOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countPreviewVideoOfMember(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(" ) ");
sql.append(" AND r.action = 'LOAD' ");
sql.append(" AND r.identifier = 'pages/videoSynthesis/buy' ");
sql.append(" AND JSONExtractString(r.params, 'share') = '' ");
if (query.getStartTime() != null) {
sql.append(" AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append(" AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
} }
@Override @Override
public Integer countScanCodeOfMember(CommonQueryReq query) { public Integer countScanCodeOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countScanCodeOfMember(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(" ) ");
sql.append(" AND r.action = 'LAUNCH' ");
sql.append(" AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
if (query.getStartTime() != null) {
sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
} }
@Override @Override
public Integer countPushOfMember(CommonQueryReq query) { public Integer countPushOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countPushOfMember(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(" ) ");
sql.append(" AND r.action = 'PERM_REQ' ");
sql.append(" AND r.identifier = 'NOTIFY' ");
if (query.getStartTime() != null) {
sql.append(" AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append(" AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
} }
@Override @Override
public Integer countUploadFaceOfMember(CommonQueryReq query) { public Integer countUploadFaceOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countUploadFaceOfMember(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(" ) ");
sql.append(" AND r.action = 'FACE_UPLOAD' ");
if (query.getStartTime() != null) {
sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
private List<String> listFaceIdsWithUpload(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT DISTINCT r.identifier FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(") ");
sql.append("AND r.action = 'FACE_UPLOAD' ");
if (query.getStartTime() != null) {
sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
return getJdbcTemplate().queryForList(sql.toString(), String.class);
} }
@Override @Override
public Integer countCompleteVideoOfMember(CommonQueryReq query) { public Integer countCompleteVideoOfMember(CommonQueryReq query) {
// 从 ClickHouse 获取 face_id 列表,然后在 MySQL 中查询完成的任务 List<String> faceIds = listFaceIdsWithUpload(query);
List<String> faceIds = clickHouseStatsMapper.listFaceIdsWithUpload(query);
if (faceIds == null || faceIds.isEmpty()) { if (faceIds == null || faceIds.isEmpty()) {
return 0; return 0;
} }
// 在 MySQL 中统计已完成任务的用户数
return taskMapper.countCompletedTaskMembersByFaceIds(faceIds); return taskMapper.countCompletedTaskMembersByFaceIds(faceIds);
} }
@Override @Override
public Integer countCompleteOfVideo(CommonQueryReq query) { public Integer countCompleteOfVideo(CommonQueryReq query) {
// 从 ClickHouse 获取 face_id 列表,然后在 MySQL 中查询完成的任务数 List<String> faceIds = listFaceIdsWithUpload(query);
List<String> faceIds = clickHouseStatsMapper.listFaceIdsWithUpload(query);
if (faceIds == null || faceIds.isEmpty()) { if (faceIds == null || faceIds.isEmpty()) {
return 0; return 0;
} }
// 在 MySQL 中统计已完成的任务数
return taskMapper.countCompletedTasksByFaceIds(faceIds); return taskMapper.countCompletedTasksByFaceIds(faceIds);
} }
@Override @Override
public Integer countTotalVisitorOfMember(CommonQueryReq query) { public Integer countTotalVisitorOfMember(CommonQueryReq query) {
return clickHouseStatsMapper.countTotalVisitorOfMember(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(" ) ");
sql.append(" AND r.action = 'LAUNCH' ");
if (query.getStartTime() != null) {
sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
} }
@Override @Override
public Integer countPreviewOfVideo(CommonQueryReq query) { public Integer countPreviewOfVideo(CommonQueryReq query) {
return clickHouseStatsMapper.countPreviewOfVideo(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
sql.append(" ) ");
sql.append(" AND r.action = 'LOAD' ");
sql.append(" AND r.identifier = 'pages/videoSynthesis/buy' ");
sql.append(" AND JSONExtractString(r.params, 'id') != '' ");
sql.append(" AND JSONExtractString(r.params, 'share') = '' ");
if (query.getStartTime() != null) {
sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY JSONExtractString(r.params, 'id') ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
} }
@Override @Override
public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) { public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
return clickHouseStatsMapper.getBrokerIdListForUser(memberId, startTime, endTime); StringBuilder sql = new StringBuilder();
sql.append("SELECT toInt64(sub.identifier) AS identifier FROM ( ");
sql.append(" SELECT identifier, max(r.create_time) AS createTime ");
sql.append(" FROM t_stats_record r ");
sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append(" WHERE r.action = 'CODE_SCAN' ");
sql.append(" AND s.member_id = ").append(memberId).append(" ");
if (startTime != null) {
sql.append(" AND r.create_time >= ").append(formatDateTime(startTime)).append(" ");
}
if (endTime != null) {
sql.append(" AND r.create_time <= ").append(formatDateTime(endTime)).append(" ");
}
sql.append(" GROUP BY identifier ");
sql.append(") sub ORDER BY sub.createTime DESC");
return getJdbcTemplate().queryForList(sql.toString(), Long.class);
} }
@Override @Override
public Long getUserRecentEnterType(Long memberId, Date endTime) { public Long getUserRecentEnterType(Long memberId, Date endTime) {
return clickHouseStatsMapper.getUserRecentEnterType(memberId, endTime); StringBuilder sql = new StringBuilder();
sql.append("SELECT JSONExtractInt(r.params, 'scene') AS scene ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.action = 'LAUNCH' ");
sql.append(" AND s.member_id = ").append(memberId).append(" ");
if (endTime != null) {
sql.append(" AND r.create_time <= ").append(formatDateTime(endTime)).append(" ");
}
sql.append("ORDER BY r.create_time DESC LIMIT 1");
try {
return getJdbcTemplate().queryForObject(sql.toString(), Long.class);
} catch (Exception e) {
return null;
}
} }
@Override @Override
public List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime) { public List<Long> getProjectIdListForUser(Long memberId, Date startTime, Date endTime) {
return clickHouseStatsMapper.getProjectIdListForUser(memberId, startTime, endTime); StringBuilder sql = new StringBuilder();
sql.append("SELECT toInt64(r.identifier) AS identifier ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE s.member_id = ").append(memberId).append(" ");
sql.append(" AND r.action = 'ENTER_PROJECT' ");
sql.append(" AND r.create_time < ").append(formatDateTime(endTime)).append(" ");
sql.append(" AND r.create_time > ").append(formatDateTime(startTime)).append(" ");
sql.append("ORDER BY r.create_time DESC LIMIT 1");
return getJdbcTemplate().queryForList(sql.toString(), Long.class);
} }
@Override @Override
public Integer countBrokerScanCount(Long brokerId) { public Integer countBrokerScanCount(Long brokerId) {
return clickHouseStatsMapper.countBrokerScanCount(brokerId); String sql = "SELECT count(1) AS count FROM t_stats_record " +
"WHERE action = 'CODE_SCAN' AND identifier = '" + brokerId + "'";
return getJdbcTemplate().queryForObject(sql, Integer.class);
} }
@Override @Override
public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) { public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
return clickHouseStatsMapper.getDailyScanStats(brokerId, startTime, endTime); String startDateStr = DATE_FORMAT.format(startTime);
String endDateStr = DATE_FORMAT.format(endTime);
String sql = "SELECT toDate(create_time) AS date, count(DISTINCT id) AS scanCount " +
"FROM t_stats_record " +
"WHERE action = 'CODE_SCAN' " +
" AND identifier = '" + brokerId + "' " +
" AND toDate(create_time) BETWEEN '" + startDateStr + "' AND '" + endDateStr + "' " +
"GROUP BY toDate(create_time)";
return getJdbcTemplate().query(sql, (rs, rowNum) -> {
HashMap<String, Object> map = new HashMap<>();
map.put("date", rs.getDate("date"));
map.put("scanCount", rs.getLong("scanCount"));
return map;
});
} }
@Override @Override
public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) { public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
return clickHouseStatsMapper.scanCodeMemberChartByHour(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT formatDateTime(s.create_time, '%m-%d %H') AS t, ");
sql.append(" count(DISTINCT s.member_id) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' ");
if (query.getScenicId() != null) {
sql.append(" AND identifier = '").append(query.getScenicId()).append("' ");
}
sql.append(") ");
sql.append("AND r.action = 'LAUNCH' ");
sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime()));
sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" ");
sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d %H')");
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
HashMap<String, String> map = new HashMap<>();
map.put("t", rs.getString("t"));
map.put("count", rs.getString("count"));
return map;
});
} }
@Override @Override
public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) { public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
return clickHouseStatsMapper.scanCodeMemberChartByDate(query); StringBuilder sql = new StringBuilder();
sql.append("SELECT formatDateTime(s.create_time, '%m-%d') AS t, ");
sql.append(" count(DISTINCT s.member_id) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN ( ");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' ");
if (query.getScenicId() != null) {
sql.append(" AND identifier = '").append(query.getScenicId()).append("' ");
}
sql.append(") ");
sql.append("AND r.action = 'LAUNCH' ");
sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime()));
sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" ");
sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d')");
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
HashMap<String, String> map = new HashMap<>();
map.put("t", rs.getString("t"));
map.put("count", rs.getString("count"));
return map;
});
} }
} }

View File

@@ -1,56 +1,37 @@
package com.ycwl.basic.config; package com.ycwl.basic.config;
import com.zaxxer.hikari.HikariDataSource; import com.zaxxer.hikari.HikariDataSource;
import org.apache.ibatis.session.SqlSessionFactory;
import org.mybatis.spring.SqlSessionFactoryBean;
import org.mybatis.spring.SqlSessionTemplate;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver; import org.springframework.jdbc.core.namedparam.NamedParameterJdbcTemplate;
import javax.sql.DataSource; import javax.sql.DataSource;
/** /**
* ClickHouse 数据源配置 * ClickHouse 数据源配置
* 用于 t_stats 和 t_stats_record 表的查询 * 用于 t_stats 和 t_stats_record 表的查询
*
* 使用 NamedParameterJdbcTemplate 而非 MyBatis,以避免干扰 MyBatis-Plus 的自动配置
*/ */
@Configuration @Configuration
@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true") @ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true")
@MapperScan(
basePackages = "com.ycwl.basic.clickhouse.mapper",
sqlSessionFactoryRef = "clickHouseSqlSessionFactory"
)
public class ClickHouseDataSourceConfig { public class ClickHouseDataSourceConfig {
/**
* ClickHouse 数据源(非 Primary)
*/
@Bean(name = "clickHouseDataSource") @Bean(name = "clickHouseDataSource")
@ConfigurationProperties(prefix = "clickhouse.datasource") @ConfigurationProperties(prefix = "clickhouse.datasource")
public DataSource clickHouseDataSource() { public DataSource clickHouseDataSource() {
return new HikariDataSource(); return new HikariDataSource();
} }
@Bean(name = "clickHouseSqlSessionFactory") @Bean(name = "clickHouseJdbcTemplate")
public SqlSessionFactory clickHouseSqlSessionFactory( public NamedParameterJdbcTemplate clickHouseJdbcTemplate(
@Qualifier("clickHouseDataSource") DataSource dataSource) throws Exception { @Qualifier("clickHouseDataSource") DataSource dataSource) {
SqlSessionFactoryBean factoryBean = new SqlSessionFactoryBean(); return new NamedParameterJdbcTemplate(dataSource);
factoryBean.setDataSource(dataSource);
factoryBean.setMapperLocations(
new PathMatchingResourcePatternResolver()
.getResources("classpath:mapper/clickhouse/*.xml")
);
// 配置 MyBatis 设置
org.apache.ibatis.session.Configuration configuration = new org.apache.ibatis.session.Configuration();
configuration.setMapUnderscoreToCamelCase(true);
factoryBean.setConfiguration(configuration);
return factoryBean.getObject();
}
@Bean(name = "clickHouseSqlSessionTemplate")
public SqlSessionTemplate clickHouseSqlSessionTemplate(
@Qualifier("clickHouseSqlSessionFactory") SqlSessionFactory sqlSessionFactory) {
return new SqlSessionTemplate(sqlSessionFactory);
} }
} }

View File

@@ -0,0 +1,41 @@
package com.ycwl.basic.config;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.jdbc.DataSourceProperties;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.Primary;
import javax.sql.DataSource;
/**
* MySQL 主数据源配置
*
* 当 ClickHouse 启用时,需要显式配置 MySQL 数据源并标记为 @Primary,
* 以确保 MyBatis-Plus 和其他组件使用正确的数据源
*/
@Configuration
@ConditionalOnProperty(prefix = "clickhouse", name = "enabled", havingValue = "true")
public class MySqlPrimaryDataSourceConfig {
/**
* MySQL 数据源属性
*/
@Primary
@Bean
@ConfigurationProperties(prefix = "spring.datasource")
public DataSourceProperties mysqlDataSourceProperties() {
return new DataSourceProperties();
}
/**
* MySQL 主数据源
* 使用 @Primary 确保这是默认数据源
*/
@Primary
@Bean(name = "dataSource")
public DataSource mysqlDataSource(DataSourceProperties properties) {
return properties.initializeDataSourceBuilder().build();
}
}

View File

@@ -1,269 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.ycwl.basic.clickhouse.mapper.ClickHouseStatsMapper">
<!-- 统计预览视频人数 -->
<select id="countPreviewVideoOfMember" resultType="java.lang.Integer">
SELECT ifNull(count(1), 0) AS count
FROM (
SELECT 1
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'LOAD'
AND r.identifier = 'pages/videoSynthesis/buy'
AND JSONExtractString(r.params, 'share') = ''
<if test="startTime != null">
AND r.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND r.create_time &lt;= #{endTime}
</if>
GROUP BY s.member_id
)
</select>
<!-- 统计扫码访问人数 -->
<select id="countScanCodeOfMember" resultType="java.lang.Integer">
SELECT ifNull(count(1), 0) AS count
FROM (
SELECT 1
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'LAUNCH'
AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049)
<if test="startTime != null">
AND s.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND s.create_time &lt;= #{endTime}
</if>
GROUP BY s.member_id
)
</select>
<!-- 统计推送订阅人数 -->
<select id="countPushOfMember" resultType="java.lang.Integer">
SELECT ifNull(count(1), 0) AS count
FROM (
SELECT 1
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'PERM_REQ'
AND r.identifier = 'NOTIFY'
<if test="startTime != null">
AND r.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND r.create_time &lt;= #{endTime}
</if>
GROUP BY s.member_id
)
</select>
<!-- 统计上传头像人数 -->
<select id="countUploadFaceOfMember" resultType="java.lang.Integer">
SELECT ifNull(count(1), 0) AS count
FROM (
SELECT 1
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'FACE_UPLOAD'
<if test="startTime != null">
AND s.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND s.create_time &lt;= #{endTime}
</if>
GROUP BY s.member_id
)
</select>
<!-- 获取上传人脸的 face_id 列表(用于与 MySQL task 表关联) -->
<select id="listFaceIdsWithUpload" resultType="java.lang.String">
SELECT DISTINCT r.identifier
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'FACE_UPLOAD'
<if test="startTime != null">
AND s.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND s.create_time &lt;= #{endTime}
</if>
</select>
<!-- 统计总访问人数 -->
<select id="countTotalVisitorOfMember" resultType="java.lang.Integer">
SELECT ifNull(count(1), 0) AS count
FROM (
SELECT 1
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'LAUNCH'
<if test="startTime != null">
AND s.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND s.create_time &lt;= #{endTime}
</if>
GROUP BY s.member_id
)
</select>
<!-- 统计预览视频条数 -->
<select id="countPreviewOfVideo" resultType="java.lang.Integer">
SELECT ifNull(count(1), 0) AS count
FROM (
SELECT 1
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC' AND identifier = toString(#{scenicId})
)
AND r.action = 'LOAD'
AND r.identifier = 'pages/videoSynthesis/buy'
AND JSONExtractString(r.params, 'id') != ''
AND JSONExtractString(r.params, 'share') = ''
<if test="startTime != null">
AND s.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND s.create_time &lt;= #{endTime}
</if>
GROUP BY JSONExtractString(r.params, 'id')
)
</select>
<!-- 获取用户分销员 ID 列表 -->
<select id="getBrokerIdListForUser" resultType="java.lang.Long">
SELECT toInt64(r.identifier) AS identifier
FROM (
SELECT identifier, max(r.create_time) AS createTime
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.action = 'CODE_SCAN'
AND s.member_id = #{memberId}
<if test="startTime != null">
AND r.create_time >= #{startTime}
</if>
<if test="endTime != null">
AND r.create_time &lt;= #{endTime}
</if>
GROUP BY identifier
) sub
ORDER BY createTime DESC
</select>
<!-- 获取用户最近进入类型 -->
<select id="getUserRecentEnterType" resultType="java.lang.Long">
SELECT JSONExtractInt(r.params, 'scene') AS scene
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.action = 'LAUNCH'
AND s.member_id = #{memberId}
<if test="endTime != null">
AND r.create_time &lt;= #{endTime}
</if>
ORDER BY r.create_time DESC
LIMIT 1
</select>
<!-- 获取用户项目 ID 列表 -->
<select id="getProjectIdListForUser" resultType="java.lang.Long">
SELECT toInt64(r.identifier) AS identifier
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE s.member_id = #{memberId}
AND r.action = 'ENTER_PROJECT'
AND r.create_time &lt; #{endTime}
AND r.create_time > #{startTime}
ORDER BY r.create_time DESC
LIMIT 1
</select>
<!-- 统计分销员扫码次数 -->
<select id="countBrokerScanCount" resultType="java.lang.Integer">
SELECT count(1) AS count
FROM t_stats_record
WHERE action = 'CODE_SCAN'
AND identifier = toString(#{brokerId})
</select>
<!-- 按日期统计分销员扫码数据 -->
<select id="getDailyScanStats" resultType="java.util.HashMap">
SELECT
toDate(create_time) AS date,
count(DISTINCT id) AS scanCount
FROM t_stats_record
WHERE action = 'CODE_SCAN'
AND identifier = toString(#{brokerId})
AND toDate(create_time) BETWEEN toDate(#{startTime}) AND toDate(#{endTime})
GROUP BY toDate(create_time)
</select>
<!-- 按小时统计扫码人数 -->
<select id="scanCodeMemberChartByHour" resultType="java.util.HashMap">
SELECT
formatDateTime(s.create_time, '%m-%d %H') AS t,
count(DISTINCT s.member_id) AS count
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC'
<if test="scenicId != null">
AND identifier = toString(#{scenicId})
</if>
)
AND r.action = 'LAUNCH'
AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049)
AND s.create_time BETWEEN #{startTime} AND #{endTime}
GROUP BY formatDateTime(s.create_time, '%m-%d %H')
</select>
<!-- 按日期统计扫码人数 -->
<select id="scanCodeMemberChartByDate" resultType="java.util.HashMap">
SELECT
formatDateTime(s.create_time, '%m-%d') AS t,
count(DISTINCT s.member_id) AS count
FROM t_stats_record r
INNER JOIN t_stats s ON r.trace_id = s.trace_id
WHERE r.trace_id IN (
SELECT trace_id FROM t_stats_record
WHERE action = 'ENTER_SCENIC'
<if test="scenicId != null">
AND identifier = toString(#{scenicId})
</if>
)
AND r.action = 'LAUNCH'
AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049)
AND s.create_time BETWEEN #{startTime} AND #{endTime}
GROUP BY formatDateTime(s.create_time, '%m-%d')
</select>
</mapper>