q792602257/FrameTour-BE
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(clickhouse): 优化统计查询SQL性能和代码结构

Browse Source 
- 提取进入景区trace_id子查询逻辑到独立方法appendEnterScenicTraceIdSubQuery
- 将count函数替换为uniqExact以提高去重统计性能
- 优化视频预览统计查询，使用WITH子句提取JSON字段减少重复计算
- 简化经纪人ID列表查询，移除不必要的子查询包装
- 修复每日扫码统计查询的时间范围过滤条件
- 优化按小时和按日期的扫码会员图表查询，使用ClickHouse内置时间函数
- 在子查询中添加时间范围过滤以减少数据扫描量
This commit is contained in:
Jerry Yan
2026-01-04 13:53:37 +08:00
parent d1962ed615
commit cc62fb4c18
1 changed files with 111 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java
View File

@@ -61,15 +61,33 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
return date != null ? "'" + DATE_FORMAT.format(date) + "'" : null;
}
/**
* 拼接“进入景区”的 trace_id 子查询。
* <p>
* ClickHouse 上 t_stats_record 往往按时间分区/排序；给子查询补充时间范围可显著减少扫描量。
*/
private void appendEnterScenicTraceIdSubQuery(StringBuilder sql, Long scenicId, Date startTime, Date endTime) {
sql.append("SELECT DISTINCT trace_id FROM t_stats_record ");
sql.append("WHERE action = 'ENTER_SCENIC' ");
if (scenicId != null) {
sql.append("AND identifier = '").append(scenicId).append("' ");
}
if (startTime != null) {
sql.append("AND create_time >= ").append(formatDateTime(startTime)).append(" ");
}
if (endTime != null) {
sql.append("AND create_time <= ").append(formatDateTime(endTime)).append(" ");
}
}
@Override
public Integer countPreviewVideoOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'LOAD' ");
sql.append("AND r.identifier = 'pages/videoSynthesis/buy' ");
@@ -80,8 +98,6 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
if (query.getEndTime() != null) {
sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -89,12 +105,11 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countScanCodeOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'LAUNCH' ");
sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
@@ -104,8 +119,6 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
if (query.getEndTime() != null) {
sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -113,12 +126,11 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countPushOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'PERM_REQ' ");
sql.append("AND r.identifier = 'NOTIFY' ");
@@ -128,8 +140,6 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
if (query.getEndTime() != null) {
sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -137,12 +147,11 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countUploadFaceOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'FACE_UPLOAD' ");
if (query.getStartTime() != null) {
@@ -151,8 +160,6 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
if (query.getEndTime() != null) {
sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -162,8 +169,7 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
sql.append("SELECT DISTINCT r.identifier FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'FACE_UPLOAD' ");
if (query.getStartTime() != null) {
@@ -197,12 +203,11 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countTotalVisitorOfMember(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'LAUNCH' ");
if (query.getStartTime() != null) {
@@ -211,8 +216,6 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
if (query.getEndTime() != null) {
sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY s.member_id ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -220,25 +223,24 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public Integer countPreviewOfVideo(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT ifNull(count(1), 0) AS count FROM ( ");
sql.append(" SELECT 1 FROM t_stats_record r ");
sql.append("WITH JSONExtractString(params, 'id') AS videoId, ");
sql.append(" JSONExtractString(params, 'share') AS share ");
sql.append("SELECT toInt32(uniqExact(videoId)) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' ");
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'LOAD' ");
sql.append("AND r.identifier = 'pages/videoSynthesis/buy' ");
sql.append(" AND JSONExtractString(r.params, 'id') != '' ");
sql.append(" AND JSONExtractString(r.params, 'share') = '' ");
sql.append("AND videoId != '' ");
sql.append("AND share = '' ");
if (query.getStartTime() != null) {
sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" ");
}
if (query.getEndTime() != null) {
sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" ");
}
sql.append(" GROUP BY JSONExtractString(r.params, 'id') ");
sql.append(")");
return getJdbcTemplate().queryForObject(sql.toString(), Integer.class);
}
@@ -246,8 +248,7 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public List<Long> getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT toInt64(sub.identifier) AS identifier FROM ( ");
sql.append(" SELECT identifier, max(r.create_time) AS createTime ");
sql.append("SELECT toInt64(r.identifier) AS identifier ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.action = 'CODE_SCAN' ");
@@ -258,8 +259,8 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
if (endTime != null) {
sql.append(" AND r.create_time <= ").append(formatDateTime(endTime)).append(" ");
}
sql.append(" GROUP BY identifier ");
sql.append(") sub ORDER BY sub.createTime DESC");
sql.append("GROUP BY r.identifier ");
sql.append("ORDER BY max(r.create_time) DESC");
return getJdbcTemplate().queryForList(sql.toString(), Long.class);
}
@@ -311,13 +312,17 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
public List<HashMap<String, Object>> getDailyScanStats(Long brokerId, Date startTime, Date endTime) {
String startDateStr = DATE_FORMAT.format(startTime);
String endDateStr = DATE_FORMAT.format(endTime);
String startDateTimeStr = "'" + startDateStr + " 00:00:00'";
String endDateTimeStr = "'" + endDateStr + " 23:59:59'";
String sql = "SELECT toDate(create_time) AS date, count(DISTINCT id) AS scanCount " +
"FROM t_stats_record " +
"WHERE action = 'CODE_SCAN' " +
" AND identifier = '" + brokerId + "' " +
" AND toDate(create_time) BETWEEN '" + startDateStr + "' AND '" + endDateStr + "' " +
"GROUP BY toDate(create_time)";
" AND create_time >= " + startDateTimeStr + " " +
" AND create_time <= " + endDateTimeStr + " " +
"GROUP BY toDate(create_time) " +
"ORDER BY toDate(create_time)";
return getJdbcTemplate().query(sql, (rs, rowNum) -> {
HashMap<String, Object> map = new HashMap<>();
@@ -330,22 +335,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public List<HashMap<String, String>> scanCodeMemberChartByHour(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT formatDateTime(s.create_time, '%m-%d %H') AS t, ");
sql.append(" count(DISTINCT s.member_id) AS count ");
sql.append("SELECT formatDateTime(toStartOfHour(s.create_time), '%m-%d %H') AS t, ");
sql.append(" uniqExact(s.member_id) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' ");
if (query.getScenicId() != null) {
sql.append(" AND identifier = '").append(query.getScenicId()).append("' ");
}
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'LAUNCH' ");
sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime()));
sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" ");
sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d %H')");
sql.append("GROUP BY toStartOfHour(s.create_time) ");
sql.append("ORDER BY toStartOfHour(s.create_time)");
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
HashMap<String, String> map = new HashMap<>();
@@ -358,22 +360,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService {
@Override
public List<HashMap<String, String>> scanCodeMemberChartByDate(CommonQueryReq query) {
StringBuilder sql = new StringBuilder();
sql.append("SELECT formatDateTime(s.create_time, '%m-%d') AS t, ");
sql.append(" count(DISTINCT s.member_id) AS count ");
sql.append("SELECT formatDateTime(toStartOfDay(s.create_time), '%m-%d') AS t, ");
sql.append(" uniqExact(s.member_id) AS count ");
sql.append("FROM t_stats_record r ");
sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id ");
sql.append("WHERE r.trace_id IN (");
sql.append(" SELECT trace_id FROM t_stats_record ");
sql.append(" WHERE action = 'ENTER_SCENIC' ");
if (query.getScenicId() != null) {
sql.append(" AND identifier = '").append(query.getScenicId()).append("' ");
}
appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime());
sql.append(") ");
sql.append("AND r.action = 'LAUNCH' ");
sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) ");
sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime()));
sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" ");
sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d')");
sql.append("GROUP BY toStartOfDay(s.create_time) ");
sql.append("ORDER BY toStartOfDay(s.create_time)");
return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> {
HashMap<String, String> map = new HashMap<>();