From cc62fb4c182da471666e5e65d2c44458bfa82e75 Mon Sep 17 00:00:00 2001 From: Jerry Yan <792602257@qq.com> Date: Sun, 4 Jan 2026 13:53:37 +0800 Subject: [PATCH] =?UTF-8?q?refactor(clickhouse):=20=E4=BC=98=E5=8C=96?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E6=9F=A5=E8=AF=A2SQL=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E5=92=8C=E4=BB=A3=E7=A0=81=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 提取进入景区trace_id子查询逻辑到独立方法appendEnterScenicTraceIdSubQuery - 将count函数替换为uniqExact以提高去重统计性能 - 优化视频预览统计查询，使用WITH子句提取JSON字段减少重复计算 - 简化经纪人ID列表查询，移除不必要的子查询包装 - 修复每日扫码统计查询的时间范围过滤条件 - 优化按小时和按日期的扫码会员图表查询，使用ClickHouse内置时间函数 - 在子查询中添加时间范围过滤以减少数据扫描量 --- .../impl/ClickHouseStatsQueryServiceImpl.java | 223 +++++++++--------- 1 file changed, 111 insertions(+), 112 deletions(-) diff --git a/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java b/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java index 7c5a2b48..df12d4da 100644 --- a/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java +++ b/src/main/java/com/ycwl/basic/clickhouse/service/impl/ClickHouseStatsQueryServiceImpl.java @@ -61,27 +61,43 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { return date != null ? "'" + DATE_FORMAT.format(date) + "'" : null; } + /** + * 拼接“进入景区”的 trace_id 子查询。 + *

+ * ClickHouse 上 t_stats_record 往往按时间分区/排序；给子查询补充时间范围可显著减少扫描量。 + */ + private void appendEnterScenicTraceIdSubQuery(StringBuilder sql, Long scenicId, Date startTime, Date endTime) { + sql.append("SELECT DISTINCT trace_id FROM t_stats_record "); + sql.append("WHERE action = 'ENTER_SCENIC' "); + if (scenicId != null) { + sql.append("AND identifier = '").append(scenicId).append("' "); + } + if (startTime != null) { + sql.append("AND create_time >= ").append(formatDateTime(startTime)).append(" "); + } + if (endTime != null) { + sql.append("AND create_time <= ").append(formatDateTime(endTime)).append(" "); + } + } + @Override public Integer countPreviewVideoOfMember(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT ifNull(count(1), 0) AS count FROM ( "); - sql.append(" SELECT 1 FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); - sql.append(" ) "); - sql.append(" AND r.action = 'LOAD' "); - sql.append(" AND r.identifier = 'pages/videoSynthesis/buy' "); - sql.append(" AND JSONExtractString(r.params, 'share') = '' "); + sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); + sql.append(") "); + sql.append("AND r.action = 'LOAD' "); + sql.append("AND r.identifier = 'pages/videoSynthesis/buy' "); + sql.append("AND JSONExtractString(r.params, 'share') = '' "); if (query.getStartTime() != null) { - sql.append(" AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); + sql.append("AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); } if (query.getEndTime() != null) { - sql.append(" AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); + sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); } - sql.append(" GROUP BY s.member_id "); - sql.append(")"); return getJdbcTemplate().queryForObject(sql.toString(), Integer.class); } @@ -89,23 +105,20 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public Integer countScanCodeOfMember(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT ifNull(count(1), 0) AS count FROM ( "); - sql.append(" SELECT 1 FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); - sql.append(" ) "); - sql.append(" AND r.action = 'LAUNCH' "); - sql.append(" AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) "); + sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); + sql.append(") "); + sql.append("AND r.action = 'LAUNCH' "); + sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) "); if (query.getStartTime() != null) { - sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); + sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); } if (query.getEndTime() != null) { - sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); + sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); } - sql.append(" GROUP BY s.member_id "); - sql.append(")"); return getJdbcTemplate().queryForObject(sql.toString(), Integer.class); } @@ -113,23 +126,20 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public Integer countPushOfMember(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT ifNull(count(1), 0) AS count FROM ( "); - sql.append(" SELECT 1 FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); - sql.append(" ) "); - sql.append(" AND r.action = 'PERM_REQ' "); - sql.append(" AND r.identifier = 'NOTIFY' "); + sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); + sql.append(") "); + sql.append("AND r.action = 'PERM_REQ' "); + sql.append("AND r.identifier = 'NOTIFY' "); if (query.getStartTime() != null) { - sql.append(" AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); + sql.append("AND r.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); } if (query.getEndTime() != null) { - sql.append(" AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); + sql.append("AND r.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); } - sql.append(" GROUP BY s.member_id "); - sql.append(")"); return getJdbcTemplate().queryForObject(sql.toString(), Integer.class); } @@ -137,22 +147,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public Integer countUploadFaceOfMember(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT ifNull(count(1), 0) AS count FROM ( "); - sql.append(" SELECT 1 FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); - sql.append(" ) "); - sql.append(" AND r.action = 'FACE_UPLOAD' "); + sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); + sql.append(") "); + sql.append("AND r.action = 'FACE_UPLOAD' "); if (query.getStartTime() != null) { - sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); + sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); } if (query.getEndTime() != null) { - sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); + sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); } - sql.append(" GROUP BY s.member_id "); - sql.append(")"); return getJdbcTemplate().queryForObject(sql.toString(), Integer.class); } @@ -161,9 +168,8 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { StringBuilder sql = new StringBuilder(); sql.append("SELECT DISTINCT r.identifier FROM t_stats_record r "); sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append("WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); sql.append(") "); sql.append("AND r.action = 'FACE_UPLOAD' "); if (query.getStartTime() != null) { @@ -197,22 +203,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public Integer countTotalVisitorOfMember(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT ifNull(count(1), 0) AS count FROM ( "); - sql.append(" SELECT 1 FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); - sql.append(" ) "); - sql.append(" AND r.action = 'LAUNCH' "); + sql.append("SELECT toInt32(uniqExact(s.member_id)) AS count "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); + sql.append(") "); + sql.append("AND r.action = 'LAUNCH' "); if (query.getStartTime() != null) { - sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); + sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); } if (query.getEndTime() != null) { - sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); + sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); } - sql.append(" GROUP BY s.member_id "); - sql.append(")"); return getJdbcTemplate().queryForObject(sql.toString(), Integer.class); } @@ -220,25 +223,24 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public Integer countPreviewOfVideo(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT ifNull(count(1), 0) AS count FROM ( "); - sql.append(" SELECT 1 FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' AND identifier = '").append(query.getScenicId()).append("' "); - sql.append(" ) "); - sql.append(" AND r.action = 'LOAD' "); - sql.append(" AND r.identifier = 'pages/videoSynthesis/buy' "); - sql.append(" AND JSONExtractString(r.params, 'id') != '' "); - sql.append(" AND JSONExtractString(r.params, 'share') = '' "); + sql.append("WITH JSONExtractString(params, 'id') AS videoId, "); + sql.append(" JSONExtractString(params, 'share') AS share "); + sql.append("SELECT toInt32(uniqExact(videoId)) AS count "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); + sql.append(") "); + sql.append("AND r.action = 'LOAD' "); + sql.append("AND r.identifier = 'pages/videoSynthesis/buy' "); + sql.append("AND videoId != '' "); + sql.append("AND share = '' "); if (query.getStartTime() != null) { - sql.append(" AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); + sql.append("AND s.create_time >= ").append(formatDateTime(query.getStartTime())).append(" "); } if (query.getEndTime() != null) { - sql.append(" AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); + sql.append("AND s.create_time <= ").append(formatDateTime(query.getEndTime())).append(" "); } - sql.append(" GROUP BY JSONExtractString(r.params, 'id') "); - sql.append(")"); return getJdbcTemplate().queryForObject(sql.toString(), Integer.class); } @@ -246,20 +248,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public List getBrokerIdListForUser(Long memberId, Date startTime, Date endTime) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT toInt64(sub.identifier) AS identifier FROM ( "); - sql.append(" SELECT identifier, max(r.create_time) AS createTime "); - sql.append(" FROM t_stats_record r "); - sql.append(" INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append(" WHERE r.action = 'CODE_SCAN' "); - sql.append(" AND s.member_id = ").append(memberId).append(" "); + sql.append("SELECT toInt64(r.identifier) AS identifier "); + sql.append("FROM t_stats_record r "); + sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); + sql.append("WHERE r.action = 'CODE_SCAN' "); + sql.append(" AND s.member_id = ").append(memberId).append(" "); if (startTime != null) { - sql.append(" AND r.create_time >= ").append(formatDateTime(startTime)).append(" "); + sql.append(" AND r.create_time >= ").append(formatDateTime(startTime)).append(" "); } if (endTime != null) { - sql.append(" AND r.create_time <= ").append(formatDateTime(endTime)).append(" "); + sql.append(" AND r.create_time <= ").append(formatDateTime(endTime)).append(" "); } - sql.append(" GROUP BY identifier "); - sql.append(") sub ORDER BY sub.createTime DESC"); + sql.append("GROUP BY r.identifier "); + sql.append("ORDER BY max(r.create_time) DESC"); return getJdbcTemplate().queryForList(sql.toString(), Long.class); } @@ -311,13 +312,17 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { public List> getDailyScanStats(Long brokerId, Date startTime, Date endTime) { String startDateStr = DATE_FORMAT.format(startTime); String endDateStr = DATE_FORMAT.format(endTime); + String startDateTimeStr = "'" + startDateStr + " 00:00:00'"; + String endDateTimeStr = "'" + endDateStr + " 23:59:59'"; String sql = "SELECT toDate(create_time) AS date, count(DISTINCT id) AS scanCount " + "FROM t_stats_record " + "WHERE action = 'CODE_SCAN' " + " AND identifier = '" + brokerId + "' " + - " AND toDate(create_time) BETWEEN '" + startDateStr + "' AND '" + endDateStr + "' " + - "GROUP BY toDate(create_time)"; + " AND create_time >= " + startDateTimeStr + " " + + " AND create_time <= " + endDateTimeStr + " " + + "GROUP BY toDate(create_time) " + + "ORDER BY toDate(create_time)"; return getJdbcTemplate().query(sql, (rs, rowNum) -> { HashMap map = new HashMap<>(); @@ -330,22 +335,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public List> scanCodeMemberChartByHour(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT formatDateTime(s.create_time, '%m-%d %H') AS t, "); - sql.append(" count(DISTINCT s.member_id) AS count "); + sql.append("SELECT formatDateTime(toStartOfHour(s.create_time), '%m-%d %H') AS t, "); + sql.append(" uniqExact(s.member_id) AS count "); sql.append("FROM t_stats_record r "); sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append("WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' "); - if (query.getScenicId() != null) { - sql.append(" AND identifier = '").append(query.getScenicId()).append("' "); - } + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); sql.append(") "); sql.append("AND r.action = 'LAUNCH' "); sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) "); sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime())); sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" "); - sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d %H')"); + sql.append("GROUP BY toStartOfHour(s.create_time) "); + sql.append("ORDER BY toStartOfHour(s.create_time)"); return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> { HashMap map = new HashMap<>(); @@ -358,22 +360,19 @@ public class ClickHouseStatsQueryServiceImpl implements StatsQueryService { @Override public List> scanCodeMemberChartByDate(CommonQueryReq query) { StringBuilder sql = new StringBuilder(); - sql.append("SELECT formatDateTime(s.create_time, '%m-%d') AS t, "); - sql.append(" count(DISTINCT s.member_id) AS count "); + sql.append("SELECT formatDateTime(toStartOfDay(s.create_time), '%m-%d') AS t, "); + sql.append(" uniqExact(s.member_id) AS count "); sql.append("FROM t_stats_record r "); sql.append("INNER JOIN t_stats s ON r.trace_id = s.trace_id "); - sql.append("WHERE r.trace_id IN ( "); - sql.append(" SELECT trace_id FROM t_stats_record "); - sql.append(" WHERE action = 'ENTER_SCENIC' "); - if (query.getScenicId() != null) { - sql.append(" AND identifier = '").append(query.getScenicId()).append("' "); - } + sql.append("WHERE r.trace_id IN ("); + appendEnterScenicTraceIdSubQuery(sql, query.getScenicId(), query.getStartTime(), query.getEndTime()); sql.append(") "); sql.append("AND r.action = 'LAUNCH' "); sql.append("AND JSONExtractInt(r.params, 'scene') IN (1047, 1048, 1049) "); sql.append("AND s.create_time BETWEEN ").append(formatDateTime(query.getStartTime())); sql.append(" AND ").append(formatDateTime(query.getEndTime())).append(" "); - sql.append("GROUP BY formatDateTime(s.create_time, '%m-%d')"); + sql.append("GROUP BY toStartOfDay(s.create_time) "); + sql.append("ORDER BY toStartOfDay(s.create_time)"); return getJdbcTemplate().query(sql.toString(), (rs, rowNum) -> { HashMap map = new HashMap<>();